diff --git a/.gitignore b/.gitignore index 2b6bcd5b..dcaedafb 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,9 @@ dist/ .vscode/ .idea/ .claude/settings.local.json +.claude/worktrees .sdk-under-test/ .sync-schema-tmp/ +.valtown-stage/ +.serve-*.ts +.env diff --git a/.prettierignore b/.prettierignore index a354d838..5496d2f4 100644 --- a/.prettierignore +++ b/.prettierignore @@ -10,6 +10,10 @@ src/seps/traceability.json src/spec-types/*.ts src/spec-types/*.schema.json +# Generated by `npm run hosted:bundle-requirements` (verbatim yaml text as +# JSON string literals); a test checks it against requirements/*.yaml. +examples/hosted/requirements-bundle.ts + # Local tooling workspaces (not part of the repo). .claude/ .sdk-under-test/ diff --git a/examples/clients/typescript/everything-client.ts b/examples/clients/typescript/everything-client.ts index b3ccbed3..f3f676f7 100644 --- a/examples/clients/typescript/everything-client.ts +++ b/examples/clients/typescript/everything-client.ts @@ -219,9 +219,15 @@ async function runListToolsOnlyClient(serverUrl: string): Promise { await client.connect(transport); logger.debug('Successfully connected to MCP server'); - await client.listTools(); + const list = await client.listTools(); logger.debug('Successfully listed tools'); + const tool = list.tools[0]; + if (tool) { + await client.callTool({ name: tool.name, arguments: { a: 2, b: 3 } }); + logger.debug('Successfully called tool'); + } + await transport.close(); logger.debug('Connection closed successfully'); } diff --git a/examples/hosted/bundle-requirements.test.ts b/examples/hosted/bundle-requirements.test.ts new file mode 100644 index 00000000..b3c5c3b7 --- /dev/null +++ b/examples/hosted/bundle-requirements.test.ts @@ -0,0 +1,38 @@ +import { describe, it, expect } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { + BUNDLE_PATH, + readRequirementSources, + renderBundle +} from './bundle-requirements'; +import { REQUIREMENT_SOURCES } from './requirements-bundle'; +import { + listRequirementRevisions, + loadRequirements, + registerRequirementSources +} from '../../src/requirements'; + +describe('bundled requirement sets', () => { + const fromDisk = readRequirementSources(); + + it('the committed bundle matches requirements/*.yaml (run `npm run hosted:bundle-requirements`)', () => { + expect(REQUIREMENT_SOURCES).toEqual(fromDisk); + expect(readFileSync(BUNDLE_PATH, 'utf8')).toBe(renderBundle(fromDisk)); + // val.town caps files at 80,000 characters. + expect(readFileSync(BUNDLE_PATH, 'utf8').length).toBeLessThan(80_000); + }); + + it('round-trips a bundled revision through registerRequirementSources()', () => { + const revision = Object.keys(REQUIREMENT_SOURCES)[0]; + const fromFile = loadRequirements(revision); + // Register under a name the disk does not have to prove the registered + // text is what gets parsed, then under its own name. + registerRequirementSources({ [revision]: REQUIREMENT_SOURCES[revision] }); + expect(loadRequirements(revision)).toEqual(fromFile); + expect(listRequirementRevisions()).toContain(revision); + // Registered text goes through the same validation as a file. + registerRequirementSources({ '2025-06-18': 'sever:\n - x\n' }); + expect(listRequirementRevisions()).toContain('2025-06-18'); + expect(() => loadRequirements('2025-06-18')).toThrow(/unknown key "sever"/); + }); +}); diff --git a/examples/hosted/bundle-requirements.ts b/examples/hosted/bundle-requirements.ts new file mode 100644 index 00000000..f03e902a --- /dev/null +++ b/examples/hosted/bundle-requirements.ts @@ -0,0 +1,57 @@ +/** + * Bundle requirements/*.yaml into a TypeScript module. + * + * `loadRequirements()` reads the yaml files from disk next to the package, + * but a serverless deploy (examples/hosted/deploy-valtown.ts) stages only + * the TypeScript import closure, so the live val has no yaml and the matrix + * would have no columns. This script writes examples/hosted/requirements- + * bundle.ts with the verbatim text of every requirement set; valtown.ts + * registers it via registerRequirementSources() before building the matrix. + * + * npm run hosted:bundle-requirements + * + * The generated file is committed; bundle-requirements.test.ts fails when it + * drifts from the yaml files. + */ + +import { readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)); +export const REQUIREMENTS_DIR = resolve(SCRIPT_DIR, '../../requirements'); +export const BUNDLE_PATH = join(SCRIPT_DIR, 'requirements-bundle.ts'); + +/** revision → verbatim yaml text, in file-name order. */ +export function readRequirementSources( + dir: string = REQUIREMENTS_DIR +): Record { + const out: Record = {}; + for (const file of readdirSync(dir) + .filter((f) => f.endsWith('.yaml')) + .sort()) { + out[file.replace(/\.yaml$/, '')] = readFileSync(join(dir, file), 'utf8'); + } + return out; +} + +/** The module text: one JSON string literal per revision, nothing to escape by hand. */ +export function renderBundle(sources: Record): string { + const entries = Object.entries(sources) + .map(([rev, text]) => ` ${JSON.stringify(rev)}: ${JSON.stringify(text)}`) + .join(',\n'); + return `// Generated by \`npm run hosted:bundle-requirements\` — do not edit. +// Verbatim text of requirements/*.yaml for deployments that ship the import +// closure only (see examples/hosted/valtown.ts). A test keeps it in sync. + +export const REQUIREMENT_SOURCES: Record = { +${entries} +}; +`; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const sources = readRequirementSources(); + writeFileSync(BUNDLE_PATH, renderBundle(sources)); + console.log(`wrote ${BUNDLE_PATH} (${Object.keys(sources).join(', ')})`); +} diff --git a/examples/hosted/deploy-valtown.test.ts b/examples/hosted/deploy-valtown.test.ts new file mode 100644 index 00000000..12c01347 --- /dev/null +++ b/examples/hosted/deploy-valtown.test.ts @@ -0,0 +1,58 @@ +import { describe, it, expect } from 'vitest'; +import { spawnSync } from 'child_process'; +import { readdirSync, readFileSync, statSync } from 'fs'; +import { join } from 'path'; + +// Stage-only run of the deploy script (no --push, no token needed). Guards +// the val.town per-file cap and the generated JSON/spec-type modules so a +// deploy can never be rejected halfway through uploading a closure. +const REPO_ROOT = join(__dirname, '../..'); +const STAGE = join(REPO_ROOT, '.valtown-stage/rs'); +const MAX_FILE_CHARS = 80_000; + +function walk(dir: string): string[] { + return readdirSync(dir).flatMap((name) => { + const p = join(dir, name); + return statSync(p).isDirectory() ? walk(p) : [p]; + }); +} + +describe('deploy-valtown staging', () => { + it('stages the rs closure with every file under the val.town size cap', async () => { + const r = spawnSync( + 'npx', + ['tsx', 'examples/hosted/deploy-valtown.ts', 'rs'], + { cwd: REPO_ROOT, encoding: 'utf8', timeout: 120_000 } + ); + expect(r.status, r.stderr).toBe(0); + + const files = walk(STAGE); + expect(files.length).toBeGreaterThan(50); + const oversized = files.filter( + (f) => readFileSync(f, 'utf8').length > MAX_FILE_CHARS + ); + expect(oversized).toEqual([]); + + // JSON schema imports became generated TS modules that round-trip. + for (const n of ['2025-03-26', '2025-06-18', '2025-11-25', 'draft']) { + const orig = JSON.parse( + readFileSync(join(REPO_ROOT, `src/spec-types/${n}.schema.json`), 'utf8') + ); + const mod = await import( + join(STAGE, `src/spec-types/${n}.schema.json.ts`) + ); + expect(mod.default).toEqual(orig); + } + const wire = readFileSync( + join(STAGE, 'src/validation/wire-schema.ts'), + 'utf8' + ); + expect(wire).toContain("'../spec-types/draft.schema.json.ts'"); + expect(wire).not.toMatch(/schema\.json';/); + + // Comment-stripped spec-type module still exports its runtime constants. + const draft = await import(join(STAGE, 'src/spec-types/draft.ts')); + expect(draft.HEADER_MISMATCH).toBeDefined(); + expect(draft.MISSING_REQUIRED_CLIENT_CAPABILITY).toBeDefined(); + }, 150_000); +}); diff --git a/examples/hosted/deploy-valtown.ts b/examples/hosted/deploy-valtown.ts new file mode 100644 index 00000000..d2f3c4d1 --- /dev/null +++ b/examples/hosted/deploy-valtown.ts @@ -0,0 +1,475 @@ +/** + * Deploy the hosted conformance server to val.town as readable source. + * + * val.town's runtime (Deno) needs import specifiers the repo's Node toolchain + * doesn't use: `npm:` prefixes for packages, `node:` prefixes for builtins, + * and explicit `.ts` extensions on relative imports. Rather than fork the + * source, this script stages a copy of the import closure of + * examples/hosted/valtown.ts with those specifiers rewritten, then uploads + * the files via the val.town v2 API (same approach as a plain + * "create val + upsert files" deploy script). + * + * Two vals are deployed: + * rs — the conformance resource server (entry: examples/hosted/valtown.ts) + * relay — the second-origin auth relay (entry: examples/hosted/valtown-relay.ts) + * + * Usage: + * npx tsx examples/hosted/deploy-valtown.ts # stage only (.valtown-stage/) + * npx tsx examples/hosted/deploy-valtown.ts --push # stage + upload both vals + * npx tsx examples/hosted/deploy-valtown.ts --push rs # upload a single val + * + * Token: VAL_TOWN_TOKEN env var (or a .env file next to this script / repo root). + * Val ids are persisted to examples/hosted/valtown-manifest.json on first push. + * + * After the first push, set env vars on the vals (val.town UI → val → Environment): + * rs val: CONFORMANCE_AS_ORIGIN=, CONFORMANCE_RELAY_SECRET= + * relay val: CONFORMANCE_RS_ORIGIN=, CONFORMANCE_RELAY_SECRET=, + * CONFORMANCE_RELAY_ROLE=as + */ + +import { + existsSync, + mkdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync +} from 'node:fs'; +import { dirname, join, relative, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import ts from 'typescript'; + +const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(SCRIPT_DIR, '../..'); +const STAGE_ROOT = join(REPO_ROOT, '.valtown-stage'); +const MANIFEST_PATH = join(SCRIPT_DIR, 'valtown-manifest.json'); +const API = 'https://api.val.town/v2'; +/** val.town rejects a file body over this many characters (HTTP 400). */ +const MAX_FILE_CHARS = 80_000; +/** Chunk budget for generated JSON part modules, leaving escaping headroom. */ +const CHUNK_TARGET_CHARS = 70_000; + +const NODE_BUILTINS = new Set([ + 'assert', + 'async_hooks', + 'buffer', + 'child_process', + 'crypto', + 'dns', + 'events', + 'fs', + 'http', + 'https', + 'net', + 'os', + 'path', + 'process', + 'querystring', + 'stream', + 'string_decoder', + 'timers', + 'tls', + 'url', + 'util', + 'zlib' +]); + +interface ValInfo { + id?: string; + name: string; + entry: string; + privacy: 'public' | 'unlisted' | 'private'; +} +interface Manifest { + vals: Record; +} + +const pkg = JSON.parse(readFileSync(join(REPO_ROOT, 'package.json'), 'utf8')); +const versions: Record = { + ...pkg.devDependencies, + ...pkg.dependencies +}; + +// --------------------------------------------------------------------------- +// Specifier rewriting +// --------------------------------------------------------------------------- + +/** Resolve a relative specifier from `fromFile` to an existing repo file. */ +function resolveRelative(fromFile: string, spec: string): string { + const base = resolve(dirname(fromFile), spec); + const candidates = [ + base, + `${base}.ts`, + `${base}.tsx`, + base.replace(/\.js$/, '.ts'), + join(base, 'index.ts') + ]; + for (const c of candidates) { + if (existsSync(c) && statSync(c).isFile()) return c; + } + throw new Error( + `cannot resolve '${spec}' from ${relative(REPO_ROOT, fromFile)}` + ); +} + +/** Rewrite one specifier to a val.town/Deno-compatible form. */ +function rewriteSpec( + fromFile: string, + spec: string, + discovered: Set +): string { + if ( + spec.startsWith('node:') || + spec.startsWith('npm:') || + spec.startsWith('http://') || + spec.startsWith('https://') + ) { + return spec; + } + if (spec.startsWith('.')) { + const target = resolveRelative(fromFile, spec); + discovered.add(target); + let rel = relative(dirname(fromFile), target).replace(/\\/g, '/'); + if (!rel.startsWith('.')) rel = `./${rel}`; + // JSON modules are staged as generated TS (see stageJsonModule): no import + // attribute needed, and oversized schemas can be split across files. + return target.endsWith('.json') ? `${rel}.ts` : rel; + } + if (NODE_BUILTINS.has(spec.split('/')[0])) return `node:${spec}`; + // npm package (possibly scoped, possibly with a subpath) + const parts = spec.split('/'); + const name = spec.startsWith('@') ? parts.slice(0, 2).join('/') : parts[0]; + const subpath = spec.slice(name.length); // '' or '/server/index.js' + const version = versions[name]; + if (!version) + throw new Error( + `no version for '${name}' in package.json (imported by ${relative(REPO_ROOT, fromFile)})` + ); + return `npm:${name}@${version}${subpath}`; +} + +/** Collect the string-literal module specifiers of a source file via the TS AST. */ +function collectSpecifiers(sourceFile: ts.SourceFile): ts.StringLiteral[] { + const specs: ts.StringLiteral[] = []; + const visit = (node: ts.Node) => { + if ( + (ts.isImportDeclaration(node) || ts.isExportDeclaration(node)) && + node.moduleSpecifier && + ts.isStringLiteral(node.moduleSpecifier) + ) { + specs.push(node.moduleSpecifier); + } else if ( + ts.isCallExpression(node) && + node.expression.kind === ts.SyntaxKind.ImportKeyword && + node.arguments.length > 0 && + ts.isStringLiteral(node.arguments[0]) + ) { + specs.push(node.arguments[0]); + } + ts.forEachChild(node, visit); + }; + visit(sourceFile); + return specs; +} + +/** Rewrite all import/export specifiers in a file; returns new source. */ +function rewriteFile(file: string, discovered: Set): string { + const src = readFileSync(file, 'utf8'); + const sourceFile = ts.createSourceFile( + file, + src, + ts.ScriptTarget.Latest, + true, + ts.ScriptKind.TS + ); + // Replace back-to-front so earlier positions stay valid. + const specs = collectSpecifiers(sourceFile).sort( + (a, b) => b.getStart(sourceFile) - a.getStart(sourceFile) + ); + let out = src; + for (const lit of specs) { + const rewritten = rewriteSpec(file, lit.text, discovered); + if (rewritten === lit.text) continue; + // getStart()/getEnd() include the quotes; keep them as-is. + const start = lit.getStart(sourceFile) + 1; + const end = lit.getEnd() - 1; + out = out.slice(0, start) + rewritten + out.slice(end); + } + return out; +} + +/** + * Keep a rewritten TS module under val.town's per-file size cap. The generated + * spec-type modules (src/spec-types/*.ts) are ~80% JSDoc, so dropping comments + * is enough; anything still over the cap is a hard error rather than a partial + * upload later. + */ +function fitTsModule(path: string, content: string): string { + if (content.length <= MAX_FILE_CHARS) return content; + const sourceFile = ts.createSourceFile( + path, + content, + ts.ScriptTarget.Latest, + true, + ts.ScriptKind.TS + ); + const stripped = ts + .createPrinter({ removeComments: true }) + .printFile(sourceFile); + if (stripped.length > MAX_FILE_CHARS) { + throw new Error( + `${path} is ${stripped.length} chars even without comments; val.town caps files at ${MAX_FILE_CHARS}` + ); + } + console.log( + ` (stripped comments from ${path}: ${content.length} → ${stripped.length} chars)` + ); + return stripped; +} + +/** + * Stage a .json import as `.json.ts`. Small documents become a literal + * default export; large ones (the spec JSON schemas are 90–180K) are split + * into `.json.part.ts` string chunks that the main module + * reassembles with JSON.parse. + */ +function stageJsonModule( + repoRelPath: string, + raw: string, + staged: Map +): void { + const minified = JSON.stringify(JSON.parse(raw)); + const modulePath = `${repoRelPath}.ts`; + const literal = `export default ${minified};\n`; + if (literal.length <= MAX_FILE_CHARS) { + staged.set(modulePath, literal); + return; + } + const parts: string[] = []; + let start = 0; + while (start < minified.length) { + // Grow the chunk until its escaped form would exceed the budget. + let end = Math.min(minified.length, start + CHUNK_TARGET_CHARS); + while ( + end > start + 1 && + JSON.stringify(minified.slice(start, end)).length > CHUNK_TARGET_CHARS + ) { + end -= 1000; + } + parts.push(minified.slice(start, end)); + start = end; + } + const base = repoRelPath.split('/').pop()!; + const imports: string[] = []; + const names: string[] = []; + parts.forEach((chunk, i) => { + const name = `p${i}`; + names.push(name); + imports.push(`import ${name} from './${base}.part${i}.ts';`); + staged.set( + `${repoRelPath}.part${i}.ts`, + `export default ${JSON.stringify(chunk)};\n` + ); + }); + staged.set( + modulePath, + `${imports.join('\n')}\nexport default JSON.parse(${names.join(' + ')});\n` + ); + console.log( + ` (split ${repoRelPath}: ${minified.length} chars → ${parts.length} parts)` + ); +} + +/** Crawl the import closure of `entry`, rewriting as we go. */ +function stageVal(key: string, entry: string): Map { + const staged = new Map(); // repo-relative path -> content + const queue = [resolve(REPO_ROOT, entry)]; + const seen = new Set(queue); + + while (queue.length > 0) { + const file = queue.shift()!; + const repoRel = relative(REPO_ROOT, file).replace(/\\/g, '/'); + if (file.endsWith('.json')) { + stageJsonModule(repoRel, readFileSync(file, 'utf8'), staged); + continue; + } + const discovered = new Set(); + const content = fitTsModule(repoRel, rewriteFile(file, discovered)); + staged.set(repoRel, content); + for (const dep of discovered) { + if (!seen.has(dep)) { + seen.add(dep); + queue.push(dep); + } + } + } + + // Entry point: val.town serves the root http.ts as the HTTP handler. + staged.set('http.ts', `export { default } from './${entry}';\n`); + + // Write the staging dir for inspection / local Deno testing. + const dir = join(STAGE_ROOT, key); + rmSync(dir, { recursive: true, force: true }); + for (const [path, content] of staged) { + const out = join(dir, path); + mkdirSync(dirname(out), { recursive: true }); + writeFileSync(out, content); + } + console.log( + `staged ${key}: ${staged.size} files → ${relative(REPO_ROOT, dir)}/` + ); + return staged; +} + +// --------------------------------------------------------------------------- +// val.town v2 API +// --------------------------------------------------------------------------- + +function getToken(): string { + if (process.env.VAL_TOWN_TOKEN) return process.env.VAL_TOWN_TOKEN; + for (const envPath of [join(SCRIPT_DIR, '.env'), join(REPO_ROOT, '.env')]) { + if (existsSync(envPath)) { + const m = readFileSync(envPath, 'utf8').match(/VAL_TOWN_TOKEN=(.+)/); + if (m) return m[1].trim(); + } + } + throw new Error('VAL_TOWN_TOKEN not set (env var or .env file)'); +} + +async function api( + token: string, + method: string, + path: string, + body?: unknown +): Promise { + return fetch(`${API}${path}`, { + method, + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json' + }, + body: body ? JSON.stringify(body) : undefined + }); +} + +async function createVal(token: string, info: ValInfo): Promise { + const res = await api(token, 'POST', '/vals', { + name: info.name, + privacy: info.privacy + }); + if (!res.ok) { + throw new Error( + `create ${info.name}: HTTP ${res.status} ${await res.text()}` + ); + } + const { id } = (await res.json()) as { id: string }; + return id; +} + +async function upsertFile( + token: string, + valId: string, + path: string, + content: string, + type: 'http' | 'script' +): Promise { + const q = `/vals/${valId}/files?path=${encodeURIComponent(path)}`; + let res = await api(token, 'PUT', q, { content, type }); + if (res.status === 404) { + res = await api(token, 'POST', q, { content, type }); + } + if (!res.ok) { + throw new Error(`upsert ${path}: HTTP ${res.status} ${await res.text()}`); + } +} + +async function pushVal( + token: string, + key: string, + info: ValInfo, + staged: Map +): Promise { + console.log(`\n── ${key} (${info.name}) ──`); + let created = false; + if (!info.id) { + info.id = await createVal(token, info); + created = true; + console.log(` created: ${info.id}`); + } + for (const [path, content] of staged) { + const type = path === 'http.ts' ? 'http' : 'script'; + await upsertFile(token, info.id, path, content, type); + console.log(` ↑ ${path} (${content.length} bytes)`); + } + return created; +} + +// --------------------------------------------------------------------------- +// main +// --------------------------------------------------------------------------- + +async function main() { + const args = process.argv.slice(2); + const push = args.includes('--push'); + const targets = args.filter((a) => !a.startsWith('--')); + + const manifest: Manifest = JSON.parse(readFileSync(MANIFEST_PATH, 'utf8')); + const keys = targets.length > 0 ? targets : Object.keys(manifest.vals); + + const stagedByKey = new Map>(); + for (const key of keys) { + const info = manifest.vals[key]; + if (!info) + throw new Error( + `unknown val '${key}' (manifest has: ${Object.keys(manifest.vals).join(', ')})` + ); + stagedByKey.set(key, stageVal(key, info.entry)); + } + + // Check every file of every val *before* the first upload: a mid-closure + // rejection would leave the live val half old, half new. + const oversized: string[] = []; + for (const [key, files] of stagedByKey) { + for (const [path, content] of files) { + if (content.length > MAX_FILE_CHARS) + oversized.push(`${key}:${path} (${content.length} chars)`); + } + } + if (oversized.length) { + throw new Error( + `staged files exceed val.town's ${MAX_FILE_CHARS}-char cap:\n ${oversized.join('\n ')}` + ); + } + + if (!push) { + console.log('\nstage only (pass --push to upload). Local check, e.g.:'); + console.log( + ' deno serve --port 3203 --allow-net --allow-env .valtown-stage/rs/http.ts' + ); + return; + } + + const token = getToken(); + let dirty = false; + for (const key of keys) { + const created = await pushVal( + token, + key, + manifest.vals[key], + stagedByKey.get(key)! + ); + dirty = dirty || created; + } + if (dirty) { + writeFileSync(MANIFEST_PATH, JSON.stringify(manifest, null, 2) + '\n'); + console.log('\nvaltown-manifest.json updated with new val ids'); + } + console.log( + '\n✓ done — remember the env vars (see header comment) if this was the first push.' + ); +} + +main().catch((e: unknown) => { + console.error(e instanceof Error ? e.message : e); + process.exit(1); +}); diff --git a/examples/hosted/fetch-bridge.ts b/examples/hosted/fetch-bridge.ts new file mode 100644 index 00000000..f1896e33 --- /dev/null +++ b/examples/hosted/fetch-bridge.ts @@ -0,0 +1,111 @@ +/** + * Web fetch ↔ Node bridge: adapt a Node RequestListener (express app) to a + * fetch-style handler for serverless runtimes (val.town, Deno Deploy, Bun). + * + * Intercepts the user-facing write surface (writeHead/setHeader/write/end) + * so we never touch ServerResponse's socket-coupled internals — the approach + * serverless-http and light-my-request take. + */ + +import { IncomingMessage, ServerResponse } from 'node:http'; +import { Socket } from 'node:net'; + +/** + * The hosted layer reads request bodies without consuming them (see + * src/hosted/body.ts). A bridge has the whole body before the listener runs, + * so it publishes it under this symbol instead of being tapped. + */ +const BUFFERED_BODY = Symbol.for('mcp-conformance.hosted.bufferedBody'); + +type NodeListener = (req: IncomingMessage, res: ServerResponse) => void; + +export function toFetchHandler( + listener: NodeListener +): (request: Request) => Promise { + return async (request: Request): Promise => { + const url = new URL(request.url); + + // --- web Request → Node IncomingMessage --- + const body = request.body + ? Buffer.from(await request.arrayBuffer()) + : undefined; + // Express's req.protocol/req.ip read socket.encrypted/.remoteAddress, + // and IncomingMessage._destroy calls socket.destroy(), so a real + // (unconnected) Socket with the encrypted flag patched on is the path + // of least surprise. + const socket = Object.assign(new Socket(), { encrypted: false }); + const nodeReq = new IncomingMessage(socket); + nodeReq.method = request.method; + nodeReq.url = url.pathname + url.search; + nodeReq.httpVersion = '1.1'; + nodeReq.httpVersionMajor = 1; + nodeReq.httpVersionMinor = 1; + nodeReq.headers = Object.fromEntries(request.headers); + nodeReq.headers.host ??= url.host; + if (body?.length) nodeReq.headers['content-length'] = String(body.length); + // The SDK's StreamableHTTPServerTransport converts Node→Web via + // @hono/node-server, which reads rawHeaders (the [k,v,k,v,...] array), + // not the parsed headers object. Deno's node:http compat exposes + // rawHeaders as a getter-only accessor, so shadow it with an own + // property instead of assigning. + Object.defineProperty(nodeReq, 'rawHeaders', { + value: Object.entries(nodeReq.headers).flat() as string[], + writable: true, + configurable: true + }); + if (body?.length) nodeReq.push(body); + nodeReq.push(null); + (nodeReq as unknown as Record)[BUFFERED_BODY] = + body ?? Buffer.alloc(0); + + // --- Node ServerResponse → web Response --- + const nodeRes = new ServerResponse(nodeReq); + const chunks: Buffer[] = []; + let status = 200; + const headers = new Headers(); + + const captureHeaders = (h?: Record) => { + for (const [k, v] of Object.entries(h ?? {})) { + headers.set(k, Array.isArray(v) ? v.join(', ') : String(v)); + } + }; + nodeRes.setHeader = ((k: string, v: string | string[] | number) => { + headers.set(k, Array.isArray(v) ? v.join(', ') : String(v)); + return nodeRes; + }) as ServerResponse['setHeader']; + nodeRes.getHeader = (k: string) => + headers.get(k.toLowerCase()) ?? undefined; + nodeRes.removeHeader = (k: string) => headers.delete(k); + nodeRes.writeHead = ((code: number, h?: Record) => { + status = code; + captureHeaders(h); + return nodeRes; + }) as ServerResponse['writeHead']; + nodeRes.write = ((c: string | Buffer, enc?: BufferEncoding) => { + if (c) chunks.push(typeof c === 'string' ? Buffer.from(c, enc) : c); + return true; + }) as ServerResponse['write']; + nodeRes.flushHeaders = () => {}; + Object.defineProperty(nodeRes, 'statusCode', { + get: () => status, + set: (v: number) => { + status = v; + } + }); + + return new Promise((resolve) => { + nodeRes.end = ((c?: string | Buffer, enc?: BufferEncoding) => { + if (c) chunks.push(typeof c === 'string' ? Buffer.from(c, enc) : c); + resolve( + new Response(chunks.length ? Buffer.concat(chunks) : null, { + status, + headers + }) + ); + return nodeRes; + }) as ServerResponse['end']; + + listener(nodeReq, nodeRes); + }); + }; +} diff --git a/examples/hosted/local-relay.ts b/examples/hosted/local-relay.ts new file mode 100644 index 00000000..1195cfd6 --- /dev/null +++ b/examples/hosted/local-relay.ts @@ -0,0 +1,36 @@ +#!/usr/bin/env -S npx tsx +/** + * Run the val.town relay locally for end-to-end testing without deploying. + * Thin Node http.Server → fetch-handler bridge around valtown-relay.ts. + * + * CONFORMANCE_RS_ORIGIN=http://localhost:3000 \ + * CONFORMANCE_RELAY_SECRET=dev \ + * npx tsx examples/hosted/local-relay.ts 3001 + */ +import http from 'node:http'; +import handler from './valtown-relay'; + +const port = Number(process.argv[2] ?? 3001); + +http + .createServer(async (req, res) => { + const chunks: Buffer[] = []; + for await (const c of req) chunks.push(c as Buffer); + const body = chunks.length ? Buffer.concat(chunks) : undefined; + const url = `http://${req.headers.host}${req.url}`; + const out = await handler( + new Request(url, { + method: req.method, + headers: req.headers as Record, + body: body ? new Uint8Array(body) : undefined + }) + ); + res.writeHead(out.status, Object.fromEntries(out.headers)); + res.end(Buffer.from(await out.arrayBuffer())); + }) + .listen(port, () => { + console.error( + `relay[${process.env.CONFORMANCE_RELAY_ROLE ?? 'as'}] ` + + `listening on http://localhost:${port} → ${process.env.CONFORMANCE_RS_ORIGIN}/__aux` + ); + }); diff --git a/examples/hosted/requirements-bundle.ts b/examples/hosted/requirements-bundle.ts new file mode 100644 index 00000000..e7eb8f2b --- /dev/null +++ b/examples/hosted/requirements-bundle.ts @@ -0,0 +1,8 @@ +// Generated by `npm run hosted:bundle-requirements` — do not edit. +// Verbatim text of requirements/*.yaml for deployments that ship the import +// closure only (see examples/hosted/valtown.ts). A test keeps it in sync. + +export const REQUIREMENT_SOURCES: Record = { + "2025-11-25": "# Conformance requirements for the 2025-11-25 specification revision.\n#\n# This file is the canonical answer to \"which scenarios must my implementation pass\n# to conform to 2025-11-25\".\n#\n# Anchor, and READ THIS BEFORE TRUSTING IT AS A SNAPSHOT: unlike the 2026-07-28 set,\n# this one could not be frozen at its own ship date. The release current on 2025-11-25\n# was 0.1.7 (published 2025-11-20), which had no --spec-version flag and no concept of\n# which revision a scenario belonged to, so there is nothing to snapshot. This set is\n# instead derived from @modelcontextprotocol/conformance@0.2.0-alpha.10 filtered to\n# 2025-11-25, and therefore contains scenarios written after 2025-11-25 shipped. It is\n# frozen from here on, but it is a reconstruction rather than a contemporaneous record.\n#\n# `server` and `client` are named for the subcommand that runs them, and list what\n# conformance to this revision requires. Scenarios run at THIS revision's wire\n# version: the dated revisions through 2025-11-25 use the stateful initialize\n# handshake and 2026-07-28 is stateless with per-request _meta, so a scenario that\n# applies to both must be run once under each and one run does not cover the other.\n#\n# There is deliberately no authorization-server section: the MCP specification puts\n# authorization-server implementation beyond its own scope, so those scenarios serve\n# people deploying an authorization server, not implementations of MCP itself.\n#\n# `not_scored` is run and reported but never counts toward a pass rate; each entry\n# says why. `extension` is optional by definition (SEP-1730: \"Experimental features\n# and protocol extensions ... are not required for any tier\"). `added-after-release`\n# means added to the suite AFTER THE ANCHOR RELEASE this file is derived from\n# (0.2.0-alpha.10) — not after the revision's own ship date. This file is a\n# reconstruction (see above), so several SCORED scenarios also postdate\n# 2025-11-25 itself (ping, dns-rebinding-protection, the token-endpoint-auth\n# trio, auth/pre-registration); they are scored because they entered the suite\n# well before the anchor and every current implementation passes them. The line\n# this file freezes is the anchor, and it draws it consistently. Promoting an\n# entry into the lists above is a deliberate, reviewable change.\n#\n# Scenarios that were PENDING in the source release are never scored — SEP-1730\n# scores \"applicable required tests\" only, and pending means the suite's own\n# reference fixture cannot pass them yet. They still RUN, under not_scored with\n# reason: pending, because the implementation under test may well pass what the\n# reference fixture cannot, and invisible coverage is how gaps hide. (The tasks\n# extension suite attaches to 2026-07-28 only and has no entries here.)\n\nserver:\n - server-initialize\n - logging-set-level\n - ping\n - completion-complete\n - tools-list\n - tools-call-simple-text\n - tools-call-image\n - tools-call-audio\n - tools-call-embedded-resource\n - tools-call-mixed-content\n - tools-call-with-logging\n - tools-call-error\n - tools-call-with-progress\n - tools-call-sampling\n - tools-call-elicitation\n - elicitation-sep1034-defaults\n - server-sse-multiple-streams\n - elicitation-sep1330-enums\n - resources-list\n - resources-read-text\n - resources-read-binary\n - resources-templates-read\n - resources-subscribe\n - resources-unsubscribe\n - prompts-list\n - prompts-get-simple\n - prompts-get-with-args\n - prompts-get-embedded-resource\n - prompts-get-with-image\n - dns-rebinding-protection\n\nclient:\n - initialize\n - tools_call\n - elicitation-sep1034-client-defaults\n - sse-retry\n - auth/metadata-default\n - auth/metadata-var1\n - auth/metadata-var2\n - auth/metadata-var3\n - auth/basic-cimd\n - auth/scope-from-www-authenticate\n - auth/scope-from-scopes-supported\n - auth/scope-omitted-when-undefined\n - auth/scope-step-up\n - auth/scope-retry-limit\n - auth/token-endpoint-auth-basic\n - auth/token-endpoint-auth-post\n - auth/token-endpoint-auth-none\n - auth/pre-registration\n\nnot_scored:\n - scenario: auth/client-credentials-jwt\n leg: client\n reason: extension\n - scenario: auth/client-credentials-basic\n leg: client\n reason: extension\n - scenario: auth/enterprise-managed-authorization\n leg: client\n reason: extension\n - scenario: auth/dpop\n leg: client\n reason: extension\n - scenario: auth/dpop-nonce\n leg: client\n reason: extension\n - scenario: auth/wif-jwt-bearer\n leg: client\n reason: extension\n - scenario: server-session-lifecycle\n leg: server\n reason: added-after-release\n - scenario: json-schema-2020-12-preservation\n leg: client\n reason: added-after-release\n - scenario: json-schema-2020-12\n leg: server\n reason: pending\n note: >-\n the reference fixture cannot pass it yet; the implementation under test might\n - scenario: server-sse-polling\n leg: server\n reason: pending\n note: >-\n on hold pending server-side SSE improvements in the reference fixture\n", + "2026-07-28": "# Conformance requirements for the 2026-07-28 specification revision.\n#\n# This file is the canonical answer to \"which scenarios must my implementation pass\n# to conform to 2026-07-28\". It is FROZEN: the lists below were fixed when the\n# revision shipped and must not be edited afterwards. An implementation is measured\n# against the suite as it stood when it was expected to conform, not against whatever\n# the suite has accumulated since.\n#\n# Anchor: @modelcontextprotocol/conformance@0.2.0-alpha.10, published 2026-07-27, the\n# release current when this revision shipped. That release could express spec-version\n# applicability, so this set is a faithful snapshot of what was required at ship.\n#\n# `server` and `client` are named for the subcommand that runs them, and list what\n# conformance to this revision requires. Scenarios run at THIS revision's wire\n# version: the dated revisions through 2025-11-25 use the stateful initialize\n# handshake and 2026-07-28 is stateless with per-request _meta, so a scenario that\n# applies to both must be run once under each and one run does not cover the other.\n#\n# There is deliberately no authorization-server section: the MCP specification puts\n# authorization-server implementation beyond its own scope, so those scenarios serve\n# people deploying an authorization server, not implementations of MCP itself.\n#\n# `not_scored` is run and reported but never counts toward a pass rate; each entry\n# says why. `extension` is optional by definition (SEP-1730: \"Experimental features\n# and protocol extensions ... are not required for any tier\"). `added-after-release`\n# means added to the suite after the anchor release this file is derived from\n# (0.2.0-alpha.10, published the day before this revision shipped), so no\n# implementation pinning a published referee could have been running it. Promoting an entry into the lists above is a deliberate, reviewable\n# change.\n#\n# Scenarios that were PENDING in the source release are never scored — SEP-1730\n# scores \"applicable required tests\" only, and pending means the suite's own\n# reference fixture cannot pass them yet. They still RUN, under not_scored with\n# reason: pending (or extension for the tasks suite), because the implementation\n# under test may well pass what the reference fixture cannot, and invisible\n# coverage is how gaps hide.\n\nserver:\n - server-stateless\n - completion-complete\n - tools-list\n - tools-call-simple-text\n - tools-call-image\n - tools-call-audio\n - tools-call-embedded-resource\n - tools-call-mixed-content\n - tools-call-error\n - tools-call-with-progress\n - server-sse-multiple-streams\n - resources-list\n - resources-read-text\n - resources-read-binary\n - resources-templates-read\n - sep-2164-resource-not-found\n - prompts-list\n - prompts-get-simple\n - prompts-get-with-args\n - prompts-get-embedded-resource\n - prompts-get-with-image\n - dns-rebinding-protection\n - caching\n - input-required-result-basic-elicitation\n - input-required-result-basic-sampling\n - input-required-result-basic-list-roots\n - input-required-result-request-state\n - input-required-result-multiple-input-requests\n - input-required-result-multi-round\n - input-required-result-missing-input-response\n - input-required-result-non-tool-request\n - input-required-result-result-type\n - input-required-result-unsupported-methods\n - input-required-result-tampered-state\n - input-required-result-capability-check\n - input-required-result-ignore-extra-params\n - input-required-result-validate-input\n\nclient:\n - tools_call\n - request-metadata\n - auth/metadata-default\n - auth/metadata-var1\n - auth/metadata-var2\n - auth/metadata-var3\n - auth/basic-cimd\n - auth/scope-from-www-authenticate\n - auth/scope-from-scopes-supported\n - auth/scope-omitted-when-undefined\n - auth/scope-step-up\n - auth/scope-retry-limit\n - auth/token-endpoint-auth-basic\n - auth/token-endpoint-auth-post\n - auth/token-endpoint-auth-none\n - auth/pre-registration\n - auth/resource-mismatch\n - auth/offline-access-scope\n - auth/offline-access-not-supported\n - auth/authorization-server-migration\n - auth/iss-supported\n - auth/iss-not-advertised\n - auth/iss-supported-missing\n - auth/iss-wrong-issuer\n - auth/iss-unexpected\n - auth/iss-normalized\n - auth/metadata-issuer-mismatch\n - sep-2322-client-request-state\n - http-standard-headers\n - http-custom-headers\n - http-invalid-tool-headers\n - json-schema-ref-no-deref\n\nnot_scored:\n - scenario: auth/client-credentials-jwt\n leg: client\n reason: extension\n - scenario: auth/client-credentials-basic\n leg: client\n reason: extension\n - scenario: auth/enterprise-managed-authorization\n leg: client\n reason: extension\n - scenario: auth/dpop\n leg: client\n reason: extension\n - scenario: auth/dpop-nonce\n leg: client\n reason: extension\n - scenario: auth/wif-jwt-bearer\n leg: client\n reason: extension\n - scenario: json-schema-2020-12-preservation\n leg: client\n reason: added-after-release\n - scenario: tasks-lifecycle\n leg: server\n reason: extension\n note: >-\n io.modelcontextprotocol/tasks (SEP-2663); pending against the reference fixture, run for visibility\n - scenario: tasks-capability-negotiation\n leg: server\n reason: extension\n note: >-\n io.modelcontextprotocol/tasks (SEP-2663); pending against the reference fixture, run for visibility\n - scenario: tasks-wire-fields\n leg: server\n reason: extension\n note: >-\n io.modelcontextprotocol/tasks (SEP-2663); pending against the reference fixture, run for visibility\n - scenario: tasks-request-state-removal\n leg: server\n reason: extension\n note: >-\n io.modelcontextprotocol/tasks (SEP-2663); pending against the reference fixture, run for visibility\n - scenario: tasks-mrtr-input\n leg: server\n reason: extension\n note: >-\n io.modelcontextprotocol/tasks (SEP-2663); pending against the reference fixture, run for visibility\n - scenario: tasks-request-headers\n leg: server\n reason: extension\n note: >-\n io.modelcontextprotocol/tasks (SEP-2663); pending against the reference fixture, run for visibility\n - scenario: tasks-dispatch-and-envelope\n leg: server\n reason: extension\n note: >-\n io.modelcontextprotocol/tasks (SEP-2663); pending against the reference fixture, run for visibility\n - scenario: tasks-status-notifications\n leg: server\n reason: extension\n note: >-\n io.modelcontextprotocol/tasks (SEP-2663); pending against the reference fixture, run for visibility\n - scenario: tasks-required-task-error\n leg: server\n reason: extension\n note: >-\n io.modelcontextprotocol/tasks (SEP-2663); pending against the reference fixture, run for visibility\n - scenario: tasks-mrtr-composition\n leg: server\n reason: extension\n note: >-\n io.modelcontextprotocol/tasks (SEP-2663); pending against the reference fixture, run for visibility\n - scenario: json-schema-2020-12\n leg: server\n reason: pending\n note: >-\n the reference fixture cannot pass it yet; the implementation under test might\n - scenario: http-header-validation\n leg: server\n reason: pending\n note: >-\n SEP-2243; pending against the reference fixture\n - scenario: http-custom-header-server-validation\n leg: server\n reason: pending\n note: >-\n SEP-2243; pending against the reference fixture\n" +}; diff --git a/examples/hosted/valtown-manifest.json b/examples/hosted/valtown-manifest.json new file mode 100644 index 00000000..0bdf326e --- /dev/null +++ b/examples/hosted/valtown-manifest.json @@ -0,0 +1,16 @@ +{ + "vals": { + "rs": { + "name": "mcp-client-conformance", + "entry": "examples/hosted/valtown.ts", + "privacy": "public", + "id": "7dcb6042-d92f-40c5-9cc8-0f661f96e58b" + }, + "relay": { + "name": "mcp-client-conformance-as", + "entry": "examples/hosted/valtown-relay.ts", + "privacy": "public", + "id": "ec6df808-1fce-4b69-884e-7fe2708988ac" + } + } +} diff --git a/examples/hosted/valtown-relay.test.ts b/examples/hosted/valtown-relay.test.ts new file mode 100644 index 00000000..7a7aaab1 --- /dev/null +++ b/examples/hosted/valtown-relay.test.ts @@ -0,0 +1,111 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import http from 'http'; +import { gzipSync } from 'zlib'; +import type { Server } from 'http'; + +// The relay reads its env at import time, so point it at a mock RS first and +// import lazily. +const SECRET = 'relay-test-secret'; +let upstream: Server; +let upstreamOrigin: string; +let seen: { url?: string; headers: http.IncomingHttpHeaders }[] = []; +let handler: (req: Request) => Promise; + +const METADATA = { + issuer: 'https://as.example/r/run-1', + authorization_endpoint: 'https://as.example/r/run-1/authorize', + token_endpoint: 'https://as.example/r/run-1/token', + registration_endpoint: 'https://as.example/r/run-1/register', + response_types_supported: ['code'], + code_challenge_methods_supported: ['S256'], + padding: 'x'.repeat(600) +}; + +beforeAll(async () => { + upstream = http.createServer((req, res) => { + seen.push({ url: req.url, headers: req.headers }); + if (req.headers['x-relay-secret'] !== SECRET) { + res.writeHead(403).end('{"error":"forbidden"}'); + return; + } + if (req.url?.startsWith('/__aux/as/.well-known/')) { + // Simulate an edge that gzips: content-length is the *compressed* size. + const gz = gzipSync(Buffer.from(JSON.stringify(METADATA))); + res.writeHead(200, { + 'content-type': 'application/json', + 'content-encoding': 'gzip', + 'content-length': String(gz.length) + }); + res.end(gz); + return; + } + if (req.url?.startsWith('/__aux/as/r/run-1/authorize')) { + res.writeHead(302, { location: 'http://localhost:3000/callback?code=c' }); + res.end(); + return; + } + res.writeHead(404, { 'content-type': 'application/json' }); + res.end('{"error":"nope"}'); + }); + await new Promise((r) => upstream.listen(0, r)); + const addr = upstream.address(); + if (addr && typeof addr === 'object') + upstreamOrigin = `http://localhost:${addr.port}`; + process.env.CONFORMANCE_RS_ORIGIN = upstreamOrigin; + process.env.CONFORMANCE_RELAY_SECRET = SECRET; + process.env.CONFORMANCE_RELAY_ROLE = 'as'; + handler = (await import('./valtown-relay')).default; +}); + +afterAll(async () => { + delete process.env.CONFORMANCE_RS_ORIGIN; + delete process.env.CONFORMANCE_RELAY_SECRET; + delete process.env.CONFORMANCE_RELAY_ROLE; + await new Promise((r) => upstream.close(() => r())); +}); + +describe('val.town AS relay', () => { + it('forwards to /__aux/ with the shared secret', async () => { + seen = []; + const res = await handler( + new Request( + 'https://as.example/.well-known/oauth-authorization-server/r/run-1', + { headers: { accept: 'application/json', 'x-relay-secret': 'spoof' } } + ) + ); + expect(res.status).toBe(200); + expect(seen[0].url).toBe( + '/__aux/as/.well-known/oauth-authorization-server/r/run-1' + ); + expect(seen[0].headers['x-relay-secret']).toBe(SECRET); // not the spoof + expect(seen[0].headers['x-relay-host']).toBe('as.example'); + expect(seen[0].headers['accept-encoding']).toBe('identity'); + }); + + it('re-frames a compressed upstream body so content-length matches the bytes sent', async () => { + const res = await handler( + new Request( + 'https://as.example/.well-known/oauth-authorization-server/r/run-1' + ) + ); + expect(res.headers.get('content-encoding')).toBeNull(); + const text = await res.text(); + // The whole document arrives — this is what a stale compressed + // content-length used to truncate. + expect(JSON.parse(text)).toEqual(METADATA); + const cl = res.headers.get('content-length'); + if (cl !== null) { + expect(Number(cl)).toBe(Buffer.byteLength(text)); + } + }); + + it('passes redirects through without following them', async () => { + const res = await handler( + new Request('https://as.example/r/run-1/authorize?client_id=x') + ); + expect(res.status).toBe(302); + expect(res.headers.get('location')).toBe( + 'http://localhost:3000/callback?code=c' + ); + }); +}); diff --git a/examples/hosted/valtown-relay.ts b/examples/hosted/valtown-relay.ts new file mode 100644 index 00000000..f9fc42b9 --- /dev/null +++ b/examples/hosted/valtown-relay.ts @@ -0,0 +1,108 @@ +/** + * MCP conformance — auxiliary-origin relay (val.town). + * + * The hosted RS app owns all scenario state, but auth scenarios need a second + * public origin so OAuth `.well-known/*` discovery and issuer validation work + * (those are origin-rooted by RFC 8414/9728 — they can't live under the + * `/s///` prefix). This val IS that origin: it forwards + * every request to the RS app's `/__aux//*` backchannel and adds a + * shared secret so the backchannel can't be spoofed by hitting the RS + * directly. + * + * One val per role: deploy this once for `as`, and again for `as2` / `idp` + * if you need the three-origin scenarios (authorization-server-migration, + * enterprise-managed-authorization). + * + * val.town env (Project → Settings → Environment variables): + * CONFORMANCE_RS_ORIGIN https://.val.run + * CONFORMANCE_RELAY_SECRET + * CONFORMANCE_RELAY_ROLE as | as2 | idp (default: as) + * + * Deploy: create an HTTP val and paste: + * + * import handler from "https://esm.sh/@modelcontextprotocol/conformance/examples/hosted/valtown-relay.ts"; + * export default handler; + * + * No scenario logic lives here, so you only redeploy this when the relay + * contract changes — adding/editing scenarios only touches the RS val. + */ + +declare const process: { env: Record }; + +const RS_ORIGIN = process.env.CONFORMANCE_RS_ORIGIN; +const RELAY_SECRET = process.env.CONFORMANCE_RELAY_SECRET; +const ROLE = process.env.CONFORMANCE_RELAY_ROLE ?? 'as'; + +/** + * Headers we forward from the client. Everything else is dropped so a client + * can't smuggle x-relay-secret / x-forwarded-* through us, and so the + * upstream sees a stable shape regardless of what the edge added. + */ +const FORWARD_HEADERS = [ + 'accept', + 'authorization', + 'content-type', + 'content-length', + 'user-agent' +] as const; + +export default async function handler(req: Request): Promise { + if (!RS_ORIGIN || !RELAY_SECRET) { + return Response.json( + { + error: + 'relay misconfigured: set CONFORMANCE_RS_ORIGIN and CONFORMANCE_RELAY_SECRET' + }, + { status: 500 } + ); + } + + const url = new URL(req.url); + const target = `${RS_ORIGIN}/__aux/${ROLE}${url.pathname}${url.search}`; + + const headers = new Headers(); + for (const h of FORWARD_HEADERS) { + const v = req.headers.get(h); + if (v) headers.set(h, v); + } + headers.set('x-relay-secret', RELAY_SECRET); + // Don't invite the RS edge to compress: the bytes are re-framed below anyway. + headers.set('accept-encoding', 'identity'); + // The aux handler reconstructs absolute URLs (issuer, endpoints) from + // getAuxBaseUrl() which the RS app already knows, so it doesn't strictly + // need this — but it's useful for logging/debugging on the RS side. + headers.set('x-relay-host', url.host); + + const upstream = await fetch(target, { + method: req.method, + headers, + body: + req.method === 'GET' || req.method === 'HEAD' + ? undefined + : await req.arrayBuffer(), + // /authorize 302s to the client's redirect_uri — pass it through, don't + // follow it ourselves. + redirect: 'manual' + }); + + // fetch() transparently decompresses a gzip/br upstream body but leaves the + // upstream's content-length (the *compressed* size) in place. Forwarding + // that header with the decompressed bytes makes the client truncate the + // body ("Unterminated string in JSON at position N" on AS metadata). So: + // buffer the body, drop every framing/encoding header, and let the runtime + // derive content-length from the bytes we actually send. + const body = await upstream.arrayBuffer(); + const outHeaders = new Headers(upstream.headers); + for (const h of [ + 'content-encoding', + 'content-length', + 'transfer-encoding', + 'connection' + ]) { + outHeaders.delete(h); + } + return new Response(body.byteLength ? body : null, { + status: upstream.status, + headers: outHeaders + }); +} diff --git a/examples/hosted/valtown-store.ts b/examples/hosted/valtown-store.ts new file mode 100644 index 00000000..1df56b84 --- /dev/null +++ b/examples/hosted/valtown-store.ts @@ -0,0 +1,167 @@ +/** + * RunStore backed by val.town's per-account SQLite (REST API). + * + * val.town injects an API token into every val as the `valtown` env var; the + * SQLite API is `POST /v1/sqlite/execute {statement:{sql,args}}`. Two tables, + * created lazily once per isolate. Old runs are swept on new-run creation, + * throttled per isolate, so the database stays bounded without a cron. + */ + +import type { ConformanceCheck } from '../../src/types'; +import type { RunStore } from '../../src/hosted/store'; + +const API = 'https://api.val.town/v1/sqlite/execute'; + +export interface SqliteRunStoreOptions { + token?: string; + /** Runs older than this are swept. Default 6h. */ + retentionMs?: number; + /** Cap on checks persisted per (run, writer). Default 1000. */ + maxChecks?: number; +} + +type Row = unknown[]; + +export class SqliteRunStore implements RunStore { + private readonly token: string; + private readonly retentionMs: number; + private readonly maxChecks: number; + private ready: Promise | undefined; + private lastSweep = 0; + + constructor(opts: SqliteRunStoreOptions = {}) { + const token = opts.token ?? process.env.valtown; + if (!token) + throw new Error('SqliteRunStore: no val.town token (env valtown)'); + this.token = token; + this.retentionMs = + opts.retentionMs ?? + Number(process.env.CONFORMANCE_RUN_RETENTION_MS ?? 6 * 3600_000); + this.maxChecks = opts.maxChecks ?? 1000; + } + + private async exec(sql: string, args: unknown[] = []): Promise { + const res = await fetch(API, { + method: 'POST', + headers: { + Authorization: `Bearer ${this.token}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ statement: { sql, args } }) + }); + if (!res.ok) { + throw new Error( + `sqlite ${res.status}: ${(await res.text()).slice(0, 200)}` + ); + } + const body = (await res.json()) as { rows?: Row[] }; + return body.rows ?? []; + } + + private init(): Promise { + this.ready ??= (async () => { + await this.exec( + `CREATE TABLE IF NOT EXISTS hosted_runs_v2 ( + id TEXT PRIMARY KEY, scenario TEXT NOT NULL, created_at INTEGER NOT NULL)` + ); + await this.exec( + `CREATE TABLE IF NOT EXISTS hosted_checks_v2 ( + run_id TEXT NOT NULL, writer TEXT NOT NULL, checks TEXT NOT NULL, + updated_at INTEGER NOT NULL, PRIMARY KEY (run_id, writer))` + ); + })().catch((e) => { + this.ready = undefined; + throw e; + }); + return this.ready; + } + + async saveRun(id: string, scenarioName: string): Promise { + await this.init(); + await this.exec( + `INSERT INTO hosted_runs_v2 (id, scenario, created_at) VALUES (?, ?, ?) + ON CONFLICT(id) DO UPDATE SET scenario = excluded.scenario`, + [id, scenarioName, Date.now()] + ); + void this.sweep().catch(() => {}); + } + + async loadRun(id: string): Promise { + await this.init(); + const rows = await this.exec( + `SELECT scenario FROM hosted_runs_v2 WHERE id = ?`, + [id] + ); + return rows[0]?.[0] as string | undefined; + } + + async listRuns( + prefix: string + ): Promise> { + await this.init(); + // LIKE treats % and _ as wildcards; escape them (and the escape char) so + // the prefix matches literally. + const like = prefix.replace(/[\\%_]/g, (c) => `\\${c}`) + '%'; + const rows = await this.exec( + `SELECT id, scenario FROM hosted_runs_v2 WHERE id LIKE ? ESCAPE '\\'`, + [like] + ); + return rows.map(([id, scenario]) => ({ + id: id as string, + scenarioName: scenario as string + })); + } + + async saveChecks( + id: string, + writer: string, + checks: ConformanceCheck[] + ): Promise { + await this.init(); + await this.exec( + `INSERT INTO hosted_checks_v2 (run_id, writer, checks, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(run_id, writer) DO UPDATE + SET checks = excluded.checks, updated_at = excluded.updated_at`, + [id, writer, JSON.stringify(checks.slice(-this.maxChecks)), Date.now()] + ); + } + + async loadChecks(id: string): Promise> { + await this.init(); + const rows = await this.exec( + `SELECT writer, checks FROM hosted_checks_v2 WHERE run_id = ?`, + [id] + ); + const out = new Map(); + for (const [writer, checks] of rows) { + try { + out.set(writer as string, JSON.parse(checks as string)); + } catch { + // corrupt row — ignore + } + } + return out; + } + + async deleteRun(id: string): Promise { + await this.init(); + await this.exec(`DELETE FROM hosted_checks_v2 WHERE run_id = ?`, [id]); + await this.exec(`DELETE FROM hosted_runs_v2 WHERE id = ?`, [id]); + } + + private async sweep(): Promise { + const now = Date.now(); + if (now - this.lastSweep < 5 * 60_000) return; + this.lastSweep = now; + const cutoff = now - this.retentionMs; + await this.exec( + `DELETE FROM hosted_checks_v2 WHERE run_id IN + (SELECT id FROM hosted_runs_v2 WHERE created_at < ?)`, + [cutoff] + ); + await this.exec(`DELETE FROM hosted_runs_v2 WHERE created_at < ?`, [ + cutoff + ]); + } +} diff --git a/examples/hosted/valtown.test.ts b/examples/hosted/valtown.test.ts new file mode 100644 index 00000000..7f858e93 --- /dev/null +++ b/examples/hosted/valtown.test.ts @@ -0,0 +1,86 @@ +import { describe, it, expect } from 'vitest'; +import handler from './valtown'; + +describe('val.town fetch bridge', () => { + async function post(path: string, body: object) { + return handler( + new Request(`http://test${path}`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + accept: 'application/json, text/event-stream' + }, + body: JSON.stringify(body) + }) + ); + } + + it('serves a raw-http scenario and records checks', async () => { + const r = await post('/s/ft1/2025-11-25/initialize', { + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2025-06-18', + clientInfo: { name: 'ft', version: '0' }, + capabilities: {} + } + }); + expect(r.status).toBe(200); + expect(r.headers.get('link')).toContain( + '/results/ft1/2025-11-25/initialize>' + ); + const checks = await handler( + new Request('http://test/results/ft1/2025-11-25/initialize') + ).then((r) => r.json()); + expect(checks.summary.passed).toBeGreaterThanOrEqual(1); + // The bridge hands the buffered body to the identity capture. + expect( + checks.checks.find( + (c: { id: string }) => c.id === 'hosted-client-identity' + )?.details + ).toMatchObject({ + name: 'ft', + version: '0', + protocolVersions: ['2025-06-18'] + }); + }); + + it('serves an SDK-transport scenario (tools_call) statelessly', async () => { + await post('/s/ft2/2025-11-25/tools_call/mcp', { + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2025-06-18', + clientInfo: { name: 'ft', version: '0' }, + capabilities: {} + } + }).then((r) => r.text()); + const r = await post('/s/ft2/2025-11-25/tools_call/mcp', { + jsonrpc: '2.0', + id: 2, + method: 'tools/call', + params: { name: 'add_numbers', arguments: { a: 7, b: 4 } } + }); + expect(r.status).toBe(200); + expect(await r.text()).toContain('The sum of 7 and 4 is 11'); + }); + + it('marks single-process scenarios as not startable on val.town', async () => { + const r = await post('/s/x/2025-11-25/sse-retry', { jsonrpc: '2.0' }); + expect(r.status).toBe(501); + expect((await r.json()).reason).toMatch(/single-process host/); + const list = await handler(new Request('http://test/scenarios')).then((r) => + r.json() + ); + const cell = list.find( + (s: { name: string }) => s.name === 'sep-2322-client-request-state' + ).cells[1]; + expect(cell).toMatchObject({ + revision: '2026-07-28', + startable: false, + startReason: expect.stringMatching(/single-process host/) + }); + }); +}); diff --git a/examples/hosted/valtown.ts b/examples/hosted/valtown.ts new file mode 100644 index 00000000..9f1f8cde --- /dev/null +++ b/examples/hosted/valtown.ts @@ -0,0 +1,76 @@ +/** + * MCP conformance — val.town deployment. + * + * The hosted runner mounts each scenario's `handler()` (a Node + * `RequestListener`) under `/s///`. On val.town + * the entry point is a fetch handler, so we bridge web Request→Node req/res + * once and reuse the *real* scenario implementations from the package — no + * reimplementation, no loopback port. + * + * Deploy with examples/hosted/deploy-valtown.ts, or create an HTTP val and + * paste: + * + * import handler from "https://esm.sh/@modelcontextprotocol/conformance/examples/hosted/valtown.ts"; + * export default handler; + * + * Requires a runtime with Node-compat (`node:http`, `node:stream`) — + * val.town, Deno Deploy, Bun all qualify. + */ + +import { createHostedApp } from '../../src/hosted/server'; +import { registerRequirementSources } from '../../src/requirements'; +import { toFetchHandler } from './fetch-bridge'; +import { SqliteRunStore } from './valtown-store'; +import { REQUIREMENT_SOURCES } from './requirements-bundle'; + +// The deploy stages the TypeScript import closure only, so requirements/*.yaml +// is not on the val. The matrix's columns come from the bundled copies +// (regenerate with `npm run hosted:bundle-requirements`); this must run +// before createHostedApp(), which builds the matrix at construction. +registerRequirementSources(REQUIREMENT_SOURCES); + +/** + * val.town spreads one run's requests over several isolates that share no + * memory. Scenarios whose checks depend on one process seeing consecutive + * requests (SSE reconnect timing, tenant-prefixed AS state, elicitation + * round-trips, MRTR request state) cannot be judged there; the matrix shows + * them as not startable with this reason. + */ +const SINGLE_PROCESS_ONLY = + "needs a single-process host (Val Town isolates don't share in-memory state)"; +const EXCLUDE = Object.fromEntries( + [ + 'sse-retry', + 'auth/metadata-var2', + 'elicitation-sep1034-client-defaults', + 'sep-2322-client-request-state' + ].map((name) => [name, SINGLE_PROCESS_ONLY]) +); + +// Auth scenarios need a second public origin (RFC 8414 well-known is +// origin-rooted). Deploy examples/hosted/valtown-relay.ts as a separate val +// and point CONFORMANCE_AS_ORIGIN at it; both vals share +// CONFORMANCE_RELAY_SECRET so /__aux can't be hit directly. +const { app, sessions } = createHostedApp({ + auxOrigins: { + as: process.env.CONFORMANCE_AS_ORIGIN, + as2: process.env.CONFORMANCE_AS2_ORIGIN, + idp: process.env.CONFORMANCE_IDP_ORIGIN + }, + relaySecret: process.env.CONFORMANCE_RELAY_SECRET, + // val.town spreads one run's requests over several isolates; persist to + // the account's SQLite so /results is the union of what they all saw. + store: process.env.valtown ? new SqliteRunStore() : undefined, + exclude: EXCLUDE +}); + +const bridge = toFetchHandler(app); + +export default async function (request: Request): Promise { + const response = await bridge(request); + // The bridge buffers until end(), by which point the scenario has recorded + // its checks and the write-through has started; finish it before the + // isolate is allowed to go idle. + await sessions.flush(); + return response; +} diff --git a/package.json b/package.json index af723a17..f87c807c 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "tier-check": "node dist/index.js tier-check", "traceability": "tsx src/index.ts traceability", "sync-schema": "tsx scripts/sync-schema.ts", + "hosted:bundle-requirements": "tsx examples/hosted/bundle-requirements.ts", "check": "npm run typecheck && npm run lint", "typecheck": "tsgo --noEmit", "prepack": "npm run build", diff --git a/src/hosted/README.md b/src/hosted/README.md new file mode 100644 index 00000000..3d338c1b --- /dev/null +++ b/src/hosted/README.md @@ -0,0 +1,266 @@ +# Hosted conformance server + +Runs the client-testing scenarios as a single long-lived HTTP server so a +client-under-test can point at a public URL instead of being spawned by the +runner. + +```bash +npx @modelcontextprotocol/conformance hosted --port 3000 +# behind a reverse proxy: +npx @modelcontextprotocol/conformance hosted --port 3000 --public-origin https://conformance.example.com +``` + +## The matrix + +One run exercises the whole matrix: every registered client scenario (rows) +at every specification revision that ships a requirement set in +`requirements/` (columns — today `2025-11-25` and `2026-07-28`). A **cell** +is one scenario at one revision; its id `//` is +the URL path the client is pointed at, the store key and the results path. + +Each cell carries two independent facts: + +- **scoring** — what the revision's `requirements/.yaml` makes of the + scenario: `scored` (in its `client:` list), `not_scored` (listed but never + counted, with the yaml's reason), `unlisted` (applies to the revision but + the frozen set predates it) or `n/a` (does not apply: introduced later, + removed earlier, or an extension the set does not carry). `n/a` cells are + never mounted. +- **startable** — whether this deployment can mount it: the scenario has + been converted to `handler()` / `authHandlers()`, every relay origin it + needs is configured, and the deployment has not excluded it + (`HostedServerOptions.exclude`). A cell that cannot start answers 501 with + the reason. + +Each cell speaks its column's wire: the `2025-11-25` column serves the +stateful mock (initialize handshake), the `2026-07-28` column the stateless +one (per-request `_meta`, `MCP-Protocol-Version` on every request) — exactly +what `conformance client --spec-version ` would run. + +## Routes + +| Route | Purpose | +| --------------------------------------------- | ------------------------------------------------------------------------------------------------ | +| `GET /` | Landing page: the static matrix (scoring, startability, steps) | +| `GET /scenarios` | JSON rows with a cell per revision | +| `GET /s` | Mints a run id, `303 → /s/` | +| `GET /s/` | Config for every startable cell of the run | +| `GET /s//` | Config for one column | +| `GET /s///` | Config for one cell (a page request, see below) | +| `ALL /s///[/]` | The cell's server. The MCP endpoint is the cell URL plus `/mcp`, for every scenario (see below). | +| `GET /results/` | Verdict per cell, `scored X of N` per column, client identity | +| `GET /results//` | One column | +| `GET /results///` | One cell: `{runId, revision, scenario, scoring, verdict, summary, checks}` (see below) | +| `DELETE /results/` | Tear down every cell of the run | + +Run ids match `[A-Za-z0-9_-]{1,64}`; pick your own or take the minted one. +Cells are created lazily on first request. Scenario names may contain `/` +and sit at the end of the path, so they are resolved by longest registered +name (`auth/metadata-var2/tenant1` → scenario `auth/metadata-var2`, suffix +`/tenant1`). + +**Every cell's MCP URL ends in `/mcp`.** Scenarios that serve MCP at their +handler root (`mcpPath` `''`) are reached at `/mcp` as well: the +server rewrites that suffix to `/` before dispatch (and +`/.well-known/oauth-protected-resource/s//mcp` to the bare well-known +path), so the config, the matrix pages and `/scenarios` show one URL shape. + +**Cell results** answer 200 for every cell of the matrix, exercised or not: +`verdict` is `incomplete` with a zero `summary` and empty `checks` for a +cell nobody has hit (plus `startable: false, startReason` for one this +deployment cannot start) and `n/a` with the `reason` for a scenario that +does not apply to the revision. Only an unknown revision or scenario is +a 404. + +**Representation.** Config and results answer HTML when the request prefers +`text/html` and JSON otherwise; `?format=html|json` overrides. At a cell URL +a GET that accepts `text/html` (and not `text/event-stream`) or carries +`?format=` is a page/config request; every other request — POST, an SSE GET, +DELETE, well-known paths — is dispatched to the scenario. Dispatched +responses carry `link: <…/results///>; +rel="conformance-results"`. + +**Config JSON** (run, column or cell scope): + +```json +{ + "runId": "…", "revision": "2026-07-28", "scenario": "tools_call", + "resultsUrl": "…/results//2026-07-28/tools_call", + "mcpServers": { "2026-07-28/tools_call": { "type": "http", "url": "…/s//2026-07-28/tools_call/mcp" } }, + "cells": [{ + "scenario": "tools_call", "revision": "2026-07-28", "url": "…", "resultsUrl": "…", + "scoring": "scored", "steps": [{ "op": "tools/list" }, …], + "env": { + "MCP_CONFORMANCE_SCENARIO": "tools_call", + "MCP_CONFORMANCE_PROTOCOL_VERSION": "2026-07-28", + "MCP_CONFORMANCE_CONTEXT": "{\"name\":\"tools_call\",\"steps\":[…]}" + } + }] +} +``` + +`env` is what the CLI runner would set for the client under test; `context` +is the scenario's context (credentials, `steps`) tagged with `name`, as a +JSON string. The HTML pages have copy-to-clipboard buttons for the same data. + +**Report.** A cell's verdict is `pass` (checks recorded, no FAILURE), `fail` +(any FAILURE), `incomplete` (never hit, or hit but nothing recorded) or +`n/a`. Per column, `scored: { passed, total, startable }` counts passes +among every cell the revision's requirement set scores — `total` is the +yaml's count whether or not this deployment can start the cell, `startable` +how many of those it can (the HTML says "3 of 32 scored (11 startable +here)"); `not_scored`/`unlisted` results are listed next to the score, never +inside it. The header names each client once — by `clientInfo` name and +version — with every protocol version it negotiated, read off accepted +exchanges only: on the stateful wire the `initialize` params and the +`protocolVersion` the server answered with, on the stateless wire +`_meta['io.modelcontextprotocol/clientInfo']` and the accepted request's +`MCP-Protocol-Version` header. Recorded as an INFO check +`hosted-client-identity` on the cell with `details.protocolVersions`. + +The hosted layer also records two FAILUREs of its own about requests to a +cell's MCP endpoint, so a cell cannot read green when the wire turned every +request away (`src/hosted/wire.ts`): + +- `hosted-wire-rejected` — a 4xx whose body is a lifecycle rejection + (JSON-RPC `-32020`/`-32022`, `-32602` naming `_meta`, or `-32000` + "Unsupported protocol version"); once per distinct (code, message). An + unsupported-version rejection of a request whose header already names + the cell's revision is a scenario's deliberate probe (`request-metadata` + rejects a run's first request once), not a wire rejection. +- `hosted-wrong-revision` — the client spoke a revision other than the + cell's: on the `2026-07-28` column any request whose `MCP-Protocol-Version` + is not the column's, or any `initialize`; on a dated column any + post-`initialize` request whose header names another revision + (`initialize` itself negotiates and is exempt); once per distinct + (method, header version). On a dated column a foreign-revision request + the wire turned away (a 4xx lifecycle rejection, or `-32601` for a + method the dated wire lacks) is version negotiation — a dual-era client + probes with `server/discover` at `2026-07-28`, then falls back to + `initialize` — and records neither check; one the wire accepted is + still a wrong revision. + +Both decide the verdict like any FAILURE. The `auth/*` resource server +records the same rejection in the scenario's own log as +`stateless-request-rejected`. + +## How it works + +Each scenario implements `handler(): RequestListener` (see `HandlerScenario` +in `src/types.ts`). The hosted server instantiates a fresh scenario per cell +with a `ScenarioContext` for the column's revision, mounts its handler under +`/s///`, and rewrites `req.url` to strip the prefix — +**no loopback port, no proxy**. The CLI runner's `start()`/`stop()` are thin +wrappers around the same `handler()`, so both modes exercise identical code. + +The run id lives in the **URL path**, not the `mcp-session-id` header, so +correlation works for stateless-transport clients: a client that never +echoes a session id still hits the same cell and its checks accumulate there. + +Each cell gets its own scenario instance, built from the registry entry with +a no-arg constructor. A scenario whose constructor takes parameters (one +class registered under several names, e.g. `skills/verification-*`) +implements `Scenario.fresh()` to carry them into the per-cell copy. + +The hosted layer reads JSON request bodies without consuming them +(`src/hosted/body.ts` intercepts the parser's `push()`), so the client +identity can be recorded while the scenario still reads the stream itself. + +## Auth scenarios — second-origin relay + +`auth/*` scenarios stand up two cross-referencing HTTP apps: a resource +server (the MCP endpoint + PRM) and an OAuth authorization server. The +`.well-known/*` discovery paths and RFC 8414 `issuer` validation are +**origin-rooted**, so the AS can't live under the cell prefix — it needs its +own public origin. + +``` +client RS origin AS-relay origin + │ POST /s//mcp │ │ + │───────────────────────────▶│ 401 + WWW-Authenticate │ + │ GET /.well-known/oauth-protected-resource/s//mcp │ + │───────────────────────────▶│ {authorization_servers: │ + │ │ [/r/]} │ + │ GET /.well-known/oauth-authorization-server/r/ │ + │─────────────────────────────────────────────────────────────────▶│ + │ │◀── /__aux/as/.well-known/…/r/ │ + │ │ (x-relay-secret) │ +``` + +The AS relay (`examples/hosted/valtown-relay.ts`) is **stateless** — it just +forwards every request to `/__aux/` with a shared +secret. All scenario state (closures, checks) stays on the RS process; the +per-cell AS issuer is `/r///` so the cell +is recoverable from any path the client constructs from it. The RS app +locates that `/r/` segment run, strips it, and dispatches to the cell's +AS handler with the path `createAuthServer()` registered. A cell is rebuilt +from its id alone when a process has never seen it, so an AS request that +arrives before the RS was ever hit still lands. + +```bash +# CLI — also reads CONFORMANCE_RELAY_SECRET from env +npx @modelcontextprotocol/conformance hosted \ + --port 3000 \ + --as-origin https://conformance-as.example.com \ + --relay-secret "$(openssl rand -hex 32)" +``` + +Two extra routes appear when `--as-origin` is set: + +| Route | Purpose | +| --------------------------------------------------- | -------------------------------------------------------------------- | +| `GET /.well-known/oauth-protected-resource/s/<...>` | RFC 9728 root-level PRM dispatch — recovers the cell from the suffix | +| `ALL /__aux//*` | Relay backchannel; 403 without `x-relay-secret` | + +Scenarios needing `as2`/`idp` origins become startable when `--as2-origin` / +`--idp-origin` are set; deploy one more relay per role with +`CONFORMANCE_RELAY_ROLE=as2|idp`. (No registered scenario has been converted +to `authHandlers()` with those roles yet.) + +**Fidelity note:** the hosted AS issuer always carries a `/r/` path +component, so scenarios that locally test root-issuer discovery +(`auth/metadata-default`, `auth/metadata-var1`) become path-issuer tests when +hosted. The RFC 8414 mechanics are identical. + +## Serverless / val.town + +`examples/hosted/valtown.ts` wraps `createHostedApp()` in a +`(Request) => Promise` bridge so the **same scenarios** run on +fetch-based runtimes (val.town, Deno Deploy, Bun, Workers with +`nodejs_compat`). Deploy with `examples/hosted/deploy-valtown.ts`; the vals +are listed in `examples/hosted/valtown-manifest.json`. + +val.town spreads one run's requests over several isolates that share no +memory, so `valtown.ts` excludes the scenarios whose checks depend on one +process seeing consecutive requests (`sse-retry`, `auth/metadata-var2`, +`elicitation-sep1034-client-defaults`, `sep-2322-client-request-state`); the +matrix shows them as not startable with that reason. Everything else +persists its raw check log to the account's SQLite (`RunStore`, +`examples/hosted/valtown-store.ts`) and `/results` re-judges the merged log. + +An isolate that has never seen a cell is **hydrated** before it dispatches +its first request to it: the scenario's `checks` array is seeded with the +merged log the store holds for the cell (`SessionManager.acquire`), so a +scenario that keys its behaviour on its own log — `request-metadata` rejects +the run's first request exactly once — sees the run's history rather than +just this isolate's. Seeded checks are persisted by the isolate that wrote +them; an isolate's row holds only what it recorded or rewrote itself. + +### Two-val auth setup + +| Val | File | Env | +| ------- | ---------------------------------- | ------------------------------------------------------------------------------------------------------------- | +| `rs` | `examples/hosted/valtown.ts` | `CONFORMANCE_AS_ORIGIN=https://.web.val.run`, `CONFORMANCE_RELAY_SECRET` | +| `relay` | `examples/hosted/valtown-relay.ts` | `CONFORMANCE_RS_ORIGIN=https://.web.val.run`, `CONFORMANCE_RELAY_SECRET`, `CONFORMANCE_RELAY_ROLE=as` | + +Same `CONFORMANCE_RELAY_SECRET` on both. + +## Example + +```bash +$ RUN=$(curl -sI https://conformance.example.com/s | sed -n 's#^location: /s/##Ip' | tr -d '\r') +$ npx @modelcontextprotocol/inspector https://conformance.example.com/s/$RUN/2025-11-25/tools_call/mcp +$ curl https://conformance.example.com/results/$RUN/2025-11-25/tools_call | jq .summary +{ "passed": 1, "failed": 0, "warnings": 0, "info": 5, "skipped": 0, "total": 6 } +$ curl https://conformance.example.com/results/$RUN | jq '.columns[] | {revision, scored}' +``` diff --git a/src/hosted/body.ts b/src/hosted/body.ts new file mode 100644 index 00000000..d3e93042 --- /dev/null +++ b/src/hosted/body.ts @@ -0,0 +1,122 @@ +/** + * Non-consuming capture of JSON request bodies. + * + * The hosted layer wants to read what the client under test sent (its + * `initialize` params or per-request `_meta`) without taking the body away + * from the scenario, whose listener reads the stream itself (express.json(), + * raw `data` events, the SDK's Node→Web conversion). Consuming and replaying + * would need a second IncomingMessage and change 'close' semantics, so + * instead we intercept `push()` — the parser's entry point into the + * Readable — and copy each chunk as it arrives. Flow control, listeners and + * consumption are untouched. + * + * Fetch-style bridges (examples/hosted/fetch-bridge.ts) already hold the + * whole body before the listener runs; they publish it under BUFFERED_BODY + * and the tap is skipped. + */ + +import type { IncomingMessage } from 'http'; +import type { RequestHandler } from 'express'; + +/** Set by a bridge that has the complete body up front. */ +export const BUFFERED_BODY = Symbol.for('mcp-conformance.hosted.bufferedBody'); +const TAP = Symbol.for('mcp-conformance.hosted.bodyTap'); + +/** Bodies above this are not captured (the identity we look for is small). */ +export const BODY_CAP = 256 * 1024; + +interface Tap { + body: Buffer | undefined; + done: boolean; + waiters: Array<(body: Buffer | undefined) => void>; +} + +type Tapped = IncomingMessage & { + [BUFFERED_BODY]?: Buffer; + [TAP]?: Tap; +}; + +function isJsonPost(req: IncomingMessage): boolean { + if (req.method !== 'POST') return false; + const type = req.headers['content-type'] ?? ''; + return /^application\/json\b/i.test(type); +} + +/** Express middleware: start capturing JSON POST bodies as they flow in. */ +export function tapJsonBody(): RequestHandler { + return (req, _res, next) => { + if (isJsonPost(req)) installTap(req); + next(); + }; +} + +export function installTap(req: IncomingMessage): void { + const r = req as Tapped; + if (r[BUFFERED_BODY] !== undefined || r[TAP] !== undefined) return; + const tap: Tap = { body: undefined, done: false, waiters: [] }; + r[TAP] = tap; + const chunks: Buffer[] = []; + let size = 0; + let overflow = false; + const push = r.push.bind(r); + r.push = ((chunk: unknown, encoding?: BufferEncoding) => { + if (chunk === null) { + tap.done = true; + tap.body = overflow ? undefined : Buffer.concat(chunks); + for (const w of tap.waiters.splice(0)) w(tap.body); + } else if (!overflow) { + const buf = Buffer.isBuffer(chunk) + ? chunk + : Buffer.from(String(chunk), encoding); + size += buf.length; + if (size > BODY_CAP) overflow = true; + else chunks.push(buf); + } + return push(chunk as Buffer | null, encoding); + }) as IncomingMessage['push']; +} + +/** + * Call `cb` with the request body once it is complete — immediately when it + * already is (bridged requests, or a body that arrived with the headers) — + * or never, if the body was not captured (not a JSON POST, over the cap, or + * the client never finished sending it). + */ +export function onBody(req: IncomingMessage, cb: (body: Buffer) => void): void { + const r = req as Tapped; + if (r[BUFFERED_BODY] !== undefined) { + if (isJsonPost(req)) cb(r[BUFFERED_BODY]); + return; + } + const tap = r[TAP]; + if (!tap) return; + const deliver = (body: Buffer | undefined) => { + if (body !== undefined) cb(body); + }; + if (tap.done) deliver(tap.body); + else tap.waiters.push(deliver); +} + +/** + * Like onBody(), but always settles: `cb` gets the body when it was captured + * and `undefined` as soon as it is known there will be none — the request is + * not a JSON POST, was never tapped, or ran over the cap. Callers that must + * judge every request (accepted or not) wait on this rather than on onBody(). + */ +export function onBodySettled( + req: IncomingMessage, + cb: (body: Buffer | undefined) => void +): void { + const r = req as Tapped; + if (r[BUFFERED_BODY] !== undefined) { + cb(isJsonPost(req) ? r[BUFFERED_BODY] : undefined); + return; + } + const tap = r[TAP]; + if (!tap) { + cb(undefined); + return; + } + if (tap.done) cb(tap.body); + else tap.waiters.push(cb); +} diff --git a/src/hosted/hosted-auth.test.ts b/src/hosted/hosted-auth.test.ts new file mode 100644 index 00000000..7ecf6835 --- /dev/null +++ b/src/hosted/hosted-auth.test.ts @@ -0,0 +1,335 @@ +/** + * Hosted auth scenarios — RS app + a local relay simulating the AS origin. + * + * Mirrors the production topology (RS val.town app + AS relay val) on two + * ephemeral localhost ports, then walks the OAuth discovery → DCR → + * authorize → token → MCP flow by hand to prove the path-rewrite and + * relay-secret guard work end to end. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import express from 'express'; +import type { Server } from 'http'; +import { createHostedApp } from './server'; +import { SessionManager, rawChecksOf, finalizeChecks } from './session'; +import { buildMatrix } from './matrix'; + +const RELAY_SECRET = 'test-relay-secret-do-not-use-in-prod'; + +describe('hosted auth scenarios (RS + AS relay)', () => { + let rsSrv: Server; + let relaySrv: Server; + let sessions: SessionManager; + let rs: string; // RS origin + let asOrigin: string; // relay origin + + beforeAll(async () => { + // Relay first so we know its origin before configuring the RS app. + const relay = express(); + relay.use(express.raw({ type: '*/*' })); + relay.all(/.*/, async (req, res) => { + const headers: Record = { + 'x-relay-secret': RELAY_SECRET, + 'x-relay-host': req.headers.host ?? '' + }; + for (const h of ['accept', 'authorization', 'content-type']) { + const v = req.headers[h]; + if (typeof v === 'string') headers[h] = v; + } + const search = req.url.includes('?') + ? req.url.slice(req.url.indexOf('?')) + : ''; + const body = ['GET', 'HEAD'].includes(req.method) + ? undefined + : new Uint8Array(req.body as Buffer); + const upstream = await fetch(`${rs}/__aux/as${req.path}${search}`, { + method: req.method, + headers, + body, + redirect: 'manual' + }); + res.status(upstream.status); + upstream.headers.forEach((v, k) => res.setHeader(k, v)); + res.send(Buffer.from(await upstream.arrayBuffer())); + }); + asOrigin = await listen(relay, (s) => (relaySrv = s)); + + const hosted = createHostedApp({ + auxOrigins: { as: asOrigin }, + relaySecret: RELAY_SECRET + }); + sessions = hosted.sessions; + rs = await listen(hosted.app, (s) => (rsSrv = s)); + }); + + afterAll(async () => { + await sessions.close(); + await Promise.all( + [rsSrv, relaySrv].map((s) => new Promise((r) => s.close(() => r()))) + ); + }); + + it('makes auth/* cells startable when as-origin is configured', () => { + const matrix = buildMatrix({ auxOrigins: { as: asOrigin } }); + for (const name of [ + 'auth/basic-cimd', + 'auth/metadata-default', + 'auth/pre-registration' + ]) { + expect(matrix.cell(name, '2025-11-25')!.startable).toBe(true); + } + // Scenarios without authHandlers() stay unstartable regardless. + expect( + matrix.cell('auth/authorization-server-migration', '2026-07-28') + ).toMatchObject({ + startable: false, + startReason: 'not converted for hosting yet' + }); + }); + + it('rejects /__aux/* without the relay secret', async () => { + const res = await fetch( + `${rs}/__aux/as/.well-known/oauth-authorization-server/r/nope` + ); + expect(res.status).toBe(403); + }); + + it('locates the /r/// segments in an /__aux path', async () => { + const hdr = { headers: { 'x-relay-secret': RELAY_SECRET } }; + const missing = /missing \/r\/\/\//; + // Illegal run id. + let res = await fetch( + `${rs}/__aux/as/tenant/r/bad!id/2025-11-25/auth/basic-cimd/token`, + hdr + ); + expect(res.status).toBe(404); + expect((await res.json()).error).toMatch(missing); + // Unknown revision, unknown scenario. + res = await fetch( + `${rs}/__aux/as/r/run/2024-01-01/auth/basic-cimd/token`, + hdr + ); + expect(res.status).toBe(404); + expect((await res.json()).error).toMatch(missing); + res = await fetch(`${rs}/__aux/as/r/run/2025-11-25/no-such/token`, hdr); + expect(res.status).toBe(404); + expect((await res.json()).error).toMatch(missing); + // A cell that does not apply to the revision is never mounted. + res = await fetch(`${rs}/__aux/as/r/run/2026-07-28/initialize/token`, hdr); + expect(res.status).toBe(404); + expect((await res.json()).scoring).toBe('n/a'); + // Adversarial input that would make a regex backtrack. + const evil = '/r/-'.repeat(2000) + '/r/x/token'; + res = await fetch(`${rs}/__aux/as${evil}`, hdr); + expect(res.status).toBe(404); + }); + + it('rebuilds a cell from its id when the aux origin is hit first', async () => { + // A client given the whole-run config may fetch AS metadata before it + // ever touches the RS; on a multi-process host that request can land on + // a process that never saw the run. The cell id carries everything. + const cell = 'cold/2026-07-28/auth/metadata-default'; + const meta = await fetch( + `${asOrigin}/.well-known/oauth-authorization-server/r/${cell}` + ).then((r) => r.json()); + expect(meta.issuer).toBe(`${asOrigin}/r/${cell}`); + expect(sessions.get(cell)?.revision).toBe('2026-07-28'); + }); + + it('walks auth/metadata-default end-to-end through the relay', async () => { + const cell = 'authflow/2025-11-25/auth/metadata-default'; + const mcpUrl = `${rs}/s/${cell}/mcp`; + + // 1. Unauthenticated MCP → 401 with WWW-Authenticate pointing at PRM + const r401 = await fetch(mcpUrl, { + method: 'POST', + headers: jsonHeaders(), + body: JSON.stringify(initBody()) + }); + expect(r401.status).toBe(401); + const www = r401.headers.get('www-authenticate') ?? ''; + expect(www).toContain('resource_metadata='); + + // 2. PRM via root well-known dispatch (RFC 9728 path-suffix derivation) + const prmUrl = `${rs}/.well-known/oauth-protected-resource/s/${cell}/mcp`; + const prm = await fetch(prmUrl).then((r) => r.json()); + expect(prm.resource).toBe(mcpUrl); + expect(prm.authorization_servers).toEqual([`${asOrigin}/r/${cell}`]); + + // 3. AS metadata — client derives well-known from issuer per RFC 8414 → + // hits the relay origin → forwarded to /__aux/as/… → run resolved. + const asMeta = await fetch( + `${asOrigin}/.well-known/oauth-authorization-server/r/${cell}` + ).then((r) => r.json()); + expect(asMeta.issuer).toBe(`${asOrigin}/r/${cell}`); + expect(asMeta.authorization_endpoint).toBe( + `${asOrigin}/r/${cell}/authorize` + ); + expect(asMeta.token_endpoint).toBe(`${asOrigin}/r/${cell}/token`); + + // 4. DCR + const reg = await fetch(asMeta.registration_endpoint, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + client_name: 'vitest', + redirect_uris: ['http://localhost:0/cb'] + }) + }).then((r) => r.json()); + expect(reg.client_id).toBeTruthy(); + + // 5. /authorize → 302 to redirect_uri with code (relay passes redirect through) + const authz = await fetch( + `${asMeta.authorization_endpoint}?` + + new URLSearchParams({ + response_type: 'code', + client_id: reg.client_id, + redirect_uri: 'http://localhost:0/cb', + code_challenge: 'E9Melhoa2OwvFrEMTJguCHaoeK1t8URWbuGJSstw-cM', + code_challenge_method: 'S256', + resource: mcpUrl + }), + { redirect: 'manual' } + ); + expect(authz.status).toBe(302); + const loc = new URL(authz.headers.get('location')!); + const code = loc.searchParams.get('code'); + expect(code).toBeTruthy(); + // RFC 9207 iss parameter should be the per-cell issuer + expect(loc.searchParams.get('iss')).toBe(`${asOrigin}/r/${cell}`); + + // 6. /token + const tok = await fetch(asMeta.token_endpoint, { + method: 'POST', + headers: { 'content-type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ + grant_type: 'authorization_code', + code: code!, + redirect_uri: 'http://localhost:0/cb', + client_id: reg.client_id, + code_verifier: + 'dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk' /* matches challenge */, + resource: mcpUrl + }) + }).then((r) => r.json()); + expect(tok.access_token).toBeTruthy(); + + // 7. Authenticated MCP initialize → 200 + const ok = await fetch(mcpUrl, { + method: 'POST', + headers: { + ...jsonHeaders(), + authorization: `Bearer ${tok.access_token}` + }, + body: JSON.stringify(initBody()) + }); + expect(ok.status).toBe(200); + + // Snapshot the raw log as a write-through store would persist it: no + // end-of-run verdicts yet (step 9 below re-judges this copy). + const raw = rawChecksOf(sessions.get(cell)!.scenario).map((c) => ({ + ...c + })); + expect(raw.some((c) => c.id.startsWith('resource-parameter-'))).toBe(false); + + // 8. Results — checks from BOTH origins accumulated on the one run. + const results = await fetch(`${rs}/results/${cell}`).then((r) => r.json()); + const ids = results.checks.map((c: { id: string }) => c.id); + expect(ids).toContain('prm-pathbased-requested'); // RS-side + expect(ids).toContain('authorization-server-metadata'); // AS-side via relay + expect(ids).toContain('client-registration'); + expect(ids).toContain('authorization-request'); + expect(ids).toContain('token-request'); + const statusOf = (id: string) => + results.checks.find((c: { id: string }) => c.id === id)?.status; + expect(statusOf('resource-parameter-in-authorization')).toBe('SUCCESS'); + expect(statusOf('resource-parameter-in-token')).toBe('SUCCESS'); + expect(statusOf('resource-parameter-matches-prm')).toBe('SUCCESS'); + + // 9. Multi-isolate: on serverless hosts the isolate serving GET /results + // is usually not the one that saw the OAuth flow. It re-judges the + // persisted raw log in a fresh scenario instance, which never observed + // the authorize/token requests directly — the RFC 8707 verdicts must be + // recoverable from the log itself. (`raw` was snapshotted before step 8, + // since getChecks() on the observing instance appends its verdicts.) + const rejudged = finalizeChecks('auth/metadata-default', raw); + const rejudgedStatus = (id: string) => + rejudged.find((c) => c.id === id)?.status; + expect(rejudgedStatus('resource-parameter-in-authorization')).toBe( + 'SUCCESS' + ); + expect(rejudgedStatus('resource-parameter-in-token')).toBe('SUCCESS'); + expect(rejudgedStatus('resource-parameter-consistency')).toBe('SUCCESS'); + expect(rejudgedStatus('resource-parameter-matches-prm')).toBe('SUCCESS'); + expect(rejudged.filter((c) => c.status === 'FAILURE')).toEqual([]); + }); + + it('exposes scenarioContext in the cell config env (pre-registration)', async () => { + const cell = 'pre/2025-11-25/auth/pre-registration'; + const config = await fetch(`${rs}/s/${cell}?format=json`).then((r) => + r.json() + ); + expect(config.cells).toHaveLength(1); + expect(config.cells[0].url).toBe(`${rs}/s/${cell}/mcp`); + expect(JSON.parse(config.cells[0].env.MCP_CONFORMANCE_CONTEXT)).toEqual({ + name: 'auth/pre-registration', + client_id: 'pre-registered-client', + client_secret: 'pre-registered-secret', + // The AS issuer this cell publishes: the relay origin + /r/. + issuer: `${asOrigin}/r/${cell}` + }); + }); + + it('routes tenant-prefixed AS metadata (auth/metadata-var2) correctly', async () => { + const cell = 'tenant/2025-11-25/auth/metadata-var2'; + // Touch RS to lazily create the cell so the aux handler exists. + await fetch(`${rs}/s/${cell}/mcp`, { + method: 'POST', + headers: jsonHeaders(), + body: JSON.stringify(initBody()) + }); + // Issuer is /r//tenant1 → well-known at + // /.well-known/oauth-authorization-server/r//tenant1; the + // scenario name is resolved by longest match, so `tenant1` is a suffix. + const meta = await fetch( + `${asOrigin}/.well-known/oauth-authorization-server/r/${cell}/tenant1` + ).then((r) => r.json()); + expect(meta.issuer).toBe(`${asOrigin}/r/${cell}/tenant1`); + expect(meta.authorization_endpoint).toBe( + `${asOrigin}/r/${cell}/tenant1/authorize` + ); + }); +}); + +function listen( + app: express.Application, + capture: (s: Server) => void +): Promise { + return new Promise((resolve) => { + const s = app.listen(0, () => { + const a = s.address(); + capture(s); + resolve(`http://localhost:${(a as { port: number }).port}`); + }); + }); +} + +function jsonHeaders() { + return { + 'content-type': 'application/json', + accept: 'application/json, text/event-stream' + }; +} + +function initBody() { + return { + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2025-06-18', + clientInfo: { name: 'vitest', version: '0' }, + capabilities: {} + } + }; +} diff --git a/src/hosted/hosted.test.ts b/src/hosted/hosted.test.ts new file mode 100644 index 00000000..dc0b13c2 --- /dev/null +++ b/src/hosted/hosted.test.ts @@ -0,0 +1,1263 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { createHostedApp } from './server'; +import { renderResults } from './html'; +import { SessionManager, cellId } from './session'; +import { MemoryRunStore } from './store'; +import type { HostedMatrix } from './matrix'; +import type { Server } from 'http'; + +const REV_STATEFUL = '2025-11-25'; +const REV_STATELESS = '2026-07-28'; + +describe('hosted server', () => { + let server: Server; + let sessions: SessionManager; + let matrix: HostedMatrix; + let base: string; + + beforeAll(async () => { + const hosted = createHostedApp({ + exclude: { 'sse-retry': 'excluded for the test' } + }); + sessions = hosted.sessions; + matrix = hosted.matrix; + await new Promise((resolve) => { + server = hosted.app.listen(0, () => { + const addr = server.address(); + if (addr && typeof addr === 'object') + base = `http://localhost:${addr.port}`; + resolve(); + }); + }); + }); + + afterAll(async () => { + await sessions.close(); + await new Promise((r) => server.close(() => r())); + }); + + async function postMcp( + path: string, + body: object, + headers: Record = {} + ) { + return fetch(`${base}${path}`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + accept: 'application/json, text/event-stream', + ...headers + }, + body: JSON.stringify(body) + }); + } + + const initBody = (clientName = 'vitest') => ({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2025-06-18', + clientInfo: { name: clientName, version: '0' }, + capabilities: {} + } + }); + + /** A SEP-2575 stateless request: version in the header and in _meta. */ + const statelessBody = (method: string, params: object = {}) => ({ + jsonrpc: '2.0', + id: 1, + method, + params: { + ...params, + _meta: { + 'io.modelcontextprotocol/protocolVersion': REV_STATELESS, + 'io.modelcontextprotocol/clientInfo': { name: 'vitest', version: '0' }, + 'io.modelcontextprotocol/clientCapabilities': {} + } + } + }); + const statelessHeaders = { 'mcp-protocol-version': REV_STATELESS }; + + it('/scenarios lists every client scenario with a cell per revision', async () => { + const list = await fetch(`${base}/scenarios`).then((r) => r.json()); + const byName = new Map[] }>( + list.map((s: { name: string; cells: Record[] }) => [ + s.name, + s + ]) + ); + expect(byName.has('initialize')).toBe(true); + expect(byName.has('auth/basic-cimd')).toBe(true); // listed even though not startable + const revisions = list[0].cells.map( + (c: { revision: string }) => c.revision + ); + expect(revisions).toEqual([REV_STATEFUL, REV_STATELESS]); + // initialize was removed in 2026-07-28 → n/a there, scored before. + expect(byName.get('initialize')!.cells).toEqual([ + { revision: REV_STATEFUL, scoring: 'scored', startable: true }, + { + revision: REV_STATELESS, + scoring: 'n/a', + reason: 'introduced in 2025-06-18, removed in 2026-07-28', + startable: false + } + ]); + // auth cells need a relay origin this instance does not have. + expect(byName.get('auth/basic-cimd')!.cells[0]).toMatchObject({ + scoring: 'scored', + startable: false, + startReason: 'needs relay origin(s) [as]' + }); + // The deployment's exclusion list shows up with its reason. + expect(byName.get('sse-retry')!.cells[0]).toMatchObject({ + startable: false, + startReason: 'excluded for the test' + }); + }); + + it('mounts a raw-http scenario at /s/// and records checks', async () => { + const res = await postMcp(`/s/t1/${REV_STATEFUL}/initialize`, initBody()); + expect(res.status).toBe(200); + expect(res.headers.get('link')).toContain( + `/results/t1/${REV_STATEFUL}/initialize>` + ); + const body = await res.json(); + expect(body.result.serverInfo.name).toBe('test-server'); + + const results = await fetch( + `${base}/results/t1/${REV_STATEFUL}/initialize` + ).then((r) => r.json()); + expect(results).toMatchObject({ + runId: 't1', + revision: REV_STATEFUL, + scenario: 'initialize' + }); + expect( + results.checks.some( + (c: { id: string }) => c.id === 'mcp-client-initialization' + ) + ).toBe(true); + }); + + it('mounts an express scenario, accumulating checks across stateless requests', async () => { + // tools_call uses StreamableHTTPServerTransport with sessionIdGenerator: undefined, + // i.e. fully stateless. Correlation must come from the path-embedded id. + const url = `/s/t2/${REV_STATEFUL}/tools_call/mcp`; + const r1 = await postMcp(url, initBody()); + expect(r1.status).toBe(200); + await r1.text(); + + const r2 = await postMcp(url, { + jsonrpc: '2.0', + id: 2, + method: 'tools/call', + params: { name: 'add_numbers', arguments: { a: 2, b: 3 } } + }); + expect(r2.status).toBe(200); + expect(await r2.text()).toContain('The sum of 2 and 3 is 5'); + + const results = await fetch( + `${base}/results/t2/${REV_STATEFUL}/tools_call` + ).then((r) => r.json()); + expect( + results.checks.some((c: { id: string }) => c.id === 'tool-add-numbers') + ).toBe(true); + }); + + it('serves the stateful mock in the 2025-11-25 column and the stateless one in 2026-07-28', async () => { + // Same scenario, two wires. The 2026-07-28 cell has no initialize + // handshake and validates the SEP-2575 header/_meta on every request. + const stateless = await postMcp( + `/s/wire/${REV_STATELESS}/tools_call/mcp`, + statelessBody('tools/call', { + name: 'add_numbers', + arguments: { a: 4, b: 6 } + }), + statelessHeaders + ); + expect(stateless.status).toBe(200); + expect(await stateless.text()).toContain('The sum of 4 and 6 is 10'); + + // A stateful-style initialize (no header, no _meta) is rejected there… + const rejected = await postMcp( + `/s/wire/${REV_STATELESS}/tools_call/mcp`, + initBody() + ); + expect(rejected.status).toBe(400); + expect((await rejected.json()).error.code).toBe(-32020); + + // …and accepted by the 2025-11-25 cell of the same run. + const stateful = await postMcp( + `/s/wire/${REV_STATEFUL}/tools_call/mcp`, + initBody() + ); + expect(stateful.status).toBe(200); + // The stateful mock answers initialize over SSE with the handshake result. + const init = await stateful.text(); + expect(init).toContain('"serverInfo"'); + expect(init).toContain('"protocolVersion"'); + + // Each cell judged independently. + const a = await fetch( + `${base}/results/wire/${REV_STATELESS}/tools_call` + ).then((r) => r.json()); + expect( + a.checks.find((c: { id: string }) => c.id === 'tool-add-numbers')?.status + ).toBe('SUCCESS'); + const b = await fetch(`${base}/results/wire`).then((r) => r.json()); + const exercised = (report: { + columns: { + cells: { revision: string; scenario: string; summary?: unknown }[]; + }[]; + }) => + report.columns.flatMap((col) => + col.cells.filter((c) => c.summary).map((c) => [c.revision, c.scenario]) + ); + expect(exercised(b)).toEqual([ + [REV_STATEFUL, 'tools_call'], + [REV_STATELESS, 'tools_call'] + ]); + }); + + it('mounts a draft scenario (request-metadata) directly', async () => { + // request-metadata simulates a version rejection on the *first* request to + // exercise client retry, then accepts. Send twice — both with no + // mcp-session-id (stateless) — and confirm checks accumulate via path id. + const init = { + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: 'DRAFT-2026-v1', + clientInfo: { name: 'vitest', version: '0' }, + capabilities: {} + } + }; + const headers = { 'mcp-protocol-version': 'DRAFT-2026-v1' }; + const url = `/s/t3/${REV_STATELESS}/request-metadata`; + const r1 = await postMcp(url, init, headers); + expect(r1.status).toBe(400); + // SEP-2575: unsupported-version rejection is -32022 (with supported/requested data). + expect((await r1.json()).error.code).toBe(-32022); + const r2 = await postMcp(url, init, headers); + expect(r2.status).toBe(200); + await r2.text(); + + const results = await fetch( + `${base}/results/t3/${REV_STATELESS}/request-metadata` + ).then((r) => r.json()); + expect(results.scenario).toBe('request-metadata'); + expect( + results.checks.some( + (c: { id: string }) => + c.id === 'sep-2575-http-client-sends-version-header' + ) + ).toBe(true); + }); + + it('json-schema-ref-deref embeds the public mounted URL in the canary $ref', async () => { + const url = `/s/t4/${REV_STATELESS}/json-schema-ref-no-deref/mcp`; + const r = await postMcp(url, statelessBody('tools/list'), statelessHeaders); + const text = await r.text(); + // Canary URL should be the *mounted* base, not localhost:randomport + expect(text).toContain( + `${base}/s/t4/${REV_STATELESS}/json-schema-ref-no-deref/canary/profile-schema.json` + ); + // The scenario rewrites the draft header to the SDK's before handing + // the request on; identity is read from the header the client sent. + const results = await fetch( + `${base}/results/t4/${REV_STATELESS}/json-schema-ref-no-deref` + ).then((r) => r.json()); + const identity = results.checks.find( + (c: { id: string }) => c.id === 'hosted-client-identity' + ); + expect(identity.details).toMatchObject({ + name: 'vitest', + protocolVersions: [REV_STATELESS] + }); + }); + + it('GET /s mints a run id and redirects to its config', async () => { + const res = await fetch(`${base}/s`, { redirect: 'manual' }); + expect(res.status).toBe(303); + const location = res.headers.get('location')!; + expect(location).toMatch(/^\/s\/[A-Za-z0-9_-]+$/); + const runId = location.slice('/s/'.length); + + const config = await fetch(`${base}${location}`).then((r) => r.json()); + expect(config.runId).toBe(runId); + expect(config.revision).toBeUndefined(); + expect(config.resultsUrl).toBe(`${base}/results/${runId}`); + // Every startable cell, keyed /, none that cannot start. + const keys = Object.keys(config.mcpServers); + expect(keys).toContain(`${REV_STATEFUL}/tools_call`); + expect(keys).toContain(`${REV_STATELESS}/tools_call`); + expect(keys).toContain(`${REV_STATEFUL}/initialize`); + expect(keys).not.toContain(`${REV_STATELESS}/initialize`); // n/a + expect(keys.some((k) => k.endsWith('/auth/basic-cimd'))).toBe(false); // no relay + expect(keys).not.toContain(`${REV_STATEFUL}/sse-retry`); // excluded + expect(config.mcpServers[`${REV_STATEFUL}/tools_call`]).toEqual({ + type: 'http', + url: `${base}/s/${runId}/${REV_STATEFUL}/tools_call/mcp` + }); + const cell = config.cells.find( + (c: { scenario: string; revision: string }) => + c.scenario === 'tools_call' && c.revision === REV_STATELESS + ); + expect(cell).toMatchObject({ + url: `${base}/s/${runId}/${REV_STATELESS}/tools_call/mcp`, + resultsUrl: `${base}/results/${runId}/${REV_STATELESS}/tools_call`, + scoring: 'scored', + env: { + MCP_CONFORMANCE_SCENARIO: 'tools_call', + MCP_CONFORMANCE_PROTOCOL_VERSION: REV_STATELESS + } + }); + expect(JSON.parse(cell.env.MCP_CONFORMANCE_CONTEXT)).toEqual({ + name: 'tools_call', + steps: cell.steps + }); + expect(cell.steps[0]).toEqual({ op: 'tools/list' }); + }); + + it('scopes config to a column or a cell', async () => { + const column = await fetch(`${base}/s/scope/${REV_STATELESS}`).then((r) => + r.json() + ); + expect(column.revision).toBe(REV_STATELESS); + expect(column.resultsUrl).toBe(`${base}/results/scope/${REV_STATELESS}`); + expect( + column.cells.every( + (c: { revision: string }) => c.revision === REV_STATELESS + ) + ).toBe(true); + expect( + column.cells.some( + (c: { scenario: string }) => c.scenario === 'initialize' + ) + ).toBe(false); + + const cell = await fetch( + `${base}/s/scope/${REV_STATEFUL}/tools_call?format=json` + ).then((r) => r.json()); + expect(cell).toMatchObject({ + runId: 'scope', + revision: REV_STATEFUL, + scenario: 'tools_call', + resultsUrl: `${base}/results/scope/${REV_STATEFUL}/tools_call` + }); + expect(cell.cells).toHaveLength(1); + expect(Object.keys(cell.mcpServers)).toEqual([ + `${REV_STATEFUL}/tools_call` + ]); + }); + + it('ignores a trailing slash on run, column and cell paths', async () => { + const run = await fetch(`${base}/s/slash/`).then((r) => r.json()); + expect(run.runId).toBe('slash'); + expect(run.revision).toBeUndefined(); + + const column = await fetch(`${base}/s/slash/${REV_STATEFUL}/`).then((r) => + r.json() + ); + expect(column.revision).toBe(REV_STATEFUL); + + const cell = await fetch( + `${base}/s/slash/${REV_STATEFUL}/tools_call/?format=json` + ).then((r) => r.json()); + expect(cell.scenario).toBe('tools_call'); + expect(cell.cells).toHaveLength(1); + + for (const path of [ + `/results/slash/`, + `/results/slash/${REV_STATEFUL}/`, + `/results/slash/${REV_STATEFUL}/tools_call/` + ]) { + const res = await fetch(`${base}${path}`); + expect(res.status, path).toBe(200); + } + }); + + it('negotiates HTML for browsers, JSON otherwise, ?format= overriding both', async () => { + const html = await fetch(`${base}/s/neg`, { + headers: { accept: 'text/html,application/xhtml+xml,*/*;q=0.8' } + }); + expect(html.headers.get('content-type')).toContain('text/html'); + const json = await fetch(`${base}/s/neg`, { + headers: { accept: '*/*' } + }); + expect(json.headers.get('content-type')).toContain('application/json'); + const forcedJson = await fetch(`${base}/s/neg?format=json`, { + headers: { accept: 'text/html' } + }); + expect(forcedJson.headers.get('content-type')).toContain( + 'application/json' + ); + const forcedHtml = await fetch(`${base}/s/neg?format=html`, { + headers: { accept: 'application/json' } + }); + expect(forcedHtml.headers.get('content-type')).toContain('text/html'); + }); + + it('at a cell URL, a browser GET is a page and everything else reaches the scenario', async () => { + const url = `${base}/s/dis/${REV_STATEFUL}/tools_call`; + const page = await fetch(url, { headers: { accept: 'text/html' } }); + expect(page.status).toBe(200); + expect(page.headers.get('content-type')).toContain('text/html'); + expect(page.headers.get('link')).toBeNull(); + + // An SSE GET is the client under test; it reaches the scenario's mount + // (whose root is not the MCP endpoint here) and carries the results link. + const sse = await fetch(url, { + headers: { accept: 'text/html, text/event-stream' } + }); + expect(sse.status).toBe(404); + expect(sse.headers.get('link')).toContain( + `/results/dis/${REV_STATEFUL}/tools_call>` + ); + }); + + it('isolates cells with the same scenario but different run ids', async () => { + await postMcp(`/s/iso-a/${REV_STATEFUL}/initialize`, initBody('a')).then( + (r) => r.text() + ); + await postMcp(`/s/iso-b/${REV_STATEFUL}/initialize`, initBody('b')).then( + (r) => r.text() + ); + + const a = await fetch( + `${base}/results/iso-a/${REV_STATEFUL}/initialize` + ).then((r) => r.json()); + const b = await fetch( + `${base}/results/iso-b/${REV_STATEFUL}/initialize` + ).then((r) => r.json()); + expect(a.checks[0].details.clientName).toBe('a'); + expect(b.checks[0].details.clientName).toBe('b'); + }); + + it('explains why a cell cannot be reached', async () => { + const body = { jsonrpc: '2.0' }; + // unknown scenario + let res = await postMcp(`/s/x/${REV_STATEFUL}/does-not-exist`, body); + expect(res.status).toBe(404); + expect((await res.json()).error).toContain( + "unknown scenario 'does-not-exist'" + ); + // bad run id / unknown revision + res = await postMcp(`/s/bad..id/${REV_STATEFUL}/initialize`, body); + expect(res.status).toBe(400); + res = await postMcp('/s/x/2024-01-01/initialize', body); + expect(res.status).toBe(404); + expect((await res.json()).revisions).toEqual([REV_STATEFUL, REV_STATELESS]); + // n/a: the scenario does not apply to the revision + res = await postMcp(`/s/x/${REV_STATELESS}/initialize`, body); + expect(res.status).toBe(404); + expect(await res.json()).toMatchObject({ + scoring: 'n/a', + reason: 'introduced in 2025-06-18, removed in 2026-07-28' + }); + // not converted for hosting yet + res = await postMcp(`/s/x/${REV_STATEFUL}/auth/scope-step-up`, body); + expect(res.status).toBe(501); + expect((await res.json()).reason).toBe('not converted for hosting yet'); + // needs a relay origin + res = await postMcp(`/s/x/${REV_STATEFUL}/auth/basic-cimd/mcp`, body); + expect(res.status).toBe(501); + expect((await res.json()).reason).toBe('needs relay origin(s) [as]'); + // excluded by the deployment + res = await postMcp(`/s/x/${REV_STATEFUL}/sse-retry`, body); + expect(res.status).toBe(501); + expect((await res.json()).reason).toBe('excluded for the test'); + // the old shapes are gone + res = await postMcp('/s/initialize/x', body); + expect(res.status).toBe(404); + res = await postMcp(`/s/x/${REV_STATEFUL}`, body); + expect(res.status).toBe(405); + }); + + it('DELETE /results/ tears down every cell of the run', async () => { + await postMcp(`/s/del/${REV_STATEFUL}/initialize`, initBody()).then((r) => + r.text() + ); + await postMcp( + `/s/del/${REV_STATELESS}/tools_call/mcp`, + statelessBody('tools/list'), + statelessHeaders + ).then((r) => r.text()); + type Cell = { + scenario: string; + summary?: { total: number }; + resultsUrl: string; + }; + const exercised = (report: { columns: { cells: Cell[] }[] }) => + report.columns.flatMap((col) => col.cells.filter((c) => c.summary)); + let run = await fetch(`${base}/results/del`).then((r) => r.json()); + expect(exercised(run)).toHaveLength(2); + expect(exercised(run)[0]).toMatchObject({ + revision: REV_STATEFUL, + scenario: 'initialize', + verdict: 'pass', + resultsUrl: `${base}/results/del/${REV_STATEFUL}/initialize` + }); + expect(exercised(run)[0].summary!.total).toBeGreaterThan(0); + + const del = await fetch(`${base}/results/del`, { method: 'DELETE' }); + expect(del.status).toBe(204); + run = await fetch(`${base}/results/del`).then((r) => r.json()); + expect(exercised(run)).toEqual([]); + // The cell is still a cell of the matrix — just nothing recorded now. + const gone = await fetch(`${base}/results/del/${REV_STATEFUL}/initialize`); + expect(gone.status).toBe(200); + expect(await gone.json()).toMatchObject({ + verdict: 'incomplete', + summary: { total: 0 }, + checks: [] + }); + }); + + it('serves every cell at /mcp, whatever the scenario mounts at its root', async () => { + // request-metadata and initialize serve MCP at their handler root; the + // config still says /mcp, and a request there is rewritten to the root. + const config = await fetch( + `${base}/s/mcp1/${REV_STATELESS}/request-metadata?format=json` + ).then((r) => r.json()); + expect(config.cells[0].url).toBe( + `${base}/s/mcp1/${REV_STATELESS}/request-metadata/mcp` + ); + const run = await fetch(`${base}/s/mcp1`).then((r) => r.json()); + const urls = Object.values(run.mcpServers).map( + (s) => (s as { url: string }).url + ); + expect(urls.length).toBeGreaterThan(0); + expect(urls.every((u) => u.endsWith('/mcp'))).toBe(true); + const list = await fetch(`${base}/scenarios`).then((r) => r.json()); + expect(list.every((s: { mcpPath: string }) => s.mcpPath === '/mcp')).toBe( + true + ); + + // Reaches the scenario: request-metadata answers its simulated + // rejection, initialize its handshake — not a 404. + const rm = await postMcp( + `/s/mcp1/${REV_STATELESS}/request-metadata/mcp`, + statelessBody('tools/list'), + statelessHeaders + ); + expect(rm.status).toBe(400); + expect((await rm.json()).error.code).toBe(-32022); + const init = await postMcp( + `/s/mcp1/${REV_STATEFUL}/initialize/mcp`, + initBody() + ); + expect(init.status).toBe(200); + expect((await init.json()).result.serverInfo.name).toBe('test-server'); + // A scenario with its own /mcp is served as before, and the bare cell + // root of a root-mounted scenario still answers (the CLI's shape). + const own = await postMcp( + `/s/mcp1/${REV_STATEFUL}/tools_call/mcp`, + initBody() + ); + expect(own.status).toBe(200); + await own.text(); + const root = await postMcp( + `/s/mcp1/${REV_STATEFUL}/initialize`, + initBody() + ); + expect(root.status).toBe(200); + await root.text(); + + // The -32022 request-metadata answered above was its own probe of a + // client that named the cell's revision — not a wire rejection. + const probed = await fetch( + `${base}/results/mcp1/${REV_STATELESS}/request-metadata` + ).then((r) => r.json()); + expect( + probed.checks.some((c: { id: string }) => c.id === 'hosted-wire-rejected') + ).toBe(false); + + // /mcp on a root-mounted scenario is its MCP endpoint for the + // hosted judgement too: a stateful initialize there is a wrong revision. + await postMcp( + `/s/mcp1/${REV_STATELESS}/request-metadata/mcp`, + initBody() + ).then((r) => r.text()); + const judged = await fetch( + `${base}/results/mcp1/${REV_STATELESS}/request-metadata` + ).then((r) => r.json()); + expect( + judged.checks.some( + (c: { id: string }) => c.id === 'hosted-wrong-revision' + ) + ).toBe(true); + }); + + it('answers results for every cell of the matrix, exercised or not', async () => { + const zeros = { + passed: 0, + failed: 0, + warnings: 0, + info: 0, + skipped: 0, + total: 0 + }; + // Untouched, startable: a valid, incomplete cell — not an unknown run. + const fresh = await fetch( + `${base}/results/fresh/${REV_STATEFUL}/tools_call` + ); + expect(fresh.status).toBe(200); + expect(await fresh.json()).toEqual({ + runId: 'fresh', + revision: REV_STATEFUL, + scenario: 'tools_call', + scoring: 'scored', + verdict: 'incomplete', + summary: zeros, + checks: [] + }); + // n/a: the scenario does not apply to the revision. + const na = await fetch(`${base}/results/fresh/${REV_STATELESS}/initialize`); + expect(na.status).toBe(200); + expect(await na.json()).toMatchObject({ + scoring: 'n/a', + verdict: 'n/a', + reason: 'introduced in 2025-06-18, removed in 2026-07-28', + summary: zeros, + checks: [] + }); + // Not startable here. + expect( + await fetch(`${base}/results/fresh/${REV_STATEFUL}/auth/basic-cimd`).then( + (r) => r.json() + ) + ).toMatchObject({ + scoring: 'scored', + verdict: 'incomplete', + startable: false, + startReason: 'needs relay origin(s) [as]' + }); + expect( + await fetch(`${base}/results/fresh/${REV_STATEFUL}/sse-retry`).then((r) => + r.json() + ) + ).toMatchObject({ startable: false, startReason: 'excluded for the test' }); + // An exercised cell says where it stands too. + await postMcp(`/s/fresh/${REV_STATEFUL}/initialize/mcp`, initBody()).then( + (r) => r.text() + ); + expect( + await fetch(`${base}/results/fresh/${REV_STATEFUL}/initialize`).then( + (r) => r.json() + ) + ).toMatchObject({ scoring: 'scored', verdict: 'pass' }); + + // HTML equivalents carry the reason text. + const page = (path: string) => + fetch(`${base}${path}`, { headers: { accept: 'text/html' } }).then((r) => + r.text() + ); + expect(await page(`/results/fresh/${REV_STATEFUL}/tools_call`)).toContain( + 'nothing recorded yet' + ); + expect(await page(`/results/fresh/${REV_STATELESS}/initialize`)).toContain( + 'does not apply to this revision: introduced in 2025-06-18, removed in 2026-07-28' + ); + expect( + await page(`/results/fresh/${REV_STATEFUL}/auth/basic-cimd`) + ).toContain('not startable here: needs relay origin(s) [as]'); + + // Only an unknown revision or scenario is a 404. + expect( + (await fetch(`${base}/results/fresh/2024-01-01/tools_call`)).status + ).toBe(404); + expect( + (await fetch(`${base}/results/fresh/${REV_STATEFUL}/no-such`)).status + ).toBe(404); + }); + + it('records the client identity on both wires without eating the body', async () => { + // Stateful: name from the initialize params, version from what the + // server answered over SSE — the SDK echoes a supported requested + // version and falls back to its latest for one it does not know. A + // second initialize by the same client adds to the one identity; a + // later header-only request adds nothing. + const url = `/s/who/${REV_STATEFUL}/tools_call/mcp`; + await postMcp(url, initBody('sdk-a'), { + 'user-agent': 'vitest-agent/1' + }).then((r) => r.text()); + await postMcp( + url, + { + ...initBody('sdk-a'), + params: { ...initBody('sdk-a').params, protocolVersion: 'bogus' } + }, + { 'user-agent': 'vitest-agent/1' } + ).then((r) => r.text()); + await postMcp( + url, + { + jsonrpc: '2.0', + id: 2, + method: 'tools/call', + params: { name: 'add_numbers', arguments: { a: 1, b: 1 } } + }, + { 'mcp-protocol-version': REV_STATEFUL, 'user-agent': 'vitest-agent/1' } + ).then((r) => r.text()); + const stateful = await fetch( + `${base}/results/who/${REV_STATEFUL}/tools_call` + ).then((r) => r.json()); + const ids = stateful.checks.filter( + (c: { id: string }) => c.id === 'hosted-client-identity' + ); + expect(ids.map((c: { details: unknown }) => c.details)).toEqual([ + { + name: 'sdk-a', + version: '0', + protocolVersions: ['2025-06-18', REV_STATEFUL], + userAgent: 'vitest-agent/1' + } + ]); + expect(ids[0].status).toBe('INFO'); + expect(ids[0].description).toContain( + `sdk-a 0 speaking protocol 2025-06-18, ${REV_STATEFUL}` + ); + // The scenario still saw and judged the body it was going to read. + expect(stateful.summary.passed).toBeGreaterThanOrEqual(1); + expect( + stateful.checks.find((c: { id: string }) => c.id === 'tool-add-numbers') + ?.status + ).toBe('SUCCESS'); + + // Stateless: identity comes from _meta on every accepted request. One + // the mock turns away (header disagreeing with _meta) is no identity. + await postMcp( + `/s/who/${REV_STATELESS}/tools_call/mcp`, + statelessBody('tools/list'), + { ...statelessHeaders, 'user-agent': 'vitest-agent/2' } + ).then((r) => r.text()); + const rejected = await postMcp( + `/s/who/${REV_STATELESS}/tools_call/mcp`, + { + ...statelessBody('tools/list'), + params: { + _meta: { + 'io.modelcontextprotocol/protocolVersion': REV_STATELESS, + 'io.modelcontextprotocol/clientInfo': { + name: 'nobody', + version: '1' + }, + 'io.modelcontextprotocol/clientCapabilities': {} + } + } + }, + { 'mcp-protocol-version': REV_STATEFUL } + ); + expect(rejected.status).toBe(400); + await rejected.text(); + const stateless = await fetch( + `${base}/results/who/${REV_STATELESS}/tools_call` + ).then((r) => r.json()); + expect( + stateless.checks + .filter((c: { id: string }) => c.id === 'hosted-client-identity') + .map((c: { details: unknown }) => c.details) + ).toEqual([ + { + name: 'vitest', + version: '0', + protocolVersions: [REV_STATELESS], + userAgent: 'vitest-agent/2' + } + ]); + }); + + it('reports a verdict per cell with scored X of N per column', async () => { + const run = 'rep'; + // pass + await postMcp( + `/s/${run}/${REV_STATEFUL}/initialize`, + initBody('rep-client') + ).then((r) => r.text()); + // fail: request-metadata's first request is rejected on purpose; stopping + // there leaves its declared checks unemitted → FAILURE on judgement. + await postMcp( + `/s/${run}/${REV_STATELESS}/request-metadata`, + { jsonrpc: '2.0', id: 1, method: 'initialize', params: {} }, + { 'mcp-protocol-version': 'DRAFT-2026-v1' } + ).then((r) => r.text()); + // incomplete (created via config, never hit): every other startable cell. + await fetch(`${base}/s/${run}`).then((r) => r.json()); + + const report = await fetch(`${base}/results/${run}`).then((r) => r.json()); + expect(report.runId).toBe(run); + expect(report.columns.map((c: { revision: string }) => c.revision)).toEqual( + [REV_STATEFUL, REV_STATELESS] + ); + const [stateful, stateless] = report.columns; + const find = (col: { cells: { scenario: string }[] }, name: string) => + col.cells.find((c) => c.scenario === name) as Record; + expect(find(stateful, 'initialize')).toMatchObject({ + verdict: 'pass', + scoring: 'scored', + resultsUrl: `${base}/results/${run}/${REV_STATEFUL}/initialize` + }); + expect(find(stateless, 'request-metadata').verdict).toBe('fail'); + expect(find(stateless, 'initialize').verdict).toBe('n/a'); + expect(find(stateful, 'tools_call').verdict).toBe('incomplete'); // configured, never hit + expect(find(stateful, 'auth/basic-cimd')).toMatchObject({ + verdict: 'incomplete', + startable: false + }); + // N is the requirement set's count of scored cells (auth/* included, + // though not startable without a relay); the startable subset alongside. + const scoredCells = (rev: string) => + matrix + .cells() + .filter((c) => c.revision === rev && c.scoring === 'scored'); + const scoredOf = (rev: string, passed: number) => ({ + passed, + total: scoredCells(rev).length, + startable: scoredCells(rev).filter((c) => c.startable).length + }); + expect(stateful.scored).toEqual(scoredOf(REV_STATEFUL, 1)); + expect(stateless.scored).toEqual(scoredOf(REV_STATELESS, 0)); + expect(stateful.scored.startable).toBeLessThan(stateful.scored.total); + // Header shows who talked to the run: the stateful client by name; the + // probe request-metadata turned away is no identity. + expect(report.identities).toEqual([ + expect.objectContaining({ + name: 'rep-client', + protocolVersions: ['2025-06-18'] + }) + ]); + expect(stateful.identities).toEqual([ + expect.objectContaining({ name: 'rep-client' }) + ]); + expect(stateless.identities).toEqual([]); + + // Column scope and HTML. + const column = await fetch(`${base}/results/${run}/${REV_STATELESS}`).then( + (r) => r.json() + ); + expect(column.revision).toBe(REV_STATELESS); + expect(column.columns).toHaveLength(1); + const html = await fetch(`${base}/results/${run}`, { + headers: { accept: 'text/html' } + }); + expect(html.headers.get('content-type')).toContain('text/html'); + const text = await html.text(); + expect(text).toContain( + `1 of ${stateful.scored.total} scored (${stateful.scored.startable} startable here)` + ); + expect(text).toContain('no client seen yet'); // the 2026-07-28 column + expect(text).toContain('rep-client'); + expect(text).toContain('>fail'); + expect(text).toContain( + `href="${base}/results/${run}/${REV_STATEFUL}/initialize"` + ); + }); + + it('fails a cell when the wire rejects the request or the client speaks another revision', async () => { + // Felix's live case: a 2025-11-25 initialize on a 2026-07-28 cell. The + // stateless mock turns it away (no _meta) and the scenario never sees a + // request it could judge — the cell must read fail, not green. + const url = `/s/rej/${REV_STATELESS}/tools_call/mcp`; + const legacyInit = { + ...initBody(), + params: { ...initBody().params, protocolVersion: REV_STATEFUL } + }; + for (let i = 0; i < 2; i++) { + const r = await postMcp(url, legacyInit, { + 'mcp-protocol-version': REV_STATEFUL + }); + expect(r.status).toBe(400); + await r.text(); + } + // …and one with no header at all (the -32020 rejection). + const bare = await postMcp(url, initBody()); + expect(bare.status).toBe(400); + await bare.text(); + + const results = await fetch( + `${base}/results/rej/${REV_STATELESS}/tools_call` + ).then((r) => r.json()); + type Check = { + id: string; + status: string; + errorMessage?: string; + details?: Record; + }; + const rejected = results.checks.filter( + (c: Check) => c.id === 'hosted-wire-rejected' + ); + // Once per distinct (code, message): the repeated -32602 is one check. + expect(rejected.map((c: Check) => c.details?.code)).toEqual([ + -32602, -32020 + ]); + expect(rejected[0]).toMatchObject({ + status: 'FAILURE', + details: { + status: 400, + method: 'initialize', + requestedVersion: REV_STATEFUL + } + }); + expect(rejected[1].details).toMatchObject({ + code: -32020, + requestedVersion: '2025-06-18' // no header: the body's version + }); + const wrong = results.checks.filter( + (c: Check) => c.id === 'hosted-wrong-revision' + ); + // Once per distinct (method, header version). + expect(wrong.map((c: Check) => c.errorMessage)).toEqual([ + `cell is served on ${REV_STATELESS}; client sent initialize`, + `cell is served on ${REV_STATELESS}; client sent initialize` + ]); + expect(wrong.map((c: Check) => c.details?.headerVersion)).toEqual([ + REV_STATEFUL, + null + ]); + const report = await fetch(`${base}/results/rej`).then((r) => r.json()); + const cellOf = (rev: string, name: string) => + report.columns + .find((c: { revision: string }) => c.revision === rev) + .cells.find((c: { scenario: string }) => c.scenario === name); + expect(cellOf(REV_STATELESS, 'tools_call').verdict).toBe('fail'); + + // On a dated revision initialize negotiates freely, but every later + // request must name the cell's revision in its header. + const stateful = `/s/rej/${REV_STATEFUL}/tools_call/mcp`; + await postMcp(stateful, initBody()).then((r) => r.text()); + const call = { + jsonrpc: '2.0', + id: 2, + method: 'tools/call', + params: { name: 'add_numbers', arguments: { a: 1, b: 2 } } + }; + await postMcp(stateful, call, { + 'mcp-protocol-version': '2025-06-18' + }).then((r) => r.text()); + const b = await fetch( + `${base}/results/rej/${REV_STATEFUL}/tools_call` + ).then((r) => r.json()); + expect( + b.checks.find((c: Check) => c.id === 'tool-add-numbers').status + ).toBe('SUCCESS'); + expect( + b.checks + .filter((c: Check) => c.id === 'hosted-wrong-revision') + .map((c: Check) => c.errorMessage) + ).toEqual([`cell is served on ${REV_STATEFUL}; client sent 2025-06-18`]); + expect(b.checks.some((c: Check) => c.id === 'hosted-wire-rejected')).toBe( + false + ); + // The scenario passed; the hosted FAILURE still decides the verdict. + expect(cellOf(REV_STATEFUL, 'tools_call')).toBeDefined(); + const report2 = await fetch(`${base}/results/rej`).then((r) => r.json()); + expect( + report2.columns[0].cells.find( + (c: { scenario: string }) => c.scenario === 'tools_call' + ).verdict + ).toBe('fail'); + + // A client that speaks the cell's revision records neither check, and a + // non-MCP path under the cell (the canary) is never judged. + const ok = `/s/rej/${REV_STATELESS}/json-schema-ref-no-deref/mcp`; + await postMcp(ok, statelessBody('tools/list'), statelessHeaders).then((r) => + r.text() + ); + await fetch( + `${base}/s/rej/${REV_STATELESS}/json-schema-ref-no-deref/canary/profile-schema.json` + ).then((r) => r.text()); + const clean = await fetch( + `${base}/results/rej/${REV_STATELESS}/json-schema-ref-no-deref` + ).then((r) => r.json()); + expect( + clean.checks.filter((c: Check) => c.id.startsWith('hosted-w')) + ).toEqual([]); + }); + + it('treats a rejected foreign-revision probe on a dated cell as negotiation', async () => { + type Check = { id: string; status: string; errorMessage?: string }; + const hostedChecks = (checks: Check[]) => + checks.filter((c) => c.id.startsWith('hosted-w')); + const resultsOf = (run: string, rev: string, name: string) => + fetch(`${base}/results/${run}/${rev}/${name}`).then((r) => r.json()); + const verdictOf = async (run: string, rev: string, name: string) => { + const report = await fetch(`${base}/results/${run}`).then((r) => + r.json() + ); + return report.columns + .find((c: { revision: string }) => c.revision === rev) + .cells.find((c: { scenario: string }) => c.scenario === name).verdict; + }; + const negotiatedInit = { + ...initBody(), + params: { ...initBody().params, protocolVersion: REV_STATEFUL } + }; + const call = { + jsonrpc: '2.0', + id: 2, + method: 'tools/call', + params: { name: 'add_numbers', arguments: { a: 1, b: 2 } } + }; + + // Felix's live case: Claude Code opens a dated cell with server/discover + // at 2026-07-28, the SDK transport turns the header away, and the client + // falls back to initialize at 2025-11-25 and carries on there. The + // rejected probe is negotiation: neither hosted check, verdict pass. + const url = `/s/neg/${REV_STATEFUL}/tools_call/mcp`; + const probe = await postMcp( + url, + statelessBody('server/discover'), + statelessHeaders + ); + expect(probe.status).toBe(400); + expect((await probe.json()).error.message).toContain( + 'Unsupported protocol version' + ); + const init = await postMcp(url, negotiatedInit); + expect(init.status).toBe(200); + expect(await init.text()).toContain(`"protocolVersion":"${REV_STATEFUL}"`); + const dated = { 'mcp-protocol-version': REV_STATEFUL }; + await postMcp( + url, + { jsonrpc: '2.0', id: 3, method: 'tools/list' }, + dated + ).then((r) => r.text()); + await postMcp(url, call, dated).then((r) => r.text()); + const negotiated = await resultsOf('neg', REV_STATEFUL, 'tools_call'); + expect(hostedChecks(negotiated.checks)).toEqual([]); + expect( + negotiated.checks.find((c: Check) => c.id === 'tool-add-numbers').status + ).toBe('SUCCESS'); + expect(await verdictOf('neg', REV_STATEFUL, 'tools_call')).toBe('pass'); + + // A client that negotiated and then carried on at 2026-07-28: the wire + // accepted the request, so it is a wrong revision, not negotiation. + const initCell = `/s/neg/${REV_STATEFUL}/initialize/mcp`; + await postMcp(initCell, negotiatedInit).then((r) => r.text()); + const carriedOn = await postMcp( + initCell, + { jsonrpc: '2.0', id: 3, method: 'tools/list' }, + statelessHeaders + ); + expect(carriedOn.status).toBe(200); + await carriedOn.text(); + const continued = await resultsOf('neg', REV_STATEFUL, 'initialize'); + expect(hostedChecks(continued.checks).map((c) => c.errorMessage)).toEqual([ + `cell is served on ${REV_STATEFUL}; client sent ${REV_STATELESS}` + ]); + expect(await verdictOf('neg', REV_STATEFUL, 'initialize')).toBe('fail'); + + // …and one whose 2026-07-28 tools/call the wire rejected after + // negotiation records no hosted check, but never called the tool. + const other = `/s/neg2/${REV_STATEFUL}/tools_call/mcp`; + await postMcp( + other, + statelessBody('server/discover'), + statelessHeaders + ).then((r) => r.text()); + await postMcp(other, negotiatedInit).then((r) => r.text()); + await postMcp( + other, + { jsonrpc: '2.0', id: 3, method: 'tools/list' }, + dated + ).then((r) => r.text()); + const rejectedCall = await postMcp( + other, + statelessBody('tools/call', call.params), + statelessHeaders + ); + expect(rejectedCall.status).toBe(400); + await rejectedCall.text(); + const never = await resultsOf('neg2', REV_STATEFUL, 'tools_call'); + expect(hostedChecks(never.checks)).toEqual([]); + expect( + never.checks.find((c: Check) => c.id === 'tool-add-numbers').status + ).toBe('FAILURE'); + expect(await verdictOf('neg2', REV_STATEFUL, 'tools_call')).toBe('fail'); + + // On the 2026-07-28 cell nothing is negotiation: a 2025-11-25 header the + // scenario accepted is a wrong revision, as before. + const stateless = `/s/neg/${REV_STATELESS}/http-standard-headers/mcp`; + const accepted = await postMcp( + stateless, + { jsonrpc: '2.0', id: 1, method: 'tools/list' }, + dated + ); + expect(accepted.status).toBe(200); + await accepted.text(); + const wrong = await resultsOf( + 'neg', + REV_STATELESS, + 'http-standard-headers' + ); + expect(hostedChecks(wrong.checks).map((c) => c.errorMessage)).toEqual([ + `cell is served on ${REV_STATELESS}; client sent ${REV_STATEFUL}` + ]); + }); + + it('HTML-escapes the run id in the results report', () => { + const html = renderResults( + { + runId: '">', + revision: REV_STATEFUL, + scenarioName: 'initialize' + }, + [] + ); + expect(html).not.toContain('' + }) + } + })); + return { + runId, + ...scope, + resultsUrl: `http://x/results/${runId}`, + mcpServers: Object.fromEntries( + cells.map((c) => [ + `${c.revision}/${c.scenario}`, + { type: 'http', url: c.url } + ]) + ), + cells + }; +} + +describe('hosted HTML', () => { + it('landing shows the static matrix with scoring, startability and steps, no run links', () => { + const html = renderLanding('http://x', matrix); + expect(html).toContain('Start a run'); + for (const r of matrix.revisions) expect(html).toContain(`${r}`); + expect(html).toContain('tools_call'); + expect(html).toContain('not startable: needs relay origin(s) [as]'); + expect(html).toContain( + 'n/a — introduced in 2025-06-18, removed in 2026-07-28' + ); + expect(html).toContain('steps (2)'); + expect(html).not.toContain('copy config'); + expect(html).not.toContain('href="/s/'); + // Exclusion reasons are request-independent but still escaped. + expect(html).toContain('excluded <here>'); + }); + + it('run page links every startable cell and embeds the config for the copy buttons', () => { + const config = configFor('run1'); + const html = renderConfig('http://x', matrix, config); + expect(html).toContain('open'); + expect(html).toContain( + 'results' + ); + expect(html).toContain('data-copy="2026-07-28/tools_call"'); + expect(html).toContain('data-copy="all"'); + expect(html).not.toContain('data-copy="2026-07-28/initialize"'); // n/a + // Embedded JSON cannot break out of its "}'); + expect(html).toContain('\\u003c/script>'); + expect(html).toContain('navigator.clipboard.writeText'); + }); + + it('column page filters to one revision', () => { + const html = renderConfig( + 'http://x', + matrix, + configFor('run2', { revision: '2025-11-25' }) + ); + expect(html).toContain( + '2025-11-25' + ); + expect(html).not.toContain('2026-07-28'); + expect(html).toContain('href="/s/run2/2025-11-25/initialize"'); + }); + + it('cell page shows the endpoint, env and steps with a copy button', () => { + const html = renderConfig( + 'http://x', + matrix, + configFor('run3', { revision: '2026-07-28', scenario: 'tools_call' }) + ); + expect(html).toContain( + '
http://x/s/run3/2026-07-28/tools_call/mcp
' + ); + expect(html).toContain( + 'MCP_CONFORMANCE_PROTOCOL_VERSION="2026-07-28"' + ); + expect(html).toContain('

Steps

'); + expect(html).toContain('data-copy="2026-07-28/tools_call"'); + expect(html).toContain( + 'href="http://x/results/run3/2026-07-28/tools_call"' + ); + }); + + it('escapes request-derived values', () => { + const evil = '">'; + const html = renderConfig('http://x', matrix, configFor(evil)); + expect(html).not.toContain('' })).toBe( + '{"a":"\\u003c/script>\\u003cb>"}' + ); + }); +}); diff --git a/src/hosted/html.ts b/src/hosted/html.ts new file mode 100644 index 00000000..022c86a8 --- /dev/null +++ b/src/hosted/html.ts @@ -0,0 +1,500 @@ +/** + * HTML for the hosted server: the matrix (landing and run/column/cell config + * pages) and the per-cell check report. Everything interpolated goes through + * escapeHtml(); JSON embedded for the copy buttons goes through jsonForScript(). + */ + +import { ConformanceCheck, CheckStatus } from '../types'; +import type { HostedMatrix, MatrixCell } from './matrix'; +import type { CellConfig, CellStatus, RunConfig } from './server'; +import type { CellRef } from './session'; +import type { CellReport, RunReport, Verdict } from './report'; +import type { ClientIdentity } from './identity'; + +const VERDICT_STYLE: Record = { + pass: 'background:#d1fae5;color:#065f46', + fail: 'background:#fee2e2;color:#991b1b', + incomplete: 'background:#f3f4f6;color:#6b7280', + 'n/a': 'background:#f3f4f6;color:#9ca3af' +}; + +const STATUS_STYLE: Record = { + SUCCESS: 'background:#d1fae5;color:#065f46', + FAILURE: 'background:#fee2e2;color:#991b1b', + WARNING: 'background:#fef3c7;color:#92400e', + SKIPPED: 'background:#e5e7eb;color:#374151', + INFO: 'background:#dbeafe;color:#1e40af' +}; + +const SCORING_STYLE: Record = { + scored: 'background:#dbeafe;color:#1e40af', + not_scored: 'background:#ede9fe;color:#5b21b6', + unlisted: 'background:#f3f4f6;color:#374151', + 'n/a': 'background:#f3f4f6;color:#9ca3af' +}; + +const SCORING_LABEL: Record = { + scored: 'scored', + not_scored: 'not scored', + unlisted: 'not in the requirement set', + 'n/a': 'n/a' +}; + +const css = ` + body{font:14px/1.5 ui-sans-serif,system-ui,sans-serif;max-width:1100px; + margin:2rem auto;padding:0 1rem;color:#111} + code,pre{font:12px/1.4 ui-monospace,SFMono-Regular,Menlo,monospace} + pre{background:#f6f8fa;padding:.75rem;border-radius:6px;overflow:auto;margin:.4rem 0} + .pill{display:inline-block;padding:2px 8px;border-radius:10px; + font-size:11px;font-weight:600;white-space:nowrap} + .check{border:1px solid #e5e7eb;border-radius:6px;padding:.75rem; + margin:.5rem 0} + .check h3{margin:0 0 .25rem;font-size:14px} + details>summary{cursor:pointer;color:#6b7280;font-size:12px} + table{border-collapse:collapse;width:100%} + td,th{text-align:left;padding:.4rem .6rem;border-bottom:1px solid #eee; + vertical-align:top} + td.cell{min-width:14rem} + td.na{color:#9ca3af} + .muted{color:#6b7280;font-size:12px} + .crumbs{color:#6b7280;margin:0 0 1rem} + .crumbs a{margin-right:.25rem} + .actions{margin:.25rem 0} + button.copy{font:inherit;font-size:11px;padding:1px 8px;border:1px solid #d1d5db; + border-radius:10px;background:#fff;cursor:pointer} + button.copy:hover{background:#f3f4f6} + a{color:#2563eb} + h1 code,h2 code{font-size:inherit} +`; + +/** Escape a string for interpolation into HTML text or a quoted attribute. */ +export function escapeHtml(s: string): string { + return s.replace( + /[&<>"']/g, + (c) => + ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[ + c + ]! + ); +} +const esc = escapeHtml; + +/** JSON safe inside a `. */ +export function jsonForScript(value: unknown): string { + return JSON.stringify(value).replace(/ +${esc(title)} +${body}`; +} + +function scoringPill(cell: MatrixCell): string { + return `${SCORING_LABEL[cell.scoring]}`; +} + +function stepsDetails(cell: Pick): string { + if (!cell.steps) return ''; + return ( + `
steps (${cell.steps.length})` + + `
${esc(JSON.stringify(cell.steps, null, 1))}
` + ); +} + +interface TableOptions { + origin: string; + /** When set, startable cells link to their page and results. */ + runId?: string; + /** Column filter. */ + revision?: string; + /** Row filter. */ + scenario?: string; +} + +/** + * The matrix as a table: scenarios down, revisions across. Without a run id + * it is the static overview (what is scored, what can start, what steps a + * cell wants); with one, every startable cell links into that run. + */ +export function renderMatrixTable( + matrix: HostedMatrix, + opts: TableOptions +): string { + const revisions = matrix.revisions.filter( + (r) => opts.revision === undefined || r === opts.revision + ); + const rows = matrix.rows.filter( + (r) => opts.scenario === undefined || r.scenario === opts.scenario + ); + const head = + `scenario` + + revisions + .map((r) => + opts.runId + ? `${esc(r)}` + : `${esc(r)}` + ) + .join('') + + ''; + const body = rows + .map((row) => { + const cells = row.cells + .filter((c) => revisions.includes(c.revision)) + .map((c) => renderMatrixCell(c, opts)) + .join(''); + return ( + `${esc(row.scenario)}` + + `
${esc(row.description)}
${cells}` + ); + }) + .join(''); + return `${head}${body}
`; +} + +function renderMatrixCell(cell: MatrixCell, opts: TableOptions): string { + if (cell.scoring === 'n/a') { + return `n/a — ${esc(cell.reason ?? '')}`; + } + const key = `${cell.revision}/${cell.scenario}`; + let lines = `
${scoringPill(cell)}
`; + if (cell.scoring !== 'scored' && cell.reason) { + lines += `
${esc(cell.reason)}
`; + } + if (!cell.startable) { + lines += `
not startable: ${esc(cell.startReason ?? '')}
`; + } else if (opts.runId) { + const base = `/s/${esc(opts.runId)}/${esc(key)}`; + lines += + `
open · ` + + `results · ` + + `
`; + } + lines += stepsDetails(cell); + return `${lines}`; +} + +export function renderLanding(origin: string, matrix: HostedMatrix): string { + const startable = matrix.cells().filter((c) => c.startable).length; + return page( + 'MCP Conformance — hosted', + `

MCP Conformance — hosted

+

Client conformance as a service. One run exercises the whole matrix below: +every client scenario at every specification revision that ships a +requirement set (${matrix.revisions.map((r) => `${esc(r)}`).join(', ')}). +Each cell is its own MCP server speaking that revision's wire, at +${esc(origin)}/s/<run-id>/<revision>/<scenario> +(plus the scenario's MCP path); results mirror the shape under +/results/<run-id>.

+

Start a run — mints a run id and shows this matrix +with a link and a copyable config per cell. Cells are created lazily on first +request; cells that show steps tell a generic client what to do +(MCP_CONFORMANCE_CONTEXT.steps).

+

${matrix.rows.length} scenarios × ${matrix.revisions.length} revisions, +${startable} startable cells here. JSON.

+${renderMatrixTable(matrix, { origin })}` + ); +} + +function crumbs(config: RunConfig): string { + const parts = [ + `matrix`, + `run ${esc(config.runId)}` + ]; + if (config.revision) { + parts.push( + `${esc(config.revision)}` + ); + } + if (config.scenario) parts.push(`${esc(config.scenario)}`); + return `

${parts.join(' › ')} · results

`; +} + +/** + * The copy-to-clipboard script. The config is embedded as JSON in a + * `; + +function envPre(cell: CellConfig): string { + const lines = Object.entries(cell.env).map( + ([k, v]) => `${k}=${JSON.stringify(v)}` + ); + return `
${esc(lines.join('\n'))}
`; +} + +/** Config page for a run, a column or a cell. */ +export function renderConfig( + origin: string, + matrix: HostedMatrix, + config: RunConfig +): string { + const title = config.scenario + ? `${config.scenario} @ ${config.revision} — run ${config.runId}` + : config.revision + ? `run ${config.runId} @ ${config.revision}` + : `run ${config.runId}`; + const embedded = ``; + + let body: string; + if (config.scenario && config.revision) { + const cell = config.cells[0]; + const key = `${config.revision}/${config.scenario}`; + const row = matrix.rows.find((r) => r.scenario === config.scenario); + body = `

${esc(config.scenario)} @ ${esc( + config.revision + )}

+${crumbs(config)} +

${esc(row?.description ?? '')}

+

${scoringPill(matrix.cell(config.scenario, config.revision)!)}${ + cell.reason ? ` ${esc(cell.reason)}` : '' + }

+

MCP endpoint

+
${esc(cell.url)}
+
+— an mcpServers entry plus the env the CLI runner would set
+

Environment

+${envPre(cell)} +${ + cell.steps + ? `

Steps

What a generic client should do here (also in MCP_CONFORMANCE_CONTEXT.steps).

${esc(
+        JSON.stringify(cell.steps, null, 1)
+      )}
` + : '' +} +

results for this cell

`; + } else { + const scope = config.revision + ? `revision ${esc(config.revision)}` + : 'every revision'; + body = `

run ${esc(config.runId)}${ + config.revision ? ` @ ${esc(config.revision)}` : '' + }

+${crumbs(config)} +

${config.cells.length} startable cell${config.cells.length === 1 ? '' : 's'} at ${scope}. +Point your client at a cell's MCP URL (open it for the env the CLI runner +would set), then read the results. +

+${renderMatrixTable(matrix, { + origin, + runId: config.runId, + revision: config.revision +})}`; + } + return page(title, `${body}\n${embedded}\n${copyScript}`); +} + +/** One line saying where the cell stands, for the cell results page. */ +function statusLine(status: CellStatus): string { + const pill = `${status.verdict}`; + const scoring = `${SCORING_LABEL[status.scoring]}`; + let note = ''; + if (status.verdict === 'n/a') { + note = `the scenario does not apply to this revision: ${esc(status.reason ?? '')}`; + } else if (status.startable === false) { + note = `not startable here: ${esc(status.startReason ?? '')}`; + } else if (status.verdict === 'incomplete') { + note = 'nothing recorded yet — point the client at the MCP endpoint'; + } else if (status.reason) { + note = esc(status.reason); + } + return `

${pill} ${scoring}${note ? ` — ${note}` : ''}

`; +} + +export function renderResults( + ref: CellRef, + checks: ConformanceCheck[], + status?: CellStatus +): string { + const items = checks + .map((c) => { + const pill = `${c.status}`; + const refs = (c.specReferences ?? []) + .map((r) => + r.url + ? `${esc(r.id)}` + : `${esc(r.id)}` + ) + .join(' · '); + const details = + c.details || c.errorMessage + ? `
details
${esc(
+              JSON.stringify(
+                { errorMessage: c.errorMessage, ...c.details },
+                null,
+                2
+              )
+            )}
` + : ''; + return `

${pill} ${esc(c.id)} — ${esc( + c.name + )}

${esc(c.description)}

${refs}

${details}
`; + }) + .join(''); + const passed = checks.filter((c) => c.status === 'SUCCESS').length; + const failed = checks.filter((c) => c.status === 'FAILURE').length; + return page( + `${ref.scenarioName} @ ${ref.revision} — ${ref.runId}`, + `

${esc(ref.scenarioName)} @ ${esc(ref.revision)}

+

run ${esc( + ref.runId + )} › ${esc(ref.revision)} › ${esc(ref.scenarioName)} · config

+${status ? statusLine(status) : ''}

${passed} passed, ${failed} failed, ${checks.length} total

${items}` + ); +} + +function identityLine(identities: ClientIdentity[]): string { + if (!identities.length) return 'no client seen yet'; + return identities + .map((i) => { + const who = i.name + ? `${esc(i.name)}${i.version ? ` ${esc(i.version)}` : ''}` + : 'unnamed client'; + const proto = i.protocolVersions.length + ? ` · protocol ${i.protocolVersions + .map((v) => `${esc(v)}`) + .join(', ')}` + : ''; + const ua = i.userAgent + ? ` (${esc( + i.userAgent.length > 40 + ? i.userAgent.slice(0, 40) + '…' + : i.userAgent + )})` + : ''; + return `${who}${proto}${ua}`; + }) + .join('
'); +} + +function verdictCell(cell: CellReport): string { + if (cell.verdict === 'n/a') { + return `n/a — ${esc(cell.reason ?? '')}`; + } + const pill = `${cell.verdict}`; + let lines = `
${pill} ${scoringPillFor(cell)}
`; + if (cell.summary) { + const s = cell.summary; + lines += `
${s.passed} passed, ${s.failed} failed${ + s.warnings ? `, ${s.warnings} warning${s.warnings === 1 ? '' : 's'}` : '' + }, ${s.total} total
`; + } else if (!cell.startable) { + lines += `
not startable: ${esc(cell.startReason ?? '')}
`; + } else { + lines += `
nothing recorded
`; + } + return `${lines}`; +} + +function scoringPillFor(cell: CellReport): string { + return `${SCORING_LABEL[cell.scoring]}`; +} + +/** Report page for a run or one of its columns. */ +export function renderReport( + origin: string, + matrix: HostedMatrix, + report: RunReport +): string { + const title = report.revision + ? `results — run ${report.runId} @ ${report.revision}` + : `results — run ${report.runId}`; + const head = + `scenario` + + report.columns + .map( + (col) => + `${esc( + col.revision + )}
${col.scored.passed} of ${col.scored.total} scored (${col.scored.startable} startable here)
` + + `
${identityLine(col.identities)}
` + ) + .join('') + + ''; + const rows = matrix.rows + .map((row) => { + const cells = report.columns + .map((col) => col.cells.find((c) => c.scenario === row.scenario)!) + .map(verdictCell) + .join(''); + return `${esc(row.scenario)}${cells}`; + }) + .join(''); + const notScored = report.columns + .map((col) => { + if (!col.notScored.length) return ''; + const items = col.notScored + .map( + (c) => + `
  • ${esc(c.scenario)} ${c.verdict} ${esc( + SCORING_LABEL[c.scoring] + )}${c.reason ? ` — ${esc(c.reason)}` : ''} · checks
    • ` + ) + .join(''); + return `

    ${esc(col.revision)}: run but not scored

      ${items}
    `; + }) + .join(''); + const crumbs = [ + `matrix`, + `run ${esc(report.runId)}` + ]; + if (report.revision) crumbs.push(`${esc(report.revision)}`); + crumbs.push( + `config` + ); + return page( + title, + `

    results — run ${esc(report.runId)}${ + report.revision ? ` @ ${esc(report.revision)}` : '' + }

    +

    ${crumbs.join(' › ')}

    +

    Client: ${identityLine(report.identities)}

    +

    A cell passes when checks were recorded and none is a FAILURE; +X of N scored counts passes among every cell the revision's requirement +set scores (N is the set's count; the cells this deployment can start are +given alongside). Not-scored and unlisted cells are listed below the table. +JSON.

    +${head}${rows}
    +${notScored}` + ); +} diff --git a/src/hosted/identity.test.ts b/src/hosted/identity.test.ts new file mode 100644 index 00000000..d8741974 --- /dev/null +++ b/src/hosted/identity.test.ts @@ -0,0 +1,168 @@ +import { describe, it, expect } from 'vitest'; +import { + addProtocolVersion, + identitiesIn, + identityCheck, + identityChecksIn, + identityFrom, + identityOf, + mergeIdentities +} from './identity'; +import type { CapturedResponse } from './wire'; + +const ok = ( + body?: object, + contentType = 'application/json' +): CapturedResponse => + ({ + status: 200, + contentType, + ...(body && { body: JSON.stringify(body) }) + }) as CapturedResponse; + +describe('client identity capture', () => { + const init = JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2025-06-18', + clientInfo: { name: 'sdk-client', version: '1.2.3' }, + capabilities: {} + } + }); + + it('reads initialize params and the version the server answered with', () => { + // JSON response: the negotiated version is result.protocolVersion, not + // what the client asked for. + expect( + identityFrom( + { 'user-agent': 'node' }, + init, + ok({ jsonrpc: '2.0', id: 1, result: { protocolVersion: '2025-11-25' } }) + ) + ).toEqual({ + name: 'sdk-client', + version: '1.2.3', + protocolVersion: '2025-11-25', + userAgent: 'node' + }); + // The SDK transport answers as SSE. + const sse: CapturedResponse = { + status: 200, + contentType: 'text/event-stream', + body: 'event: message\ndata: {"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-03-26","capabilities":{}}}\n\n' + }; + expect(identityFrom({}, init, sse)).toEqual({ + name: 'sdk-client', + version: '1.2.3', + protocolVersion: '2025-03-26' + }); + // No usable response body: the requested version is the best we know. + expect(identityFrom({}, init, { status: 200 })).toMatchObject({ + protocolVersion: '2025-06-18' + }); + }); + + it('reads per-request _meta on the 2026-07-28 wire, the accepted header being the version', () => { + const meta = { + 'io.modelcontextprotocol/protocolVersion': '2026-07-28', + 'io.modelcontextprotocol/clientInfo': { name: 'stateless', version: '9' }, + 'io.modelcontextprotocol/clientCapabilities': {} + }; + const body = (params: object) => + JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/list', params }); + expect( + identityFrom( + { 'mcp-protocol-version': '2026-07-28' }, + body({ _meta: meta }), + ok() + ) + ).toEqual({ + name: 'stateless', + version: '9', + protocolVersion: '2026-07-28' + }); + // clientInfo is a SHOULD: version from _meta, no name. + const noInfo = Object.fromEntries( + Object.entries(meta).filter( + ([k]) => k !== 'io.modelcontextprotocol/clientInfo' + ) + ); + expect(identityFrom({}, body({ _meta: noInfo }), ok())).toEqual({ + protocolVersion: '2026-07-28' + }); + // Batch: the first member speaks for the client. + expect( + identityFrom( + {}, + JSON.stringify([JSON.parse(body({ _meta: meta })), { jsonrpc: '2.0' }]), + ok() + ) + ).toMatchObject({ name: 'stateless' }); + }); + + it('records nothing from a rejected request or a header-only stateful request', () => { + const call = JSON.stringify({ + jsonrpc: '2.0', + id: 2, + method: 'tools/call', + params: { name: 'x' } + }); + // A later stateful request repeats what initialize established. + expect( + identityFrom({ 'mcp-protocol-version': '2025-11-25' }, call, ok()) + ).toBeUndefined(); + // Turned away: says nothing about who the client is. + expect(identityFrom({}, init, { status: 400 })).toBeUndefined(); + expect(identityFrom({ 'user-agent': 'curl' }, call, ok())).toBeUndefined(); + expect(identityFrom({}, 'not json', ok())).toBeUndefined(); + expect(identityFrom({}, undefined, ok())).toBeUndefined(); + }); + + it('is one INFO check per client, pooling the protocol versions it spoke', () => { + const a = identityCheck( + identityOf({ name: 'a', version: '1', protocolVersion: 'v1' }) + ); + expect(a).toMatchObject({ + id: 'hosted-client-identity', + status: 'INFO', + details: { name: 'a', version: '1', protocolVersions: ['v1'] } + }); + expect(a.description).toContain('a 1 speaking protocol v1'); + expect(addProtocolVersion(a, 'v2')).toBe(true); + expect(addProtocolVersion(a, 'v2')).toBe(false); + expect(a.details?.protocolVersions).toEqual(['v1', 'v2']); + expect(a.description).toContain('speaking protocol v1, v2'); + + // Rows from two processes, each with its own view of the same client + // (and a different User-Agent — not part of who the client is). + const fromB = identityCheck( + identityOf({ + name: 'a', + version: '1', + protocolVersion: 'v3', + userAgent: 'ua' + }) + ); + const anon = identityCheck(identityOf({ protocolVersion: 'v1' })); + const merged = identityChecksIn([a, fromB, anon, { ...anon }]); + expect(merged).toHaveLength(2); + expect(merged[0].details).toEqual({ + name: 'a', + version: '1', + protocolVersions: ['v1', 'v2', 'v3'] + }); + expect(identitiesIn([a, fromB, anon])).toEqual([ + { name: 'a', version: '1', protocolVersions: ['v1', 'v2', 'v3'] }, + { protocolVersions: ['v1'] } + ]); + + const into = new Map(); + mergeIdentities(into, identitiesIn([a])); + mergeIdentities(into, identitiesIn([fromB])); + expect(Array.from(into.values())).toEqual([ + { name: 'a', version: '1', protocolVersions: ['v1', 'v2', 'v3'] } + ]); + }); +}); diff --git a/src/hosted/identity.ts b/src/hosted/identity.ts new file mode 100644 index 00000000..b500c7e1 --- /dev/null +++ b/src/hosted/identity.ts @@ -0,0 +1,222 @@ +/** + * Who is talking to a cell. The hosted report's header names the client and + * the protocol version(s) it negotiated, read off accepted exchanges the way + * the mock servers see them: on the stateful wire the `initialize` request's + * `params.clientInfo` with the version the server answered in + * `result.protocolVersion`; on the stateless wire every request's + * `_meta['io.modelcontextprotocol/clientInfo']` with the accepted request's + * `MCP-Protocol-Version` header. A request the cell turned away (4xx) says + * nothing about who the client is, and a later stateful request that only + * carries the header repeats what `initialize` already established, so + * neither is recorded. One client (name, version) is one identity, however + * many protocol versions it spoke. + */ + +import type { IncomingHttpHeaders } from 'http'; +import type { ConformanceCheck } from '../types'; +import { jsonRpcMessages, type CapturedResponse } from './wire'; + +export const IDENTITY_CHECK_ID = 'hosted-client-identity'; + +/** One client as the report shows it. */ +export interface ClientIdentity { + name?: string; + version?: string; + /** Protocol versions negotiated with this client, first seen first. */ + protocolVersions: string[]; + userAgent?: string; +} + +/** What one accepted exchange said about the client. */ +export interface IdentityObservation { + name?: string; + version?: string; + protocolVersion?: string; + userAgent?: string; +} + +const META_CLIENT_INFO = 'io.modelcontextprotocol/clientInfo'; +const META_PROTOCOL_VERSION = 'io.modelcontextprotocol/protocolVersion'; + +function asRecord(v: unknown): Record | undefined { + return typeof v === 'object' && v !== null && !Array.isArray(v) + ? (v as Record) + : undefined; +} + +function str(v: unknown): string | undefined { + return typeof v === 'string' && v.length > 0 ? v : undefined; +} + +/** + * The identity one accepted exchange establishes, or undefined when it + * establishes nothing new: the request was turned away, carries neither + * `initialize` params nor `_meta`, or is not JSON. + */ +export function identityFrom( + headers: IncomingHttpHeaders, + body: Buffer | string | undefined, + response: CapturedResponse +): IdentityObservation | undefined { + if (response.status >= 400 || body === undefined) return undefined; + const header = str(headers['mcp-protocol-version']); + const userAgent = str(headers['user-agent']); + + let message: Record | undefined; + try { + const parsed: unknown = JSON.parse(body.toString()); + // A JSON-RPC batch: any member carries the same identity. + message = asRecord(Array.isArray(parsed) ? parsed[0] : parsed); + } catch { + return undefined; + } + const params = asRecord(message?.params); + const meta = asRecord(params?._meta); + + let info: Record | undefined; + let protocolVersion: string | undefined; + if (meta && (meta[META_CLIENT_INFO] || meta[META_PROTOCOL_VERSION])) { + // Stateless wire: the header the cell accepted is the negotiated version. + info = asRecord(meta[META_CLIENT_INFO]); + protocolVersion = header ?? str(meta[META_PROTOCOL_VERSION]); + } else if (message?.method === 'initialize') { + // Stateful wire: the version the server answered with is the one + // negotiated — the SDK transport may answer as SSE, hence both parsers. + info = asRecord(params?.clientInfo); + protocolVersion = + negotiatedVersion(response) ?? str(params?.protocolVersion) ?? header; + } else { + return undefined; + } + + return { + ...(str(info?.name) && { name: str(info?.name) }), + ...(str(info?.version) && { version: str(info?.version) }), + ...(protocolVersion && { protocolVersion }), + ...(userAgent && { userAgent }) + }; +} + +/** `result.protocolVersion` of the initialize response, JSON or SSE body. */ +function negotiatedVersion(response: CapturedResponse): string | undefined { + for (const m of jsonRpcMessages(response.body, response.contentType)) { + const v = str(asRecord(m.result)?.protocolVersion); + if (v) return v; + } + return undefined; +} + +/** One client is one (name, version); the versions it spoke accumulate. */ +export function identityKey( + identity: Pick +): string { + return JSON.stringify([identity.name, identity.version]); +} + +function describe(identity: ClientIdentity): string { + const who = identity.name + ? `${identity.name}${identity.version ? ` ${identity.version}` : ''}` + : 'unnamed client'; + const spoke = identity.protocolVersions.length + ? ` speaking protocol ${identity.protocolVersions.join(', ')}` + : ''; + return `${who}${spoke} — as the client under test identified itself to this cell`; +} + +export function identityCheck(identity: ClientIdentity): ConformanceCheck { + return { + id: IDENTITY_CHECK_ID, + name: 'Client identity', + description: describe(identity), + status: 'INFO', + timestamp: new Date().toISOString(), + details: { ...identity, protocolVersions: [...identity.protocolVersions] } + }; +} + +/** The identity one observation establishes on its own. */ +export function identityOf(observation: IdentityObservation): ClientIdentity { + const { protocolVersion, ...rest } = observation; + return { + ...rest, + protocolVersions: protocolVersion ? [protocolVersion] : [] + }; +} + +/** + * Fold a protocol version into a recorded identity check: appended to its + * `protocolVersions` (once) and reflected in its description. Returns + * whether the check changed. + */ +export function addProtocolVersion( + check: ConformanceCheck, + protocolVersion: string | undefined +): boolean { + const identity = identityIn(check); + if (!identity || !protocolVersion) return false; + if (identity.protocolVersions.includes(protocolVersion)) return false; + identity.protocolVersions.push(protocolVersion); + check.details = { ...identity }; + check.description = describe(identity); + return true; +} + +function identityIn(check: ConformanceCheck): ClientIdentity | undefined { + if (check.id !== IDENTITY_CHECK_ID || !check.details) return undefined; + const d = check.details as Partial; + return { + ...(d.name !== undefined && { name: d.name }), + ...(d.version !== undefined && { version: d.version }), + ...(d.userAgent !== undefined && { userAgent: d.userAgent }), + protocolVersions: Array.isArray(d.protocolVersions) + ? [...d.protocolVersions] + : [] + }; +} + +/** + * The identity checks in a list collapsed to one per client, in order of + * first appearance, each carrying every protocol version any of them saw. + * Rows from several processes each hold their own view of a client; this + * is what the results view shows instead. + */ +export function identityChecksIn( + checks: ConformanceCheck[] +): ConformanceCheck[] { + const byKey = new Map(); + for (const c of checks) { + const identity = identityIn(c); + if (!identity) continue; + const key = identityKey(identity); + const kept = byKey.get(key); + if (!kept) { + byKey.set(key, { ...c, details: { ...identity } }); + continue; + } + for (const v of identity.protocolVersions) addProtocolVersion(kept, v); + } + return Array.from(byKey.values()); +} + +/** The clients a check list names, one per (name, version). */ +export function identitiesIn(checks: ConformanceCheck[]): ClientIdentity[] { + return identityChecksIn(checks).map((c) => identityIn(c) as ClientIdentity); +} + +/** Merge `seen` into `into` by client, accumulating protocol versions. */ +export function mergeIdentities( + into: Map, + seen: ClientIdentity[] +): void { + for (const i of seen) { + const key = identityKey(i); + const kept = into.get(key); + if (!kept) { + into.set(key, { ...i, protocolVersions: [...i.protocolVersions] }); + continue; + } + for (const v of i.protocolVersions) { + if (!kept.protocolVersions.includes(v)) kept.protocolVersions.push(v); + } + } +} diff --git a/src/hosted/index.ts b/src/hosted/index.ts new file mode 100644 index 00000000..44fb82ad --- /dev/null +++ b/src/hosted/index.ts @@ -0,0 +1,67 @@ +import { createHostedApp } from './server'; +import { AuxOriginRole } from '../types'; + +export { createHostedApp } from './server'; +export { buildMatrix } from './matrix'; + +export interface HostedCliOptions { + port: number; + publicOrigin?: string; + ttlMs?: number; + auxOrigins?: Partial>; + relaySecret?: string; +} + +export async function runHostedServer(opts: HostedCliOptions): Promise { + const auxOrigins = opts.auxOrigins ?? {}; + const haveAux = (Object.keys(auxOrigins) as AuxOriginRole[]).filter( + (r) => auxOrigins[r] + ); + if (haveAux.length && !opts.relaySecret) { + console.error( + 'Refusing to start with --as-origin but no --relay-secret: the /__aux ' + + 'backchannel would be open to direct check-forgery. Set ' + + '--relay-secret (or CONFORMANCE_RELAY_SECRET) to the same value the ' + + 'relay sends.' + ); + process.exit(1); + } + + const { app, sessions, matrix } = createHostedApp({ + publicOrigin: opts.publicOrigin, + ttlMs: opts.ttlMs, + auxOrigins, + relaySecret: opts.relaySecret + }); + + const server = app.listen(opts.port, () => { + const origin = opts.publicOrigin ?? `http://localhost:${opts.port}`; + const startable = matrix.cells().filter((c) => c.startable).length; + console.error(`MCP conformance hosted server listening on ${origin}`); + console.error( + ` ${matrix.rows.length} scenarios × ${matrix.revisions.length} revisions ` + + `(${matrix.revisions.join(', ')}); ${startable} startable cells under ` + + `${origin}/s///` + ); + console.error(` GET ${origin}/s mints a run id`); + if (haveAux.length) { + for (const r of haveAux) { + console.error(` aux[${r}] relay origin: ${auxOrigins[r]}`); + } + } else { + console.error( + ' (auth/* cells not startable — pass --as-origin to enable them)' + ); + } + }); + + const shutdown = async () => { + console.error('\nshutting down...'); + await sessions.close(); + server.close(() => process.exit(0)); + }; + process.on('SIGINT', shutdown); + process.on('SIGTERM', shutdown); + + await new Promise(() => {}); +} diff --git a/src/hosted/matrix.test.ts b/src/hosted/matrix.test.ts new file mode 100644 index 00000000..9fa7e64e --- /dev/null +++ b/src/hosted/matrix.test.ts @@ -0,0 +1,98 @@ +import { describe, it, expect } from 'vitest'; +import { buildMatrix, notApplicableReason, scoringFor } from './matrix'; +import { loadRequirements } from '../requirements'; +import { getScenario } from '../scenarios'; + +describe('hosted matrix', () => { + const requirements = loadRequirements('2026-07-28'); + + it("classifies cells against the revision's requirement set", () => { + const at = (name: string) => scoringFor(getScenario(name)!, requirements); + expect(at('tools_call')).toEqual({ scoring: 'scored' }); + expect(at('auth/dpop')).toEqual({ + scoring: 'not_scored', + reason: 'extension' + }); + expect(at('initialize')).toEqual({ + scoring: 'n/a', + reason: 'introduced in 2025-06-18, removed in 2026-07-28' + }); + // Off-timeline extension the set does not list. + expect(at('sep-2640-client-no-prefetch')).toEqual({ + scoring: 'n/a', + reason: 'extension, not on the spec timeline' + }); + // Applicable but absent from the frozen set. + expect( + scoringFor( + { name: 'brand-new-scenario', source: { introducedIn: '2025-11-25' } }, + requirements + ) + ).toEqual({ scoring: 'unlisted', reason: 'not in the requirement set' }); + }); + + it('carries the not_scored note into the reason', () => { + const cell = scoringFor( + { name: 'x', source: { introducedIn: '2025-11-25' } }, + { + revision: '2026-07-28', + server: [], + client: [], + notScored: [ + { scenario: 'x', leg: 'client', reason: 'pending', note: 'why' } + ] + } + ); + expect(cell).toEqual({ scoring: 'not_scored', reason: 'pending: why' }); + }); + + it('words applicability like the CLI', () => { + expect(notApplicableReason({ introducedIn: '2026-07-28' })).toBe( + 'introduced in 2026-07-28' + ); + expect( + notApplicableReason({ + extensionId: 'io.modelcontextprotocol/skills' + }) + ).toBe('extension, not on the spec timeline'); + }); + + it('derives startability from handlers, relay origins and exclusions', () => { + const bare = buildMatrix(); + expect(bare.revisions).toEqual(['2025-11-25', '2026-07-28']); + expect(bare.rows.map((r) => r.scenario)).toContain('auth/basic-cimd'); + expect(bare.cell('auth/basic-cimd', '2025-11-25')).toMatchObject({ + scoring: 'scored', + startable: false, + startReason: 'needs relay origin(s) [as]' + }); + expect(bare.cell('auth/scope-step-up', '2025-11-25')).toMatchObject({ + startable: false, + startReason: 'not converted for hosting yet' + }); + // n/a cells are never startable and carry no start reason. + expect(bare.cell('initialize', '2026-07-28')).toMatchObject({ + scoring: 'n/a', + startable: false + }); + expect(bare.cell('initialize', '2026-07-28')!.startReason).toBeUndefined(); + expect(bare.cell('tools_call', '2026-07-28')).toMatchObject({ + startable: true, + mcpPath: '/mcp' + }); + expect(bare.cell('tools_call', '2026-07-28')!.steps).toBeDefined(); + + const withRelay = buildMatrix({ + auxOrigins: { as: 'https://as.example' }, + exclude: { tools_call: 'nope' } + }); + expect(withRelay.cell('auth/basic-cimd', '2025-11-25')!.startable).toBe( + true + ); + expect(withRelay.cell('tools_call', '2026-07-28')).toMatchObject({ + startable: false, + startReason: 'nope' + }); + expect(bare.cell('nope', '2026-07-28')).toBeUndefined(); + }); +}); diff --git a/src/hosted/matrix.ts b/src/hosted/matrix.ts new file mode 100644 index 00000000..a3291d8c --- /dev/null +++ b/src/hosted/matrix.ts @@ -0,0 +1,185 @@ +/** + * The hosted matrix: every registered client scenario (rows) against every + * specification revision that ships a requirement set (columns). + * + * A cell says two independent things about (scenario, revision): + * + * - scoring — what the revision's requirement set makes of the scenario: + * `scored` (in its `client:` list), `not_scored` (listed but never counted, + * with the set's reason), `unlisted` (applies to the revision but the + * frozen set predates it), or `n/a` (does not apply: introduced later, + * removed earlier, or an extension the set does not carry). The CLI + * answers the same question with `--requirements` / `--spec-version`. + * - startable — whether this deployment can mount the cell: the scenario + * has been converted to `handler()` / `authHandlers()`, every relay + * origin it needs is configured, and the deployment has not excluded it. + * + * `n/a` cells are never mounted; the other three are, when startable. + */ + +import { isScenarioApplicableAt, scenarios } from '../scenarios'; +import { + listRequirementRevisions, + loadRequirements, + type RequirementSet +} from '../requirements'; +import { + AuthHandlerScenario, + type AuxOriginRole, + type Scenario, + type ScenarioSource, + type SpecVersion +} from '../types'; +import type { Step } from '../steps'; + +export type CellScoring = 'scored' | 'not_scored' | 'unlisted' | 'n/a'; + +/** + * Every cell's MCP endpoint is the cell URL plus this. A scenario that + * serves MCP at its handler root (`mcpPath` '') is reached at `/mcp` + * too: the hosted server rewrites that suffix to `/` before dispatch, so + * clients see one URL shape across the matrix. + */ +export const MCP_PATH = '/mcp'; + +/** The public MCP sub-path of a scenario's cells: its mcpPath, or /mcp. */ +export function publicMcpPath(scenario: Pick): string { + return scenario.mcpPath || MCP_PATH; +} + +export interface MatrixCell { + scenario: string; + revision: SpecVersion; + scoring: CellScoring; + /** + * For `not_scored`: the requirement set's reason (and note). For `n/a`: why + * the scenario does not apply to the revision. For `unlisted`: a fixed + * explanation. Absent for `scored`. + */ + reason?: string; + /** False when this deployment cannot mount the cell; see `startReason`. */ + startable: boolean; + /** Why the cell cannot be started. Absent when startable or `n/a`. */ + startReason?: string; + /** The scenario's declarative client choreography, when it has one. */ + steps?: readonly Step[]; + /** Sub-path of the MCP endpoint under the cell URL; always ends in /mcp. */ + mcpPath: string; +} + +export interface MatrixRow { + scenario: string; + description: string; + source: ScenarioSource; + cells: MatrixCell[]; +} + +export interface HostedMatrix { + /** Columns: revisions with a requirement set, in timeline order. */ + revisions: SpecVersion[]; + /** Rows: every registered client scenario, in registry order. */ + rows: MatrixRow[]; + cell(scenario: string, revision: string): MatrixCell | undefined; + cells(): MatrixCell[]; +} + +export interface MatrixOptions { + /** Relay origins this deployment has, keyed by role. */ + auxOrigins?: Partial>; + /** Scenario name → why this deployment refuses to mount it. */ + exclude?: Record; +} + +/** The CLI's wording for a scenario outside its applicability window. */ +export function notApplicableReason(source: ScenarioSource): string { + if ('introducedIn' in source) { + return ( + `introduced in ${source.introducedIn}` + + (source.removedIn !== undefined ? `, removed in ${source.removedIn}` : '') + ); + } + return 'extension, not on the spec timeline'; +} + +export function scoringFor( + scenario: Pick, + requirements: RequirementSet +): { scoring: CellScoring; reason?: string } { + if (requirements.client.includes(scenario.name)) return { scoring: 'scored' }; + const entry = requirements.notScored.find( + (e) => e.scenario === scenario.name && e.leg === 'client' + ); + if (entry) { + return { + scoring: 'not_scored', + reason: entry.note ? `${entry.reason}: ${entry.note}` : entry.reason + }; + } + if ( + isScenarioApplicableAt( + scenario.source, + requirements.revision as SpecVersion + ) + ) { + return { scoring: 'unlisted', reason: 'not in the requirement set' }; + } + return { scoring: 'n/a', reason: notApplicableReason(scenario.source) }; +} + +/** Whether this deployment can mount `scenario` at all, and if not, why. */ +export function startability( + scenario: Scenario, + opts: MatrixOptions +): { startable: true } | { startable: false; reason: string } { + if (scenario instanceof AuthHandlerScenario) { + const missing = scenario.auxRoles.filter((r) => !opts.auxOrigins?.[r]); + if (missing.length) { + return { + startable: false, + reason: `needs relay origin(s) [${missing.join(', ')}]` + }; + } + } else if (typeof scenario.handler !== 'function') { + return { startable: false, reason: 'not converted for hosting yet' }; + } + const excluded = opts.exclude?.[scenario.name]; + if (excluded) return { startable: false, reason: excluded }; + return { startable: true }; +} + +export function buildMatrix(opts: MatrixOptions = {}): HostedMatrix { + const revisions = listRequirementRevisions(); + const requirements = revisions.map((r) => loadRequirements(r)); + const rows: MatrixRow[] = Array.from(scenarios.values()).map((scenario) => { + const start = startability(scenario, opts); + const cells = requirements.map((req): MatrixCell => { + const { scoring, reason } = scoringFor(scenario, req); + const applicable = scoring !== 'n/a'; + return { + scenario: scenario.name, + revision: req.revision as SpecVersion, + scoring, + ...(reason !== undefined && { reason }), + startable: applicable && start.startable, + ...(applicable && !start.startable && { startReason: start.reason }), + ...(scenario.steps && { steps: scenario.steps }), + mcpPath: publicMcpPath(scenario) + }; + }); + return { + scenario: scenario.name, + description: scenario.description, + source: scenario.source, + cells + }; + }); + const byKey = new Map( + rows.flatMap((r) => r.cells.map((c) => [`${c.revision}/${c.scenario}`, c])) + ); + return { + revisions, + rows, + cell: (scenario, revision) => byKey.get(`${revision}/${scenario}`), + cells: () => rows.flatMap((r) => r.cells) + }; +} diff --git a/src/hosted/report.test.ts b/src/hosted/report.test.ts new file mode 100644 index 00000000..29bb23ae --- /dev/null +++ b/src/hosted/report.test.ts @@ -0,0 +1,139 @@ +import { describe, it, expect } from 'vitest'; +import { buildMatrix } from './matrix'; +import { buildReport, verdictFor } from './report'; +import { cellId, type CellRef } from './session'; +import { identityCheck, identityOf } from './identity'; +import type { ConformanceCheck } from '../types'; + +const check = (status: ConformanceCheck['status']): ConformanceCheck => ({ + id: 'c', + name: 'c', + description: '', + status, + timestamp: new Date().toISOString() +}); + +describe('verdicts', () => { + it('follows FAILURE only', () => { + expect(verdictFor({ scoring: 'scored' }, undefined)).toBe('incomplete'); + expect(verdictFor({ scoring: 'scored' }, [])).toBe('incomplete'); + expect(verdictFor({ scoring: 'scored' }, [check('SUCCESS')])).toBe('pass'); + expect( + verdictFor({ scoring: 'scored' }, [check('WARNING'), check('INFO')]) + ).toBe('pass'); + expect( + verdictFor({ scoring: 'not_scored' }, [ + check('SUCCESS'), + check('FAILURE') + ]) + ).toBe('fail'); + expect(verdictFor({ scoring: 'n/a' }, [check('FAILURE')])).toBe('n/a'); + // Judgement-added failures on a cell that recorded nothing. + expect(verdictFor({ scoring: 'scored' }, [check('FAILURE')], 0)).toBe( + 'incomplete' + ); + }); + + it("scores a column over the requirement set's cells and lists the rest apart", async () => { + const matrix = buildMatrix({ exclude: { 'sse-retry': 'x' } }); + const rev = '2025-11-25'; + const results = new Map([ + [ + `r/${rev}/tools_call`, + [ + check('SUCCESS'), + identityCheck(identityOf({ name: 'c1', protocolVersion: rev })) + ] + ], + [ + `r/${rev}/initialize`, + [ + check('FAILURE'), + // Same client, another negotiated version: one identity. + identityCheck( + identityOf({ name: 'c1', protocolVersion: '2025-06-18' }) + ) + ] + ], + [`r/2026-07-28/tools_call`, [check('SUCCESS')]], + [`r/${rev}/json-schema-2020-12-preservation`, [check('SUCCESS')]], // not_scored; not startable but exercised + [`r/${rev}/elicitation-sep1034-client-defaults`, []] // created, nothing recorded + ]); + const report = await buildReport(matrix, 'r', undefined, { + listCells: async () => + Array.from(results.keys()).map((id) => { + const [runId, revision, ...rest] = id.split('/'); + return { + runId, + revision: revision as CellRef['revision'], + scenarioName: rest.join('/') + }; + }), + results: async (id) => { + const checks = results.get(id); + return checks ? { checks, recorded: checks.length } : undefined; + }, + resultsUrl: (ref) => `http://x/results/${cellId(ref)}` + }); + + expect(report.columns.map((c) => c.revision)).toEqual([rev, '2026-07-28']); + const col = report.columns[0]; + // N is the yaml's count — every scored cell, startable here or not + // (auth/* cells are not, with no relay origin); the startable subset is + // reported alongside. + const scored = matrix + .cells() + .filter((c) => c.revision === rev && c.scoring === 'scored'); + const startable = scored.filter((c) => c.startable).length; + expect(startable).toBeLessThan(scored.length); + expect(col.scored).toEqual({ + passed: 1, + total: scored.length, + startable + }); + const by = (name: string) => col.cells.find((c) => c.scenario === name)!; + expect(by('tools_call')).toMatchObject({ + verdict: 'pass', + summary: { passed: 1, info: 1, total: 2 }, + identities: [{ name: 'c1', protocolVersions: [rev] }], + resultsUrl: `http://x/results/r/${rev}/tools_call` + }); + expect(by('initialize').verdict).toBe('fail'); + expect(by('elicitation-sep1034-client-defaults')).toMatchObject({ + verdict: 'incomplete', + summary: { total: 0 } + }); + expect(by('request-metadata')).toMatchObject({ verdict: 'n/a' }); + expect(by('request-metadata').summary).toBeUndefined(); + expect(by('sse-retry')).toMatchObject({ + verdict: 'incomplete', + startable: false, + startReason: 'x' + }); + // Exercised not_scored cell reported next to the score, not in it. + expect(col.notScored.map((c) => c.scenario)).toEqual([ + 'json-schema-2020-12-preservation' + ]); + expect(col.notScored[0].verdict).toBe('pass'); + // One line per client across the column and the run, versions pooled + // in row order (the initialize row precedes tools_call). + expect(col.identities).toEqual([ + { name: 'c1', protocolVersions: ['2025-06-18', rev] } + ]); + expect(report.identities).toEqual([ + { name: 'c1', protocolVersions: ['2025-06-18', rev] } + ]); + + const column = await buildReport(matrix, 'r', '2026-07-28', { + listCells: async () => [], + results: async () => undefined, + resultsUrl: () => '' + }); + expect(column.revision).toBe('2026-07-28'); + expect(column.columns).toHaveLength(1); + expect(column.columns[0].scored.passed).toBe(0); + expect( + column.columns[0].cells.find((c) => c.scenario === 'initialize')!.verdict + ).toBe('n/a'); + }); +}); diff --git a/src/hosted/report.ts b/src/hosted/report.ts new file mode 100644 index 00000000..a0a50437 --- /dev/null +++ b/src/hosted/report.ts @@ -0,0 +1,176 @@ +/** + * Verdicts for a run: the matrix with a result per cell. + * + * pass checks recorded, none FAILURE + * fail any FAILURE + * incomplete the cell exists but nothing was recorded, or it was never hit + * n/a the scenario does not apply to the revision + * + * Per column, "scored X of N" counts passes among every cell the revision's + * requirement set scores — N is the yaml's count, whether or not this + * deployment can start the cell — and says separately how many of those N + * are startable here; not_scored and unlisted cells are reported next to the + * score, never inside it. Only FAILURE decides a verdict — INFO checks such + * as the client identity the hosted layer records never do. + */ + +import type { ConformanceCheck } from '../types'; +import type { HostedMatrix, MatrixCell } from './matrix'; +import { cellId, type CellRef, type RunResults } from './session'; +import { identitiesIn, mergeIdentities, type ClientIdentity } from './identity'; + +export type Verdict = 'pass' | 'fail' | 'incomplete' | 'n/a'; + +export interface CheckSummary { + passed: number; + failed: number; + warnings: number; + info: number; + skipped: number; + total: number; +} + +export interface CellReport { + scenario: string; + revision: string; + scoring: MatrixCell['scoring']; + reason?: string; + startable: boolean; + startReason?: string; + verdict: Verdict; + /** Absent when the cell was never exercised or does not apply. */ + summary?: CheckSummary; + resultsUrl: string; + identities?: ClientIdentity[]; +} + +export interface ColumnReport { + revision: string; + /** + * Passes among the cells the requirement set scores, out of all of them + * (`total`, the yaml's count), with how many of those this deployment + * can start (`startable`). + */ + scored: { passed: number; total: number; startable: number }; + cells: CellReport[]; + /** The not_scored / unlisted cells that were exercised, with verdicts. */ + notScored: CellReport[]; + identities: ClientIdentity[]; +} + +export interface RunReport { + runId: string; + revision?: string; + columns: ColumnReport[]; + /** Every client identity seen anywhere in the run. */ + identities: ClientIdentity[]; +} + +export function summarize(checks: ConformanceCheck[]): CheckSummary { + const counts = { SUCCESS: 0, FAILURE: 0, WARNING: 0, SKIPPED: 0, INFO: 0 }; + for (const c of checks) counts[c.status]++; + return { + passed: counts.SUCCESS, + failed: counts.FAILURE, + warnings: counts.WARNING, + info: counts.INFO, + skipped: counts.SKIPPED, + total: checks.length + }; +} + +/** + * `recorded` is what the scenario itself observed; judgement may add + * "expected but never seen" failures to `checks`, which must not turn a cell + * nobody talked to into a `fail`. + */ +export function verdictFor( + cell: Pick, + checks: ConformanceCheck[] | undefined, + recorded: number = checks?.length ?? 0 +): Verdict { + if (cell.scoring === 'n/a') return 'n/a'; + if (!checks || recorded === 0) return 'incomplete'; + return checks.some((c) => c.status === 'FAILURE') ? 'fail' : 'pass'; +} + +export interface ReportSources { + /** Cells of the run that were exercised (in memory or in the store). */ + listCells(runId: string): Promise; + results( + id: string + ): Promise | undefined>; + resultsUrl(ref: CellRef): string; +} + +export async function buildReport( + matrix: HostedMatrix, + runId: string, + revision: string | undefined, + sources: ReportSources +): Promise { + const exercised = new Set( + (await sources.listCells(runId)).map((ref) => cellId(ref)) + ); + const columns: ColumnReport[] = []; + const allIdentities = new Map(); + + for (const rev of matrix.revisions) { + if (revision !== undefined && rev !== revision) continue; + const cells: CellReport[] = []; + const identities = new Map(); + for (const row of matrix.rows) { + const cell = matrix.cell(row.scenario, rev)!; + const ref: CellRef = { + runId, + revision: cell.revision, + scenarioName: cell.scenario + }; + const id = cellId(ref); + const results = + cell.scoring !== 'n/a' && exercised.has(id) + ? await sources.results(id) + : undefined; + const seen = results ? identitiesIn(results.checks) : []; + mergeIdentities(identities, seen); + mergeIdentities(allIdentities, seen); + cells.push({ + scenario: cell.scenario, + revision: cell.revision, + scoring: cell.scoring, + ...(cell.reason !== undefined && { reason: cell.reason }), + startable: cell.startable, + ...(cell.startReason !== undefined && { + startReason: cell.startReason + }), + verdict: verdictFor(cell, results?.checks, results?.recorded), + ...(results && { summary: summarize(results.checks) }), + resultsUrl: sources.resultsUrl(ref), + ...(seen.length && { identities: seen }) + }); + } + const scoredCells = cells.filter((c) => c.scoring === 'scored'); + columns.push({ + revision: rev, + scored: { + passed: scoredCells.filter((c) => c.verdict === 'pass').length, + total: scoredCells.length, + startable: scoredCells.filter((c) => c.startable).length + }, + cells, + notScored: cells.filter( + (c) => + (c.scoring === 'not_scored' || c.scoring === 'unlisted') && + c.summary !== undefined + ), + identities: Array.from(identities.values()) + }); + } + + return { + runId, + ...(revision !== undefined && { revision }), + columns, + identities: Array.from(allIdentities.values()) + }; +} diff --git a/src/hosted/server.ts b/src/hosted/server.ts new file mode 100644 index 00000000..db0277a0 --- /dev/null +++ b/src/hosted/server.ts @@ -0,0 +1,832 @@ +/** + * Hosted conformance server — direct-mount, no loopback proxy. + * + * One run exercises the whole matrix: every client scenario at every + * specification revision that ships a requirement set (see ./matrix.ts). + * + * GET / Landing page: the static matrix + * GET /scenarios JSON rows with per-revision cells + * GET /s Mint a run id → 303 /s/ + * GET /s/ Config for every startable cell + * GET /s// … for one revision (a column) + * GET /s/// … for one cell + * ALL /s///[/] + * The cell's server. Its MCP + * endpoint is the cell URL plus + * /mcp, whatever the scenario's + * own mcpPath (see MCP_PATH). + * GET /results/[/[/]] + * Results, mirroring /s + * DELETE /results/ Tear down every cell of the run + * + * Config and results answer HTML when the request prefers text/html and JSON + * otherwise; `?format=html|json` overrides. At a cell URL a GET that accepts + * text/html (and not text/event-stream) or carries `?format=` is a page + * request; every other request is dispatched to the scenario. + * + * Scenarios are mounted via Scenario.handler() — the same RequestListener the + * CLI runner wraps in http.createServer — so there is no loopback port and + * this works on serverless hosts. Each cell gets a fresh Scenario instance, + * created lazily on first hit, built for the column's wire version. + */ + +import express, { Request, Response } from 'express'; +import { timingSafeEqual } from 'crypto'; +import { + SessionManager, + HostedRun, + CellRef, + RunResults, + RUN_ID_RE, + UnknownScenarioError, + NotHostableError, + cellId, + mintRunId +} from './session'; +import { + buildMatrix, + MCP_PATH, + type HostedMatrix, + type MatrixCell +} from './matrix'; +import { + renderLanding, + renderConfig, + renderReport, + renderResults +} from './html'; +import { onBodySettled, tapJsonBody } from './body'; +import { identityFrom } from './identity'; +import { + describeRequest, + isNegotiation, + tapResponse, + wireRejectedCheck, + wireRejection, + wrongRevision, + wrongRevisionCheck, + type CapturedResponse, + type RequestInfo +} from './wire'; +import { buildReport, summarize, verdictFor, type Verdict } from './report'; +import type { RunStore } from './store'; +import { scenarios } from '../scenarios'; +import { ConformanceCheck, AuxOriginRole, SpecVersion } from '../types'; + +export interface HostedServerOptions { + publicOrigin?: string; + ttlMs?: number; + /** + * Public origins of the AS/IdP relay deployments. When set, scenarios that + * implement `authHandlers()` become startable; their per-cell AS issuer is + * `/r///`. See + * examples/hosted/valtown-relay.ts. + */ + auxOrigins?: Partial>; + /** + * Shared secret the relay sends in `x-relay-secret`. `/__aux/*` rejects + * requests without it so the aux backchannel can't be hit directly. Set + * the same value in the relay's env. + */ + relaySecret?: string; + /** + * Persist runs so a deployment that load-balances one run's requests + * across processes (serverless isolates) still serves complete results. + * See ./store.ts. Omit for a single long-lived process. + */ + store?: RunStore; + /** + * Scenarios this deployment refuses to mount, with the reason shown in + * the matrix (e.g. scenarios that need one process's memory across + * requests on a host that has none). + */ + exclude?: Record; +} + +const AUX_ROLES: readonly AuxOriginRole[] = ['as', 'as2', 'idp']; + +/** Cell config as served under /s/[/[/]]. */ +export interface CellConfig { + scenario: string; + revision: SpecVersion; + /** The MCP endpoint URL — what the client under test connects to. */ + url: string; + resultsUrl: string; + scoring: MatrixCell['scoring']; + reason?: string; + steps?: MatrixCell['steps']; + env: { + MCP_CONFORMANCE_SCENARIO: string; + MCP_CONFORMANCE_PROTOCOL_VERSION: string; + MCP_CONFORMANCE_CONTEXT?: string; + }; +} + +export interface RunConfig { + runId: string; + revision?: SpecVersion; + scenario?: string; + resultsUrl: string; + mcpServers: Record; + cells: CellConfig[]; +} + +export function createHostedApp(opts: HostedServerOptions = {}): { + app: express.Application; + sessions: SessionManager; + matrix: HostedMatrix; +} { + const auxOrigins = opts.auxOrigins ?? {}; + const haveAux = AUX_ROLES.filter((r) => auxOrigins[r]); + const sessions = new SessionManager({ + ttlMs: opts.ttlMs, + auxOrigins, + store: opts.store + }); + const matrix = buildMatrix({ auxOrigins, exclude: opts.exclude }); + const revisions: readonly string[] = matrix.revisions; + const app = express(); + // Copy JSON POST bodies as they flow so the report can name the client + // (initialize params / per-request _meta) without consuming the stream + // the scenario is about to read. + app.use(tapJsonBody()); + + function origin(req: Request): string { + if (opts.publicOrigin) return opts.publicOrigin; + const proto = (req.header('x-forwarded-proto') ?? req.protocol) || 'http'; + const host = req.header('x-forwarded-host') ?? req.header('host'); + return `${proto}://${host}`; + } + + const cellBaseUrl = (req: Request, ref: CellRef) => + `${origin(req)}/s/${cellId(ref)}`; + const resultsUrlFor = (req: Request, ...parts: string[]) => + `${origin(req)}/results/${parts.join('/')}`; + + /** + * Path segments of a captured route tail. A trailing slash (`/s//`, + * `/results///`) is not a segment: without this it would read + * as an empty revision or scenario name and 404. + */ + function segmentsOf(tail: string): string[] { + // A loop, not /\/+$/: that regex backtracks polynomially on + // request-controlled input (CodeQL js/polynomial-redos). + while (tail.endsWith('/')) tail = tail.slice(0, -1); + return tail.split('/'); + } + + /** + * Longest registered scenario name that prefixes `segments` (names may + * contain '/'), plus whatever follows it as a path suffix ('' if nothing). + * Matches every registered client scenario, not only startable ones, so a + * cell that cannot start still answers with its reason. + */ + function resolveScenario( + segments: string[] + ): { scenarioName: string; suffix: string } | undefined { + for (let i = segments.length; i >= 1; i--) { + const candidate = segments.slice(0, i).join('/'); + if (scenarios.has(candidate)) { + return { + scenarioName: candidate, + suffix: i < segments.length ? '/' + segments.slice(i).join('/') : '' + }; + } + } + return undefined; + } + + interface ResolvedCell { + ref: CellRef; + cell: MatrixCell; + suffix: string; + } + + /** + * Resolve `[, , , ]` to a startable + * cell, or answer the request with why it isn't one and return undefined. + */ + function resolveCell( + segments: string[], + res: Response + ): ResolvedCell | undefined { + const [runId, revision, ...rest] = segments; + if (!RUN_ID_RE.test(runId ?? '')) { + res.status(400).json({ error: 'invalid run-id' }); + return undefined; + } + if (!revisions.includes(revision ?? '')) { + res + .status(404) + .json({ error: `unknown revision '${revision ?? ''}'`, revisions }); + return undefined; + } + const resolved = resolveScenario(rest); + if (!resolved) { + res.status(404).json({ error: `unknown scenario '${rest.join('/')}'` }); + return undefined; + } + const cell = matrix.cell(resolved.scenarioName, revision)!; + if (!checkStartable(cell, res)) return undefined; + return { + ref: { + runId, + revision: revision as SpecVersion, + scenarioName: resolved.scenarioName + }, + cell, + suffix: resolved.suffix + }; + } + + /** 404 for n/a cells, 501 for cells this deployment cannot start. */ + function checkStartable(cell: MatrixCell, res: Response): boolean { + if (cell.scoring === 'n/a') { + res.status(404).json({ + error: `scenario '${cell.scenario}' does not apply to ${cell.revision}: ${cell.reason}`, + scenario: cell.scenario, + revision: cell.revision, + scoring: cell.scoring, + reason: cell.reason + }); + return false; + } + if (!cell.startable) { + res.status(501).json({ + error: `scenario '${cell.scenario}' at ${cell.revision} cannot be started here: ${cell.startReason}`, + scenario: cell.scenario, + revision: cell.revision, + scoring: cell.scoring, + reason: cell.startReason + }); + return false; + } + return true; + } + + /** + * The cell, hydrated from the store when this process has never seen it + * (see SessionManager.acquire) — a request must not be dispatched before + * the scenario knows the run's history. + */ + async function createRun( + req: Request, + ref: CellRef, + res: Response + ): Promise { + try { + return await sessions.acquire(ref, (r) => cellBaseUrl(req, r)); + } catch (e) { + if (e instanceof UnknownScenarioError) { + res.status(404).json({ error: e.message }); + return undefined; + } + if (e instanceof NotHostableError) { + res.status(501).json({ error: e.message }); + return undefined; + } + throw e; + } + } + + /** + * The path the scenario sees for a request at ``: the + * suffix, except that `/mcp` on a scenario serving MCP at its root is the + * root — every cell is reachable at `/mcp` (see MCP_PATH). + */ + function scenarioPath(run: HostedRun, suffix: string): string { + if (suffix === MCP_PATH && !run.mcpPath) return ''; + return suffix; + } + + /** Whether `rewrittenUrl` (path, maybe a query) is the cell's MCP endpoint. */ + function isMcpEndpoint(run: HostedRun, rewrittenUrl: string): boolean { + const q = rewrittenUrl.indexOf('?'); + const path = q === -1 ? rewrittenUrl : rewrittenUrl.slice(0, q); + return path === (run.mcpPath || '/'); + } + + /** + * Dispatch (req, res) to `listener` after rewriting `req.url` so the + * scenario sees the path it would have under start()/stop() — i.e. with + * the cell prefix stripped and (for well-known dispatch) the well-known + * prefix re-prepended. + * + * `mcp` says the request is to the cell's MCP endpoint (not a PRM, + * canary or aux path): only those are judged for wire rejections and + * revision discipline (see ./wire.ts). + */ + function dispatch( + run: HostedRun, + listener: (req: Request, res: Response) => void, + req: Request, + res: Response, + rewrittenUrl: string, + mcp = false + ) { + res.setHeader( + 'link', + `<${resultsUrlFor(req, run.id)}>; rel="conformance-results"` + ); + req.url = rewrittenUrl; + run.touched = true; + + // The headers as the client sent them: a scenario may rewrite them + // before handing the request on (json-schema-ref-deref maps the draft + // header to the SDK's), and identity is what the client said. + const headers = { ...req.headers }; + const headerVersion = req.header('mcp-protocol-version'); + let request: RequestInfo | undefined; + let body: Buffer | undefined; + let response: CapturedResponse | undefined; + let judged = false; + + // Write this process's view through once the scenario has answered + // (hosted scenarios record their checks before calling end()). + // Serverless entry points should await sessions.flush() before + // returning the response so this write isn't abandoned. + const persist = () => { + if (sessions.store) void sessions.persist(run); + }; + + /** Once the request is parsed and the response is out, judge both. */ + const judge = (): boolean => { + if (judged || !request || !response) return false; + judged = true; + // A foreign-revision request a dated cell turned away is the client + // negotiating (it falls back to `initialize`): neither judgement. + if (mcp && !isNegotiation(run.revision, headerVersion, response)) { + for (const method of request.methods) { + const reason = wrongRevision(run.revision, method, headerVersion); + if (!reason) continue; + sessions.recordHostedCheck( + run, + `revision:${method}:${headerVersion ?? ''}`, + wrongRevisionCheck(run.revision, method, headerVersion, reason) + ); + } + const rejection = wireRejection(response, run.revision, headerVersion); + if (rejection) { + sessions.recordHostedCheck( + run, + `rejected:${rejection.code}:${rejection.message}`, + wireRejectedCheck(rejection, request, headerVersion) + ); + } + } + // Who the client is, from accepted exchanges only. + const identity = identityFrom(headers, body, response); + if (identity) sessions.recordIdentity(run, identity); + return true; + }; + + onBodySettled(req, (captured) => { + body = captured; + request = describeRequest(captured); + // The response is already out: what judge() recorded needs its own + // write-through. + if (judge() && response) persist(); + }); + tapResponse(res, (captured) => { + response = captured; + judge(); + persist(); + }); + listener(req, res); + } + + // ---------- representation ---------- + + /** `?format=` wins; otherwise the Accept header decides (JSON by default). */ + function wantsHtml(req: Request): boolean { + const format = req.query.format; + if (format === 'html') return true; + if (format === 'json') return false; + return req.accepts(['json', 'html']) === 'html'; + } + + /** + * A GET at a cell URL is a page/config request when it accepts text/html + * (and not the MCP SSE stream) or spells out `?format=`; anything else is + * the client under test talking to the scenario. + */ + function isPageRequest(req: Request): boolean { + if (req.method !== 'GET') return false; + if (typeof req.query.format === 'string') return true; + const accept = req.header('accept') ?? ''; + return ( + accept.includes('text/html') && !accept.includes('text/event-stream') + ); + } + + // ---------- config ---------- + + function cellConfig( + req: Request, + run: HostedRun, + cell: MatrixCell + ): CellConfig { + const context = run.context + ? JSON.stringify({ name: run.scenarioName, ...run.context }) + : undefined; + return { + scenario: run.scenarioName, + revision: run.revision, + url: `${cellBaseUrl(req, run)}${run.mcpPath || MCP_PATH}`, + resultsUrl: resultsUrlFor(req, run.id), + scoring: cell.scoring, + ...(cell.reason !== undefined && { reason: cell.reason }), + ...(cell.steps && { steps: cell.steps }), + env: { + MCP_CONFORMANCE_SCENARIO: run.scenarioName, + MCP_CONFORMANCE_PROTOCOL_VERSION: run.revision, + ...(context !== undefined && { MCP_CONFORMANCE_CONTEXT: context }) + } + }; + } + + /** Config for every startable cell in scope; creates the cells. */ + function runConfig( + req: Request, + runId: string, + scope: { revision?: SpecVersion; scenario?: string } + ): RunConfig { + const cells = matrix + .cells() + .filter( + (c) => + c.startable && + (scope.revision === undefined || c.revision === scope.revision) && + (scope.scenario === undefined || c.scenario === scope.scenario) + ) + .map((c) => { + const run = sessions.getOrCreate( + { runId, revision: c.revision, scenarioName: c.scenario }, + (r) => cellBaseUrl(req, r) + ); + return cellConfig(req, run, c); + }); + const parts = [runId]; + if (scope.revision) parts.push(scope.revision); + if (scope.revision && scope.scenario) parts.push(scope.scenario); + return { + runId, + ...(scope.revision && { revision: scope.revision }), + ...(scope.scenario && { scenario: scope.scenario }), + resultsUrl: resultsUrlFor(req, ...parts), + mcpServers: Object.fromEntries( + cells.map((c) => [ + `${c.revision}/${c.scenario}`, + { type: 'http' as const, url: c.url } + ]) + ), + cells + }; + } + + function sendConfig(req: Request, res: Response, config: RunConfig): void { + if (wantsHtml(req)) { + res.type('html').send(renderConfig(origin(req), matrix, config)); + } else { + res.json(config); + } + } + + // ---------- discovery ---------- + + app.get('/', (req, res) => { + res.type('html').send(renderLanding(origin(req), matrix)); + }); + + app.get('/scenarios', (_req, res) => { + res.json( + matrix.rows.map((row) => ({ + name: row.scenario, + description: row.description, + source: row.source, + mcpPath: row.cells[0]?.mcpPath ?? MCP_PATH, + ...(row.cells[0]?.steps && { steps: row.cells[0].steps }), + cells: row.cells.map( + ({ revision, scoring, reason, startable, startReason }) => ({ + revision, + scoring, + ...(reason !== undefined && { reason }), + startable, + ...(startReason !== undefined && { startReason }) + }) + ) + })) + ); + }); + + // ---------- runs ---------- + + app.get('/s', (_req, res) => { + res.redirect(303, `/s/${mintRunId()}`); + }); + + // We can't pre-register an express route per cell because run-ids are + // open-ended and scenario names contain '/'. A single catch-all resolves + // the cell, rewrites req.url to strip the /s/// + // prefix, and hands off to the cell's listener — exactly what + // app.use(prefix, fn) would do, but with a dynamic prefix. + app.all(/^\/s\/(.+)$/, async (req, res) => { + const segments = segmentsOf(req.params[0]); + const [runId, revision] = segments; + + if (segments.length <= 2) { + // Run or column scope: config only. + if (!RUN_ID_RE.test(runId)) { + res.status(400).json({ error: 'invalid run-id' }); + return; + } + if (segments.length === 2 && !revisions.includes(revision)) { + res + .status(404) + .json({ error: `unknown revision '${revision}'`, revisions }); + return; + } + if (req.method !== 'GET') { + res.status(405).json({ + error: + 'MCP endpoints live at /s///; GET here for the config.' + }); + return; + } + sendConfig( + req, + res, + runConfig(req, runId, { + revision: + segments.length === 2 ? (revision as SpecVersion) : undefined + }) + ); + return; + } + + const resolved = resolveCell(segments, res); + if (!resolved) return; + const { ref, suffix } = resolved; + + if (suffix === '' && isPageRequest(req)) { + sendConfig( + req, + res, + runConfig(req, ref.runId, { + revision: ref.revision, + scenario: ref.scenarioName + }) + ); + return; + } + + const run = await createRun(req, ref, res); + if (!run) return; + // Rewrite to the path the scenario expects (it thinks it's at root). + // The query string is preserved because we keep the express req object. + const rewritten = scenarioPath(run, suffix) || run.mcpPath || '/'; + dispatch( + run, + run.listener, + req, + res, + rewritten, + isMcpEndpoint(run, rewritten) + ); + }); + + // ---------- root well-known dispatch (RS side) ---------- + // + // RFC 9728: a client given MCP URL /s//mcp derives the PRM + // URL as /.well-known/oauth-protected-resource/s//mcp — i.e. + // at the *origin root*, not under the cell prefix. We catch that here, + // recover the cell from the path suffix, and re-dispatch to its RS handler + // with the path it would have seen on its own origin. + // + // Requests that arrive *under* the cell prefix (because the WWW-Authenticate + // header points there) already work via the /s/* mount above. + + app.get( + /^\/\.well-known\/oauth-protected-resource\/s\/(.+)$/, + async (req, res) => { + const resolved = resolveCell(req.params[0].split('/'), res); + if (!resolved) return; + const run = await createRun(req, resolved.ref, res); + if (!run) return; + // Scenario expects e.g. '/.well-known/oauth-protected-resource/mcp' + // (or the bare well-known path when its MCP endpoint is its root). + dispatch( + run, + run.listener, + req, + res, + '/.well-known/oauth-protected-resource' + + scenarioPath(run, resolved.suffix) + ); + } + ); + + // ---------- aux-origin backchannel (relay target) ---------- + // + // The AS relay (examples/hosted/valtown-relay.ts) forwards every request it + // receives to /__aux/. The per-cell AS issuer is + // /r///, so every path the client hits + // — endpoints (/r//authorize) and RFC 8414 well-known + // (/.well-known/oauth-authorization-server/r/[/tenant]) — carries + // `/r/` somewhere in it. We locate it, strip it, and dispatch to the + // cell's aux handler so it sees exactly the path createAuthServer + // registered. + // + // Guarded by a shared secret so this internal mount can't be hit directly + // to forge checks into someone else's run. + + if (haveAux.length) { + const secret = opts.relaySecret ?? process.env.CONFORMANCE_RELAY_SECRET; + const guard = (req: Request, res: Response): boolean => { + const got = req.header('x-relay-secret') ?? ''; + // Constant-time compare; mismatch length → fast 403 is fine. + const ok = + !!secret && + got.length === secret.length && + timingSafeEqual(Buffer.from(got), Buffer.from(secret)); + if (!ok) { + res + .status(403) + .json({ error: 'forbidden: /__aux is the relay backchannel' }); + } + return ok; + }; + + /** + * Find the first `/r///` in `segments` (plain + * segment scan — a regex over the whole path would backtrack + * polynomially on adversarial input) and split the path around it. + */ + function locateCell( + segments: string[] + ): { prefix: string; ref: CellRef; suffix: string } | undefined { + for (let i = 1; i < segments.length - 2; i++) { + if ( + segments[i] !== 'r' || + !RUN_ID_RE.test(segments[i + 1]) || + !revisions.includes(segments[i + 2]) + ) { + continue; + } + const resolved = resolveScenario(segments.slice(i + 3)); + if (!resolved) continue; + return { + prefix: segments.slice(0, i).join('/'), + ref: { + runId: segments[i + 1], + revision: segments[i + 2] as SpecVersion, + scenarioName: resolved.scenarioName + }, + suffix: resolved.suffix + }; + } + return undefined; + } + + app.all(/^\/__aux\/([a-z0-9]+)(\/.*)$/, async (req, res) => { + if (!guard(req, res)) return; + const role = req.params[0] as AuxOriginRole; + const path = req.params[1]; + if (!AUX_ROLES.includes(role)) { + res.status(404).json({ error: `unknown aux role '${role}'` }); + return; + } + + const located = locateCell(path.split('/')); // path starts with '/', so [0] === '' + if (!located) { + res.status(404).json({ + error: + 'aux request path missing /r/// segment' + }); + return; + } + const { prefix, ref, suffix } = located; + const cell = matrix.cell(ref.scenarioName, ref.revision)!; + if (!checkStartable(cell, res)) return; + const search = req.url.includes('?') + ? req.url.slice(req.url.indexOf('?')) + : ''; + + // On a multi-process host this may be the first request this process + // sees for the cell; the id carries everything needed to rebuild it. + const run = sessions.ensure(cellId(ref), (r) => cellBaseUrl(req, r)); + const listener = run?.auxListeners?.[role]; + if (!run || !listener) { + res.status(404).json({ error: `no aux '${role}' handler for cell` }); + return; + } + await sessions.hydrate(run); + dispatch(run, listener, req, res, (prefix + suffix || '/') + search); + }); + } + + // ---------- results ---------- + + app.get(/^\/results\/(.+)$/, async (req, res) => { + const segments = segmentsOf(req.params[0]); + const [runId, revision, ...rest] = segments; + if (!RUN_ID_RE.test(runId)) { + res.status(400).json({ error: 'invalid run-id' }); + return; + } + if (segments.length >= 2 && !revisions.includes(revision)) { + res + .status(404) + .json({ error: `unknown revision '${revision}'`, revisions }); + return; + } + + if (segments.length >= 3) { + const resolved = resolveScenario(rest); + if (!resolved || resolved.suffix !== '') { + res.status(404).json({ error: `unknown scenario '${rest.join('/')}'` }); + return; + } + const ref: CellRef = { + runId, + revision: revision as SpecVersion, + scenarioName: resolved.scenarioName + }; + const cell = matrix.cell(resolved.scenarioName, ref.revision)!; + // A cell nobody has hit yet is a valid, incomplete cell — not an + // unknown run: the config page links here before any traffic. + const r = + cell.scoring === 'n/a' + ? undefined + : await sessions.results(cellId(ref)); + const status = cellStatus(cell, r); + if (wantsHtml(req)) { + res.type('html').send(renderResults(ref, r?.checks ?? [], status)); + } else { + res.json({ ...summarise(ref, r?.checks ?? []), ...status }); + } + return; + } + + // Run or column scope: a verdict per cell of the matrix. + const scope = segments.length === 2 ? (revision as SpecVersion) : undefined; + const report = await buildReport(matrix, runId, scope, { + listCells: (id) => sessions.listCells(id), + results: (id) => sessions.results(id), + resultsUrl: (ref) => resultsUrlFor(req, cellId(ref)) + }); + if (wantsHtml(req)) { + res.type('html').send(renderReport(origin(req), matrix, report)); + } else { + res.json(report); + } + }); + + app.delete('/results/:runId', async (req, res) => { + if (!RUN_ID_RE.test(req.params.runId)) { + res.status(400).json({ error: 'invalid run-id' }); + return; + } + await sessions.destroyRun(req.params.runId); + res.status(204).end(); + }); + + return { app, sessions, matrix }; +} + +export function summarise(ref: CellRef, checks: ConformanceCheck[]) { + return { + runId: ref.runId, + revision: ref.revision, + scenario: ref.scenarioName, + summary: summarize(checks), + checks + }; +} + +/** What a cell's results say about the cell itself, next to its checks. */ +export interface CellStatus { + scoring: MatrixCell['scoring']; + verdict: Verdict; + /** For n/a (why the scenario does not apply) and not_scored/unlisted. */ + reason?: string; + /** Present, false, when this deployment cannot start the cell. */ + startable?: false; + startReason?: string; +} + +export function cellStatus( + cell: MatrixCell, + results: Pick | undefined +): CellStatus { + return { + scoring: cell.scoring, + verdict: verdictFor(cell, results?.checks, results?.recorded), + ...(cell.reason !== undefined && { reason: cell.reason }), + ...(!cell.startable && + cell.scoring !== 'n/a' && { + startable: false as const, + ...(cell.startReason !== undefined && { startReason: cell.startReason }) + }) + }; +} diff --git a/src/hosted/session.test.ts b/src/hosted/session.test.ts new file mode 100644 index 00000000..9bcf0d70 --- /dev/null +++ b/src/hosted/session.test.ts @@ -0,0 +1,178 @@ +import { describe, it, expect } from 'vitest'; +import http from 'http'; +import { + SessionManager, + cellId, + rawChecksOf, + type CellRef, + type HostedRun +} from './session'; +import { MemoryRunStore } from './store'; +import type { RequestListener } from '../types'; + +const ref: CellRef = { + runId: 'r1', + revision: '2025-11-25', + scenarioName: 'tools_call' +}; + +describe('SessionManager.persist', () => { + it('retries saveRun after a failed write so the cell reaches listRuns', async () => { + let failures = 1; + let saveRunCalls = 0; + class FlakyStore extends MemoryRunStore { + override async saveRun(id: string, scenarioName: string) { + saveRunCalls++; + if (failures-- > 0) throw new Error('sqlite 503'); + return super.saveRun(id, scenarioName); + } + } + const store = new FlakyStore(); + const sessions = new SessionManager({ store }); + try { + const run = sessions.getOrCreate(ref, () => 'http://rs.test/s/x'); + await sessions.persist(run); // saveRun rejects; the error is logged + expect(run.saved).toBe(false); + expect(await store.listRuns('r1/')).toEqual([]); + + await sessions.persist(run); + expect(run.saved).toBe(true); + expect(await store.listRuns('r1/')).toEqual([ + { id: 'r1/2025-11-25/tools_call', scenarioName: 'tools_call' } + ]); + + await sessions.persist(run); + expect(saveRunCalls).toBe(2); + } finally { + await sessions.close(); + } + }); +}); + +/** POST one JSON-RPC request straight at a cell's listener. */ +async function post( + listener: RequestListener, + body: object, + headers: Record +): Promise<{ status: number; body: any }> { + const server = http.createServer(listener); + await new Promise((r) => server.listen(0, r)); + try { + const port = (server.address() as { port: number }).port; + const res = await fetch(`http://localhost:${port}/`, { + method: 'POST', + headers: { 'content-type': 'application/json', ...headers }, + body: JSON.stringify(body) + }); + return { status: res.status, body: await res.json() }; + } finally { + server.closeAllConnections?.(); + await new Promise((r) => server.close(() => r())); + } +} + +describe('SessionManager hydration', () => { + const cold: CellRef = { + runId: 'h1', + revision: '2026-07-28', + scenarioName: 'request-metadata' + }; + const RETRY = 'sep-2575-client-retry-supported-version'; + const request = { + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { + _meta: { + 'io.modelcontextprotocol/protocolVersion': '2026-07-28', + 'io.modelcontextprotocol/clientInfo': { name: 'vitest', version: '0' }, + 'io.modelcontextprotocol/clientCapabilities': {} + } + } + }; + const headers = { 'mcp-protocol-version': '2026-07-28' }; + + it('seeds a cold process from the persisted log and persists only what it adds', async () => { + const store = new MemoryRunStore(); + const a = new SessionManager({ store }); + const b = new SessionManager({ store }); + try { + // Process A sees the first request: the one simulated rejection. + const runA = await a.acquire(cold, () => 'http://x'); + const first = await post(runA.listener, request, headers); + expect(first.status).toBe(400); + expect(first.body.error.code).toBe(-32022); + await a.persist(runA); + const rowA = (await store.loadChecks(cellId(cold))).get(a.writerId)!; + expect(rowA.map((c) => c.id)).toContain(RETRY); + + // Process B has never seen the cell. Hydrated, it knows the rejection + // already happened and answers the retry instead of rejecting again. + const runB: HostedRun = await b.acquire(cold, () => 'http://x'); + expect(runB.seeded.size).toBe(rowA.length); + expect(rawChecksOf(runB.scenario).map((c) => c.id)).toContain(RETRY); + // Seeded checks are A's to persist: B owns nothing yet. + expect(b.ownChecks(runB)).toEqual([]); + const second = await post(runB.listener, request, headers); + expect(second.status).toBe(200); + + // B's row carries its own observations only — here every id, since + // the scenario re-emits each one per request, with the retry check + // rewritten to SUCCESS. + await b.persist(runB); + const rowB = (await store.loadChecks(cellId(cold))).get(b.writerId)!; + expect(rowB).toEqual(b.ownChecks(runB)); + expect(rowB.find((c) => c.id === RETRY)?.status).toBe('SUCCESS'); + expect(rowA.find((c) => c.id === RETRY)?.status).toBe('WARNING'); + + // Judged from the merged log by either process: one retry check, the + // latest observation, and nothing declared missing. + for (const m of [a, b]) { + const results = (await m.results(cellId(cold)))!; + const retries = results.checks.filter((c) => c.id === RETRY); + expect(retries).toHaveLength(1); + expect(retries[0].status).toBe('SUCCESS'); + expect(results.checks.filter((c) => c.status === 'FAILURE')).toEqual( + [] + ); + } + } finally { + await a.close(); + await b.close(); + } + }); + + it('reloads its own evicted row as its own, and settles at once without a store', async () => { + const store = new MemoryRunStore(); + const a = new SessionManager({ store }); + try { + const run = await a.acquire(cold, () => 'http://x'); + await post(run.listener, request, headers); + await a.persist(run); + const before = (await store.loadChecks(cellId(cold))).get(a.writerId)!; + expect(before.length).toBeGreaterThan(0); + + // Evicted from memory, rebuilt on the next request: the row it wrote + // is not "seeded" — it stays in the row on the next persist. + await a.destroy(cellId(cold), false); + const again = await a.acquire(cold, () => 'http://x'); + expect(again.seeded.size).toBe(0); + expect(rawChecksOf(again.scenario)).toHaveLength(before.length); + await a.persist(again); + expect((await store.loadChecks(cellId(cold))).get(a.writerId)).toEqual( + before + ); + } finally { + await a.close(); + } + + const plain = new SessionManager(); + try { + const run = await plain.acquire(cold, () => 'http://x'); + expect(rawChecksOf(run.scenario)).toEqual([]); + expect(run.hydration).toBeDefined(); + } finally { + await plain.close(); + } + }); +}); diff --git a/src/hosted/session.ts b/src/hosted/session.ts new file mode 100644 index 00000000..ce304db6 --- /dev/null +++ b/src/hosted/session.ts @@ -0,0 +1,618 @@ +/** + * Session management for the hosted conformance server. + * + * A "cell" is one scenario exercised at one specification revision inside a + * run. Its id, `//`, is at once the URL path the + * client is pointed at, the store key and the results path. Each cell owns a + * fresh Scenario instance built for that revision's wire and the + * RequestListener it returns from handler() — no loopback port, no proxy. + * Cells are created lazily on first reference and can be rebuilt from their + * id alone, which is what lets a cold serverless isolate answer for a run it + * never saw (an aux-origin request arriving before the RS was ever hit, say). + */ + +import { randomBytes } from 'crypto'; +import { + Scenario, + ConformanceCheck, + RequestListener, + AuthHandlerScenario, + AuxOriginRole, + SpecVersion, + isSpecVersion +} from '../types'; +import { createHandlerFor, type ScenarioContext } from '../mock-server'; +import { getScenario, scenarios } from '../scenarios'; +import type { RunStore } from './store'; +import { + addProtocolVersion, + identityCheck, + identityChecksIn, + identityKey, + identityOf, + IDENTITY_CHECK_ID, + type IdentityObservation +} from './identity'; + +/** Store writer suffix for the hosted layer's own checks (client identity). */ +const HOSTED_WRITER_SUFFIX = '/hosted'; + +/** Run ids are one path segment: safe in URLs and after the relay's /r/. */ +export const RUN_ID_RE = /^[A-Za-z0-9_-]{1,64}$/; + +export function mintRunId(): string { + return randomBytes(6).toString('base64url'); +} + +/** One scenario at one revision inside one run. */ +export interface CellRef { + runId: string; + revision: SpecVersion; + scenarioName: string; +} + +/** `//` — scenario names may contain '/'. */ +export function cellId(ref: CellRef): string { + return `${ref.runId}/${ref.revision}/${ref.scenarioName}`; +} + +export function parseCellId(id: string): CellRef | undefined { + const [runId, revision, ...rest] = id.split('/'); + if (!RUN_ID_RE.test(runId ?? '') || !isSpecVersion(revision) || !rest.length) + return undefined; + return { runId, revision, scenarioName: rest.join('/') }; +} + +/** + * Per-cell context for a hosted scenario: the column's revision is the wire + * the mock speaks, exactly as `--spec-version` sets it for the CLI runner + * (src/runner/client.ts). `createServer()` would bind a loopback port, which + * serverless hosts don't allow — hosted scenarios use `createHandler()`. + */ +export function hostedScenarioContext( + specVersion: SpecVersion +): ScenarioContext { + return { + specVersion, + createServer: () => + Promise.reject( + new Error( + 'ScenarioContext.createServer() binds a loopback port and is not available when hosted; use createHandler()' + ) + ), + createHandler: (handlers) => createHandlerFor(specVersion)(handlers) + }; +} + +export interface HostedRun extends CellRef { + /** Cell id — see cellId(). */ + id: string; + scenario: Scenario; + /** The mounted RS handler — invoke directly with (req, res). */ + listener: RequestListener; + /** Aux-origin handlers (AS, IdP, …) for auth scenarios. */ + auxListeners?: Partial>; + /** Sub-path under the cell URL where the MCP endpoint lives. */ + mcpPath: string; + createdAt: number; + lastSeenAt: number; + /** Scenario-provided client context (credentials, steps, …). */ + context?: Record; + /** Whether the store has been told this cell exists. */ + saved: boolean; + /** + * Whether any request was dispatched to the cell. A cell created only to + * answer a config request has not been exercised and stays out of the + * results listing. + */ + touched: boolean; + /** + * Checks the hosted layer records about the cell (client identity, wire + * rejections, revision discipline), kept apart from the scenario's own log + * so they never enter its judgement. + */ + hostedChecks: ConformanceCheck[]; + /** The identity check per client (name, version) already recorded. */ + identities: Map; + /** Keys of hosted checks already recorded, so each finding is one check. */ + hostedKeys: Set; + /** + * Checks seeded into the scenario from other processes' persisted rows + * (see hydrate()), each with its JSON as seeded. They are those writers' + * to persist, not ours — unless the scenario has changed one since. + */ + seeded: Map; + /** Settled once the cell has been seeded from the store (or never will be). */ + hydration?: Promise; +} + +export interface SessionManagerOptions { + /** Idle ms after which a cell is evicted from memory. Default 5 minutes. */ + ttlMs?: number; + sweepIntervalMs?: number; + /** + * Public origins of the relay deployments, keyed by role. Required for any + * scenario that exposes `authHandlers()`. Each value is the relay's public + * URL (no trailing slash); the per-cell AS issuer becomes + * `/r///`. + */ + auxOrigins?: Partial>; + /** + * Optional persistence so results survive being load-balanced across + * processes (serverless isolates). Omit for a single long-lived process. + */ + store?: RunStore; +} + +/** Results view: the cell plus its judged checks. */ +export interface RunResults extends CellRef { + checks: ConformanceCheck[]; + /** + * How many checks were recorded from traffic: the scenario's own raw log + * (before judgement, which may add "expected but never seen" failures) + * plus the hosted layer's FAILUREs (a request the wire turned away is + * traffic too), but not its INFO checks. Zero means nothing was exercised. + */ + recorded: number; +} + +/** Hosted checks that count as exercise: what went wrong on the wire. */ +function hostedFailures(checks: ConformanceCheck[]): number { + return checks.filter((c) => c.status === 'FAILURE').length; +} + +/** + * The scenario's raw event log — what it actually observed — as opposed to + * getChecks(), which for most client scenarios also appends "expected X, + * never saw it" FAILUREs (and mutates). Persisting the raw log per process + * and judging the merged log once is what makes multi-process hosting work. + */ +export function rawChecksOf(scenario: Scenario): ConformanceCheck[] { + if (scenario.rawChecks) return scenario.rawChecks(); + const bag = (scenario as unknown as { checks?: unknown }).checks; + if (Array.isArray(bag)) return bag as ConformanceCheck[]; + return scenario.getChecks(); +} + +/** + * New, unstarted instance of a registered scenario. Prefers `Scenario.fresh()` + * so scenarios registered with constructor parameters keep them; falls back + * to the no-arg constructor. + */ +export function freshScenario(proto: Scenario): Scenario { + if (typeof proto.fresh === 'function') return proto.fresh(); + const Ctor = proto.constructor as new () => Scenario; + return new Ctor(); +} + +/** + * Judge a merged raw log with the scenario's own end-of-run logic by loading + * it into a fresh instance. Falls back to the raw log for scenarios that + * don't keep a plain `checks` array. + */ +export function finalizeChecks( + scenarioName: string, + merged: ConformanceCheck[] +): ConformanceCheck[] { + const proto = getScenario(scenarioName); + if (!proto) return merged; + try { + const fresh = freshScenario(proto) as unknown as { + checks?: unknown; + getChecks(): ConformanceCheck[]; + }; + if (!Array.isArray(fresh.checks)) return merged; + fresh.checks = merged.map((c) => ({ ...c })); + return fresh.getChecks(); + } catch { + return merged; + } +} + +export class SessionManager { + private runs = new Map(); + private readonly ttlMs: number; + private readonly auxOrigins: Partial>; + private sweeper: ReturnType; + readonly store: RunStore | undefined; + private pending = new Set>(); + /** Identifies this process's rows in the store. */ + readonly writerId = randomBytes(4).toString('hex'); + + constructor(opts: SessionManagerOptions = {}) { + this.ttlMs = opts.ttlMs ?? 5 * 60_000; + this.auxOrigins = opts.auxOrigins ?? {}; + this.store = opts.store; + const sweepIntervalMs = opts.sweepIntervalMs ?? 30_000; + this.sweeper = setInterval(() => this.sweep(), sweepIntervalMs); + this.sweeper.unref?.(); + } + + /** + * Get the cell, creating it on first reference. `baseUrlFor` is the public + * URL of the cell (no trailing slash) — the scenario embeds it in + * self-referential responses (PRM `resource`, canary `$ref`s). + */ + getOrCreate(ref: CellRef, baseUrlFor: (ref: CellRef) => string): HostedRun { + const id = cellId(ref); + const existing = this.runs.get(id); + if (existing) { + existing.lastSeenAt = Date.now(); + return existing; + } + + const proto = getScenario(ref.scenarioName); + if (!proto) throw new UnknownScenarioError(ref.scenarioName); + + const scenario = freshScenario(proto); + const ctx = hostedScenarioContext(ref.revision); + + let listener: RequestListener; + let auxListeners: HostedRun['auxListeners']; + let context: Record | undefined; + + if (scenario instanceof AuthHandlerScenario) { + // Multi-origin scenario: build RS + aux handlers from authHandlers(). + // Aux issuer is /r/ so the cell is recoverable + // from any RFC 8414 well-known path the client constructs from it. + const missing = scenario.auxRoles.filter((r) => !this.auxOrigins[r]); + if (missing.length) { + throw new NotHostableError( + ref.scenarioName, + `needs relay origin(s) [${missing.join(', ')}]` + ); + } + const handlers = scenario.authHandlers({ + ...ctx, + getRsBaseUrl: () => baseUrlFor(ref), + getAuxBaseUrl: (role) => `${this.auxOrigins[role]}/r/${id}` + }); + listener = handlers.rs; + auxListeners = handlers.aux; + context = ( + scenario as unknown as { + scenarioContext?: () => Record; + } + ).scenarioContext?.(); + } else if (scenario.handler) { + listener = scenario.handler(() => baseUrlFor(ref), ctx); + } else { + throw new NotHostableError(ref.scenarioName); + } + + const steps = (scenario as Scenario).steps; + if (steps) context = { ...context, steps }; + + const run: HostedRun = { + ...ref, + id, + scenario, + listener, + auxListeners, + mcpPath: scenario.mcpPath ?? '', + createdAt: Date.now(), + lastSeenAt: Date.now(), + context, + saved: false, + touched: false, + hostedChecks: [], + identities: new Map(), + hostedKeys: new Set(), + seeded: new Map() + }; + this.runs.set(id, run); + return run; + } + + /** + * getOrCreate() plus hydrate(): the cell, seeded with what other processes + * already recorded about it. What every request must go through before it + * is dispatched, so a scenario that keys its behaviour on its own log (the + * one-time rejection in request-metadata, say) sees the run's history and + * not just this process's. + */ + async acquire( + ref: CellRef, + baseUrlFor: (ref: CellRef) => string + ): Promise { + const run = this.getOrCreate(ref, baseUrlFor); + await this.hydrate(run); + return run; + } + + /** + * Seed the cell's scenario with the merged raw log the store holds for it, + * once per cell per process. Only scenarios that keep a plain `checks` + * array can be seeded (the same ones finalizeChecks() can re-judge); the + * rest are left alone. Rows this process wrote itself (a cell evicted and + * rebuilt) are loaded as its own, so they are persisted again rather than + * dropped from its row. Without a store this settles at once. + */ + hydrate(run: HostedRun): Promise { + if (run.hydration) return run.hydration; + const store = this.store; + if (!store) return (run.hydration = Promise.resolve()); + run.hydration = (async () => { + const bag = (run.scenario as unknown as { checks?: unknown }).checks; + if (!Array.isArray(bag) || run.scenario.rawChecks) return; + const byWriter = await store.loadChecks(run.id); + const merged: ConformanceCheck[] = []; + for (const [writer, checks] of byWriter) { + if (writer.endsWith(HOSTED_WRITER_SUFFIX)) continue; + for (const c of checks) { + const copy = { ...c }; + if (writer !== this.writerId) + run.seeded.set(copy, JSON.stringify(copy)); + merged.push(copy); + } + } + if (!merged.length) return; + merged.sort(byTime); + (bag as ConformanceCheck[]).unshift(...merged); + })().catch(logStoreError); + return run.hydration; + } + + /** + * This process's contribution to the cell's raw log: everything the + * scenario recorded except seeded checks it has not touched. A seeded + * check the scenario replaced or rewrote in place is ours to persist. + */ + ownChecks(run: HostedRun): ConformanceCheck[] { + const raw = rawChecksOf(run.scenario); + if (!run.seeded.size) return raw; + return raw.filter((c) => run.seeded.get(c) !== JSON.stringify(c)); + } + + /** + * Record a hosted-layer check about the cell once per `key` (what makes + * the finding distinct — e.g. the rejection's code and message). + */ + recordHostedCheck( + run: HostedRun, + key: string, + check: ConformanceCheck + ): void { + if (run.hostedKeys.has(key)) return; + run.hostedKeys.add(key); + run.hostedChecks.push(check); + } + + /** + * Record who is talking to the cell — one INFO check per client (name, + * version), accumulating the protocol versions it negotiated. + */ + recordIdentity(run: HostedRun, observed: IdentityObservation): void { + const key = identityKey(observed); + const existing = run.identities.get(key); + if (existing) { + addProtocolVersion(existing, observed.protocolVersion); + return; + } + const check = identityCheck(identityOf(observed)); + run.identities.set(key, check); + run.hostedChecks.push(check); + } + + get(id: string): HostedRun | undefined { + const r = this.runs.get(id); + if (r) r.lastSeenAt = Date.now(); + return r; + } + + /** + * Like get(), but rebuilds the cell from its id when this process has never + * seen it — the id carries everything needed. Returns undefined for an id + * that does not parse or names a scenario this deployment cannot mount. + */ + ensure( + id: string, + baseUrlFor: (ref: CellRef) => string + ): HostedRun | undefined { + const local = this.get(id); + if (local) return local; + const ref = parseCellId(id); + if (!ref) return undefined; + try { + return this.getOrCreate(ref, baseUrlFor); + } catch (e) { + if (e instanceof UnknownScenarioError || e instanceof NotHostableError) + return undefined; + throw e; + } + } + + /** + * Write this process's view of a cell's checks through to the store; the + * first write also records that the cell exists, so a cell that was only + * configured (never hit) does not show up as exercised. + */ + persist(run: HostedRun): Promise { + const store = this.store; + if (!store) return Promise.resolve(); + const p = (async () => { + if (!run.saved) { + // Mark saved only once the write landed: a failed saveRun must be + // retried on the next persist, or the cell never appears in + // listRuns() and the report shows it as never exercised even though + // its checks are in the store. + await store.saveRun(run.id, run.scenarioName); + run.saved = true; + } + await store.saveChecks( + run.id, + this.writerId, + this.ownChecks(run).map((c) => ({ ...c })) + ); + if (run.hostedChecks.length) { + await store.saveChecks( + run.id, + this.writerId + HOSTED_WRITER_SUFFIX, + run.hostedChecks.map((c) => ({ ...c })) + ); + } + })() + .catch(logStoreError) + .finally(() => this.pending.delete(p)); + this.pending.add(p); + return p; + } + + /** + * Resolve once every in-flight persist() has settled. Serverless entry + * points await this before handing back the response so the write isn't + * abandoned when the isolate is frozen after responding. + */ + async flush(): Promise { + while (this.pending.size) await Promise.all(Array.from(this.pending)); + } + + /** + * Judged checks for a cell, or undefined when neither this process nor the + * store has seen it. Without a store this is the scenario's own getChecks(). + * With a store it is every process's raw log merged (this process's live + * log wins over its own persisted row) and re-judged once. The hosted + * layer's own checks are appended after judgement, deduplicated across + * processes, so they never influence the scenario's verdicts — though a + * hosted FAILURE (wire rejection, wrong revision) does decide the cell's. + */ + async results(id: string): Promise { + const ref = parseCellId(id); + if (!ref) return undefined; + const run = this.runs.get(id); + if (!this.store) { + if (!run) return undefined; + const recorded = + rawChecksOf(run.scenario).length + hostedFailures(run.hostedChecks); + return { + ...ref, + checks: [...run.scenario.getChecks(), ...run.hostedChecks], + recorded + }; + } + let byWriter = new Map(); + let known = false; + try { + byWriter = await this.store.loadChecks(id); + known = (await this.store.loadRun(id)) !== undefined; + } catch (e) { + logStoreError(e); + } + if (run) { + byWriter.set(this.writerId, this.ownChecks(run)); + byWriter.set(this.writerId + HOSTED_WRITER_SUFFIX, run.hostedChecks); + } + if (!run && !known && byWriter.size === 0) return undefined; + const scenarioLog: ConformanceCheck[] = []; + const hostedLog: ConformanceCheck[] = []; + for (const [writer, checks] of byWriter) { + (writer.endsWith(HOSTED_WRITER_SUFFIX) ? hostedLog : scenarioLog).push( + ...checks + ); + } + // Identity checks collapse to one per client (their protocol versions + // pooled); every other hosted finding is one check per distinct details. + const seen = new Set(); + hostedLog.sort(byTime); + const hosted = [ + ...identityChecksIn(hostedLog), + ...hostedLog.filter((c) => { + if (c.id === IDENTITY_CHECK_ID) return false; + const key = `${c.id}:${JSON.stringify(c.details ?? null)}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }) + ].sort(byTime); + return { + ...ref, + checks: [ + ...finalizeChecks(ref.scenarioName, scenarioLog.sort(byTime)), + ...hosted + ], + recorded: scenarioLog.length + hostedFailures(hosted) + }; + } + + /** Exercised cells of a run: hit in this process, or saved to the store. */ + async listCells(runId: string): Promise { + const ids = new Set(); + for (const r of this.runs.values()) { + if (r.runId === runId && r.touched) ids.add(r.id); + } + if (this.store) { + try { + for (const { id } of await this.store.listRuns(`${runId}/`)) + ids.add(id); + } catch (e) { + logStoreError(e); + } + } + return Array.from(ids) + .map(parseCellId) + .filter((r): r is CellRef => r !== undefined); + } + + list(): HostedRun[] { + return Array.from(this.runs.values()); + } + + async destroy(id: string, fromStore = true): Promise { + const r = this.runs.get(id); + if (fromStore) void this.store?.deleteRun(id).catch(logStoreError); + if (!r) return; + this.runs.delete(id); + // handler() never started a server, but some scenarios hold timers/streams + // that stop() cleans up. Safe to call even though start() wasn't. + try { + await r.scenario.stop(); + } catch { + // best-effort + } + } + + /** Tear down every cell of a run, here and in the store. */ + async destroyRun(runId: string): Promise { + const cells = await this.listCells(runId); + await Promise.all(cells.map((ref) => this.destroy(cellId(ref)))); + } + + async close(): Promise { + clearInterval(this.sweeper); + await Promise.all( + Array.from(this.runs.keys()).map((id) => this.destroy(id, false)) + ); + } + + private sweep(): void { + const now = Date.now(); + for (const [id, r] of this.runs) { + // Local eviction only — the store has its own retention. + if (now - r.lastSeenAt > this.ttlMs) void this.destroy(id, false); + } + } +} + +const byTime = (a: ConformanceCheck, b: ConformanceCheck) => + (a.timestamp ?? '').localeCompare(b.timestamp ?? ''); + +function logStoreError(e: unknown): void { + console.error('[hosted] run store:', e instanceof Error ? e.message : e); +} + +export class UnknownScenarioError extends Error { + constructor(name: string) { + super( + `Unknown scenario '${name}'. Available: ${Array.from(scenarios.keys()).join(', ')}` + ); + } +} + +export class NotHostableError extends Error { + constructor(name: string, why?: string) { + super( + `Scenario '${name}' cannot run hosted` + + (why ? `: ${why}` : ' (not converted for hosting yet)') + ); + } +} diff --git a/src/hosted/store.test.ts b/src/hosted/store.test.ts new file mode 100644 index 00000000..33390793 --- /dev/null +++ b/src/hosted/store.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { MemoryRunStore } from './store'; +import { SqliteRunStore } from '../../examples/hosted/valtown-store'; + +describe('MemoryRunStore', () => { + it('lists the cells of one run by id prefix', async () => { + const store = new MemoryRunStore(); + await store.saveRun('r1/2025-11-25/tools_call', 'tools_call'); + await store.saveRun('r1/2026-07-28/auth/basic-cimd', 'auth/basic-cimd'); + await store.saveRun('r10/2025-11-25/tools_call', 'tools_call'); + expect((await store.listRuns('r1/')).map((r) => r.id).sort()).toEqual([ + 'r1/2025-11-25/tools_call', + 'r1/2026-07-28/auth/basic-cimd' + ]); + expect(await store.listRuns('r1/')).toContainEqual({ + id: 'r1/2026-07-28/auth/basic-cimd', + scenarioName: 'auth/basic-cimd' + }); + expect(await store.listRuns('nope/')).toEqual([]); + }); +}); + +describe('SqliteRunStore', () => { + afterEach(() => vi.unstubAllGlobals()); + + it('lists by prefix with LIKE, escaping the wildcard characters', async () => { + const statements: { sql: string; args: unknown[] }[] = []; + vi.stubGlobal( + 'fetch', + vi.fn(async (_url: string, init: { body: string }) => { + const { statement } = JSON.parse(init.body); + statements.push(statement); + const rows = statement.sql.includes('SELECT id, scenario') + ? [['r_1/2025-11-25/tools_call', 'tools_call']] + : []; + return new Response(JSON.stringify({ rows }), { status: 200 }); + }) + ); + const store = new SqliteRunStore({ token: 't' }); + const listed = await store.listRuns('r_1%/'); + expect(listed).toEqual([ + { id: 'r_1/2025-11-25/tools_call', scenarioName: 'tools_call' } + ]); + const select = statements.find((s) => + s.sql.includes('SELECT id, scenario') + )!; + expect(select.sql).toContain("WHERE id LIKE ? ESCAPE '\\'"); + expect(select.args).toEqual(['r\\_1\\%/%']); + }); +}); diff --git a/src/hosted/store.ts b/src/hosted/store.ts new file mode 100644 index 00000000..f2597435 --- /dev/null +++ b/src/hosted/store.ts @@ -0,0 +1,81 @@ +/** + * Run persistence for the hosted conformance server. + * + * A long-lived Node process keeps every run in memory and needs none of this. + * Serverless hosts (val.town, Deno Deploy, …) load-balance one run's requests + * across short-lived isolates, so the isolate that answers GET /results is + * often not the one that saw the MCP traffic. A RunStore lets each isolate + * write through what it observed and lets any isolate serve a merged view: + * + * - run metadata (cell id → scenario) so the results page can list which + * cells of a run were exercised, and tear them all down together; + * - checks, keyed by (run, writer): each isolate owns its own row and + * replaces it wholesale after every request, so concurrent writers never + * clobber each other and no append ordering is needed. + * + * The merged log is re-judged at results time by a fresh scenario instance + * (see SessionManager.results), which is what turns "isolate B never saw a + * tools/call" from a false FAILURE into the union of what A and B saw. + */ + +import type { ConformanceCheck } from '../types'; + +export interface RunStore { + saveRun(id: string, scenarioName: string): Promise; + /** Scenario name for a run id, or undefined if no isolate ever saw it. */ + loadRun(id: string): Promise; + /** + * Every saved run whose id starts with `prefix`. Cell ids are + * `//`, so `/` lists one run's cells. + */ + listRuns( + prefix: string + ): Promise>; + saveChecks( + id: string, + writer: string, + checks: ConformanceCheck[] + ): Promise; + /** All writers' check lists for a run, keyed by writer id. */ + loadChecks(id: string): Promise>; + deleteRun(id: string): Promise; +} + +/** In-process store — used by tests to exercise the merge path. */ +export class MemoryRunStore implements RunStore { + private runs = new Map(); + private checks = new Map>(); + + async saveRun(id: string, scenarioName: string): Promise { + if (!this.runs.has(id)) this.runs.set(id, scenarioName); + } + async loadRun(id: string): Promise { + return this.runs.get(id); + } + async listRuns( + prefix: string + ): Promise> { + return Array.from(this.runs.entries()) + .filter(([id]) => id.startsWith(prefix)) + .map(([id, scenarioName]) => ({ id, scenarioName })); + } + async saveChecks( + id: string, + writer: string, + checks: ConformanceCheck[] + ): Promise { + let byWriter = this.checks.get(id); + if (!byWriter) this.checks.set(id, (byWriter = new Map())); + byWriter.set( + writer, + checks.map((c) => ({ ...c })) + ); + } + async loadChecks(id: string): Promise> { + return new Map(this.checks.get(id) ?? []); + } + async deleteRun(id: string): Promise { + this.runs.delete(id); + this.checks.delete(id); + } +} diff --git a/src/hosted/wire.ts b/src/hosted/wire.ts new file mode 100644 index 00000000..1f7c7a92 --- /dev/null +++ b/src/hosted/wire.ts @@ -0,0 +1,312 @@ +/** + * What the wire said about a dispatched MCP request: the client's request as + * a JSON-RPC message (method, requested version), the response the cell gave + * it (status, body), and the two judgements the hosted layer records from + * them so a cell cannot read green when every request was turned away: + * + * hosted-wire-rejected a 4xx whose body is one of the lifecycle + * rejections (missing header / _meta, unsupported + * protocol version) — the scenario never saw a + * request it could judge; + * hosted-wrong-revision the client spoke a revision other than the one + * the cell is served on (a stateful `initialize` on + * the stateless wire, a header naming another + * revision). + * + * Both are FAILUREs, so they decide the cell's verdict (see report.ts). On a + * dated cell a request at a foreign revision that the wire turned away is + * version negotiation and records neither (see isNegotiation()). + */ + +import type { ServerResponse } from 'http'; +import { isStatefulVersion } from '../connection/select'; +import type { ConformanceCheck, SpecVersion } from '../types'; + +export const WIRE_REJECTED_CHECK_ID = 'hosted-wire-rejected'; +export const WRONG_REVISION_CHECK_ID = 'hosted-wrong-revision'; + +/** Response bodies above this are not captured (a rejection is small). */ +export const RESPONSE_CAP = 64 * 1024; + +const META_PROTOCOL_VERSION = 'io.modelcontextprotocol/protocolVersion'; + +function asRecord(v: unknown): Record | undefined { + return typeof v === 'object' && v !== null && !Array.isArray(v) + ? (v as Record) + : undefined; +} + +function str(v: unknown): string | undefined { + return typeof v === 'string' && v.length > 0 ? v : undefined; +} + +/** The JSON-RPC message(s) of a body: one object, or a batch. */ +function messagesOfJson(text: string): Record[] { + try { + const parsed: unknown = JSON.parse(text); + const list = Array.isArray(parsed) ? parsed : [parsed]; + return list.map(asRecord).filter((m): m is Record => !!m); + } catch { + return []; + } +} + +/** + * JSON-RPC messages in a response body: a JSON object or batch, or the + * `data:` lines of an SSE stream (the SDK transport answers a POST that + * way — `event: message\ndata: {...}\n\n`). + */ +export function jsonRpcMessages( + body: string | undefined, + contentType: string | undefined +): Record[] { + if (body === undefined) return []; + if (/^text\/event-stream\b/i.test(contentType ?? '')) { + const out: Record[] = []; + for (const event of body.split(/\r?\n\r?\n/)) { + const data = event + .split(/\r?\n/) + .filter((line) => line.startsWith('data:')) + .map((line) => line.slice('data:'.length).replace(/^ /, '')) + .join('\n'); + if (data) out.push(...messagesOfJson(data)); + } + return out; + } + return messagesOfJson(body); +} + +/** What one request says about itself, read from its JSON-RPC body. */ +export interface RequestInfo { + /** JSON-RPC methods carried (one per batch member with a `method`). */ + methods: string[]; + /** + * The protocol version the client asked for in the body: `initialize` + * params on the stateful wire, `_meta` on the stateless one. + */ + bodyVersion?: string; +} + +export function describeRequest( + body: Buffer | string | undefined +): RequestInfo { + const messages = body === undefined ? [] : messagesOfJson(body.toString()); + const methods: string[] = []; + let bodyVersion: string | undefined; + for (const m of messages) { + const method = str(m.method); + if (method) methods.push(method); + const params = asRecord(m.params); + const meta = asRecord(params?._meta); + bodyVersion ??= + str(meta?.[META_PROTOCOL_VERSION]) ?? + (method === 'initialize' ? str(params?.protocolVersion) : undefined); + } + return { methods, ...(bodyVersion && { bodyVersion }) }; +} + +/** The response a cell gave, as captured by tapResponse(). */ +export interface CapturedResponse { + status: number; + contentType?: string; + /** Undefined when the body was over RESPONSE_CAP. */ + body?: string; +} + +/** + * Wrap `res.write`/`res.end` so `onEnd` sees the status and (up to + * RESPONSE_CAP) the body once the response is complete. `onEnd` runs after + * the original `end`, synchronously, so a bridge that resolves its Response + * from `end` still sees whatever `onEnd` records before it flushes. + */ +export function tapResponse( + res: ServerResponse, + onEnd: (captured: CapturedResponse) => void +): void { + const chunks: Buffer[] = []; + let size = 0; + let overflow = false; + let ended = false; + const capture = (chunk: unknown, encoding?: unknown) => { + if (overflow) return; + let buf: Buffer; + if (Buffer.isBuffer(chunk)) buf = chunk; + else if (chunk instanceof Uint8Array) + buf = Buffer.from(chunk); // hono/node-server + else if (typeof chunk === 'string') + buf = Buffer.from( + chunk, + typeof encoding === 'string' ? (encoding as BufferEncoding) : 'utf8' + ); + else return; // end(cb), end() + size += buf.length; + if (size > RESPONSE_CAP) overflow = true; + else chunks.push(buf); + }; + const write = res.write; + const end = res.end; + res.write = function (this: ServerResponse, ...args: unknown[]) { + capture(args[0], args[1]); + return (write as (...a: unknown[]) => boolean).apply(this, args); + } as ServerResponse['write']; + res.end = function (this: ServerResponse, ...args: unknown[]) { + capture(args[0], args[1]); + const out = (end as (...a: unknown[]) => ServerResponse).apply(this, args); + if (!ended) { + ended = true; + const contentType = res.getHeader('content-type'); + onEnd({ + status: res.statusCode, + ...(typeof contentType === 'string' && { contentType }), + ...(!overflow && { body: Buffer.concat(chunks).toString('utf8') }) + }); + } + return out; + } as ServerResponse['end']; +} + +export interface WireRejection { + status: number; + code: number; + message: string; +} + +/** + * The lifecycle rejection in a 4xx response, if that is what it is: a + * JSON-RPC error with code -32020 / -32022 (protocol-version header), -32602 + * naming `_meta`, or -32000 saying "Unsupported protocol version" (the SDK + * transport's stateful negotiation failure). + * + * An unsupported-version rejection of a request whose header already names + * the cell's revision `served` is not the client's doing — it is a + * scenario's deliberate probe (request-metadata rejects a run's first + * request to exercise the client's retry) — and is not one. + */ +export function wireRejection( + response: CapturedResponse, + served: SpecVersion, + headerVersion: string | undefined +): WireRejection | undefined { + if (response.status < 400 || response.status >= 500) return undefined; + for (const m of jsonRpcMessages(response.body, response.contentType)) { + const error = asRecord(m.error); + if (!error || typeof error.code !== 'number') continue; + const code = error.code; + const message = str(error.message) ?? ''; + const unsupportedVersion = + code === -32022 || + (code === -32000 && message.includes('Unsupported protocol version')); + if (unsupportedVersion && headerVersion === served) continue; + if ( + unsupportedVersion || + code === -32020 || + (code === -32602 && message.includes('_meta')) + ) { + return { status: response.status, code, message }; + } + } + return undefined; +} + +/** + * Whether a request to a dated cell was version negotiation rather than a + * client at the wrong revision: its header named a revision `served` does + * not serve and the wire turned it away for it — a 4xx whose body is a + * lifecycle rejection (see wireRejection()) or a -32601 method-not-found + * (the probe method does not exist on the dated wire). A dual-era client + * opens a dated cell that way — `server/discover` at 2026-07-28 is + * rejected, then it falls back to `initialize` — so neither judgement + * applies to the rejected request. A foreign-revision request the wire + * accepted is still the client carrying on at the wrong revision, and on + * the stateless wire nothing is negotiation. + */ +export function isNegotiation( + served: SpecVersion, + headerVersion: string | undefined, + response: CapturedResponse +): boolean { + if (!isStatefulVersion(served)) return false; + if (headerVersion === undefined || headerVersion === served) return false; + if (response.status < 400 || response.status >= 500) return false; + if (wireRejection(response, served, headerVersion)) return true; + return jsonRpcMessages(response.body, response.contentType).some( + (m) => asRecord(m.error)?.code === -32601 + ); +} + +/** + * Why a request is not one the cell's revision `served` should receive, or + * undefined when it is. On the stateless wire every request must carry the + * cell's revision in the header and `initialize` does not exist; on a dated + * (stateful) revision `initialize` negotiates and is exempt, and every later + * request's header, when present, must name the cell's revision — unless + * the wire rejected it as negotiation (isNegotiation()), which the caller + * decides from the response. + */ +export function wrongRevision( + served: SpecVersion, + method: string, + headerVersion: string | undefined +): string | undefined { + if (isStatefulVersion(served)) { + if (method === 'initialize') return undefined; + if (headerVersion !== undefined && headerVersion !== served) + return `sent ${headerVersion}`; + return undefined; + } + if (method === 'initialize') return 'sent initialize'; + if (headerVersion !== served) + return headerVersion === undefined + ? 'sent no MCP-Protocol-Version header' + : `sent ${headerVersion}`; + return undefined; +} + +export function wireRejectedCheck( + rejection: WireRejection, + request: RequestInfo, + headerVersion: string | undefined +): ConformanceCheck { + const method = request.methods[0]; + return { + id: WIRE_REJECTED_CHECK_ID, + name: 'WireRejected', + description: + 'The cell turned a request away before the scenario could judge it', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `${response(rejection)}${method ? ` to ${method}` : ''}: ${rejection.message}`, + details: { + status: rejection.status, + code: rejection.code, + message: rejection.message, + ...(method && { method }), + requestedVersion: headerVersion ?? request.bodyVersion ?? null + } + }; +} + +function response(r: WireRejection): string { + return `HTTP ${r.status}, JSON-RPC error ${r.code}`; +} + +export function wrongRevisionCheck( + served: SpecVersion, + method: string, + headerVersion: string | undefined, + reason: string +): ConformanceCheck { + return { + id: WRONG_REVISION_CHECK_ID, + name: 'WrongRevision', + description: `The client spoke a revision other than the one this cell is served on`, + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `cell is served on ${served}; client ${reason}`, + details: { + served, + method, + headerVersion: headerVersion ?? null + } + }; +} diff --git a/src/index.ts b/src/index.ts index b376644e..9ccb9f0a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -63,6 +63,7 @@ import { createTierCheckCommand } from './tier-check'; import { createNewSepCommand } from './new-sep'; import { createSdkCommand } from './sdk-runner'; import { createTraceabilityCommand } from './traceability'; +import { runHostedServer } from './hosted'; import packageJson from '../package.json'; // Note on naming: `command` refers to which CLI command is calling this. @@ -929,6 +930,47 @@ program.addCommand(createSdkCommand()); // SEP traceability manifest command program.addCommand(createTraceabilityCommand()); +// Hosted server — mount scenarios on URL paths for remote clients +program + .command('hosted') + .description( + 'Run a long-lived HTTP server that exposes every client scenario at ' + + 'every requirement-set revision under /s/// ' + + 'and serves results at /results/. With --as-origin, auth/* ' + + 'scenarios are also mounted; deploy examples/hosted/valtown-relay.ts ' + + 'at that origin.' + ) + .option('--port ', 'Port to listen on', '3000') + .option( + '--public-origin ', + 'Origin to use in generated links (default: derived from Host header)' + ) + .option('--ttl ', 'Idle session TTL in milliseconds', '300000') + .option( + '--as-origin ', + 'Public origin of the AS relay (enables auth/* scenarios)' + ) + .option('--as2-origin ', 'Second AS relay (for migration scenario)') + .option('--idp-origin ', 'IdP relay (for EMA scenario)') + .option( + '--relay-secret ', + 'Shared secret the relay sends in x-relay-secret. Required with --as-origin. ' + + 'Defaults to $CONFORMANCE_RELAY_SECRET.' + ) + .action(async (options) => { + await runHostedServer({ + port: parseInt(options.port, 10), + publicOrigin: options.publicOrigin, + ttlMs: parseInt(options.ttl, 10), + auxOrigins: { + as: options.asOrigin, + as2: options.as2Origin, + idp: options.idpOrigin + }, + relaySecret: options.relaySecret ?? process.env.CONFORMANCE_RELAY_SECRET + }); + }); + // List scenarios command program .command('list') diff --git a/src/mock-server/index.ts b/src/mock-server/index.ts index 7c8470e6..8de6f9cd 100644 --- a/src/mock-server/index.ts +++ b/src/mock-server/index.ts @@ -9,7 +9,7 @@ * This is the client-conformance mirror of `Connection` in `../connection`. */ -import type { SpecVersion } from '../types'; +import type { RequestListener, SpecVersion } from '../types'; import type { JSONRPCRequest } from '../spec-types/2025-11-25'; /** @@ -41,6 +41,19 @@ export interface MockServer { close(): Promise; } +/** + * The same mock as `MockServer` but not bound to a port: a request listener + * the caller mounts itself — `http.createServer(listener)` in the CLI + * runner, a path-prefix mount in the hosted runner. `Scenario.handler()` + * implementations use this so one scenario body serves both. + */ +export interface MockHandler { + /** Serves `/mcp` — hand it to `http.createServer` or mount it. */ + listener: RequestListener; + /** See `MockServer.recorded`. */ + readonly recorded: JSONRPCRequest[]; +} + /** * Per-run context handed to `Scenario.start()`. The runner constructs this * from the resolved `--spec-version`. @@ -53,13 +66,19 @@ export interface ScenarioContext { * `http.createServer`. */ createServer(handlers: RequestHandlers): Promise; + /** + * Same mock, unbound. What `Scenario.handler()` implementations call so + * the scenario can be mounted without a loopback port (see src/hosted). + */ + createHandler(handlers: RequestHandlers): MockHandler; } -export { createServerStateful } from './stateful'; +export { createServerStateful, createHandlerStateful } from './stateful'; export { createServerStateless, + createHandlerStateless, validateStatelessRequest, withRequiredDraftResultFields, CACHEABLE_RESULT_METHODS } from './stateless'; -export { createServerFor } from './select'; +export { createServerFor, createHandlerFor } from './select'; diff --git a/src/mock-server/select.ts b/src/mock-server/select.ts index 2a836f1c..1ffee33e 100644 --- a/src/mock-server/select.ts +++ b/src/mock-server/select.ts @@ -1,8 +1,8 @@ import type { SpecVersion } from '../types'; -import type { MockServer, RequestHandlers } from './index'; +import type { MockHandler, MockServer, RequestHandlers } from './index'; import { isStatefulVersion } from '../connection/select'; -import { createServerStateful } from './stateful'; -import { createServerStateless } from './stateless'; +import { createHandlerStateful, createServerStateful } from './stateful'; +import { createHandlerStateless, createServerStateless } from './stateless'; export function createServerFor( specVersion: SpecVersion @@ -11,3 +11,11 @@ export function createServerFor( ? (handlers) => createServerStateful(handlers, specVersion) : (handlers) => createServerStateless(handlers, specVersion); } + +export function createHandlerFor( + specVersion: SpecVersion +): (handlers: RequestHandlers) => MockHandler { + return isStatefulVersion(specVersion) + ? (handlers) => createHandlerStateful(handlers, specVersion) + : (handlers) => createHandlerStateless(handlers, specVersion); +} diff --git a/src/mock-server/stateful.ts b/src/mock-server/stateful.ts index 0a80b7f7..7e320ae1 100644 --- a/src/mock-server/stateful.ts +++ b/src/mock-server/stateful.ts @@ -8,6 +8,7 @@ */ import express from 'express'; +import http from 'http'; import { Server } from '@modelcontextprotocol/sdk/server/index.js'; import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js'; @@ -15,7 +16,7 @@ import { z } from 'zod'; import type { JSONRPCRequest } from '../spec-types/2025-11-25'; import { isSpecVersion, type SpecVersion } from '../types'; import { validateWireMessage } from '../validation/wire-schema'; -import type { MockServer, RequestHandlers } from './index'; +import type { MockHandler, MockServer, RequestHandlers } from './index'; const CAPABILITY_BY_PREFIX: Record = { tools: 'tools', @@ -41,10 +42,10 @@ export function capabilitiesFromHandlers( return out; } -export async function createServerStateful( +export function createHandlerStateful( handlers: RequestHandlers, specVersion: SpecVersion -): Promise { +): MockHandler { const recorded: JSONRPCRequest[] = []; const capabilities = capabilitiesFromHandlers(handlers); @@ -152,24 +153,32 @@ export async function createServerStateful( } }); - return listen(app, recorded); + return { listener: app, recorded }; } -function listen( - app: express.Application, - recorded: JSONRPCRequest[] +export async function createServerStateful( + handlers: RequestHandlers, + specVersion: SpecVersion ): Promise { + return listenMockHandler(createHandlerStateful(handlers, specVersion)); +} + +/** + * Bind a `MockHandler` to an ephemeral localhost port — the CLI runner's + * path. Shared with the stateless impl. + */ +export function listenMockHandler(mock: MockHandler): Promise { return new Promise((resolve, reject) => { - const httpServer = app.listen(0); + const httpServer = http.createServer(mock.listener); httpServer.on('error', reject); - httpServer.on('listening', () => { + httpServer.listen(0, () => { const addr = httpServer.address(); const port = typeof addr === 'object' && addr ? addr.port : 0; const baseUrl = `http://localhost:${port}`; resolve({ url: `${baseUrl}/mcp`, baseUrl, - recorded, + recorded: mock.recorded, close: () => new Promise((res) => { httpServer.closeAllConnections?.(); diff --git a/src/mock-server/stateless.ts b/src/mock-server/stateless.ts index 3d06c86a..37d05da5 100644 --- a/src/mock-server/stateless.ts +++ b/src/mock-server/stateless.ts @@ -11,10 +11,10 @@ import express from 'express'; import { DRAFT_PROTOCOL_VERSION, type SpecVersion } from '../types'; import type { JSONRPCRequest } from '../spec-types/2025-11-25'; -import type { MockServer, RequestHandlers } from './index'; +import type { MockHandler, MockServer, RequestHandlers } from './index'; import { STATELESS_SPEC_VERSIONS } from '../connection/select'; import { validateWireMessage } from '../validation/wire-schema'; -import { capabilitiesFromHandlers } from './stateful'; +import { capabilitiesFromHandlers, listenMockHandler } from './stateful'; /** * The required per-request `_meta` keys. `io.modelcontextprotocol/clientInfo` @@ -182,10 +182,10 @@ export function validateStatelessRequest( * server accepts exactly that version. Without it, every known stateless * version is accepted. */ -export async function createServerStateless( +export function createHandlerStateless( handlers: RequestHandlers, specVersion?: SpecVersion -): Promise { +): MockHandler { const recorded: JSONRPCRequest[] = []; const capabilities = capabilitiesFromHandlers(handlers); const supportedVersions: readonly string[] = specVersion @@ -263,23 +263,12 @@ export async function createServerStateless( } }); - return new Promise((resolve, reject) => { - const httpServer = app.listen(0); - httpServer.on('error', reject); - httpServer.on('listening', () => { - const addr = httpServer.address(); - const port = typeof addr === 'object' && addr ? addr.port : 0; - const baseUrl = `http://localhost:${port}`; - resolve({ - url: `${baseUrl}/mcp`, - baseUrl, - recorded, - close: () => - new Promise((r) => { - httpServer.closeAllConnections?.(); - httpServer.close(() => r()); - }) - }); - }); - }); + return { listener: app, recorded }; +} + +export async function createServerStateless( + handlers: RequestHandlers, + specVersion?: SpecVersion +): Promise { + return listenMockHandler(createHandlerStateless(handlers, specVersion)); } diff --git a/src/mock-server/testing.ts b/src/mock-server/testing.ts index 1e307b8d..96b97c88 100644 --- a/src/mock-server/testing.ts +++ b/src/mock-server/testing.ts @@ -1,5 +1,5 @@ import { LATEST_SPEC_VERSION, type SpecVersion } from '../types'; -import { createServerFor } from './select'; +import { createServerFor, createHandlerFor } from './select'; import type { ScenarioContext } from './index'; /** @@ -12,6 +12,7 @@ export function testScenarioContext( ): ScenarioContext { return { specVersion, - createServer: (handlers) => createServerFor(specVersion)(handlers) + createServer: (handlers) => createServerFor(specVersion)(handlers), + createHandler: (handlers) => createHandlerFor(specVersion)(handlers) }; } diff --git a/src/requirements.ts b/src/requirements.ts index 26101cec..45dcdf50 100644 --- a/src/requirements.ts +++ b/src/requirements.ts @@ -3,6 +3,7 @@ import { dirname, join } from 'path'; import { fileURLToPath } from 'url'; import { parse as parseYaml } from 'yaml'; import { ALL_SPEC_VERSIONS } from './scenarios'; +import { SPEC_VERSION_TIMELINE, type SpecVersion } from './types'; /** * A frozen requirement set for one specification revision: the scenarios an @@ -61,13 +62,56 @@ function requirementsDir(): string { return join(dirname(fileURLToPath(import.meta.url)), '..', 'requirements'); } -export function listRequirementRevisions(): string[] { - const dir = requirementsDir(); - if (!existsSync(dir)) return []; - return readdirSync(dir) - .filter((f) => f.endsWith('.yaml')) - .map((f) => f.replace(/\.yaml$/, '')) - .sort(); +/** + * Requirement sets supplied as text, keyed by revision. A deployment that has + * no `requirements/` directory next to it (a serverless bundle of the import + * closure, say) registers the yaml texts up front; see + * examples/hosted/bundle-requirements.ts. Registered text wins over a file of + * the same revision and goes through exactly the same validation. + */ +const registeredSources = new Map(); + +export function registerRequirementSources( + sources: Record +): void { + for (const [revision, text] of Object.entries(sources)) { + registeredSources.set(revision, text); + } +} + +/** Revisions with a yaml on disk; empty when the directory is unreadable. */ +function revisionsOnDisk(): string[] { + try { + const dir = requirementsDir(); + if (!existsSync(dir)) return []; + return readdirSync(dir) + .filter((f) => f.endsWith('.yaml')) + .map((f) => f.replace(/\.yaml$/, '')); + } catch { + return []; + } +} + +/** + * Revisions that ship a requirement set — registered or on disk — in + * spec-timeline order. A yaml whose name is not a protocol version this build + * knows is ignored: it could not be loaded anyway (see loadRequirements). + */ +export function listRequirementRevisions(): SpecVersion[] { + const present = new Set([...registeredSources.keys(), ...revisionsOnDisk()]); + return SPEC_VERSION_TIMELINE.filter((v) => present.has(v)); +} + +/** The yaml text for a revision: registered first, then the bundled file. */ +function requirementSource(revision: string): string | undefined { + const registered = registeredSources.get(revision); + if (registered !== undefined) return registered; + try { + const path = join(requirementsDir(), `${revision}.yaml`); + return existsSync(path) ? readFileSync(path, 'utf-8') : undefined; + } catch { + return undefined; + } } function asNameList(value: unknown, field: string, revision: string): string[] { @@ -102,8 +146,8 @@ export function loadRequirements(revision: string): RequirementSet { ); } - const path = join(requirementsDir(), `${revision}.yaml`); - if (!existsSync(path)) { + const source = requirementSource(revision); + if (source === undefined) { const known = listRequirementRevisions(); throw new Error( `No requirement set for ${revision}.` + @@ -113,7 +157,7 @@ export function loadRequirements(revision: string): RequirementSet { ); } - const parsed = parseYaml(readFileSync(path, 'utf-8')) ?? {}; + const parsed = parseYaml(source) ?? {}; // A frozen contract must fail loudly on anything it does not recognise: a // typo'd key ("sever:") would otherwise silently empty a leg and the gate diff --git a/src/runner/client.ts b/src/runner/client.ts index af10657f..9b0111c0 100644 --- a/src/runner/client.ts +++ b/src/runner/client.ts @@ -9,7 +9,11 @@ import { DRAFT_PROTOCOL_VERSION } from '../types'; import { getScenario, isScenarioApplicableAt } from '../scenarios'; -import { createServerFor, type ScenarioContext } from '../mock-server'; +import { + createServerFor, + createHandlerFor, + type ScenarioContext +} from '../mock-server'; import { resetWireValidation, wireSchemaChecks @@ -182,16 +186,23 @@ export async function runConformanceTest( ); const ctx: ScenarioContext = { specVersion: resolvedVersion, - createServer: (handlers) => createServerFor(resolvedVersion)(handlers) + createServer: (handlers) => createServerFor(resolvedVersion)(handlers), + createHandler: (handlers) => createHandlerFor(resolvedVersion)(handlers) }; console.error(`Starting scenario: ${scenarioName}`); resetWireValidation(); const urls = await scenario.start(ctx); + // Steering steps ride in the same context blob as credentials etc. + const context: Record | undefined = + scenario.steps || urls.context + ? { ...urls.context, ...(scenario.steps && { steps: scenario.steps }) } + : undefined; + console.error(`Executing client: ${clientCommand} ${urls.serverUrl}`); - if (urls.context) { - console.error(`With context: ${JSON.stringify(urls.context)}`); + if (context) { + console.error(`With context: ${JSON.stringify(context)}`); } try { @@ -200,7 +211,7 @@ export async function runConformanceTest( scenarioName, urls.serverUrl, timeout, - urls.context, + context, resolvedVersion ); @@ -368,7 +379,8 @@ export async function runInteractiveMode( ); const ctx: ScenarioContext = { specVersion: resolvedVersion, - createServer: (handlers) => createServerFor(resolvedVersion)(handlers) + createServer: (handlers) => createServerFor(resolvedVersion)(handlers), + createHandler: (handlers) => createHandlerFor(resolvedVersion)(handlers) }; console.log(`Starting scenario: ${scenarioName}`); diff --git a/src/scenarios/client/auth/basic-cimd.ts b/src/scenarios/client/auth/basic-cimd.ts index 1e4b37c5..d0c63805 100644 --- a/src/scenarios/client/auth/basic-cimd.ts +++ b/src/scenarios/client/auth/basic-cimd.ts @@ -1,9 +1,11 @@ -import type { ScenarioContext } from '../../../mock-server'; -import type { Scenario, ConformanceCheck } from '../../../types'; -import { ScenarioUrls } from '../../../types'; +import { + AuthHandlerScenario, + AuthHandlerContext, + AuthHandlers, + ConformanceCheck +} from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; import { createServer } from './helpers/createServer'; -import { ServerLifecycle } from './helpers/serverLifecycle'; import { SpecReferences } from './spec-references'; /** @@ -21,19 +23,18 @@ export const CIMD_CLIENT_METADATA_URL = * clients SHOULD use a URL as their client_id instead of using dynamic client * registration. */ -export class AuthBasicCIMDScenario implements Scenario { +export class AuthBasicCIMDScenario extends AuthHandlerScenario { name = 'auth/basic-cimd'; readonly source = { introducedIn: '2025-11-25' } as const; description = 'Tests OAuth flow with Client ID Metadata Documents (SEP-991/URL-based client IDs). Server advertises client_id_metadata_document_supported=true and client should use URL as client_id instead of DCR.'; - private authServer = new ServerLifecycle(); - private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(ctx: ScenarioContext): Promise { + authHandlers(ctx: AuthHandlerContext): AuthHandlers { this.checks = []; + const getAsUrl = () => ctx.getAuxBaseUrl('as'); - const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, getAsUrl, { clientIdMetadataDocumentSupported: true, onAuthorizationRequest: (data) => { // Check if client used URL-based client ID @@ -58,23 +59,9 @@ export class AuthBasicCIMDScenario implements Scenario { } }); - await this.authServer.start(authApp); + const rsApp = createServer(ctx, this.checks, ctx.getRsBaseUrl, getAsUrl); - const app = createServer( - ctx, - this.checks, - this.server.getUrl, - this.authServer.getUrl - ); - - await this.server.start(app); - - return { serverUrl: `${this.server.getUrl()}/mcp` }; - } - - async stop() { - await this.authServer.stop(); - await this.server.stop(); + return { rs: rsApp, aux: { as: authApp } }; } getChecks(): ConformanceCheck[] { diff --git a/src/scenarios/client/auth/discovery-metadata.ts b/src/scenarios/client/auth/discovery-metadata.ts index c5623a92..1ba3303a 100644 --- a/src/scenarios/client/auth/discovery-metadata.ts +++ b/src/scenarios/client/auth/discovery-metadata.ts @@ -6,13 +6,18 @@ * generated from them. */ -import type { ScenarioContext } from '../../../mock-server'; -import type { Scenario, ConformanceCheck } from '../../../types'; -import { ScenarioUrls } from '../../../types'; +import { + AuthHandlerScenario, + AuthHandlerContext, + AuthHandlers, + ConformanceCheck +} from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; import { createServer } from './helpers/createServer'; -import { ServerLifecycle } from './helpers/serverLifecycle'; -import { addResourceParameterChecks } from './helpers/resourceParameterChecks'; +import { + addResourceParameterChecks, + observeResourceParameters +} from './helpers/resourceParameterChecks'; import { SpecReferences } from './spec-references'; import { Request, Response } from 'express'; @@ -71,184 +76,199 @@ const SCENARIO_CONFIGS: MetadataScenarioConfig[] = [ ]; /** - * Creates a metadata discovery scenario from configuration. + * Base for the table-driven discovery scenarios. Each subclass binds a row + * of SCENARIO_CONFIGS; we use real classes (not factory-returned literals) + * so the hosted runner can do `new Ctor()` for a fresh instance per run. */ -function createMetadataScenario(config: MetadataScenarioConfig): Scenario { - const authServer = new ServerLifecycle(); - const server = new ServerLifecycle(); - let checks: ConformanceCheck[] = []; +abstract class MetadataDiscoveryScenario extends AuthHandlerScenario { + protected abstract readonly config: MetadataScenarioConfig; + readonly source = { introducedIn: '2025-11-25' } as const; + private checks: ConformanceCheck[] = []; // Track resource parameters for RFC 8707 validation. metadata-var2 serves // the PRM at the root, so its `resource` is a bare origin: the case a URL // parser rewrites with a trailing slash. - let authorizationResource: string | undefined; - let tokenResource: string | undefined; - let prmResource: string | undefined; + private authorizationResource: string | undefined; + private tokenResource: string | undefined; + private prmResource: string | undefined; - const routePrefix = config.authRoutePrefix || ''; - const isOpenIdConfiguration = config.oauthMetadataLocation.includes( - 'openid-configuration' - ); + get name() { + return `auth/${this.config.name}`; + } + get description() { + return `Tests Basic OAuth metadata discovery flow. - // Determine if PRM is at path-based location - const isPathBasedPrm = - config.prmLocation === '/.well-known/oauth-protected-resource/mcp'; +**PRM:** ${this.config.prmLocation}${this.config.inWwwAuth ? '' : ' (not in WWW-Authenticate)'} +**OAuth metadata:** ${this.config.oauthMetadataLocation} +`; + } - return { - name: `auth/${config.name}`, - source: { introducedIn: '2025-11-25' }, - description: `Tests Basic OAuth metadata discovery flow. + authHandlers(ctx: AuthHandlerContext): AuthHandlers { + this.checks = []; + this.authorizationResource = undefined; + this.tokenResource = undefined; + this.prmResource = undefined; + const config = this.config; + const routePrefix = config.authRoutePrefix || ''; + const isOpenIdConfiguration = config.oauthMetadataLocation.includes( + 'openid-configuration' + ); + const getAsUrl = () => ctx.getAuxBaseUrl('as'); -**PRM:** ${config.prmLocation}${config.inWwwAuth ? '' : ' (not in WWW-Authenticate)'} -**OAuth metadata:** ${config.oauthMetadataLocation} -`, + const authApp = createAuthServer(ctx, this.checks, getAsUrl, { + metadataPath: config.oauthMetadataLocation, + isOpenIdConfiguration, + ...(routePrefix && { routePrefix }), + onAuthorizationRequest: ({ resource }) => { + this.authorizationResource = resource; + }, + onTokenRequest: ({ body }) => { + this.tokenResource = body.resource; + // Same token the auth server mints by default; these scenarios + // request no scopes. + return { token: `test-token-${Date.now()}`, scopes: [] }; + } + }); - async start(ctx: ScenarioContext): Promise { - checks = []; - authorizationResource = undefined; - tokenResource = undefined; - prmResource = undefined; + // If path-based OAuth metadata, trap root requests + if (routePrefix) { + authApp.get('/.well-known/oauth-authorization-server', (req, res) => { + this.checks.push({ + id: 'authorization-server-metadata-wrong-path', + name: 'AuthorizationServerMetadataWrongPath', + description: + 'Client requested authorization server at the root path when the AS URL has a path-based location', + status: 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [ + SpecReferences.RFC_AUTH_SERVER_METADATA_REQUEST, + SpecReferences.MCP_AUTH_DISCOVERY + ], + details: { + url: req.url + } + }); + res.status(404).send('Not Found'); + }); + } + + const getAuthServerUrl = routePrefix + ? () => `${getAsUrl()}${routePrefix}` + : getAsUrl; - const authApp = createAuthServer(ctx, checks, authServer.getUrl, { - metadataPath: config.oauthMetadataLocation, - isOpenIdConfiguration, - ...(routePrefix && { routePrefix }), - onAuthorizationRequest: ({ resource }) => { - authorizationResource = resource; - }, - onTokenRequest: ({ body }) => { - tokenResource = body.resource; - // Same token the auth server mints by default; these scenarios - // request no scopes. - return { token: `test-token-${Date.now()}`, scopes: [] }; + const rsApp = createServer( + ctx, + this.checks, + ctx.getRsBaseUrl, + getAuthServerUrl, + { + prmPath: config.prmLocation, + includePrmInWwwAuth: config.inWwwAuth, + onPrmRequest: ({ resource }) => { + this.prmResource = resource; } - }); + } + ); - // If path-based OAuth metadata, trap root requests - if (routePrefix) { - authApp.get('/.well-known/oauth-authorization-server', (req, res) => { - checks.push({ - id: 'authorization-server-metadata-wrong-path', - name: 'AuthorizationServerMetadataWrongPath', + // Add trap for root PRM requests if configured + if (config.trapRootPrm) { + rsApp.get( + '/.well-known/oauth-protected-resource', + (req: Request, res: Response) => { + this.checks.push({ + id: 'prm-priority-order', + name: 'PRM Priority Order', description: - 'Client requested authorization server at the root path when the AS URL has a path-based location', + 'Client requested PRM metadata at root location on a server with path-based PRM', status: 'FAILURE', timestamp: new Date().toISOString(), specReferences: [ - SpecReferences.RFC_AUTH_SERVER_METADATA_REQUEST, - SpecReferences.MCP_AUTH_DISCOVERY + SpecReferences.RFC_PRM_DISCOVERY, + SpecReferences.MCP_PRM_DISCOVERY ], details: { - url: req.url + url: req.url, + path: req.path } }); - res.status(404).send('Not Found'); - }); - } - - await authServer.start(authApp); - - const getAuthServerUrl = routePrefix - ? () => `${authServer.getUrl()}${routePrefix}` - : authServer.getUrl; - const app = createServer(ctx, checks, server.getUrl, getAuthServerUrl, { - prmPath: config.prmLocation, - includePrmInWwwAuth: config.inWwwAuth, - onPrmRequest: ({ resource }) => { - prmResource = resource; + res.status(404).json({ + error: 'not_found', + error_description: 'PRM metadata not available at root location' + }); } - }); - - // Add trap for root PRM requests if configured - if (config.trapRootPrm) { - app.get( - '/.well-known/oauth-protected-resource', - (req: Request, res: Response) => { - checks.push({ - id: 'prm-priority-order', - name: 'PRM Priority Order', - description: - 'Client requested PRM metadata at root location on a server with path-based PRM', - status: 'FAILURE', - timestamp: new Date().toISOString(), - specReferences: [ - SpecReferences.RFC_PRM_DISCOVERY, - SpecReferences.MCP_PRM_DISCOVERY - ], - details: { - url: req.url, - path: req.path - } - }); - - res.status(404).json({ - error: 'not_found', - error_description: 'PRM metadata not available at root location' - }); - } - ); - } - - await server.start(app); + ); + } - return { serverUrl: `${server.getUrl()}/mcp` }; - }, + return { rs: rsApp, aux: { as: authApp } }; + } - async stop() { - await authServer.stop(); - await server.stop(); - }, + rawChecks(): ConformanceCheck[] { + return this.checks; + } - getChecks(): ConformanceCheck[] { - const expectedSlugs = [ - ...(isPathBasedPrm ? ['prm-pathbased-requested'] : []), - 'authorization-server-metadata', - 'client-registration', - 'authorization-request', - 'token-request' - ]; + getChecks(): ConformanceCheck[] { + const isPathBasedPrm = + this.config.prmLocation === '/.well-known/oauth-protected-resource/mcp'; + const expectedSlugs = [ + ...(isPathBasedPrm ? ['prm-pathbased-requested'] : []), + 'authorization-server-metadata', + 'client-registration', + 'authorization-request', + 'token-request' + ]; - for (const slug of expectedSlugs) { - if (!checks.find((c) => c.id === slug)) { - checks.push({ - id: slug, - name: `Expected Check Missing: ${slug}`, - description: `Expected Check Missing: ${slug}`, - status: 'FAILURE', - timestamp: new Date().toISOString() - }); - } + for (const slug of expectedSlugs) { + if (!this.checks.find((c) => c.id === slug)) { + this.checks.push({ + id: slug, + name: `Expected Check Missing: ${slug}`, + description: `Expected Check Missing: ${slug}`, + status: 'FAILURE', + timestamp: new Date().toISOString() + }); } + } - // RFC 8707 Resource Parameter Validation Checks - addResourceParameterChecks( - checks, - { authorizationResource, tokenResource, prmResource }, - new Date().toISOString() - ); + // RFC 8707 Resource Parameter Validation Checks. The private fields are + // empty when a fresh instance re-judges a persisted log (hosted server), + // so fall back to what the request logger recorded. + const observed = observeResourceParameters(this.checks); + addResourceParameterChecks( + this.checks, + { + authorizationResource: + this.authorizationResource ?? observed.authorizationResource, + tokenResource: this.tokenResource ?? observed.tokenResource, + prmResource: this.prmResource ?? observed.prmResource + }, + new Date().toISOString() + ); - return checks; - } - }; + return this.checks; + } } -// Generate scenario instances from configurations -export const AuthMetadataDefaultScenario = createMetadataScenario( - SCENARIO_CONFIGS[0] -); -export const AuthMetadataVar1Scenario = createMetadataScenario( - SCENARIO_CONFIGS[1] -); -export const AuthMetadataVar2Scenario = createMetadataScenario( - SCENARIO_CONFIGS[2] -); -export const AuthMetadataVar3Scenario = createMetadataScenario( - SCENARIO_CONFIGS[3] -); +export class AuthMetadataDefaultScenario extends MetadataDiscoveryScenario { + protected readonly config = SCENARIO_CONFIGS[0]; +} +export class AuthMetadataVar1Scenario extends MetadataDiscoveryScenario { + protected readonly config = SCENARIO_CONFIGS[1]; +} +export class AuthMetadataVar2Scenario extends MetadataDiscoveryScenario { + protected readonly config = SCENARIO_CONFIGS[2]; +} +export class AuthMetadataVar3Scenario extends MetadataDiscoveryScenario { + protected readonly config = SCENARIO_CONFIGS[3]; +} // Export all scenarios as an array for convenience -export const metadataScenarios = SCENARIO_CONFIGS.map(createMetadataScenario); +export const metadataScenarios = [ + new AuthMetadataDefaultScenario(), + new AuthMetadataVar1Scenario(), + new AuthMetadataVar2Scenario(), + new AuthMetadataVar3Scenario() +]; // Export function to list metadata scenario names (for suite support) export function listMetadataScenarios(): string[] { diff --git a/src/scenarios/client/auth/helpers/createAuthServer.ts b/src/scenarios/client/auth/helpers/createAuthServer.ts index 162f93aa..9197b569 100644 --- a/src/scenarios/client/auth/helpers/createAuthServer.ts +++ b/src/scenarios/client/auth/helpers/createAuthServer.ts @@ -13,6 +13,43 @@ import { type TokenIssuerKey } from './dpopToken'; +/** + * The authorization code is opaque to the client, so we use it to carry the + * per-flow state (PKCE challenge, requested scopes) from /authorize to + * /token. This keeps the AS stateless across processes — on serverless hosts + * (val.town) the two requests can land on different isolates, where closure + * state from /authorize doesn't exist. The closure variables remain as a + * fallback for flows that don't round-trip our code (e.g. hand-rolled tests). + * + * The code is a plain base64url JSON envelope — no secret, no signature, no + * verification: it is a state carrier for a test fixture, not a credential + * check, and the helper names say so (they perform no authorization). + */ +interface FlowCodeState { + challenge?: string; + scopes?: string[]; +} + +const AUTH_CODE_PREFIX = 'test-auth-code'; + +function packFlowCode(state: FlowCodeState): string { + return `${AUTH_CODE_PREFIX}.${Buffer.from(JSON.stringify(state)).toString('base64url')}`; +} + +function unpackFlowCode(code: string | undefined): FlowCodeState | undefined { + if (!code?.startsWith(`${AUTH_CODE_PREFIX}.`)) return undefined; + try { + return JSON.parse( + Buffer.from( + code.slice(AUTH_CODE_PREFIX.length + 1), + 'base64url' + ).toString() + ) as FlowCodeState; + } catch { + return undefined; + } +} + /** * Compute S256 code challenge from a code verifier. * BASE64URL(SHA256(code_verifier)) @@ -471,7 +508,13 @@ export function createAuthServer( const redirectUri = req.query.redirect_uri as string; const state = req.query.state as string; const redirectUrl = new URL(redirectUri); - redirectUrl.searchParams.set('code', 'test-auth-code'); + redirectUrl.searchParams.set( + 'code', + packFlowCode({ + challenge: codeChallenge, + scopes: lastAuthorizationScopes + }) + ); if (state) { redirectUrl.searchParams.set('state', state); } @@ -499,6 +542,13 @@ export function createAuthServer( const requestedScope = req.body.scope; const grantType = req.body.grant_type; + // Recover per-flow state from the code itself (survives process changes + // on serverless hosts); fall back to closure state for codes we didn't + // mint via packFlowCode. + const codeState = unpackFlowCode(req.body.code as string | undefined); + const flowChallenge = codeState?.challenge ?? storedCodeChallenge; + const flowScopes = codeState?.scopes ?? lastAuthorizationScopes; + checks.push({ id: 'token-request', name: 'TokenRequest', @@ -529,18 +579,17 @@ export function createAuthServer( // PKCE: Validate code_verifier matches code_challenge (S256) // Fail if either is missing const computedChallenge = - codeVerifier && storedCodeChallenge + codeVerifier && flowChallenge ? computeS256Challenge(codeVerifier) : undefined; const matches = - computedChallenge !== undefined && - computedChallenge === storedCodeChallenge; + computedChallenge !== undefined && computedChallenge === flowChallenge; let description: string; - if (!storedCodeChallenge && !codeVerifier) { + if (!flowChallenge && !codeVerifier) { description = 'Neither code_challenge nor code_verifier were sent - PKCE is required'; - } else if (!storedCodeChallenge) { + } else if (!flowChallenge) { description = 'code_challenge was not sent in authorization request - PKCE is required'; } else if (!codeVerifier) { @@ -561,7 +610,7 @@ export function createAuthServer( specReferences: [SpecReferences.MCP_PKCE], details: { matches, - storedChallenge: storedCodeChallenge || 'not sent', + storedChallenge: flowChallenge || 'not sent', computedChallenge: computedChallenge || 'not computed' } }); @@ -684,7 +733,7 @@ export function createAuthServer( } let token = `test-token-${Date.now()}`; - let scopes: string[] = lastAuthorizationScopes; + let scopes: string[] = flowScopes; if (onTokenRequest) { const result = await onTokenRequest({ diff --git a/src/scenarios/client/auth/helpers/createServer.test.ts b/src/scenarios/client/auth/helpers/createServer.test.ts new file mode 100644 index 00000000..12313bb6 --- /dev/null +++ b/src/scenarios/client/auth/helpers/createServer.test.ts @@ -0,0 +1,80 @@ +import { describe, it, expect } from 'vitest'; +import http from 'http'; +import { createServer } from './createServer'; +import { testScenarioContext } from '../../../../mock-server/testing'; +import { + DRAFT_PROTOCOL_VERSION, + type ConformanceCheck +} from '../../../../types'; + +describe('auth helper createServer — stateless /mcp', () => { + it('records a FAILURE in the scenario log when the stateless wire rejects a request', async () => { + const checks: ConformanceCheck[] = []; + const app = createServer( + testScenarioContext(DRAFT_PROTOCOL_VERSION), + checks, + () => 'http://rs.test', + () => 'http://as.test', + { authMiddleware: (_req, _res, next) => next() } + ); + const server = http.createServer(app); + await new Promise((r) => server.listen(0, r)); + const port = (server.address() as { port: number }).port; + try { + // A stateful initialize on the stateless wire: no header, no _meta. + const res = await fetch(`http://localhost:${port}/mcp`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { protocolVersion: '2025-11-25', capabilities: {} } + }) + }); + expect(res.status).toBe(400); + expect((await res.json()).error.code).toBe(-32020); + const rejected = checks.filter( + (c) => c.id === 'stateless-request-rejected' + ); + expect(rejected).toHaveLength(1); + expect(rejected[0]).toMatchObject({ + status: 'FAILURE', + errorMessage: 'Missing MCP-Protocol-Version header', + details: { + status: 400, + code: -32020, + method: 'initialize', + headerVersion: null + } + }); + + // A well-formed stateless request records nothing of the kind. + const ok = await fetch(`http://localhost:${port}/mcp`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION + }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 2, + method: 'tools/list', + params: { + _meta: { + 'io.modelcontextprotocol/protocolVersion': DRAFT_PROTOCOL_VERSION, + 'io.modelcontextprotocol/clientCapabilities': {} + } + } + }) + }); + expect(ok.status).toBe(200); + expect( + checks.filter((c) => c.id === 'stateless-request-rejected') + ).toHaveLength(1); + } finally { + server.closeAllConnections?.(); + await new Promise((r) => server.close(() => r())); + } + }); +}); diff --git a/src/scenarios/client/auth/helpers/createServer.ts b/src/scenarios/client/auth/helpers/createServer.ts index 6d308d64..2a6be686 100644 --- a/src/scenarios/client/auth/helpers/createServer.ts +++ b/src/scenarios/client/auth/helpers/createServer.ts @@ -108,6 +108,15 @@ export function createServer( if (prmPath !== null) { app.get(prmPath, (req: Request, res: Response) => { + // Resource is usually $baseUrl/mcp, but if PRM is at the root, + // the resource identifier is the root. + // Can be overridden via prmResourceOverride for testing resource mismatch. + const resource = + prmResourceOverride ?? + (prmPath === '/.well-known/oauth-protected-resource' + ? getBaseUrl() + : `${getBaseUrl()}/mcp`); + checks.push({ id: 'prm-pathbased-requested', name: 'PRMPathBasedRequested', @@ -120,19 +129,13 @@ export function createServer( ], details: { url: req.url, - path: req.path + path: req.path, + // Recorded so the RFC 8707 checks can be re-derived from the log + // (see observeResourceParameters). + resource } }); - // Resource is usually $baseUrl/mcp, but if PRM is at the root, - // the resource identifier is the root. - // Can be overridden via prmResourceOverride for testing resource mismatch. - const resource = - prmResourceOverride ?? - (prmPath === '/.well-known/oauth-protected-resource' - ? getBaseUrl() - : `${getBaseUrl()}/mcp`); - onPrmRequest?.({ resource, timestamp: new Date().toISOString() }); const prmResponse: any = { @@ -208,7 +211,37 @@ export function createServer( // version-independent. function handleStateless(req: Request, res: Response) { const v = validateStatelessRequest(req, { tools: {} }, [ctx.specVersion]); - if (v.kind !== 'route') { + if (v.kind === 'reject') { + // The client never reached the tools handlers: a stateful initialize, + // a missing header or _meta. Recorded here, in the scenario's own log, + // so a cell where every request was turned away cannot read green on + // the strength of its OAuth checks alone. + const error = (v.body as { error?: { code?: number; message?: string } }) + .error; + checks.push({ + id: 'stateless-request-rejected', + name: 'StatelessRequestRejected', + description: + 'The stateless MCP endpoint turned the request away before it reached the scenario', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: error?.message, + specReferences: [ + { + id: 'SEP-2575', + url: 'https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header' + } + ], + details: { + status: v.status, + code: error?.code, + method: (req.body as { method?: unknown } | undefined)?.method, + headerVersion: req.headers['mcp-protocol-version'] ?? null + } + }); + return res.status(v.status).json(v.body); + } + if (v.kind === 'handled') { return res.status(v.status).json(v.body); } const { id, method } = v; diff --git a/src/scenarios/client/auth/helpers/resourceParameterChecks.ts b/src/scenarios/client/auth/helpers/resourceParameterChecks.ts index e7223f1e..5c436509 100644 --- a/src/scenarios/client/auth/helpers/resourceParameterChecks.ts +++ b/src/scenarios/client/auth/helpers/resourceParameterChecks.ts @@ -15,6 +15,43 @@ export interface ResourceParameterObservation { prmResource?: string; } +/** + * Recover a ResourceParameterObservation from the raw check log alone. + * + * Scenarios observe the `resource` parameter through createAuthServer / + * createServer callbacks into private fields. The hosted server re-judges a + * run's merged log in a *fresh* scenario instance (possibly in a different + * process/isolate from the one that served the OAuth flow), where those + * fields are empty. The request logger already records every authorize query + * and token body, and the PRM route records the identifier it served, so the + * same facts can be read back from the log. + */ +export function observeResourceParameters( + checks: ConformanceCheck[] +): ResourceParameterObservation { + const observed: ResourceParameterObservation = {}; + const str = (v: unknown): string | undefined => + typeof v === 'string' ? v : undefined; + for (const c of checks) { + const d = c.details as Record | undefined; + if (!d) continue; + if (c.id === 'incoming-auth-request') { + const path = str(d.path) ?? ''; + if (path.endsWith('/authorize')) { + const q = d.query as Record | undefined; + observed.authorizationResource = + str(q?.resource) ?? observed.authorizationResource; + } else if (path.endsWith('/token')) { + const b = d.body as Record | undefined; + observed.tokenResource = str(b?.resource) ?? observed.tokenResource; + } + } else if (c.id === 'prm-pathbased-requested') { + observed.prmResource = str(d.resource) ?? observed.prmResource; + } + } + return observed; +} + /** * RFC 8707 resource-parameter checks, shared by every client-auth scenario * whose mock servers observe the authorization and token requests. The check diff --git a/src/scenarios/client/auth/pre-registration.ts b/src/scenarios/client/auth/pre-registration.ts index 7e4ff545..895dcb30 100644 --- a/src/scenarios/client/auth/pre-registration.ts +++ b/src/scenarios/client/auth/pre-registration.ts @@ -1,8 +1,11 @@ -import type { ScenarioContext } from '../../../mock-server'; -import type { Scenario, ConformanceCheck, ScenarioUrls } from '../../../types'; +import { + AuthHandlerScenario, + AuthHandlerContext, + AuthHandlers, + ConformanceCheck +} from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; import { createServer } from './helpers/createServer'; -import { ServerLifecycle } from './helpers/serverLifecycle'; import { SpecReferences } from './spec-references'; import { MockTokenVerifier } from './helpers/mockTokenVerifier'; @@ -23,21 +26,22 @@ const PRE_REGISTERED_CLIENT_SECRET = 'pre-registered-secret'; * This tests the pre-registration approach described in the MCP spec: * https://modelcontextprotocol.io/specification/draft/basic/authorization#preregistration */ -export class PreRegistrationScenario implements Scenario { +export class PreRegistrationScenario extends AuthHandlerScenario { name = 'auth/pre-registration'; readonly source = { introducedIn: '2025-11-25' } as const; description = 'Tests OAuth flow with pre-registered client credentials. Server does not support DCR.'; - private authServer = new ServerLifecycle(); - private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; + private getAsUrl: (() => string) | null = null; - async start(ctx: ScenarioContext): Promise { + authHandlers(ctx: AuthHandlerContext): AuthHandlers { this.checks = []; + const getAsUrl = () => ctx.getAuxBaseUrl('as'); + this.getAsUrl = getAsUrl; const tokenVerifier = new MockTokenVerifier(this.checks, []); - const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, getAsUrl, { tokenVerifier, disableDynamicRegistration: true, tokenEndpointAuthMethodsSupported: ['client_secret_basic'], @@ -108,40 +112,26 @@ export class PreRegistrationScenario implements Scenario { } }); - await this.authServer.start(authApp); - - const app = createServer( - ctx, - this.checks, - this.server.getUrl, - this.authServer.getUrl, - { - prmPath: '/.well-known/oauth-protected-resource/mcp', - requiredScopes: [], - tokenVerifier - } - ); + const rsApp = createServer(ctx, this.checks, ctx.getRsBaseUrl, getAsUrl, { + prmPath: '/.well-known/oauth-protected-resource/mcp', + requiredScopes: [], + tokenVerifier + }); - await this.server.start(app); + return { rs: rsApp, aux: { as: authApp } }; + } + protected scenarioContext() { return { - serverUrl: `${this.server.getUrl()}/mcp`, - context: { - client_id: PRE_REGISTERED_CLIENT_ID, - client_secret: PRE_REGISTERED_CLIENT_SECRET, - // The `issuer` the mock AS publishes in its metadata — the scenario - // sets neither `metadataIssuer` nor `routePrefix`, so createAuthServer - // resolves the published issuer to exactly this URL. - issuer: this.authServer.getUrl() - } + client_id: PRE_REGISTERED_CLIENT_ID, + client_secret: PRE_REGISTERED_CLIENT_SECRET, + // The `issuer` the mock AS publishes in its metadata — the scenario + // sets neither `metadataIssuer` nor `routePrefix`, so createAuthServer + // resolves the published issuer to exactly the AS base URL. + issuer: this.getAsUrl?.() }; } - async stop() { - await this.authServer.stop(); - await this.server.stop(); - } - getChecks(): ConformanceCheck[] { // Ensure we have the pre-registration check const hasPreRegCheck = this.checks.some( diff --git a/src/scenarios/client/auth/token-endpoint-auth.ts b/src/scenarios/client/auth/token-endpoint-auth.ts index ae046e22..ee53a3e2 100644 --- a/src/scenarios/client/auth/token-endpoint-auth.ts +++ b/src/scenarios/client/auth/token-endpoint-auth.ts @@ -6,7 +6,10 @@ import { createServer } from './helpers/createServer.js'; import { ServerLifecycle } from './helpers/serverLifecycle.js'; import { SpecReferences } from './spec-references.js'; import { MockTokenVerifier } from './helpers/mockTokenVerifier.js'; -import { addResourceParameterChecks } from './helpers/resourceParameterChecks.js'; +import { + addResourceParameterChecks, + observeResourceParameters +} from './helpers/resourceParameterChecks.js'; type AuthMethod = 'client_secret_basic' | 'client_secret_post' | 'none'; @@ -178,13 +181,17 @@ class TokenEndpointAuthScenario implements Scenario { }); } - // RFC 8707 Resource Parameter Validation Checks + // RFC 8707 Resource Parameter Validation Checks. The private fields are + // empty when a fresh instance re-judges a persisted log (hosted server), + // so fall back to what the request logger recorded. + const observed = observeResourceParameters(this.checks); addResourceParameterChecks( this.checks, { - authorizationResource: this.authorizationResource, - tokenResource: this.tokenResource, - prmResource: this.prmResource + authorizationResource: + this.authorizationResource ?? observed.authorizationResource, + tokenResource: this.tokenResource ?? observed.tokenResource, + prmResource: this.prmResource ?? observed.prmResource }, timestamp ); diff --git a/src/scenarios/client/elicitation-defaults.ts b/src/scenarios/client/elicitation-defaults.ts index d4170295..ee547a1b 100644 --- a/src/scenarios/client/elicitation-defaults.ts +++ b/src/scenarios/client/elicitation-defaults.ts @@ -1,4 +1,3 @@ -import type { ScenarioContext } from '../../mock-server'; /** * SEP-1034: Elicitation defaults test * Validates that clients properly apply default values for omitted fields @@ -12,9 +11,10 @@ import { ListToolsRequestSchema, ElicitResultSchema } from '@modelcontextprotocol/sdk/types.js'; -import type { Scenario, ConformanceCheck } from '../../types'; +import type { ConformanceCheck, RequestListener } from '../../types'; +import { HandlerScenario } from '../../types'; import express, { Request, Response } from 'express'; -import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../types'; +import { DRAFT_PROTOCOL_VERSION } from '../../types'; import { createRequestLogger } from '../request-logger'; import { randomUUID } from 'crypto'; @@ -473,7 +473,7 @@ function createServer(checks: ConformanceCheck[]): { return { app, cleanup }; } -export class ElicitationClientDefaultsScenario implements Scenario { +export class ElicitationClientDefaultsScenario extends HandlerScenario { name = 'elicitation-sep1034-client-defaults'; readonly source = { introducedIn: '2025-11-25', @@ -481,33 +481,36 @@ export class ElicitationClientDefaultsScenario implements Scenario { } as const; description = 'Tests client applies default values for omitted elicitation fields (SEP-1034)'; - private app: express.Application | null = null; - private httpServer: any = null; + mcpPath = '/mcp'; private checks: ConformanceCheck[] = []; private cleanup: (() => void) | null = null; - async start(_ctx: ScenarioContext): Promise { + handler(_getBaseUrl: () => string): RequestListener { this.checks = []; const { app, cleanup } = createServer(this.checks); - this.app = app; this.cleanup = cleanup; - this.httpServer = this.app.listen(0); - const port = this.httpServer.address().port; - return { serverUrl: `http://localhost:${port}/mcp` }; + return app; } async stop() { - if (this.cleanup) { - this.cleanup(); - this.cleanup = null; - } - if (this.httpServer) { - await new Promise((resolve) => this.httpServer.close(resolve)); - this.httpServer = null; - } - this.app = null; + this.cleanup?.(); + this.cleanup = null; + await super.stop(); } + /** + * The tool call triggers elicitation/create; the interpreter's standing + * default (accept with schema defaults) is exactly the behaviour under test. + */ + readonly steps = [ + { op: 'tools/list' }, + { + op: 'tools/call', + name: 'test_client_elicitation_defaults', + arguments: {} + } + ] as const; + getChecks(): ConformanceCheck[] { const expectedSlugs = [ 'client-elicitation-sep1034-string-default', diff --git a/src/scenarios/client/http-base.ts b/src/scenarios/client/http-base.ts index 6751dedd..80ccd024 100644 --- a/src/scenarios/client/http-base.ts +++ b/src/scenarios/client/http-base.ts @@ -1,7 +1,4 @@ -import { - withRequiredDraftResultFields, - type ScenarioContext -} from '../../mock-server'; +import { withRequiredDraftResultFields } from '../../mock-server'; /** * Shared HTTP test-server scaffold for client-under-test SEP-2243 scenarios. * @@ -13,8 +10,8 @@ import { import http from 'http'; import { - Scenario, - ScenarioUrls, + HandlerScenario, + RequestListener, ConformanceCheck, ScenarioSource, DRAFT_PROTOCOL_VERSION @@ -43,49 +40,18 @@ const EMPTY_LIST_RESULTS: ReadonlyMap = new Map([ ['tasks/list', { tasks: [] }] ]); -export abstract class BaseHttpScenario implements Scenario { +export abstract class BaseHttpScenario extends HandlerScenario { abstract name: string; abstract description: string; readonly source: ScenarioSource = { introducedIn: DRAFT_PROTOCOL_VERSION }; - allowClientError?: boolean; - protected server: http.Server | null = null; protected checks: ConformanceCheck[] = []; - protected port: number = 0; protected sessionId: string = `session-${Date.now()}`; - async start(_ctx: ScenarioContext): Promise { - return new Promise((resolve, reject) => { - this.server = http.createServer((req, res) => { - this.handleRequest(req, res); - }); - this.server.on('error', reject); - this.server.listen(0, () => { - const address = this.server!.address(); - if (address && typeof address === 'object') { - this.port = address.port; - resolve({ serverUrl: `http://localhost:${this.port}` }); - } else { - reject(new Error('Failed to get server address')); - } - }); - }); - } - - async stop(): Promise { - return new Promise((resolve, reject) => { - if (this.server) { - this.server.close((err) => { - if (err) reject(err); - else { - this.server = null; - resolve(); - } - }); - } else { - resolve(); - } - }); + handler(_getBaseUrl: () => string): RequestListener { + this.checks = []; + this.sessionId = `session-${Date.now()}`; + return (req, res) => this.handleRequest(req, res); } abstract getChecks(): ConformanceCheck[]; diff --git a/src/scenarios/client/http-custom-headers.test.ts b/src/scenarios/client/http-custom-headers.test.ts index 28341918..0cbdea2a 100644 --- a/src/scenarios/client/http-custom-headers.test.ts +++ b/src/scenarios/client/http-custom-headers.test.ts @@ -6,6 +6,7 @@ import { CUSTOM_HEADERS_DECLARED_CHECK_IDS, INVALID_TOOL_DECLARED_CHECK_IDS } from './http-custom-headers'; +import { finalizeChecks, rawChecksOf } from '../../hosted/session'; /** * Pins the SEP-2243 requirement-level check IDs emitted by the custom-header @@ -242,6 +243,55 @@ describe('HttpCustomHeadersScenario (SEP-2243) check IDs', () => { }); }); +describe('HttpInvalidToolHeadersScenario judged from its raw log', () => { + it('FAILs the constraint when the tool was called in another process', async () => { + // The hosted server judges a merged log in a fresh instance (see + // src/hosted/session.ts finalizeChecks): what the observing instance + // saw must be in its raw log, not in instance fields, or a tool the + // client did call reads as never called. + const observer = new HttpInvalidToolHeadersScenario(); + const { serverUrl } = await observer.start(testScenarioContext()); + try { + await post(serverUrl, { jsonrpc: '2.0', id: 1, method: 'tools/list' }); + await post( + serverUrl, + { + jsonrpc: '2.0', + id: 2, + method: 'tools/call', + params: { name: 'invalid_number_header', arguments: { score: 1.5 } } + }, + { 'Mcp-Param-Score': '1.5' } + ); + } finally { + await observer.stop(); + } + const raw = rawChecksOf(observer); + expect(raw.map((c) => c.id)).toEqual([ + 'sep-2243-invalid-tool-tools-list-gate', + 'sep-2243-invalid-tool-call' + ]); + expect(raw[1].details).toMatchObject({ + tool: 'invalid_number_header', + mcpParamHeaders: { 'mcp-param-score': '1.5' } + }); + + const judged = finalizeChecks('http-invalid-tool-headers', raw); + expect( + statusesFor(judged, 'sep-2243-x-mcp-header-primitive-only') + ).toContain('FAILURE'); + expect(statusesFor(judged, 'sep-2243-client-reject-invalid-tool')).toEqual([ + 'FAILURE' + ]); // valid_tool never called + expect( + statusesFor(judged, 'sep-2243-invalid-tool-tools-list-gate') + ).toEqual(['SUCCESS']); + // Idempotent: judging leaves the raw log alone. + expect(observer.getChecks()).toHaveLength(observer.getChecks().length); + expect(rawChecksOf(observer)).toHaveLength(2); + }); +}); + describe('HttpInvalidToolHeadersScenario (SEP-2243) check IDs', () => { it('emits every x-mcp-header constraint ID, SUCCESS when only valid_tool is called', async () => { const scenario = new HttpInvalidToolHeadersScenario(); diff --git a/src/scenarios/client/http-custom-headers.ts b/src/scenarios/client/http-custom-headers.ts index 062efd72..cd45e411 100644 --- a/src/scenarios/client/http-custom-headers.ts +++ b/src/scenarios/client/http-custom-headers.ts @@ -697,19 +697,35 @@ export class HttpCustomHeadersScenario extends BaseHttpScenario { // HttpInvalidToolHeadersScenario - tests that clients reject invalid tools // ───────────────────────────────────────────────────────────────────────────── +/** Raw event: the client sent tools/list (SUCCESS) — or never did (FAILURE). */ +const TOOLS_LIST_GATE_ID = 'sep-2243-invalid-tool-tools-list-gate'; +/** Raw event, one per tools/call: which tool, with the headers it carried. */ +const TOOL_CALL_EVENT_ID = 'sep-2243-invalid-tool-call'; + export class HttpInvalidToolHeadersScenario extends BaseHttpScenario { name = 'http-invalid-tool-headers'; description = 'Tests that client rejects tools with invalid x-mcp-header annotations (SEP-2243)'; allowClientError = true; - private calledTools: Set = new Set(); - private toolsListSent = false; - + /** + * Verdicts are derived from the raw events in `this.checks` (tools/list + * sent, each tools/call with its header observations), never from + * instance state: the hosted server judges a merged log in a fresh + * instance, and anything only an instance field knew would be lost — a + * tool the client did call would read as never called. + */ getChecks(): ConformanceCheck[] { - if (!this.toolsListSent) { - this.checks.push({ - id: 'sep-2243-invalid-tool-tools-list-gate', + const calledTools = new Set( + this.checks + .filter((c) => c.id === TOOL_CALL_EVENT_ID) + .map((c) => c.details?.tool) + .filter((t): t is string => typeof t === 'string') + ); + const verdicts: ConformanceCheck[] = []; + if (!this.checks.some((c) => c.id === TOOLS_LIST_GATE_ID)) { + verdicts.push({ + id: TOOLS_LIST_GATE_ID, name: 'ClientInvalidToolHeadersToolsList', description: 'Client requests tools/list', status: 'FAILURE', @@ -720,8 +736,8 @@ export class HttpInvalidToolHeadersScenario extends BaseHttpScenario { } // Check that valid_tool WAS called — proves client kept valid tools - const validToolCalled = this.calledTools.has('valid_tool'); - this.checks.push({ + const validToolCalled = calledTools.has('valid_tool'); + verdicts.push({ id: 'sep-2243-client-reject-invalid-tool', name: 'ClientKeepsValidTool', description: 'Client MUST keep valid tools while excluding invalid ones', @@ -740,8 +756,8 @@ export class HttpInvalidToolHeadersScenario extends BaseHttpScenario { for (const [toolName, constraintId] of Object.entries( INVALID_TOOL_CONSTRAINT_IDS )) { - const called = this.calledTools.has(toolName); - this.checks.push({ + const called = calledTools.has(toolName); + verdicts.push({ id: constraintId, name: `ClientRejectsInvalidTool_${toolName}`, description: `Client MUST NOT call tool '${toolName}' with invalid x-mcp-header`, @@ -754,11 +770,13 @@ export class HttpInvalidToolHeadersScenario extends BaseHttpScenario { }); } - return this.checks; + // Events first, verdicts after; `this.checks` itself is left as the raw + // log so this is idempotent and the hosted server can re-judge it. + return [...this.checks, ...verdicts]; } protected handlePost( - _req: http.IncomingMessage, + req: http.IncomingMessage, res: http.ServerResponse, request: any ): void { @@ -767,7 +785,7 @@ export class HttpInvalidToolHeadersScenario extends BaseHttpScenario { } else if (request.method === 'tools/list') { this.handleToolsList(res, request); } else if (request.method === 'tools/call') { - this.handleToolsCall(res, request); + this.handleToolsCall(req, res, request); } else if (request.id === undefined) { this.sendNotificationAck(res); } else { @@ -776,7 +794,16 @@ export class HttpInvalidToolHeadersScenario extends BaseHttpScenario { } private handleToolsList(res: http.ServerResponse, request: any): void { - this.toolsListSent = true; + if (!this.checks.some((c) => c.id === TOOLS_LIST_GATE_ID)) { + this.checks.push({ + id: TOOLS_LIST_GATE_ID, + name: 'ClientInvalidToolHeadersToolsList', + description: 'Client requests tools/list', + status: 'SUCCESS', + timestamp: new Date().toISOString(), + specReferences: [SPEC_REFERENCE_TOOL_DEF] + }); + } this.sendJson(res, { jsonrpc: '2.0', @@ -972,9 +999,32 @@ export class HttpInvalidToolHeadersScenario extends BaseHttpScenario { }); } - private handleToolsCall(res: http.ServerResponse, request: any): void { + private handleToolsCall( + req: http.IncomingMessage, + res: http.ServerResponse, + request: any + ): void { const toolName = request.params?.name; - if (toolName) this.calledTools.add(toolName); + if (typeof toolName === 'string') { + const mcpParamHeaders = Object.fromEntries( + Object.entries(req.headers).filter(([k]) => + k.toLowerCase().startsWith('mcp-param-') + ) + ); + this.checks.push({ + id: TOOL_CALL_EVENT_ID, + name: 'ClientCalledTool', + description: `Client called tool '${toolName}'`, + status: 'INFO', + timestamp: new Date().toISOString(), + specReferences: [SPEC_REFERENCE_TOOL_DEF], + details: { + tool: toolName, + arguments: request.params?.arguments, + mcpParamHeaders + } + }); + } this.sendJson(res, { jsonrpc: '2.0', diff --git a/src/scenarios/client/http-standard-headers.test.ts b/src/scenarios/client/http-standard-headers.test.ts index 94674562..86a08c15 100644 --- a/src/scenarios/client/http-standard-headers.test.ts +++ b/src/scenarios/client/http-standard-headers.test.ts @@ -1,6 +1,7 @@ import { testScenarioContext } from '../../mock-server/testing'; import { describe, it, expect } from 'vitest'; import { HttpStandardHeadersScenario } from './http-standard-headers'; +import { finalizeChecks, rawChecksOf } from '../../hosted/session'; /** * Negative test for SEP-2243 standard-header checks: a client that omits @@ -79,4 +80,102 @@ describe('HttpStandardHeadersScenario (SEP-2243) — negative', () => { await scenario.stop(); } }); + + it('judges a merged log from two instances without SUCCESS and SKIPPED for one method', async () => { + // The hosted server persists each process's raw log and judges the + // merged log in a fresh instance (src/hosted/session.ts finalizeChecks): + // a method one instance saw must not read as never sent in the re-judge. + async function drive( + requests: { body: object; headers: Record }[] + ): Promise { + const scenario = new HttpStandardHeadersScenario(); + const { serverUrl } = await scenario.start(testScenarioContext()); + try { + for (const { body, headers } of requests) { + const r = await fetch(serverUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json', ...headers }, + body: JSON.stringify(body) + }); + await r.text(); + } + } finally { + await scenario.stop(); + } + return scenario; + } + const a = await drive([ + { + body: { + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2026-07-28', + clientInfo: { name: 'split', version: '0' }, + capabilities: {} + } + }, + headers: { 'Mcp-Method': 'initialize' } + } + ]); + const b = await drive([ + { + body: { jsonrpc: '2.0', id: 2, method: 'tools/list' }, + headers: { 'Mcp-Method': 'tools/list' } + }, + { + body: { + jsonrpc: '2.0', + id: 3, + method: 'tools/call', + params: { name: 'test_headers', arguments: {} } + }, + headers: { 'Mcp-Method': 'tools/call' } // no Mcp-Name + }, + { + body: { jsonrpc: '2.0', id: 4, method: 'tools/list' }, + headers: {} // second tools/list: the first already decided + } + ]); + expect(rawChecksOf(a).map((c) => c.name)).toEqual([ + 'ClientMcpMethodHeader_initialize' + ]); + expect(rawChecksOf(b).map((c) => c.name)).toEqual([ + 'ClientMcpMethodHeader_tools_list', + 'ClientMcpMethodHeader_tools_call', + 'ClientMcpNameHeader_tools_call' + ]); + // Each instance alone skips what it never saw. + expect( + a.getChecks().find((c) => c.name === 'ClientMcpMethodHeader_tools_list') + ?.status + ).toBe('SKIPPED'); + + const judged = finalizeChecks('http-standard-headers', [ + ...rawChecksOf(a), + ...rawChecksOf(b) + ]); + const byName = new Map(); + for (const c of judged) + byName.set(c.name, [...(byName.get(c.name) ?? []), c.status]); + // One row per method, never SUCCESS and SKIPPED for the same one. + expect(byName.get('ClientMcpMethodHeader_initialize')).toEqual(['SUCCESS']); + expect(byName.get('ClientMcpMethodHeader_tools_list')).toEqual(['SUCCESS']); + expect(byName.get('ClientMcpMethodHeader_tools_call')).toEqual(['SUCCESS']); + expect(byName.get('ClientMcpNameHeader_tools_call')).toEqual(['FAILURE']); + expect(byName.get('ClientMcpMethodHeader_prompts_get')).toEqual([ + 'SKIPPED' + ]); + expect(judged).toHaveLength(8 + 3); + // A log merged twice over (two processes that both saw initialize) + // still yields one row, the first recorded; judging leaves logs alone. + const twice = finalizeChecks('http-standard-headers', [ + ...rawChecksOf(a), + ...rawChecksOf(a) + ]); + expect(twice).toHaveLength(8 + 3); + expect(rawChecksOf(a)).toHaveLength(1); + expect(rawChecksOf(b)).toHaveLength(3); + }); }); diff --git a/src/scenarios/client/http-standard-headers.ts b/src/scenarios/client/http-standard-headers.ts index 227546b2..01c98948 100644 --- a/src/scenarios/client/http-standard-headers.ts +++ b/src/scenarios/client/http-standard-headers.ts @@ -25,15 +25,36 @@ export class HttpStandardHeadersScenario extends BaseHttpScenario { description = 'Tests that client includes Mcp-Method and Mcp-Name headers on HTTP POST requests (SEP-2243)'; - // Track which header checks have been recorded - private methodHeaderChecks = new Map(); - // Track which Mcp-Name checks have been recorded - private nameHeaderChecks = new Map(); + /** + * Which methods have a recorded check is read off `this.checks` itself + * (one row per check name), never off instance state: the hosted server + * judges a merged log in a fresh instance, and a method only an instance + * field remembered would read as never sent — SUCCESS and SKIPPED for + * the same method. + */ + private recorded(name: string): boolean { + return this.checks.some((c) => c.name === name); + } + + private static methodCheckName(method: string): string { + return `ClientMcpMethodHeader_${method.replace(/\//g, '_')}`; + } + + private static nameCheckName(method: string): string { + return `ClientMcpNameHeader_${method.replace(/\//g, '_')}`; + } getChecks(): ConformanceCheck[] { // Build a fresh array each call so getChecks() is idempotent — the runner - // may call it more than once and we must not accumulate duplicates. - const result = [...this.checks]; + // may call it more than once and we must not accumulate duplicates. One + // row per check name, the first recorded (a merged log can hold the same + // method from several processes); `this.checks` is left as the raw log. + const seen = new Set(); + const result = this.checks.filter((c) => { + if (seen.has(c.name)) return false; + seen.add(c.name); + return true; + }); // SEP-2243 requires Mcp-Method on "all requests and notifications". A // client that never sent prompts/list isn't violating SEP-2243 — it just @@ -51,10 +72,10 @@ export class HttpStandardHeadersScenario extends BaseHttpScenario { ]; for (const method of expectedMethods) { - if (!this.methodHeaderChecks.has(method)) { + if (!seen.has(HttpStandardHeadersScenario.methodCheckName(method))) { result.push({ id: 'sep-2243-client-includes-standard-headers', - name: `ClientMcpMethodHeader_${method.replace(/\//g, '_')}`, + name: HttpStandardHeadersScenario.methodCheckName(method), description: `Client sends correct Mcp-Method header on ${method} request`, status: 'SKIPPED', timestamp: new Date().toISOString(), @@ -66,10 +87,10 @@ export class HttpStandardHeadersScenario extends BaseHttpScenario { const expectedNameMethods = ['tools/call', 'resources/read', 'prompts/get']; for (const method of expectedNameMethods) { - if (!this.nameHeaderChecks.has(method)) { + if (!seen.has(HttpStandardHeadersScenario.nameCheckName(method))) { result.push({ id: 'sep-2243-client-includes-standard-headers', - name: `ClientMcpNameHeader_${method.replace(/\//g, '_')}`, + name: HttpStandardHeadersScenario.nameCheckName(method), description: `Client sends correct Mcp-Name header on ${method} request`, status: 'SKIPPED', timestamp: new Date().toISOString(), @@ -121,7 +142,8 @@ export class HttpStandardHeadersScenario extends BaseHttpScenario { if (!method) return; // Already recorded a check for this method - if (this.methodHeaderChecks.has(method)) return; + const name = HttpStandardHeadersScenario.methodCheckName(method); + if (this.recorded(name)) return; // Header names are lowercased by Node.js http parser const mcpMethodHeader = req.headers['mcp-method'] as string | undefined; @@ -138,11 +160,9 @@ export class HttpStandardHeadersScenario extends BaseHttpScenario { ); } - this.methodHeaderChecks.set(method, errors.length === 0); - this.checks.push({ id: 'sep-2243-client-includes-standard-headers', - name: `ClientMcpMethodHeader_${method.replace(/\//g, '_')}`, + name, description: `Client sends correct Mcp-Method header on ${method} request`, status: errors.length === 0 ? 'SUCCESS' : 'FAILURE', timestamp: new Date().toISOString(), @@ -165,7 +185,8 @@ export class HttpStandardHeadersScenario extends BaseHttpScenario { // Same de-dup guard as checkMcpMethodHeader: the harness advertises two // tools and two resources, so a client that calls both would otherwise // produce duplicate check rows for the same id. - if (this.nameHeaderChecks.has(method)) return; + const name = HttpStandardHeadersScenario.nameCheckName(method); + if (this.recorded(name)) return; const expectedValue = sourceField === 'params.uri' ? request.params?.uri : request.params?.name; @@ -183,11 +204,9 @@ export class HttpStandardHeadersScenario extends BaseHttpScenario { ); } - this.nameHeaderChecks.set(method, errors.length === 0); - this.checks.push({ id: 'sep-2243-client-includes-standard-headers', - name: `ClientMcpNameHeader_${method.replace(/\//g, '_')}`, + name, description: `Client sends correct Mcp-Name header on ${method} request`, status: errors.length === 0 ? 'SUCCESS' : 'FAILURE', timestamp: new Date().toISOString(), diff --git a/src/scenarios/client/initialize.test.ts b/src/scenarios/client/initialize.test.ts new file mode 100644 index 00000000..4361fbc4 --- /dev/null +++ b/src/scenarios/client/initialize.test.ts @@ -0,0 +1,100 @@ +import { describe, it, expect } from 'vitest'; +import { testScenarioContext } from '../../mock-server/testing'; +import { InitializeScenario } from './initialize'; + +/** + * The 2025-11-25 initialize mock answers the lifecycle it serves and turns + * away methods it does not have: a dual-era client's 2026-07-28 + * `server/discover` probe must see an error, not an empty result it could + * read as a discovery, so it falls back to `initialize`. + */ +describe('initialize scenario', () => { + async function post( + url: string, + body: object, + headers: Record = {} + ): Promise { + return fetch(url, { + method: 'POST', + headers: { 'content-type': 'application/json', ...headers }, + body: JSON.stringify(body) + }); + } + + it('rejects unknown request methods with -32601 and still serves its lifecycle', async () => { + const scenario = new InitializeScenario(); + const { serverUrl } = await scenario.start(testScenarioContext()); + try { + const probe = await post( + serverUrl, + { + jsonrpc: '2.0', + id: 0, + method: 'server/discover', + params: { + _meta: { + 'io.modelcontextprotocol/protocolVersion': '2026-07-28', + 'io.modelcontextprotocol/clientInfo': { + name: 'dual', + version: '0' + }, + 'io.modelcontextprotocol/clientCapabilities': {} + } + } + }, + { 'mcp-protocol-version': '2026-07-28' } + ); + expect(probe.status).toBe(404); + expect(await probe.json()).toMatchObject({ + id: 0, + error: { code: -32601, message: 'Method not found: server/discover' } + }); + + const init = await post(serverUrl, { + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2025-11-25', + clientInfo: { name: 'dual', version: '0' }, + capabilities: {} + } + }); + expect(init.status).toBe(200); + expect((await init.json()).result.protocolVersion).toBe('2025-11-25'); + + const initialized = await post(serverUrl, { + jsonrpc: '2.0', + method: 'notifications/initialized' + }); + expect(initialized.status).toBe(202); + + // Notifications the mock does not know, and ping, behave as before. + const unknownNotification = await post(serverUrl, { + jsonrpc: '2.0', + method: 'notifications/whatever' + }); + expect(unknownNotification.status).toBe(200); + const ping = await post(serverUrl, { + jsonrpc: '2.0', + id: 2, + method: 'ping' + }); + expect(ping.status).toBe(200); + expect(await ping.json()).toEqual({ jsonrpc: '2.0', id: 2, result: {} }); + + const list = await post(serverUrl, { + jsonrpc: '2.0', + id: 3, + method: 'tools/list' + }); + expect(list.status).toBe(200); + expect((await list.json()).result.tools).toEqual([]); + + const checks = scenario.getChecks(); + expect(checks.map((c) => c.status)).toEqual(['SUCCESS', 'INFO']); + } finally { + await scenario.stop(); + } + }); +}); diff --git a/src/scenarios/client/initialize.ts b/src/scenarios/client/initialize.ts index 99066220..91f805ed 100644 --- a/src/scenarios/client/initialize.ts +++ b/src/scenarios/client/initialize.ts @@ -1,8 +1,7 @@ -import type { ScenarioContext } from '../../mock-server'; import http from 'http'; import { - Scenario, - ScenarioUrls, + HandlerScenario, + RequestListener, ConformanceCheck, LATEST_SPEC_VERSION, NEGOTIABLE_PROTOCOL_VERSIONS, @@ -10,7 +9,7 @@ import { } from '../../types'; import { clientChecks } from '../../checks/index'; -export class InitializeScenario implements Scenario { +export class InitializeScenario extends HandlerScenario { name = 'initialize'; readonly source = { introducedIn: '2025-06-18', @@ -18,49 +17,16 @@ export class InitializeScenario implements Scenario { } as const; description = 'Tests MCP client initialization handshake'; - private server: http.Server | null = null; private checks: ConformanceCheck[] = []; - private port: number = 0; - - async start(_ctx: ScenarioContext): Promise { - return new Promise((resolve, reject) => { - this.server = http.createServer((req, res) => { - this.handleRequest(req, res); - }); - - this.server.on('error', reject); - - this.server.listen(0, () => { - const address = this.server!.address(); - if (address && typeof address === 'object') { - this.port = address.port; - resolve({ - serverUrl: `http://localhost:${this.port}` - }); - } else { - reject(new Error('Failed to get server address')); - } - }); - }); - } - async stop(): Promise { - return new Promise((resolve, reject) => { - if (this.server) { - this.server.close((err) => { - if (err) { - reject(err); - } else { - this.server = null; - resolve(); - } - }); - } else { - resolve(); - } - }); + handler(_getBaseUrl: () => string): RequestListener { + this.checks = []; + return (req, res) => this.handleRequest(req, res); } + /** Plumbing only: connect (implicit) and make one ordinary request. */ + readonly steps = [{ op: 'tools/list' }] as const; + getChecks(): ConformanceCheck[] { return this.checks; } @@ -89,7 +55,8 @@ export class InitializeScenario implements Scenario { // the server MUST return HTTP status code 202 Accepted with no body." res.writeHead(202); res.end(); - } else { + } else if (request.method === 'ping' || request.id === undefined) { + // Empty result for ping; notifications and responses get none. res.writeHead(200, { 'Content-Type': 'application/json' }); res.end( JSON.stringify({ @@ -98,6 +65,22 @@ export class InitializeScenario implements Scenario { result: {} }) ); + } else { + // A method this dated server does not have — notably the + // 2026-07-28 `server/discover` probe of a dual-era client, which + // must be turned away so the client falls back to `initialize` + // rather than read an empty result as a discovery. + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + jsonrpc: '2.0', + id: request.id, + error: { + code: -32601, + message: `Method not found: ${request.method}` + } + }) + ); } } catch (error) { res.writeHead(400, { 'Content-Type': 'application/json' }); diff --git a/src/scenarios/client/json-schema-ref-deref.ts b/src/scenarios/client/json-schema-ref-deref.ts index aff791d3..a08de662 100644 --- a/src/scenarios/client/json-schema-ref-deref.ts +++ b/src/scenarios/client/json-schema-ref-deref.ts @@ -1,13 +1,12 @@ -import type { ScenarioContext } from '../../mock-server'; import { Server } from '@modelcontextprotocol/sdk/server/index.js'; import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; import { ListToolsRequestSchema, LATEST_PROTOCOL_VERSION as SDK_LATEST_PROTOCOL_VERSION } from '@modelcontextprotocol/sdk/types.js'; -import type { Scenario, ConformanceCheck } from '../../types'; +import type { ConformanceCheck, RequestListener } from '../../types'; import express, { Request, Response } from 'express'; -import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../types'; +import { HandlerScenario, DRAFT_PROTOCOL_VERSION } from '../../types'; /** * Scenario: JSON Schema network $ref dereferencing (SEP-2106) @@ -29,6 +28,8 @@ import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../types'; const TOOL_NAME = 'lookup_user'; const CANARY_PATH = '/canary/profile-schema.json'; +const TOOLS_LISTED_EVENT = '_state/tools-listed'; +const CANARY_EVENT = '_state/canary-fetched'; const CHECK_ID = 'sep-2106-no-network-ref-deref'; const SPEC_REFERENCES = [ @@ -77,21 +78,40 @@ function createMcpServer(canaryUrl: string, onToolsListed: () => void): Server { return server; } -export class JsonSchemaRefDerefScenario implements Scenario { +export class JsonSchemaRefDerefScenario extends HandlerScenario { name = 'json-schema-ref-no-deref'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; description = `Tests that a client does not automatically dereference a network-URI \`$ref\` in a tool's inputSchema (SEP-2106). The scenario advertises a tool whose inputSchema contains a \`$ref\` pointing at a canary URL. The client should list tools (and may otherwise process the schema), but must not fetch the canary URL. Same-document refs (\`#/$defs/...\`) remain safe to resolve.`; + mcpPath = '/mcp'; + /** List only — the point is what the client does NOT fetch afterwards. */ + readonly steps = [{ op: 'tools/list' }] as const; - private app: express.Application | null = null; - private httpServer: ReturnType | null = null; - private canaryRequests: Array<{ method: string; userAgent?: string }> = []; - private toolsListed = false; + /** + * Raw event log. What the scenario observed is kept as INFO events here + * (rather than in private fields) so a host that spreads one run over + * several processes can merge the logs and judge once — see rawChecks(). + */ + checks: ConformanceCheck[] = []; - async start(_ctx: ScenarioContext): Promise { - this.canaryRequests = []; - this.toolsListed = false; + private record(id: string, details?: Record): void { + this.checks.push({ + id, + name: id, + description: id, + status: 'INFO', + timestamp: new Date().toISOString(), + details + }); + } + + rawChecks(): ConformanceCheck[] { + return this.checks; + } + + handler(getBaseUrl: () => string): RequestListener { + this.checks = []; const app = express(); app.use(express.json()); @@ -100,7 +120,7 @@ The scenario advertises a tool whose inputSchema contains a \`$ref\` pointing at // network $ref. Return a valid schema so a dereferencing client gets a // realistic response rather than an error it might silently swallow. app.all(CANARY_PATH, (req: Request, res: Response) => { - this.canaryRequests.push({ + this.record(CANARY_EVENT, { method: req.method, userAgent: req.headers['user-agent'] }); @@ -112,6 +132,44 @@ The scenario advertises a tool whose inputSchema contains a \`$ref\` pointing at }); app.post('/mcp', async (req: Request, res: Response) => { + // The bundled SDK server below predates the 2026-07-28 lifecycle and + // does not implement server/discover; answer it directly so a client + // that negotiates first can proceed to tools/list. + if ( + (req.body as Record | undefined)?.method === + 'server/discover' + ) { + return res.json({ + jsonrpc: '2.0', + id: (req.body as Record).id ?? null, + result: { + resultType: 'complete', + ttlMs: 0, + cacheScope: 'private', + supportedVersions: [DRAFT_PROTOCOL_VERSION], + capabilities: { tools: {} }, + serverInfo: { + name: 'json-schema-ref-deref-server', + version: '1.0.0' + } + } + }); + } + // Second half of the same workaround: the pinned SDK transport + // whitelists MCP-Protocol-Version headers and would reject the draft + // version that the server/discover response above advertises with an + // HTTP 400. Rewrite it to the newest version the SDK understands so a + // client that honors the negotiated version can reach tools/list. + if (req.headers['mcp-protocol-version'] === DRAFT_PROTOCOL_VERSION) { + req.headers['mcp-protocol-version'] = SDK_LATEST_PROTOCOL_VERSION; + // The SDK's Node adapter rebuilds its web-standard Request from + // rawHeaders, not the parsed headers object, so patch those too. + for (let i = 0; i < req.rawHeaders.length; i += 2) { + if (req.rawHeaders[i].toLowerCase() === 'mcp-protocol-version') { + req.rawHeaders[i + 1] = SDK_LATEST_PROTOCOL_VERSION; + } + } + } // The bundled SDK server below predates the 2026-07-28 lifecycle and // does not implement server/discover; answer it directly so a client // that negotiates first can proceed to tools/list. @@ -152,8 +210,9 @@ The scenario advertises a tool whose inputSchema contains a \`$ref\` pointing at } try { // Stateless: fresh server and transport per request - const server = createMcpServer(this.canaryUrl(), () => { - this.toolsListed = true; + const canaryUrl = `${getBaseUrl()}${CANARY_PATH}`; + const server = createMcpServer(canaryUrl, () => { + this.record(TOOLS_LISTED_EVENT); }); const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined @@ -174,38 +233,20 @@ The scenario advertises a tool whose inputSchema contains a \`$ref\` pointing at } }); - this.app = app; - this.httpServer = app.listen(0); - return { serverUrl: `${this.baseUrl()}/mcp` }; - } - - private baseUrl(): string { - const address = this.httpServer?.address(); - if (!address || typeof address === 'string') { - throw new Error('Scenario server is not listening'); - } - return `http://localhost:${address.port}`; - } - - private canaryUrl(): string { - return `${this.baseUrl()}${CANARY_PATH}`; - } - - async stop() { - if (this.httpServer) { - await new Promise((resolve) => this.httpServer!.close(resolve)); - this.httpServer = null; - } - this.app = null; + return app; } getChecks(): ConformanceCheck[] { // Built fresh on every call so getChecks() is idempotent — the runner may // call it more than once and we must not accumulate duplicates. const timestamp = new Date().toISOString(); - const fetched = this.canaryRequests.length > 0; + const canaryRequests = this.checks + .filter((c) => c.id === CANARY_EVENT) + .map((c) => c.details ?? {}); + const toolsListed = this.checks.some((c) => c.id === TOOLS_LISTED_EVENT); + const fetched = canaryRequests.length > 0; - if (!this.toolsListed) { + if (!toolsListed) { return [ { id: CHECK_ID, @@ -232,13 +273,13 @@ The scenario advertises a tool whose inputSchema contains a \`$ref\` pointing at status: fetched ? 'FAILURE' : 'SUCCESS', timestamp, errorMessage: fetched - ? `Canary URL ${CANARY_PATH} was fetched ${this.canaryRequests.length} time(s)` + ? `Canary URL ${CANARY_PATH} was fetched ${canaryRequests.length} time(s)` : undefined, specReferences: SPEC_REFERENCES, details: { toolsListed: true, - canaryRequestCount: this.canaryRequests.length, - canaryRequests: this.canaryRequests + canaryRequestCount: canaryRequests.length, + canaryRequests } } ]; diff --git a/src/scenarios/client/mrtr-client.ts b/src/scenarios/client/mrtr-client.ts index 2bd5692e..7f882173 100644 --- a/src/scenarios/client/mrtr-client.ts +++ b/src/scenarios/client/mrtr-client.ts @@ -1,4 +1,3 @@ -import type { ScenarioContext } from '../../mock-server'; /** * SEP-2322: MRTR Client Conformance Tests * @@ -11,8 +10,8 @@ import type { ScenarioContext } from '../../mock-server'; * fulfills the elicitation, and retries. The server verifies correct client behavior. */ -import type { Scenario, ConformanceCheck } from '../../types'; -import { DRAFT_PROTOCOL_VERSION, ScenarioUrls } from '../../types'; +import type { ConformanceCheck, RequestListener } from '../../types'; +import { HandlerScenario, DRAFT_PROTOCOL_VERSION } from '../../types'; import express, { Request, Response } from 'express'; import { randomUUID } from 'crypto'; @@ -463,30 +462,17 @@ function createMRTRServer(checks: ConformanceCheck[]): express.Application { return app; } -export class MRTRClientScenario implements Scenario { +export class MRTRClientScenario extends HandlerScenario { name = 'sep-2322-client-request-state'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; description = 'Tests client MRTR behavior: requestState echo, no-state omission, and JSON-RPC id uniqueness (SEP-2322)'; - private app: express.Application | null = null; - private httpServer: ReturnType | null = null; + mcpPath = '/mcp'; private checks: ConformanceCheck[] = []; - async start(_ctx: ScenarioContext): Promise { + handler(_getBaseUrl: () => string): RequestListener { this.checks = []; - this.app = createMRTRServer(this.checks); - this.httpServer = this.app.listen(0); - const addr = this.httpServer.address(); - const port = typeof addr === 'object' && addr ? addr.port : 0; - return { serverUrl: `http://localhost:${port}/mcp` }; - } - - async stop() { - if (this.httpServer) { - await new Promise((resolve) => this.httpServer!.close(resolve)); - this.httpServer = null; - } - this.app = null; + return createMRTRServer(this.checks); } getChecks(): ConformanceCheck[] { diff --git a/src/scenarios/client/request-metadata.ts b/src/scenarios/client/request-metadata.ts index 54673b00..33f9065e 100644 --- a/src/scenarios/client/request-metadata.ts +++ b/src/scenarios/client/request-metadata.ts @@ -1,11 +1,8 @@ -import { - withRequiredDraftResultFields, - type ScenarioContext -} from '../../mock-server'; +import { withRequiredDraftResultFields } from '../../mock-server'; import http from 'http'; import { - Scenario, - ScenarioUrls, + HandlerScenario, + RequestListener, ConformanceCheck, CheckStatus, DRAFT_PROTOCOL_VERSION @@ -40,48 +37,38 @@ export const DECLARED_CHECK_IDS = [ 'sep-2575-client-retry-supported-version' ] as const; -export class RequestMetadataScenario implements Scenario { +/** + * Recorded the moment the simulated version rejection is issued, and only + * then — so its presence in the log is the record that the rejection + * happened. Deriving that from the log rather than from an instance flag is + * what lets a run split across processes (the hosted server seeds a cold + * process with the persisted log) reject the client's first request once, + * not once per process. + */ +const RETRY_CHECK_ID = 'sep-2575-client-retry-supported-version'; + +export class RequestMetadataScenario extends HandlerScenario { name = 'request-metadata'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; description = 'Per-request _meta and MCP-Protocol-Version header obligations (SEP-2575)'; - private server: http.Server | null = null; private checks: ConformanceCheck[] = []; - private hasSimulatedRejection = false; private requestsObserved = 0; - async start(_ctx: ScenarioContext): Promise { - this.hasSimulatedRejection = false; + handler(_getBaseUrl: () => string): RequestListener { this.checks = []; this.requestsObserved = 0; - return new Promise((resolve, reject) => { - this.server = http.createServer((req, res) => { - this.handleRequest(req, res); - }); - this.server.on('error', reject); - this.server.listen(0, () => { - const address = this.server!.address(); - if (address && typeof address === 'object') { - resolve({ serverUrl: `http://localhost:${address.port}` }); - } - }); - }); + return (req, res) => this.handleRequest(req, res); } - async stop(): Promise { - return new Promise((resolve) => { - if (this.server) { - this.server.close(() => { - resolve(); - }); - } else { - resolve(); - } - }); + /** Whether this run has already issued its one simulated rejection. */ + private hasSimulatedRejection(): boolean { + return this.checks.some((c) => c.id === RETRY_CHECK_ID); } getChecks(): ConformanceCheck[] { + this.collapseDuplicateIds(); // Declared but never emitted -> FAILURE. A check that is legitimately not // applicable must be emitted as SKIPPED explicitly to avoid this. for (const id of DECLARED_CHECK_IDS) { @@ -109,6 +96,30 @@ export class RequestMetadataScenario implements Scenario { return this.checks; } + /** + * A log merged from several processes (see src/hosted/session.ts) can carry + * one id more than once, one per process that observed it. Collapse each + * id the way addOrUpdateCheck() would have as the requests arrived: the + * worst status wins. The retry check is the exception — it is rewritten by + * every retry the client makes, so its latest observation supersedes. + */ + private collapseDuplicateIds(): void { + if (new Set(this.checks.map((c) => c.id)).size === this.checks.length) + return; + const byId = new Map(); + for (const check of this.checks) { + const kept = byId.get(check.id); + if ( + !kept || + check.id === RETRY_CHECK_ID || + STATUS_SEVERITY[check.status] >= STATUS_SEVERITY[kept.status] + ) { + byId.set(check.id, check); + } + } + this.checks = Array.from(byId.values()); + } + private addOrUpdateCheck(check: ConformanceCheck): void { const index = this.checks.findIndex((c) => c.id === check.id); if (index === -1) { @@ -303,12 +314,12 @@ export class RequestMetadataScenario implements Scenario { 'ClientDeclaresElicitationCapability' ); - // 5. Simulated Version Negotiation Retry Check - if (!this.hasSimulatedRejection) { - this.hasSimulatedRejection = true; - + // 5. Simulated Version Negotiation Retry Check — issued once per run; + // the retry check recorded here is the record that it was (see + // RETRY_CHECK_ID). + if (!this.hasSimulatedRejection()) { this.addOrUpdateCheck({ - id: 'sep-2575-client-retry-supported-version', + id: RETRY_CHECK_ID, name: 'ClientRetrySupportedVersion', description: 'Client retries with a supported version when first choice is rejected', @@ -342,9 +353,7 @@ export class RequestMetadataScenario implements Scenario { return; } - const retryCheck = this.checks.find( - (c) => c.id === 'sep-2575-client-retry-supported-version' - ); + const retryCheck = this.checks.find((c) => c.id === RETRY_CHECK_ID); if (retryCheck) { if ( headerVersion === DRAFT_PROTOCOL_VERSION && @@ -354,6 +363,9 @@ export class RequestMetadataScenario implements Scenario { } else { retryCheck.status = 'WARNING'; } + // Re-stamped so that, in a log merged across processes, this + // observation is the latest one for the id. + retryCheck.timestamp = new Date().toISOString(); retryCheck.details = { ...retryCheck.details, retryHeaderVersion: headerVersion, diff --git a/src/scenarios/client/skills/verification.ts b/src/scenarios/client/skills/verification.ts index ee6d2d11..8e7c4bd4 100644 --- a/src/scenarios/client/skills/verification.ts +++ b/src/scenarios/client/skills/verification.ts @@ -103,6 +103,11 @@ export class SkillsVerificationScenario extends BaseHttpScenario { this.description = MODES[mode].description; } + /** One class, three registry entries: a per-run copy must keep its mode. */ + fresh(): SkillsVerificationScenario { + return new SkillsVerificationScenario(this.mode); + } + /** The entry as advertised. Always internally consistent with SKILL_MD. */ private entry() { const resources = [ diff --git a/src/scenarios/client/sse-retry.ts b/src/scenarios/client/sse-retry.ts index 3cee1e12..014110fd 100644 --- a/src/scenarios/client/sse-retry.ts +++ b/src/scenarios/client/sse-retry.ts @@ -13,7 +13,8 @@ import { Scenario, ScenarioUrls, ConformanceCheck, - DRAFT_PROTOCOL_VERSION + DRAFT_PROTOCOL_VERSION, + RequestListener } from '../../types.js'; export class SSERetryScenario implements Scenario { @@ -45,14 +46,24 @@ export class SSERetryScenario implements Scenario { // Tolerance for timing validation (early side only; lateness is not gated) private readonly EARLY_TOLERANCE = 50; // Allow 50ms early for scheduler variance - async start(_ctx: ScenarioContext): Promise { - return new Promise((resolve, reject) => { - this.server = http.createServer((req, res) => { - this.handleRequest(req, res); - }); + handler(_getBaseUrl: () => string): RequestListener { + this.checks = []; + this.toolStreamCloseTime = null; + this.getReconnectionTime = null; + this.getConnectionCount = 0; + this.lastEventIds = []; + this.eventIdCounter = 0; + this.sessionId = `session-${Date.now()}`; + this.pendingToolCallId = null; + this.getResponseStream = null; + return (req, res) => this.handleRequest(req, res); + } + async start(_ctx?: ScenarioContext): Promise { + const listener = this.handler(() => `http://localhost:${this.port}`); + return new Promise((resolve, reject) => { + this.server = http.createServer(listener); this.server.on('error', reject); - this.server.listen(0, () => { const address = this.server!.address(); if (address && typeof address === 'object') { diff --git a/src/scenarios/client/tools_call.test.ts b/src/scenarios/client/tools_call.test.ts index ad6a58e6..8eae6fa9 100644 --- a/src/scenarios/client/tools_call.test.ts +++ b/src/scenarios/client/tools_call.test.ts @@ -4,6 +4,7 @@ import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; import { ToolsCallScenario } from './tools_call'; import { DRAFT_PROTOCOL_VERSION } from '../../types'; +import { finalizeChecks, rawChecksOf } from '../../hosted/session'; describe('tools_call scenario', () => { it('emits a single FAILURE check when the tool was never called', async () => { @@ -77,6 +78,65 @@ describe('tools_call scenario', () => { } }); + it('judges from a raw log a client split across two instances left behind', async () => { + // The hosted server persists each process's raw log and judges the + // merged log in a fresh instance; the mock's `recorded` never leaves + // the process that saw the request. + const meta = { + 'io.modelcontextprotocol/protocolVersion': DRAFT_PROTOCOL_VERSION, + 'io.modelcontextprotocol/clientCapabilities': {} + }; + async function drive(body: object): Promise { + const scenario = new ToolsCallScenario(); + const { serverUrl } = await scenario.start( + testScenarioContext(DRAFT_PROTOCOL_VERSION) + ); + try { + const r = await fetch(serverUrl, { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION + }, + body: JSON.stringify(body) + }); + expect(r.status).toBe(200); + await r.text(); + } finally { + await scenario.stop(); + } + return scenario; + } + const a = await drive({ + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { _meta: meta } + }); + const b = await drive({ + jsonrpc: '2.0', + id: 2, + method: 'tools/call', + params: { _meta: meta, name: 'add_numbers', arguments: { a: 2, b: 3 } } + }); + expect(rawChecksOf(a).map((c) => c.id)).toEqual(['tools-list-requested']); + expect(rawChecksOf(b).map((c) => c.id)).toEqual(['tools-call-requested']); + // Each instance alone: A never saw the call. + expect(a.getChecks()[0].status).toBe('FAILURE'); + expect(b.getChecks()[0].status).toBe('SUCCESS'); + // The merged log judged once: the CLI's single check, SUCCESS. + const judged = finalizeChecks('tools_call', [ + ...rawChecksOf(a), + ...rawChecksOf(b) + ]); + expect(judged).toHaveLength(1); + expect(judged[0]).toMatchObject({ + id: 'tool-add-numbers', + status: 'SUCCESS', + details: { a: 2, b: 3, result: 5 } + }); + }); + it('emits SUCCESS after a valid tools/call and getChecks() is idempotent', async () => { const scenario = new ToolsCallScenario(); const { serverUrl } = await scenario.start(testScenarioContext()); diff --git a/src/scenarios/client/tools_call.ts b/src/scenarios/client/tools_call.ts index b3dcb74c..e70b1c72 100644 --- a/src/scenarios/client/tools_call.ts +++ b/src/scenarios/client/tools_call.ts @@ -1,5 +1,6 @@ -import type { ScenarioContext, MockServer } from '../../mock-server'; -import type { Scenario, ConformanceCheck, ScenarioUrls } from '../../types'; +import type { ScenarioContext, MockHandler } from '../../mock-server'; +import type { ConformanceCheck, RequestListener } from '../../types'; +import { HandlerScenario } from '../../types'; import type { CallToolRequest } from '../../spec-types/2025-06-18'; const SPEC_REF = { @@ -7,32 +8,67 @@ const SPEC_REF = { url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools' }; -export class ToolsCallScenario implements Scenario { +/** Raw events, one per request the mock routed to the scenario. */ +const TOOLS_LIST_EVENT_ID = 'tools-list-requested'; +const TOOLS_CALL_EVENT_ID = 'tools-call-requested'; + +export class ToolsCallScenario extends HandlerScenario { name = 'tools_call'; readonly source = { introducedIn: '2025-06-18' } as const; description = 'Tests calling tools with various parameter types'; - private srv: MockServer | null = null; + mcpPath = '/mcp'; + private mock: MockHandler | null = null; + /** + * The raw log: an INFO event per tools/list and tools/call the client + * made. getChecks() judges from this, so the hosted server can persist it + * per process and judge the merged log once when a client's requests are + * spread across processes (see src/hosted/session.ts). + */ + private checks: ConformanceCheck[] = []; - async start(ctx: ScenarioContext): Promise { - this.srv = await ctx.createServer({ - 'tools/list': () => ({ - tools: [ - { - name: 'add_numbers', - description: 'Add two numbers together', - inputSchema: { - type: 'object', - properties: { - a: { type: 'number', description: 'First number' }, - b: { type: 'number', description: 'Second number' } - }, - required: ['a', 'b'] + handler(_getBaseUrl: () => string, ctx: ScenarioContext): RequestListener { + this.checks = []; + // The version-aware mock supplies the lifecycle scaffold; unbound so the + // same body serves the CLI runner (via HandlerScenario.start) and the + // hosted runner's path-prefix mount. + this.mock = ctx.createHandler({ + 'tools/list': () => { + this.checks.push({ + id: TOOLS_LIST_EVENT_ID, + name: 'ToolsListRequested', + description: 'Client requested tools/list', + status: 'INFO', + timestamp: new Date().toISOString(), + specReferences: [SPEC_REF] + }); + return { + tools: [ + { + name: 'add_numbers', + description: 'Add two numbers together', + inputSchema: { + type: 'object', + properties: { + a: { type: 'number', description: 'First number' }, + b: { type: 'number', description: 'Second number' } + }, + required: ['a', 'b'] + } } - } - ] - }), + ] + }; + }, 'tools/call': (params) => { const p = params as CallToolRequest['params']; + this.checks.push({ + id: TOOLS_CALL_EVENT_ID, + name: 'ToolsCallRequested', + description: `Client called tool '${p.name}'`, + status: 'INFO', + timestamp: new Date().toISOString(), + specReferences: [SPEC_REF], + details: { name: p.name, arguments: p.arguments } + }); if (p.name !== 'add_numbers') { throw new Error(`Unknown tool: ${p.name}`); } @@ -44,20 +80,23 @@ export class ToolsCallScenario implements Scenario { }; } }); - return { serverUrl: this.srv.url }; + return this.mock.listener; } - async stop() { - await this.srv?.close(); - this.srv = null; - } + readonly steps = [ + { op: 'tools/list' }, + { op: 'tools/call', name: 'add_numbers', arguments: { a: 5, b: 3 } } + ] as const; getChecks(): ConformanceCheck[] { // Built fresh on every call so getChecks() is idempotent — the runner may - // call it more than once and we must not accumulate duplicates. - const call = this.srv?.recorded.find((r) => r.method === 'tools/call'); - const args = (call?.params as CallToolRequest['params'] | undefined) - ?.arguments as { a?: unknown; b?: unknown } | undefined; + // call it more than once and we must not accumulate duplicates. Judged + // from the raw log, not the mock's `recorded`, which only this process's + // mock instance holds. + const call = this.checks.find((c) => c.id === TOOLS_CALL_EVENT_ID); + const args = call?.details?.arguments as + | { a?: unknown; b?: unknown } + | undefined; const ok = call !== undefined && typeof args?.a === 'number' && diff --git a/src/steps/index.test.ts b/src/steps/index.test.ts new file mode 100644 index 00000000..5fbf9a96 --- /dev/null +++ b/src/steps/index.test.ts @@ -0,0 +1,61 @@ +import { describe, it, expect } from 'vitest'; +import { StepsSchema, resolveFrom, resolveArguments } from './index'; +import { getScenario } from '../scenarios'; + +describe('steps', () => { + it('validates the closed op set', () => { + expect( + StepsSchema.safeParse([ + { op: 'tools/list' }, + { op: 'tools/call', name: 'x', arguments: { a: 1 } }, + { op: 'wait', ms: 10 }, + { op: 'disconnect' } + ]).success + ).toBe(true); + expect(StepsSchema.safeParse([{ op: 'resources/nuke' }]).success).toBe( + false + ); + }); + + it('resolves $from paths with key, index and filter segments', () => { + const captures = { + 'tools/list': { + tools: [ + { name: 'a', inputSchema: { type: 'object', title: 'A' } }, + { name: 'b', inputSchema: { type: 'object', title: 'B' } } + ] + } + }; + expect( + resolveFrom(captures, { + $from: 'tools/list', + path: 'tools[name=b].inputSchema.title' + }) + ).toBe('B'); + expect( + resolveFrom(captures, { $from: 'tools/list', path: 'tools[0].name' }) + ).toBe('a'); + expect( + resolveFrom(captures, { $from: 'tools/list', path: 'tools[name=z].x' }) + ).toBeUndefined(); + expect( + resolveArguments(captures, { + lit: 1, + schema: { $from: 'tools/list', path: 'tools[1].inputSchema' } + }) + ).toEqual({ lit: 1, schema: { type: 'object', title: 'B' } }); + }); + + it('every scenario that declares steps declares valid ones', () => { + for (const name of [ + 'initialize', + 'tools_call', + 'json-schema-ref-no-deref', + 'elicitation-sep1034-client-defaults' + ]) { + const s = getScenario(name); + expect(s?.steps, name).toBeDefined(); + expect(StepsSchema.safeParse(s!.steps).success, name).toBe(true); + } + }); +}); diff --git a/src/steps/index.ts b/src/steps/index.ts new file mode 100644 index 00000000..2390d48b --- /dev/null +++ b/src/steps/index.ts @@ -0,0 +1,103 @@ +/** + * Client steering steps ("generic steering"). + * + * Most client scenarios only need plumbing from the client under test: + * connect, list tools, call a tool, hang around, disconnect. Instead of every + * SDK's everything-client carrying a per-scenario dispatch table for that + * choreography, a scenario can declare it as data. The runner ships the + * steps to the client in MCP_CONFORMANCE_CONTEXT (`context.steps`); a client + * with no bespoke handler for the scenario name runs a small interpreter over + * them. Checks stay in the scenario — only the *instructions* become data. + * + * The op set is deliberately closed. Anything that needs judgement on the + * client side (credential modes, MRTR) keeps a named handler. + * + * Standing defaults an interpreter should apply without being told: + * - `initialize` is implicit (connect before the first step); + * - if the server sends elicitation/create, accept with schema defaults + * (empty content + `elicitation.applyDefaults` capability); + * - disconnect after the last step unless a `disconnect` step says when. + */ + +import { z } from 'zod'; + +/** + * `{ "$from": "tools/list", "path": "tools[name=echo].inputSchema" }` — a + * value captured from the most recent result of a previous op. The only + * dataflow form; two scenarios need it ("call B with the schema you got for + * A"), nothing needs more. + */ +export const FromRefSchema = z.object({ + $from: z.enum(['tools/list', 'tools/call']), + path: z.string() +}); +export type FromRef = z.infer; + +export const StepSchema = z.discriminatedUnion('op', [ + z.object({ op: z.literal('tools/list') }), + z.object({ + op: z.literal('tools/call'), + name: z.string(), + /** Argument values may be literals or `$from` captures. */ + arguments: z.record(z.string(), z.unknown()).optional() + }), + z.object({ op: z.literal('wait'), ms: z.number().int().nonnegative() }), + z.object({ op: z.literal('disconnect') }) +]); +export type Step = z.infer; + +export const StepsSchema = z.array(StepSchema); + +/** Results an interpreter has captured so far, keyed by op. */ +export type Captures = Partial>; + +export function isFromRef(v: unknown): v is FromRef { + return ( + typeof v === 'object' && + v !== null && + '$from' in v && + FromRefSchema.safeParse(v).success + ); +} + +/** + * Resolve a `$from` path against a captured result. Path grammar: + * dot-separated segments, each either a key (`inputSchema`), an index + * (`tools[0]`) or a filter on an array of objects (`tools[name=echo]`, + * first match). Returns undefined when anything along the way is missing — + * the interpreter should pass that through and let the scenario judge. + */ +export function resolveFrom(captures: Captures, ref: FromRef): unknown { + let cur: unknown = captures[ref.$from]; + for (const seg of ref.path.split('.').filter(Boolean)) { + const m = seg.match(/^([^[\]]*)(?:\[(?:(\d+)|([^=\]]+)=([^\]]*))\])?$/); + if (!m) return undefined; + const [, key, index, fkey, fval] = m; + if (key) cur = (cur as Record | undefined)?.[key]; + if (index !== undefined) cur = (cur as unknown[] | undefined)?.[+index]; + else if (fkey !== undefined) { + cur = Array.isArray(cur) + ? cur.find( + (x) => + typeof x === 'object' && + x !== null && + String((x as Record)[fkey]) === fval + ) + : undefined; + } + if (cur === undefined) return undefined; + } + return cur; +} + +/** Resolve every `$from` capture in a tools/call arguments object (shallow). */ +export function resolveArguments( + captures: Captures, + args: Record | undefined +): Record { + const out: Record = {}; + for (const [k, v] of Object.entries(args ?? {})) { + out[k] = isFromRef(v) ? resolveFrom(captures, v) : v; + } + return out; +} diff --git a/src/types.ts b/src/types.ts index ebe75a27..4c95b0c7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,7 @@ import type { RunContext } from './connection'; import type { ScenarioContext } from './mock-server'; import type { AuthorizationServerOptions } from './schemas'; +import type { Step } from './steps'; export type CheckStatus = | 'SUCCESS' @@ -129,6 +130,12 @@ export interface ScenarioUrls { context?: Record; } +/** A Node-style request handler — what `http.createServer` accepts. */ +export type RequestListener = ( + req: import('http').IncomingMessage, + res: import('http').ServerResponse +) => void; + export interface Scenario { name: string; description: string; @@ -138,9 +145,219 @@ export interface Scenario { * Use this for scenarios where the client is expected to error (e.g., rejecting invalid auth). */ allowClientError?: boolean; + /** + * Sub-path of the MCP endpoint relative to the handler root. The CLI runner + * appends this to the listen URL; the hosted runner appends it to the + * mounted prefix. Default: '' (handler root is the MCP endpoint). + */ + mcpPath?: string; + /** + * Return the request handler without binding a port. The hosted runner + * mounts this directly under a path prefix so scenarios can run on + * serverless hosts that don't allow loopback listeners. + * + * `getBaseUrl` returns the public URL this handler is reachable at (no + * trailing slash) — use it for scenarios that embed self-referential + * absolute URLs in responses. Called lazily so `start()` can resolve it + * after the OS assigns a port. + * + * `ctx` is the same per-run context `start()` receives (resolved spec + * version + version-aware mock factory); use `ctx.createHandler()` rather + * than `ctx.createServer()` so the scenario stays port-free. + * + * Implementations should reset per-run state here, not in `start()`. + * If omitted, the scenario only runs via `start()`/`stop()` (e.g. auth + * scenarios that need a second origin). + */ + handler?(getBaseUrl: () => string, ctx: ScenarioContext): RequestListener; + /** + * Return a new, unstarted instance configured like this one. The hosted + * runner creates one scenario instance per run (and one to re-judge a + * persisted log) from the registry entry; by default it calls the + * constructor with no arguments, so a scenario whose constructor takes + * parameters (e.g. one class registered under several names) must + * implement this to carry them over. + */ + fresh?(): Scenario; start(ctx: ScenarioContext): Promise; stop(): Promise; getChecks(): ConformanceCheck[]; + /** + * Checks recorded so far WITHOUT end-of-flow finalization. Some scenarios' + * `getChecks()` appends aggregate failures for flow steps never observed + * ("expected check missing"); those judgments are only meaningful when one + * instance saw the whole flow. The hosted server persists this view per + * process and re-judges the merged log once (see src/hosted/session.ts), + * so it reads this view when present. + */ + rawChecks?(): ConformanceCheck[]; + /** + * Client-side choreography as data (see src/steps). When present the + * runner includes it in MCP_CONFORMANCE_CONTEXT as `steps`, so a client + * with no bespoke handler for this scenario can still drive it. + */ + readonly steps?: readonly Step[]; +} + +/** + * Convenience: implement `handler()` + `mcpPath` and get `start()`/`stop()` + * for free. Covers every scenario that just needs one HTTP origin. + */ +export abstract class HandlerScenario implements Scenario { + abstract name: string; + abstract description: string; + abstract readonly source: ScenarioSource; + allowClientError?: boolean; + mcpPath = ''; + + private _server: import('http').Server | null = null; + private _baseUrl = ''; + + abstract handler( + getBaseUrl: () => string, + ctx: ScenarioContext + ): RequestListener; + abstract getChecks(): ConformanceCheck[]; + + async start(ctx: ScenarioContext): Promise { + const http = await import('http'); + const listener = this.handler(() => this._baseUrl, ctx); + return new Promise((resolve, reject) => { + this._server = http.createServer(listener); + this._server.on('error', reject); + this._server.listen(0, () => { + const addr = this._server!.address(); + if (!addr || typeof addr !== 'object') { + return reject(new Error('Failed to get server address')); + } + this._baseUrl = `http://localhost:${addr.port}`; + resolve({ serverUrl: `${this._baseUrl}${this.mcpPath}` }); + }); + }); + } + + async stop(): Promise { + if (!this._server) return; + await new Promise((resolve) => { + // closeAllConnections so hung SSE streams don't keep the process alive + this._server!.closeAllConnections?.(); + this._server!.close(() => resolve()); + }); + this._server = null; + } +} + +/** + * Named extra origins a multi-origin scenario needs beyond the resource + * server. `as` is the OAuth authorization server; `as2`/`idp` cover the + * three-origin scenarios (authorization-server-migration, EMA). + */ +export type AuxOriginRole = 'as' | 'as2' | 'idp'; + +/** + * What `authHandlers()` receives: the per-run `ScenarioContext` (spec + * version, mock factories) plus the public URLs of each origin. + */ +export interface AuthHandlerContext extends ScenarioContext { + /** Public URL of the resource-server mount (no trailing slash). */ + getRsBaseUrl: () => string; + /** + * Public URL of an aux origin for this run (no trailing slash). When + * hosted, this is `/r/` so the run-id is recoverable + * from any path the client constructs from it (RFC 8414 well-known + * insertion, endpoint paths, etc.). + */ + getAuxBaseUrl: (role: AuxOriginRole) => string; +} + +export interface AuthHandlers { + /** Resource-server handler — serves /mcp and PRM. */ + rs: RequestListener; + /** Aux-origin handlers keyed by role. */ + aux: Partial>; +} + +/** + * Convenience: implement `authHandlers()` and get `start()`/`stop()` for + * free. `start()` binds one ephemeral localhost port per origin, exactly as + * the auth scenarios did with `ServerLifecycle` before; the hosted runner + * mounts the same handlers behind path prefixes + an AS relay instead. + */ +export abstract class AuthHandlerScenario implements Scenario { + abstract name: string; + abstract description: string; + abstract readonly source: ScenarioSource; + allowClientError?: boolean; + mcpPath = '/mcp'; + + /** Aux origins this scenario needs. Override for 3-origin scenarios. */ + readonly auxRoles: readonly AuxOriginRole[] = ['as']; + + private _servers: import('http').Server[] = []; + private _urls: { rs: string; aux: Partial> } = { + rs: '', + aux: {} + }; + + abstract authHandlers(ctx: AuthHandlerContext): AuthHandlers; + abstract getChecks(): ConformanceCheck[]; + + /** Optional context to pass to the client (credentials etc). */ + protected scenarioContext?(): Record; + + async start(ctx: ScenarioContext): Promise { + const http = await import('http'); + const handlers = this.authHandlers({ + ...ctx, + getRsBaseUrl: () => this._urls.rs, + getAuxBaseUrl: (role) => { + const u = this._urls.aux[role]; + if (!u) throw new Error(`aux role '${role}' not started`); + return u; + } + }); + + const listen = (h: RequestListener): Promise => + new Promise((resolve, reject) => { + const srv = http.createServer(h); + srv.on('error', reject); + srv.listen(0, () => { + const addr = srv.address(); + if (!addr || typeof addr !== 'object') { + return reject(new Error('Failed to get server address')); + } + this._servers.push(srv); + resolve(`http://localhost:${addr.port}`); + }); + }); + + // Aux origins must come up first — RS handlers reference their URLs. + for (const role of this.auxRoles) { + const h = handlers.aux[role]; + if (!h) throw new Error(`authHandlers() missing aux role '${role}'`); + this._urls.aux[role] = await listen(h); + } + this._urls.rs = await listen(handlers.rs); + + return { + serverUrl: `${this._urls.rs}${this.mcpPath}`, + ...(this.scenarioContext && { context: this.scenarioContext() }) + }; + } + + async stop(): Promise { + await Promise.all( + this._servers.map( + (s) => + new Promise((resolve) => { + s.closeAllConnections?.(); + s.close(() => resolve()); + }) + ) + ); + this._servers = []; + this._urls = { rs: '', aux: {} }; + } } export interface ClientScenario {