From c5a2732c2c26f8913e3d9e3840a7e44ab62a43d6 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Mon, 27 Jul 2026 17:17:10 +0100 Subject: [PATCH] feat(cli): --extensions/--exclude-extensions scenario selection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extension scenarios (DPoP, WIF, client-credentials, EMA, tasks) are optional by definition, but --suite all selects them unconditionally, so SDKs that don't implement an extension end up baselining its scenarios as expected failures — a file whose semantics ("burns down per milestone") misrepresent optional surface as conformance debt. Give the runner explicit extension selection instead: - --extensions : keep only the listed extensions' scenarios in the selected suite ("none" deselects them all). Asking for an extension with no scenarios in the suite is an error, so a wrong suite or a --spec-version that already filtered them out cannot silently pass as tested. - --exclude-extensions : drop the listed extensions' scenarios and run the rest ("none" deselects nothing; excluding an absent extension is a satisfied no-op). Mutually exclusive with --extensions via commander conflicts. IDs may omit the io.modelcontextprotocol/ prefix (auth/dpop, tasks). Spec-timeline scenarios always pass through, an explicit --scenario runs unfiltered (with a note when combined with the flags), deselected scenarios are logged, an empty value errors (an unset CI variable must not silently change which scenarios run), and a selection that deselects every scenario in a non-empty suite fails loudly. The list command accepts both flags as a silent dry-run preview, and sdk-runner passes them through to the client/server invocations. The authorization command is untouched: no extension-tagged AS scenarios exist, so the flags would be a guaranteed error there. Verified end-to-end: typescript-sdk client --suite all --exclude-extensions auth/dpop,auth/wif passes 439/0 with no expected-failures file, and python-sdk server --suite all --extensions none passes 137/0 likewise. --- README.md | 2 + src/index.ts | 230 +++++++++++++++++++------ src/scenarios/extension-filter.test.ts | 105 +++++++++++ src/scenarios/index.ts | 94 +++++++++- src/sdk-runner/index.ts | 18 ++ 5 files changed, 394 insertions(+), 55 deletions(-) create mode 100644 src/scenarios/extension-filter.test.ts diff --git a/README.md b/README.md index 8fa48afc..2388f40a 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ npx @modelcontextprotocol/conformance client --command "" --scen - `--suite` - Run a suite of tests in parallel: `all`, `core`, `extensions`, `backcompat`, `auth`, `metadata`, `draft` (scenarios targeting the in-progress draft spec), or `sep-835` - `--spec-version ` - Filter scenarios by spec version (e.g., `2025-11-25`, `2026-07-28`; `draft` is accepted as an alias for the current draft identifier). The draft version selects the latest dated release plus any draft-only scenarios. When omitted, the version is inferred from the scenario's spec applicability (draft-only scenarios run at the draft version, everything else at the latest dated release); an explicitly requested version outside a scenario's applicability window skips the scenario (exit 0) unless `--force` is passed - `--force` - Run a scenario even if it is not applicable at the requested `--spec-version` +- `--extensions ` / `--exclude-extensions ` - Select which protocol extensions' scenarios run within the chosen suite (mutually exclusive; spec-timeline scenarios are unaffected, and a lone `--scenario` runs unfiltered). `--extensions` keeps only the listed extensions (`none` deselects them all); `--exclude-extensions` runs everything but the listed ones (`none` deselects nothing). IDs are comma-separated and may omit the `io.modelcontextprotocol/` prefix, e.g. `--exclude-extensions auth/dpop,auth/wif` or `--extensions oauth-client-credentials`. Deselected scenarios are listed in the run output; asking to include an extension with no scenarios in the suite is an error - `--expected-failures ` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures)) - `--timeout` - Timeout in milliseconds (default: 30000) - `--verbose` - Show verbose output @@ -83,6 +84,7 @@ npx @modelcontextprotocol/conformance server --url [--scenario ] - `--url` - URL of the server to test - `--scenario ` - Test scenario to run (e.g., "server-initialize"). Runs all available scenarios by default - `--suite ` - Suite to run: "active" (default; excludes pending and draft-spec scenarios), "all", "draft" (scenarios targeting the in-progress draft spec), or "pending" +- `--extensions ` / `--exclude-extensions ` - Select which protocol extensions' scenarios run within the chosen suite (mutually exclusive; spec-timeline scenarios are unaffected, and a lone `--scenario` runs unfiltered). `--extensions` keeps only the listed extensions (`none` deselects them all); `--exclude-extensions` runs everything but the listed ones (`none` deselects nothing). IDs are comma-separated and may omit the `io.modelcontextprotocol/` prefix, e.g. `--exclude-extensions tasks` or `--extensions tasks`. Deselected scenarios are listed in the run output; asking to include an extension with no scenarios in the suite is an error - `--expected-failures ` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures)) - `--verbose` - Show verbose output diff --git a/src/index.ts b/src/index.ts index b82093c3..d3365d92 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,6 @@ #!/usr/bin/env node -import { Command } from 'commander'; +import { Command, Option } from 'commander'; import { ZodError } from 'zod'; import { promises as fs } from 'fs'; import { @@ -33,9 +33,13 @@ import { getScenarioSpecVersions, listClientScenariosForAuthorizationServer, listClientScenariosForAuthorizationServerForSpec, - resolveSpecVersion + resolveSpecVersion, + resolveExtensionIds, + filterScenariosByExtensions, + getScenarioExtensionId } from './scenarios'; import type { SpecVersion } from './scenarios'; +import type { ExtensionSelection } from './scenarios'; import { ConformanceCheck } from './types'; import { AuthorizationServerOptionsSchema, @@ -78,6 +82,90 @@ function filterScenariosBySpecVersion( return allScenarios.filter((s) => allowed.has(s)); } +const EXTENSION_SELECTION_NOTE = + 'Note: --extensions/--exclude-extensions apply to --suite selection; a lone --scenario runs regardless.'; + +/** Add the shared --extensions/--exclude-extensions option pair to a command. */ +function withExtensionSelectionOptions(cmd: Command): Command { + return cmd + .option( + '--extensions ', + 'Comma-separated extension IDs: keep only these extensions\' scenarios in the selected suite (spec-timeline scenarios are unaffected; "none" deselects all extension scenarios)' + ) + .addOption( + new Option( + '--exclude-extensions ', + 'Comma-separated extension IDs whose scenarios are deselected from the selected suite ("none" deselects nothing; mutually exclusive with --extensions)' + ).conflicts('extensions') + ); +} + +/** + * Parse `--extensions` / `--exclude-extensions` (commander enforces their + * mutual exclusion) into an extension selection, or undefined when neither + * is given (all extension scenarios stay selected, the historical behavior). + * `none` is accepted by both: `--extensions none` deselects every extension + * scenario, `--exclude-extensions none` deselects nothing. + */ +function parseExtensionSelection(options: { + extensions?: string; + excludeExtensions?: string; +}): ExtensionSelection | undefined { + if (options.extensions !== undefined) { + return { mode: 'include', ids: resolveExtensionIds(options.extensions) }; + } + if (options.excludeExtensions !== undefined) { + return { + mode: 'exclude', + ids: resolveExtensionIds(options.excludeExtensions) + }; + } + return undefined; +} + +/** + * Apply the extension selection to a suite's scenario list, logging what was + * deselected. An empty result is an error: a run whose selection removes + * every scenario (or whose --spec-version already removed all extension + * scenarios the selection asked for) should fail loudly, not exit 0 having + * tested nothing. + */ +function applyExtensionSelection( + scenarios: string[], + selection: ExtensionSelection | undefined +): string[] { + if (!selection) return scenarios; + const { kept, dropped } = filterScenariosByExtensions(scenarios, selection); + if (dropped.length > 0) { + console.log( + `Deselected ${dropped.length} extension scenario(s): ${dropped.join(', ')}\n` + ); + } + if (selection.mode === 'include') { + // Asking to include an extension that contributes nothing to the + // selected suite (wrong suite, or --spec-version already removed its + // scenarios) must not silently pass as if it were tested. Excluding an + // absent extension, by contrast, is a satisfied no-op — including when + // the suite was already empty before the selection. + const unmatched = selection.ids.filter( + (id) => !kept.some((name) => getScenarioExtensionId(name) === id) + ); + if (unmatched.length > 0) { + console.error( + `No scenarios for extension(s) ${unmatched.join(', ')} in the selected suite; check --suite and --spec-version` + ); + process.exit(1); + } + } + if (kept.length === 0 && dropped.length > 0) { + console.error( + 'The extension selection deselected every scenario in the suite; check --suite and --extensions/--exclude-extensions' + ); + process.exit(1); + } + return kept; +} + const program = new Command(); program @@ -86,28 +174,33 @@ program .version(packageJson.version); // Client command - tests a client implementation against scenarios -program - .command('client') - .description( - 'Run conformance tests against a client implementation or start interactive mode' - ) - .option('--command ', 'Command to run the client') - .option('--scenario ', 'Scenario to test') - .option('--suite ', 'Run a suite of tests in parallel (e.g., "auth")') - .option('--timeout ', 'Timeout in milliseconds', '30000') - .option( - '--expected-failures ', - 'Path to YAML file listing expected failures (baseline)' - ) - .option('-o, --output-dir ', 'Save results to this directory') - .option( - '--spec-version ', - 'Filter scenarios by spec version (cumulative for date versions)' - ) - .option( - '--force', - 'Run a scenario even if it is not applicable at the requested --spec-version' - ) +withExtensionSelectionOptions( + program + .command('client') + .description( + 'Run conformance tests against a client implementation or start interactive mode' + ) + .option('--command ', 'Command to run the client') + .option('--scenario ', 'Scenario to test') + .option( + '--suite ', + 'Run a suite of tests in parallel (e.g., "auth")' + ) + .option('--timeout ', 'Timeout in milliseconds', '30000') + .option( + '--expected-failures ', + 'Path to YAML file listing expected failures (baseline)' + ) + .option('-o, --output-dir ', 'Save results to this directory') + .option( + '--spec-version ', + 'Filter scenarios by spec version (cumulative for date versions)' + ) + .option( + '--force', + 'Run a scenario even if it is not applicable at the requested --spec-version' + ) +) .option('--verbose', 'Show verbose output') .action(async (options) => { try { @@ -118,6 +211,8 @@ program ? resolveSpecVersion(options.specVersion) : undefined; + const extensionSelection = parseExtensionSelection(options); + // Handle suite mode if (options.suite) { if (!options.command) { @@ -152,6 +247,7 @@ program 'client' ); } + scenarios = applyExtensionSelection(scenarios, extensionSelection); console.log( `Running ${suiteName} suite (${scenarios.length} scenarios) in parallel...\n` ); @@ -270,6 +366,10 @@ program process.exit(1); } + if (extensionSelection) { + console.log(EXTENSION_SELECTION_NOTE); + } + // Validate options with Zod for single scenario mode const validated = ClientOptionsSchema.parse(options); @@ -338,32 +438,34 @@ program }); // Server command - tests a server implementation -program - .command('server') - .description('Run conformance tests against a server implementation') - .requiredOption('--url ', 'URL of the server to test') - .option( - '--scenario ', - 'Scenario to test (defaults to active suite if not specified)' - ) - .option( - '--suite ', - 'Suite to run: "active" (default, excludes pending and draft), "all", "draft", or "pending"', - 'active' - ) - .option( - '--expected-failures ', - 'Path to YAML file listing expected failures (baseline)' - ) - .option('-o, --output-dir ', 'Save results to this directory') - .option( - '--spec-version ', - 'Filter scenarios by spec version (cumulative for date versions)' - ) - .option( - '--force', - 'Run a scenario even if it is not applicable at the requested --spec-version' - ) +withExtensionSelectionOptions( + program + .command('server') + .description('Run conformance tests against a server implementation') + .requiredOption('--url ', 'URL of the server to test') + .option( + '--scenario ', + 'Scenario to test (defaults to active suite if not specified)' + ) + .option( + '--suite ', + 'Suite to run: "active" (default, excludes pending and draft), "all", "draft", or "pending"', + 'active' + ) + .option( + '--expected-failures ', + 'Path to YAML file listing expected failures (baseline)' + ) + .option('-o, --output-dir ', 'Save results to this directory') + .option( + '--spec-version ', + 'Filter scenarios by spec version (cumulative for date versions)' + ) + .option( + '--force', + 'Run a scenario even if it is not applicable at the requested --spec-version' + ) +) .option('--verbose', 'Show verbose output (JSON instead of pretty print)') .action(async (options) => { try { @@ -376,8 +478,14 @@ program ? resolveSpecVersion(options.specVersion) : undefined; + const extensionSelection = parseExtensionSelection(options); + // If a single scenario is specified, run just that one if (validated.scenario) { + if (extensionSelection) { + console.log(EXTENSION_SELECTION_NOTE); + } + const result = await runServerConformanceTest( validated.url, validated.scenario, @@ -441,6 +549,7 @@ program 'server' ); } + scenarios = applyExtensionSelection(scenarios, extensionSelection); console.log( `Running ${suite} suite (${scenarios.length} scenarios) against ${validated.url}\n` @@ -699,10 +808,26 @@ program '--spec-version ', 'Filter scenarios by spec version (cumulative for date versions)' ) + .option( + '--extensions ', + 'Preview an extension selection: list only these extensions\' scenarios plus spec-timeline scenarios ("none" hides all extension scenarios)' + ) + .addOption( + new Option( + '--exclude-extensions ', + "Preview an extension deselection: hide these extensions' scenarios (mutually exclusive with --extensions)" + ).conflicts('extensions') + ) .action((options) => { const specVersionFilter = options.specVersion ? resolveSpecVersion(options.specVersion) : undefined; + const extensionSelection = parseExtensionSelection(options); + // Listing is a preview: filter silently, no fail-loud checks. + const applySelection = (names: string[]): string[] => + extensionSelection + ? filterScenariosByExtensions(names, extensionSelection).kept + : names; if ( options.server || @@ -717,6 +842,7 @@ program 'server' ); } + serverScenarios = applySelection(serverScenarios); serverScenarios.forEach((s) => { const v = getScenarioSpecVersions(s); console.log(` - ${s}${v ? ` [${v}]` : ''}`); @@ -739,6 +865,7 @@ program 'client' ); } + clientScenarioNames = applySelection(clientScenarioNames); clientScenarioNames.forEach((s) => { const v = getScenarioSpecVersions(s); console.log(` - ${s}${v ? ` [${v}]` : ''}`); @@ -764,6 +891,9 @@ program 'authorization' ); } + authorizationServerScenarios = applySelection( + authorizationServerScenarios + ); authorizationServerScenarios.forEach((s) => { const v = getScenarioSpecVersions(s); console.log(` - ${s}${v ? ` [${v}]` : ''}`); diff --git a/src/scenarios/extension-filter.test.ts b/src/scenarios/extension-filter.test.ts new file mode 100644 index 00000000..6e7bae71 --- /dev/null +++ b/src/scenarios/extension-filter.test.ts @@ -0,0 +1,105 @@ +import { describe, it, expect } from 'vitest'; +import { + resolveExtensionIds, + getScenarioExtensionId, + filterScenariosByExtensions +} from './index'; + +describe('resolveExtensionIds', () => { + it('accepts full extension IDs', () => { + expect(resolveExtensionIds('io.modelcontextprotocol/tasks')).toEqual([ + 'io.modelcontextprotocol/tasks' + ]); + }); + + it('accepts IDs without the io.modelcontextprotocol/ prefix', () => { + expect(resolveExtensionIds('auth/dpop,tasks')).toEqual([ + 'io.modelcontextprotocol/auth/dpop', + 'io.modelcontextprotocol/tasks' + ]); + }); + + it('trims whitespace and ignores empty segments', () => { + expect(resolveExtensionIds(' auth/wif , ')).toEqual([ + 'io.modelcontextprotocol/auth/wif' + ]); + }); +}); + +describe('getScenarioExtensionId', () => { + it('returns the extension ID for extension-tagged scenarios', () => { + // Client-testing scenario (registered in the client `scenarios` map). + expect(getScenarioExtensionId('auth/dpop')).toBe( + 'io.modelcontextprotocol/auth/dpop' + ); + // Server-testing scenario (registered in the `clientScenarios` map). + expect(getScenarioExtensionId('tasks-lifecycle')).toBe( + 'io.modelcontextprotocol/tasks' + ); + }); + + it('returns undefined for spec-timeline scenarios', () => { + expect(getScenarioExtensionId('initialize')).toBeUndefined(); + expect(getScenarioExtensionId('server-stateless')).toBeUndefined(); + }); + + it('returns undefined for unknown scenarios', () => { + expect(getScenarioExtensionId('no-such-scenario')).toBeUndefined(); + }); +}); + +describe('filterScenariosByExtensions', () => { + const names = [ + 'initialize', + 'auth/dpop', + 'auth/dpop-nonce', + 'auth/wif-jwt-bearer', + 'auth/client-credentials-basic' + ]; + + it('include mode keeps spec-timeline scenarios plus only the listed extensions', () => { + const { kept, dropped } = filterScenariosByExtensions(names, { + mode: 'include', + ids: ['io.modelcontextprotocol/auth/dpop'] + }); + expect(kept).toEqual(['initialize', 'auth/dpop', 'auth/dpop-nonce']); + expect(dropped).toEqual([ + 'auth/wif-jwt-bearer', + 'auth/client-credentials-basic' + ]); + }); + + it('an empty include list (--extensions none) drops every extension scenario', () => { + const { kept, dropped } = filterScenariosByExtensions(names, { + mode: 'include', + ids: [] + }); + expect(kept).toEqual(['initialize']); + expect(dropped).toHaveLength(4); + }); + + it('exclude mode drops only the listed extensions', () => { + const { kept, dropped } = filterScenariosByExtensions(names, { + mode: 'exclude', + ids: [ + 'io.modelcontextprotocol/auth/dpop', + 'io.modelcontextprotocol/auth/wif' + ] + }); + expect(kept).toEqual(['initialize', 'auth/client-credentials-basic']); + expect(dropped).toEqual([ + 'auth/dpop', + 'auth/dpop-nonce', + 'auth/wif-jwt-bearer' + ]); + }); + + it('an empty exclude list (--exclude-extensions none) keeps everything', () => { + const { kept, dropped } = filterScenariosByExtensions(names, { + mode: 'exclude', + ids: [] + }); + expect(kept).toEqual(names); + expect(dropped).toEqual([]); + }); +}); diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index db1584ea..a9431805 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -7,7 +7,9 @@ import { DatedSpecVersion, ScenarioSpecTag, DATED_SPEC_VERSIONS, - DRAFT_PROTOCOL_VERSION + DRAFT_PROTOCOL_VERSION, + EXTENSION_IDS, + ExtensionId } from '../types'; import { InitializeScenario } from './client/initialize'; import { ToolsCallScenario } from './client/tools_call'; @@ -472,13 +474,95 @@ export function listClientScenariosForAuthorizationServerForSpec( .map((s) => s.name); } -export function getScenarioSpecVersions( +/** + * Resolve a comma-separated list of extension IDs from the CLI. IDs may be + * given in full (`io.modelcontextprotocol/tasks`) or without the + * `io.modelcontextprotocol/` prefix (`tasks`, `auth/dpop`). The sentinel + * `none` (the whole value) resolves to the empty list. + */ +export function resolveExtensionIds(input: string): ExtensionId[] { + if (input.trim().toLowerCase() === 'none') return []; + const parts = input + .split(',') + .map((part) => part.trim()) + .filter((part) => part.length > 0); + if (parts.length === 0) { + // An empty value (e.g. an unset CI variable) must not silently change + // which scenarios run; "none" is the explicit spelling for that. + console.error('Empty extension list; use "none" to select no extensions'); + console.error(`Known extensions: ${EXTENSION_IDS.join(', ')}`); + process.exit(1); + } + return parts.map((raw) => { + const full = EXTENSION_IDS.find( + (id) => id === raw || id === `io.modelcontextprotocol/${raw}` + ); + if (!full) { + console.error(`Unknown extension: ${raw}`); + console.error(`Known extensions: ${EXTENSION_IDS.join(', ')}`); + process.exit(1); + } + return full; + }); +} + +/** Look a scenario up by name across all three scenario registries. */ +function findScenarioByName( name: string -): ScenarioSpecTag[] | undefined { - const s = +): + | Scenario + | ClientScenario + | ClientScenarioForAuthorizationServer + | undefined { + return ( scenarios.get(name) ?? clientScenarios.get(name) ?? - clientScenariosForAuthorizationServer.get(name); + clientScenariosForAuthorizationServer.get(name) + ); +} + +/** The extension a scenario belongs to, or undefined for spec-timeline scenarios. */ +export function getScenarioExtensionId(name: string): ExtensionId | undefined { + const s = findScenarioByName(name); + if (!s || !('extensionId' in s.source)) return undefined; + return s.source.extensionId; +} + +/** + * The `--extensions` / `--exclude-extensions` selection: `include` keeps + * only the listed extensions' scenarios, `exclude` drops the listed + * extensions' scenarios and keeps the rest. + */ +export interface ExtensionSelection { + mode: 'include' | 'exclude'; + ids: ExtensionId[]; +} + +/** + * Apply an extension selection to a suite's scenario list. Spec-timeline + * scenarios always pass through. + */ +export function filterScenariosByExtensions( + names: string[], + selection: ExtensionSelection +): { kept: string[]; dropped: string[] } { + const kept: string[] = []; + const dropped: string[] = []; + for (const name of names) { + const extension = getScenarioExtensionId(name); + const listed = extension !== undefined && selection.ids.includes(extension); + const keep = + extension === undefined || + (selection.mode === 'include' ? listed : !listed); + (keep ? kept : dropped).push(name); + } + return { kept, dropped }; +} + +export function getScenarioSpecVersions( + name: string +): ScenarioSpecTag[] | undefined { + const s = findScenarioByName(name); if (!s) return undefined; if ('extensionId' in s.source) return ['extension']; const result: ScenarioSpecTag[] = []; diff --git a/src/sdk-runner/index.ts b/src/sdk-runner/index.ts index d4ac74d9..c5cc7be2 100644 --- a/src/sdk-runner/index.ts +++ b/src/sdk-runner/index.ts @@ -120,6 +120,8 @@ function passThrough(options: { verbose?: boolean; output?: string; specVersion?: string; + extensions?: string; + excludeExtensions?: string; }): string[] { const args: string[] = []; if (options.scenario) args.push('--scenario', options.scenario); @@ -128,6 +130,12 @@ function passThrough(options: { if (options.verbose) args.push('--verbose'); if (options.output) args.push('-o', options.output); if (options.specVersion) args.push('--spec-version', options.specVersion); + // Forward even empty values: the child's fail-loud empty-list guard must + // see them (an unset CI variable must not silently widen the run). + if (options.extensions !== undefined) + args.push('--extensions', options.extensions); + if (options.excludeExtensions !== undefined) + args.push('--exclude-extensions', options.excludeExtensions); return args; } @@ -157,6 +165,16 @@ export function createSdkCommand(): Command { ) .option('--scenario ', 'Run a single scenario (passed through)') .option('--suite ', 'Run a suite (passed through)') + .option( + '--extensions ', + 'Extension selection, passed through (see the client/server commands)' + ) + .addOption( + new Option( + '--exclude-extensions ', + 'Extension deselection, passed through (see the client/server commands)' + ).conflicts('extensions') + ) .option('--skip-build', 'Skip the SDK build step (reuse prior build)') .option('--build-cmd ', 'Override the build command from config') .option('--client-cmd ', 'Override the client command from config')