diff --git a/src/checks/checks.test.ts b/src/checks/checks.test.ts index b82f17c9..0a659cac 100644 --- a/src/checks/checks.test.ts +++ b/src/checks/checks.test.ts @@ -53,7 +53,7 @@ describe('createClientInitializationCheck', () => { const check = createClientInitializationCheck(invalidRequest); expect(check.status).toBe('FAILURE'); - expect(check.errorMessage).toContain('Client name missing'); + expect(check.errorMessage).toContain("clientInfo needs a string 'name'"); }); it('should return FAILURE when client version is missing', () => { @@ -66,7 +66,7 @@ describe('createClientInitializationCheck', () => { const check = createClientInitializationCheck(invalidRequest); expect(check.status).toBe('FAILURE'); - expect(check.errorMessage).toContain('Client version missing'); + expect(check.errorMessage).toContain("clientInfo needs a string 'version'"); }); it('should accept the current draft protocol version', () => { @@ -94,6 +94,39 @@ describe('createClientInitializationCheck', () => { } ); + // `Implementation` constrains `name` and `version` to be present strings and + // says nothing about their content, so '' is a supplied field. Failing here + // rejected a client the spec allows — and at FAILURE severity. + it.each([ + ['version', { name: 'TestClient', version: '' }], + ['name', { name: '', version: '1.0.0' }] + ])('should accept an empty %s', (_field, clientInfo) => { + const check = createClientInitializationCheck({ + protocolVersion: '2025-06-18', + clientInfo + }); + expect(check.status).toBe('SUCCESS'); + expect(check.errorMessage).toBeUndefined(); + }); + + it('should return FAILURE when a clientInfo field is not a string', () => { + const check = createClientInitializationCheck({ + protocolVersion: '2025-06-18', + clientInfo: { name: 'TestClient', version: 1 } + }); + expect(check.status).toBe('FAILURE'); + expect(check.errorMessage).toContain("clientInfo needs a string 'version'"); + }); + + it('should report both fields when clientInfo is absent', () => { + const check = createClientInitializationCheck({ + protocolVersion: '2025-06-18' + }); + expect(check.status).toBe('FAILURE'); + expect(check.errorMessage).toContain("clientInfo needs a string 'name'"); + expect(check.errorMessage).toContain("clientInfo needs a string 'version'"); + }); + it('should support custom expected spec version', () => { const request = { protocolVersion: '2024-11-05', diff --git a/src/checks/client.ts b/src/checks/client.ts index 89b11068..fbec86d6 100644 --- a/src/checks/client.ts +++ b/src/checks/client.ts @@ -48,9 +48,15 @@ export function createClientInitializationCheck( errors.push( `Version mismatch: expected ${expectedSpecVersion}, got ${protocolVersionSent}` ); - if (!initializeRequest?.clientInfo?.name) errors.push('Client name missing'); - if (!initializeRequest?.clientInfo?.version) - errors.push('Client version missing'); + // Presence and type, not truthiness. `Implementation` is + // `"required": ["name", "version"]` with both a bare `{"type": "string"}`, + // so a client sending `version: ''` has supplied the field; a falsy test + // reports it as missing, and at FAILURE severity. + const clientInfo = initializeRequest?.clientInfo; + if (typeof clientInfo?.name !== 'string') + errors.push("clientInfo needs a string 'name'"); + if (typeof clientInfo?.version !== 'string') + errors.push("clientInfo needs a string 'version'"); const status: CheckStatus = errors.length === 0 ? 'SUCCESS' : 'FAILURE'; diff --git a/src/index.ts b/src/index.ts index b82093c3..23889d92 100644 --- a/src/index.ts +++ b/src/index.ts @@ -478,7 +478,7 @@ program } } - const { totalFailed } = printServerSummary(allResults); + const { totalFailed, totalWarnings } = printServerSummary(allResults); if (options.expectedFailures) { const expectedFailuresConfig = await loadExpectedFailures( @@ -493,7 +493,7 @@ program process.exit(baselineResult.exitCode); } - process.exit(totalFailed > 0 ? 1 : 0); + process.exit(totalFailed > 0 || totalWarnings > 0 ? 1 : 0); } } catch (error) { if (error instanceof ZodError) { diff --git a/src/runner/server.test.ts b/src/runner/server.test.ts index f2a2faa7..c8e99b10 100644 --- a/src/runner/server.test.ts +++ b/src/runner/server.test.ts @@ -5,8 +5,9 @@ */ import http from 'http'; import type { AddressInfo } from 'net'; -import { afterEach, beforeEach, describe, test, expect } from 'vitest'; -import { runServerConformanceTest } from './server'; +import { afterEach, beforeEach, describe, test, expect, vi } from 'vitest'; +import { printServerSummary, runServerConformanceTest } from './server'; +import { evaluateBaseline } from '../expected-failures'; import { DRAFT_PROTOCOL_VERSION, LATEST_SPEC_VERSION } from '../types'; // The skip decision happens before any network request, so an unreachable @@ -141,3 +142,65 @@ describe('runServerConformanceTest wire selection for draft-only scenarios', () }); }, 30000); }); + +describe('printServerSummary', () => { + const check = (status: 'SUCCESS' | 'FAILURE' | 'WARNING', id: string) => ({ + id, + name: id, + description: `a ${status} check`, + status, + timestamp: '2026-07-22T00:00:00.000Z' + }); + + function capture(allResults: Parameters[0]): { + totals: ReturnType; + output: string; + } { + const spy = vi.spyOn(console, 'log').mockImplementation(() => {}); + try { + const totals = printServerSummary(allResults); + return { totals, output: spy.mock.calls.map((c) => c[0]).join('\n') }; + } finally { + spy.mockRestore(); + } + } + + test('ticks a scenario whose checks all pass', () => { + const { output } = capture([ + { scenario: 'server-stateless', checks: [check('SUCCESS', 'a')] } + ]); + expect(output).toContain('✓ server-stateless: 1 passed, 0 failed'); + }); + + // The reported defect: a warning-only scenario printed `✓ ... 0 failed` and + // then failed the baseline gate, which counts WARNING. + test('crosses a scenario whose only failing check is a warning', () => { + const { totals, output } = capture([ + { + scenario: 'server-stateless', + checks: [check('SUCCESS', 'a'), check('WARNING', 'b')] + } + ]); + expect(output).toContain( + '✗ server-stateless: 1 passed, 0 failed, 1 warnings' + ); + expect(totals.totalWarnings).toBe(1); + }); + + // The invariant that was violated: the summary's verdict and the gate's + // exit code are the same verdict, for every status. + test.each(['SUCCESS', 'FAILURE', 'WARNING', 'SKIPPED', 'INFO'] as const)( + 'agrees with the baseline gate on a %s check', + (status) => { + const results = [ + { scenario: 's', checks: [check(status as never, 'a')] } + ]; + const { totals } = capture(results); + const summarySaysFailing = + totals.totalFailed > 0 || totals.totalWarnings > 0; + expect(summarySaysFailing).toBe( + evaluateBaseline(results, []).exitCode === 1 + ); + } + ); +}); diff --git a/src/runner/server.ts b/src/runner/server.ts index d19f3405..55f25062 100644 --- a/src/runner/server.ts +++ b/src/runner/server.ts @@ -157,24 +157,33 @@ export function printServerResults( export function printServerSummary( allResults: { scenario: string; checks: ConformanceCheck[] }[] -): { totalPassed: number; totalFailed: number } { +): { totalPassed: number; totalFailed: number; totalWarnings: number } { console.log('\n\n=== SUMMARY ==='); let totalPassed = 0; let totalFailed = 0; + let totalWarnings = 0; for (const result of allResults) { const passed = result.checks.filter((c) => c.status === 'SUCCESS').length; const failed = result.checks.filter((c) => c.status === 'FAILURE').length; + const warnings = result.checks.filter((c) => c.status === 'WARNING').length; totalPassed += passed; totalFailed += failed; + totalWarnings += warnings; - const status = failed === 0 ? '✓' : '✗'; + // A WARNING fails the expected-failures gate, so it has to fail the tick + // too: counting only FAILURE here printed `✓ ... 0 failed` for a run that + // then exited 1. The client suite summary already does this. + const status = failed === 0 && warnings === 0 ? '✓' : '✗'; + const warningStr = warnings > 0 ? `, ${warnings} warnings` : ''; console.log( - `${status} ${result.scenario}: ${passed} passed, ${failed} failed` + `${status} ${result.scenario}: ${passed} passed, ${failed} failed${warningStr}` ); } - console.log(`\nTotal: ${totalPassed} passed, ${totalFailed} failed`); + console.log( + `\nTotal: ${totalPassed} passed, ${totalFailed} failed, ${totalWarnings} warnings` + ); - return { totalPassed, totalFailed }; + return { totalPassed, totalFailed, totalWarnings }; } diff --git a/src/scenarios/server/stateless.test.ts b/src/scenarios/server/stateless.test.ts index f8175d86..925b3cd1 100644 --- a/src/scenarios/server/stateless.test.ts +++ b/src/scenarios/server/stateless.test.ts @@ -777,4 +777,59 @@ describe('Stateless Server Scenario Negative Tests', () => { expect(promptsCheck?.status).toBe('WARNING'); expect(toolsCheck?.status).toBe('WARNING'); }); + + describe('sep-2575-server-identifies-in-result-meta', () => { + // Runs the scenario against a server whose discover result carries the + // given _meta serverInfo, and returns the identity check. + async function identityCheckFor(serverInfo: unknown) { + const mockUrl = mockFetchTarget((reqBody) => { + if (reqBody.method === 'server/discover') { + return { + status: 200, + body: { + jsonrpc: '2.0', + id: reqBody.id, + result: { + supportedVersions: ['2026-07-28'], + capabilities: {}, + _meta: { 'io.modelcontextprotocol/serverInfo': serverInfo } + } + } + }; + } + }); + const checks = await new ServerStatelessScenario().run( + testContext(mockUrl) + ); + return findCheck(checks, 'sep-2575-server-identifies-in-result-meta'); + } + + // `Implementation` requires `name` and `version` to be present strings and + // constrains nothing about their content, so an empty version identifies + // the server. Warning about it fails CI over a conformant payload. + it.each([ + ['a full identity', { name: 'srv', version: '1.0.0' }], + ['an empty version', { name: 'srv', version: '' }], + ['an empty name', { name: '', version: '1.0.0' }] + ])('passes a server reporting %s', async (_label, serverInfo) => { + const check = await identityCheckFor(serverInfo); + expect(check?.status).toBe('SUCCESS'); + expect(check?.errorMessage).toBeUndefined(); + }); + + it.each([ + ['no version', { name: 'srv' }], + ['neither field', {}], + ['non-string fields', { name: 1, version: 2 }] + ])( + 'warns, without claiming absence, when the stamp has %s', + async (_label, serverInfo) => { + const check = await identityCheckFor(serverInfo); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toBe( + "serverInfo in the discover result _meta needs a string 'name' and 'version'" + ); + } + ); + }); }); diff --git a/src/scenarios/server/stateless.ts b/src/scenarios/server/stateless.ts index 849437a0..62d0ee73 100644 --- a/src/scenarios/server/stateless.ts +++ b/src/scenarios/server/stateless.ts @@ -509,14 +509,23 @@ export class ServerStatelessScenario implements ClientScenario { }; const metaServerInfo = discoverResult?._meta?.['io.modelcontextprotocol/serverInfo']; - if (!metaServerInfo?.name || !metaServerInfo?.version) { + // Presence and type, not truthiness. `Implementation` is + // `"required": ["name", "version"]` with both a bare + // `{"type": "string"}`, so `version: ''` identifies the server just as + // `'1.0.0'` does; a falsy test reports a present identity as absent. + if ( + typeof metaServerInfo?.name !== 'string' || + typeof metaServerInfo?.version !== 'string' + ) { const bodyServerInfo = discoverResult?.serverInfo; return { // SHOULD-level: WARNING, never FAILURE. warning: true, - error: bodyServerInfo - ? 'serverInfo found only in the pre-#3002 result body, not in _meta' - : 'No serverInfo in the discover result _meta', + error: metaServerInfo + ? "serverInfo in the discover result _meta needs a string 'name' and 'version'" + : bodyServerInfo + ? 'serverInfo found only in the pre-#3002 result body, not in _meta' + : 'No serverInfo in the discover result _meta', details: { result: discoverResult } }; }