From b2c564d5d53c9219ca440c9466b8d0ebc095b56d Mon Sep 17 00:00:00 2001 From: Ryan Alberts <25306145+RyanAlberts@users.noreply.github.com> Date: Mon, 20 Jul 2026 20:05:38 -0500 Subject: [PATCH 1/2] feat: add MCP App validator (library + CLI) with spec requirements catalogue Adds @modelcontextprotocol/ext-apps/validator and a mcp-app-validator bin implementing the official app validator proposed in #673: - rules.ts: machine-readable catalogue of app-side spec requirements (stable APP-NNN ids, MUST->error / SHOULD->warning severity, spec section + normative sentence per rule; direction field reserved for host-side rules per #674) - static.ts: no-render checks over tools/list, resources/list, and resources/read (ui:// scheme, mcp-app mimeType, text-or-blob content, HTML sanity, resource existence, _meta.ui shapes via the generated Zod schemas, CSP origin well-formedness, visibility values, deprecated flat-key usage) - harness.ts: behavioral checks under a minimal Playwright mock host speaking the postMessage JSON-RPC protocol (initialize handshake, appCapabilities, initialized notification, JSON-RPC well-formedness, size-changed emission, tools/list answer when the tools capability is declared, display-mode discipline); playwright resolved lazily so static validation runs without it - CLI: streamable-HTTP / --stdio / --html targets, pretty and --json reporters, exit codes 0/1/2 Validated against the in-repo examples: all Node example servers pass; pdf-server correctly draws the APP-104 (size-changed) warning. Refs #673 Co-Authored-By: Claude Fable 5 --- build.bun.ts | 12 ++ package.json | 7 + src/validator/cli.ts | 73 +++++++ src/validator/harness.ts | 351 ++++++++++++++++++++++++++++++++ src/validator/index.ts | 194 ++++++++++++++++++ src/validator/report.ts | 92 +++++++++ src/validator/rules.ts | 232 +++++++++++++++++++++ src/validator/static.ts | 334 ++++++++++++++++++++++++++++++ src/validator/validator.test.ts | 347 +++++++++++++++++++++++++++++++ 9 files changed, 1642 insertions(+) create mode 100644 src/validator/cli.ts create mode 100644 src/validator/harness.ts create mode 100644 src/validator/index.ts create mode 100644 src/validator/report.ts create mode 100644 src/validator/rules.ts create mode 100644 src/validator/static.ts create mode 100644 src/validator/validator.test.ts diff --git a/build.bun.ts b/build.bun.ts index a129d5ddd..cfa567d92 100644 --- a/build.bun.ts +++ b/build.bun.ts @@ -61,4 +61,16 @@ await Promise.all([ outdir: "dist/src/server", external: PEER_EXTERNALS, }), + // The validator runs in Node (CLI + library), not the browser; playwright + // stays external because it is an optional requirement for behavioral checks. + buildJs("src/validator/index.ts", { + outdir: "dist/src/validator", + target: "node", + external: [...PEER_EXTERNALS, "playwright"], + }), + buildJs("src/validator/cli.ts", { + outdir: "dist/src/validator", + target: "node", + external: [...PEER_EXTERNALS, "playwright"], + }), ]); diff --git a/package.json b/package.json index 8c6ead57b..f13d04328 100644 --- a/package.json +++ b/package.json @@ -38,8 +38,15 @@ "types": "./dist/src/server/index.d.ts", "default": "./dist/src/server/index.js" }, + "./validator": { + "types": "./dist/src/validator/index.d.ts", + "default": "./dist/src/validator/index.js" + }, "./schema.json": "./dist/src/generated/schema.json" }, + "bin": { + "mcp-app-validator": "./dist/src/validator/cli.js" + }, "files": [ "dist" ], diff --git a/src/validator/cli.ts b/src/validator/cli.ts new file mode 100644 index 000000000..46d027221 --- /dev/null +++ b/src/validator/cli.ts @@ -0,0 +1,73 @@ +#!/usr/bin/env node +/** + * CLI for the MCP App validator. + * + * Usage: + * mcp-app-validator [flags] + * mcp-app-validator --stdio [args...] [flags] + * mcp-app-validator --html [flags] + * + * Flags: + * --json Emit the report as JSON + * --no-behavioral Skip behavioral (headless browser) checks + * + * Exit codes: 0 = no errors (warnings allowed), 1 = errors found, + * 2 = usage or connection failure. + * + * @module validator + */ + +import { readFile } from "node:fs/promises"; + +import { + errorCount, + formatJson, + formatPretty, + validateApp, + type ValidationTarget, +} from "./index.js"; + +function usage(): never { + console.error( + [ + "Usage:", + " mcp-app-validator [--json] [--no-behavioral]", + " mcp-app-validator --stdio [args...] [--json] [--no-behavioral]", + " mcp-app-validator --html [--json]", + ].join("\n"), + ); + process.exit(2); +} + +async function main(): Promise { + const args = process.argv.slice(2); + const json = args.includes("--json"); + const behavioral = !args.includes("--no-behavioral"); + const positional = args.filter((a) => !a.startsWith("--")); + + let target: ValidationTarget; + if (args.includes("--html")) { + const path = args[args.indexOf("--html") + 1]; + if (!path) usage(); + target = { html: await readFile(path, "utf-8"), label: path }; + } else if (args.includes("--stdio")) { + const command = args + .slice(args.indexOf("--stdio") + 1) + .filter((a) => !a.startsWith("--")); + if (command.length === 0) usage(); + target = { command }; + } else if (positional.length === 1 && /^https?:\/\//.test(positional[0])) { + target = { url: positional[0] }; + } else { + usage(); + } + + const report = await validateApp(target, { behavioral }); + console.log(json ? formatJson(report) : formatPretty(report)); + process.exit(errorCount(report) > 0 ? 1 : 0); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(2); +}); diff --git a/src/validator/harness.ts b/src/validator/harness.ts new file mode 100644 index 000000000..f2526b0f9 --- /dev/null +++ b/src/validator/harness.ts @@ -0,0 +1,351 @@ +/** + * Behavioral validation: loads an app's HTML in a headless browser under a + * minimal mock host that speaks the MCP Apps postMessage JSON-RPC protocol, + * and checks the app's observable behavior against the catalogue rules. + * + * Playwright is resolved lazily (it is a devDependency of this repo and an + * optional requirement for validator users); static validation never needs it. + * + * @module validator + */ + +import { makeFinding, type Finding } from "./report.js"; +import type { RuleId } from "./rules.js"; + +/** Rule ids exercised by {@link validateAppBehavior}. */ +export const BEHAVIORAL_RULE_IDS: RuleId[] = [ + "APP-100", + "APP-101", + "APP-102", + "APP-103", + "APP-104", + "APP-105", + "APP-106", +]; + +export interface BehavioralOptions { + /** Max time to wait for the ui/initialize request (ms). */ + initializeTimeoutMs?: number; + /** Observation window after initialization for notifications (ms). */ + observeMs?: number; +} + +interface RecordedMessage { + direction: "app-to-host"; + message: Record; +} + +/** Display modes the mock host declares; APP-106 checks requests against it. */ +const HOST_DISPLAY_MODES = ["inline"]; + +const KNOWN_APP_TO_HOST_METHODS = new Set([ + "ui/initialize", + "ui/notifications/initialized", + "ui/notifications/size-changed", + "ui/notifications/request-teardown", + "ui/open-link", + "ui/download-file", + "ui/message", + "ui/update-model-context", + "ui/request-display-mode", + "tools/call", + "tools/list", + "resources/read", + "notifications/message", + "notifications/tools/list_changed", + "sampling/createMessage", + "ping", +]); + +/** + * The mock host page. It embeds the app in a sandboxed iframe via srcdoc, + * answers the protocol handshake, records every message the app sends, and + * exposes the log on `window.__mcpAppValidatorLog`. + * + * Kept as a self-contained template so the harness needs no asset pipeline. + */ +function mockHostPage(): string { + const hostScript = ` + const log = []; + window.__mcpAppValidatorLog = log; + window.__mcpAppValidatorToolsListResponse = null; + let toolsListRequestId = null; + + const iframe = document.getElementById("app"); + + function send(message) { + iframe.contentWindow.postMessage(message, "*"); + } + + window.addEventListener("message", (event) => { + if (event.source !== iframe.contentWindow) return; + const message = event.data; + log.push({ direction: "app-to-host", message }); + if (!message || message.jsonrpc !== "2.0") return; + + if (message.method === "ui/initialize" && message.id !== undefined) { + send({ + jsonrpc: "2.0", + id: message.id, + result: { + protocolVersion: "2025-06-18", + hostInfo: { name: "mcp-app-validator", version: "0.1.0" }, + hostCapabilities: { + openLinks: {}, + serverTools: { listChanged: true }, + serverResources: {}, + logging: {}, + message: { text: {} }, + }, + hostContext: { + theme: "light", + displayMode: "inline", + availableDisplayModes: ${JSON.stringify(HOST_DISPLAY_MODES)}, + containerDimensions: { maxHeight: 600 }, + locale: "en-US", + platform: "web", + }, + }, + }); + return; + } + + if (message.method === "ui/notifications/initialized") { + // Per spec the host sends tool input after initialization completes. + send({ + jsonrpc: "2.0", + method: "ui/notifications/tool-input", + params: { arguments: {} }, + }); + // Probe app-registered tools (APP-105); harmless if undeclared. + toolsListRequestId = "validator-tools-list"; + send({ jsonrpc: "2.0", id: toolsListRequestId, method: "tools/list", params: {} }); + return; + } + + if (message.id === toolsListRequestId && (message.result || message.error)) { + window.__mcpAppValidatorToolsListResponse = message; + return; + } + + // Answer app-initiated requests benignly so apps don't wedge. + if (message.id !== undefined && message.method) { + if (message.method === "tools/call") { + send({ + jsonrpc: "2.0", + id: message.id, + result: { content: [{ type: "text", text: "" }] }, + }); + } else if (message.method === "tools/list") { + send({ jsonrpc: "2.0", id: message.id, result: { tools: [] } }); + } else if (message.method === "ui/request-display-mode") { + send({ jsonrpc: "2.0", id: message.id, result: { mode: "inline" } }); + } else if (message.method === "ping") { + send({ jsonrpc: "2.0", id: message.id, result: {} }); + } else { + send({ jsonrpc: "2.0", id: message.id, result: {} }); + } + } + }); + + `; + return [ + '', + '', + `" sequences that + // would terminate an inline script block. + await page.evaluate((html) => { + (document.getElementById("app") as HTMLIFrameElement).srcdoc = html; + }, appHtml); + + await page + .waitForFunction( + () => + ( + window as unknown as { + __mcpAppValidatorLog: RecordedMessage[]; + } + ).__mcpAppValidatorLog?.some( + (entry) => + (entry.message as { method?: string }).method === "ui/initialize", + ), + undefined, + { timeout: initializeTimeoutMs }, + ) + .catch(() => { + // Absence of ui/initialize is reported by evaluateLog, not a crash. + }); + + // Give the app a window to finish the handshake and emit notifications. + await page.waitForTimeout(observeMs); + + const log = (await page.evaluate( + () => + (window as unknown as { __mcpAppValidatorLog: RecordedMessage[] }) + .__mcpAppValidatorLog, + )) as RecordedMessage[]; + const toolsListResponse = (await page.evaluate( + () => + ( + window as unknown as { + __mcpAppValidatorToolsListResponse: Record | null; + } + ).__mcpAppValidatorToolsListResponse, + )) as Record | null; + + return evaluateLog(log ?? [], toolsListResponse, subject); + } finally { + await browser.close(); + } +} diff --git a/src/validator/index.ts b/src/validator/index.ts new file mode 100644 index 000000000..7144aef1b --- /dev/null +++ b/src/validator/index.ts @@ -0,0 +1,194 @@ +/** + * MCP App validator: checks an MCP Apps server (or a standalone app HTML + * document) against the app-side requirements of the MCP Apps specification. + * + * Static checks inspect `tools/list` / `resources/list` / `resources/read` + * responses; behavioral checks load the app under a mock host in a headless + * browser (requires playwright). See `rules.ts` for the requirements + * catalogue; every finding cites the rule and spec section it derives from. + * + * @example + * ```ts + * import { validateApp } from "@modelcontextprotocol/ext-apps/validator"; + * + * const report = await validateApp({ url: "http://localhost:3001/mcp" }); + * console.log(report.findings); + * ``` + * + * @module validator + */ + +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; + +import { RESOURCE_MIME_TYPE } from "../app.js"; +import { EXTENSION_ID } from "../server/index.js"; +import { + BEHAVIORAL_RULE_IDS, + validateAppBehavior, + type BehavioralOptions, +} from "./harness.js"; +import type { ValidationReport } from "./report.js"; +import { + STATIC_RULE_IDS, + validateServerStatically, + looksLikeHtmlDocument, + type FetchedUiResource, +} from "./static.js"; + +export * from "./rules.js"; +export * from "./report.js"; +export { + STATIC_RULE_IDS, + BEHAVIORAL_RULE_IDS, + validateServerStatically, + validateAppBehavior, +}; + +/** What to validate. Exactly one of the members must be provided. */ +export type ValidationTarget = + | { /** Streamable HTTP endpoint of an MCP server. */ url: string } + | { + /** Command (argv) to spawn an MCP server over stdio. */ command: string[]; + } + | { + /** Raw app HTML, validated behaviorally only. */ html: string; + label?: string; + }; + +export interface ValidateAppOptions { + /** Run behavioral checks (default true; skipped with a note if playwright is unavailable). */ + behavioral?: boolean; + behavioralOptions?: BehavioralOptions; +} + +async function connectClient( + target: { url: string } | { command: string[] }, +): Promise { + const client = new Client( + { name: "mcp-app-validator", version: "0.1.0" }, + { + capabilities: { + // Advertise MCP Apps support so servers register their UI-enabled + // tools (spec: servers SHOULD check client capabilities first). + extensions: { + [EXTENSION_ID]: { mimeTypes: [RESOURCE_MIME_TYPE] }, + }, + } as never, + }, + ); + + if ("url" in target) { + const { StreamableHTTPClientTransport } = + await import("@modelcontextprotocol/sdk/client/streamableHttp.js"); + await client.connect( + new StreamableHTTPClientTransport(new URL(target.url)), + ); + } else { + const { StdioClientTransport } = + await import("@modelcontextprotocol/sdk/client/stdio.js"); + const [command, ...args] = target.command; + await client.connect(new StdioClientTransport({ command, args })); + } + return client; +} + +async function runBehavioral( + resources: FetchedUiResource[], + report: ValidationReport, + options: ValidateAppOptions, +): Promise { + if (options.behavioral === false) { + report.skippedRules.push( + ...BEHAVIORAL_RULE_IDS.map((id) => ({ + id, + reason: "behavioral checks disabled", + })), + ); + return; + } + for (const resource of resources) { + if (!resource.html) continue; + try { + report.findings.push( + ...(await validateAppBehavior( + resource.html, + resource.uri, + options.behavioralOptions, + )), + ); + } catch (error) { + report.skippedRules.push( + ...BEHAVIORAL_RULE_IDS.map((id) => ({ + id, + reason: error instanceof Error ? error.message : String(error), + })), + ); + return; + } + } + report.checkedRules.push(...BEHAVIORAL_RULE_IDS); +} + +/** Validate an MCP Apps server or a standalone app HTML document. */ +export async function validateApp( + target: ValidationTarget, + options: ValidateAppOptions = {}, +): Promise { + if ("html" in target) { + const report: ValidationReport = { + target: target.label ?? "", + findings: [], + checkedRules: [], + skippedRules: STATIC_RULE_IDS.map((id) => ({ + id, + reason: "server-level rule; target is a standalone HTML document", + })), + }; + if (!looksLikeHtmlDocument(target.html)) { + report.findings.push({ + rule: (await import("./rules.js")).getRule("APP-004"), + message: + "content does not look like an HTML5 document (no or in the first 1024 bytes)", + subject: report.target, + }); + } + report.checkedRules.push("APP-004"); + await runBehavioral( + [{ uri: report.target, html: target.html }], + report, + options, + ); + return report; + } + + const targetLabel = "url" in target ? target.url : target.command.join(" "); + const client = await connectClient(target); + try { + const { findings, resources } = await validateServerStatically(client); + const report: ValidationReport = { + target: targetLabel, + findings, + checkedRules: [...STATIC_RULE_IDS], + skippedRules: [ + { + id: "APP-010", + reason: + "requires invoking tools (potential side effects); not run automatically", + }, + ], + }; + if (resources.length === 0) { + report.skippedRules.push( + ...BEHAVIORAL_RULE_IDS.map((id) => ({ + id, + reason: "no UI resources fetched from the server", + })), + ); + } else { + await runBehavioral(resources, report, options); + } + return report; + } finally { + await client.close(); + } +} diff --git a/src/validator/report.ts b/src/validator/report.ts new file mode 100644 index 000000000..f008fa47f --- /dev/null +++ b/src/validator/report.ts @@ -0,0 +1,92 @@ +/** + * Finding and report types shared by the validator's checks and reporters. + * + * @module validator + */ + +import { getRule, type Rule, type RuleId } from "./rules.js"; + +/** A single rule violation (or advisory) found during validation. */ +export interface Finding { + rule: Rule; + /** What was wrong, with enough context to locate it. */ + message: string; + /** The tool, resource URI, or message the finding is about, if any. */ + subject?: string; +} + +/** Result of a validation run. */ +export interface ValidationReport { + /** The target that was validated (URL, command, or file path). */ + target: string; + /** Rule violations, in detection order. */ + findings: Finding[]; + /** Rule ids that were checked (a finding's absence is only meaningful for these). */ + checkedRules: RuleId[]; + /** Rule ids that were skipped, with the reason (e.g. behavioral checks disabled). */ + skippedRules: { id: RuleId; reason: string }[]; +} + +export function makeFinding( + id: RuleId, + message: string, + subject?: string, +): Finding { + return { rule: getRule(id), message, subject }; +} + +export function errorCount(report: ValidationReport): number { + return report.findings.filter((f) => f.rule.severity === "error").length; +} + +export function warningCount(report: ValidationReport): number { + return report.findings.filter((f) => f.rule.severity === "warning").length; +} + +/** Render a human-readable report. */ +export function formatPretty(report: ValidationReport): string { + const lines: string[] = []; + lines.push(`MCP App validation: ${report.target}`); + lines.push(""); + if (report.findings.length === 0) { + lines.push("No issues found."); + } + for (const finding of report.findings) { + const label = finding.rule.severity === "error" ? "ERROR" : "WARN "; + const subject = finding.subject ? ` [${finding.subject}]` : ""; + lines.push(`${label} ${finding.rule.id}${subject} ${finding.message}`); + lines.push(` spec: ${finding.rule.specSection}`); + } + lines.push(""); + lines.push( + `${errorCount(report)} error(s), ${warningCount(report)} warning(s); ` + + `${report.checkedRules.length} rule(s) checked, ${report.skippedRules.length} skipped.`, + ); + for (const skipped of report.skippedRules) { + lines.push(` skipped ${skipped.id}: ${skipped.reason}`); + } + return lines.join("\n"); +} + +/** Render the report as JSON (stable shape for CI consumption). */ +export function formatJson(report: ValidationReport): string { + return JSON.stringify( + { + target: report.target, + findings: report.findings.map((f) => ({ + ruleId: f.rule.id, + severity: f.rule.severity, + title: f.rule.title, + message: f.message, + subject: f.subject, + specSection: f.rule.specSection, + })), + checkedRules: report.checkedRules, + skippedRules: report.skippedRules, + errors: errorCount(report), + warnings: warningCount(report), + }, + null, + 2, + ); +} diff --git a/src/validator/rules.ts b/src/validator/rules.ts new file mode 100644 index 000000000..b0f360f2a --- /dev/null +++ b/src/validator/rules.ts @@ -0,0 +1,232 @@ +/** + * Machine-readable catalogue of app-side conformance requirements extracted + * from the MCP Apps specification (`specification/draft/apps.mdx`). + * + * Each rule carries a stable id, the normative keyword it is derived from + * (MUST → `error`, SHOULD → `warning`), and the spec section it comes from, + * so findings are traceable back to spec text. Host-side requirements are out + * of scope here (see the platform conformance effort in issue #674); the + * `direction` field exists so the catalogue can grow host rules later without + * re-keying. + * + * @module validator + */ + +/** Which side of the protocol a requirement constrains. */ +export type RuleDirection = "app" | "server" | "host"; + +/** Severity, derived from the spec's normative keyword. */ +export type RuleSeverity = "error" | "warning"; + +/** How the rule is checked. */ +export type RuleKind = "static" | "behavioral"; + +export interface Rule { + /** Stable identifier, e.g. "APP-001". Never re-used or renumbered. */ + id: string; + /** Short human-readable title. */ + title: string; + /** `error` for MUST-level requirements, `warning` for SHOULD-level. */ + severity: RuleSeverity; + /** Which participant the requirement constrains. */ + direction: RuleDirection; + /** Whether the rule is checked statically or by driving the app. */ + kind: RuleKind; + /** Section heading in specification/draft/apps.mdx the rule derives from. */ + specSection: string; + /** The normative sentence (or close paraphrase) the rule enforces. */ + specText: string; +} + +export const RULES = [ + { + id: "APP-001", + title: "UI resource URIs use the ui:// scheme", + severity: "error", + direction: "server", + kind: "static", + specSection: "UI Resource Format > Content Requirements", + specText: "URI MUST start with `ui://` scheme", + }, + { + id: "APP-002", + title: "UI resource content mimeType is text/html;profile=mcp-app", + severity: "error", + direction: "server", + kind: "static", + specSection: "UI Resource Format > Content Requirements", + specText: + "`mimeType` MUST be `text/html;profile=mcp-app` (other types reserved for future extensions)", + }, + { + id: "APP-003", + title: "UI resource content is provided via text or blob", + severity: "error", + direction: "server", + kind: "static", + specSection: "UI Resource Format > Content Requirements", + specText: + "Content MUST be provided via either `text` (string) or `blob` (base64-encoded)", + }, + { + id: "APP-004", + title: "UI resource content is a valid HTML5 document", + severity: "error", + direction: "server", + kind: "static", + specSection: "UI Resource Format > Content Requirements", + specText: "Content MUST be valid HTML5 document", + }, + { + id: "APP-005", + title: "Tool-referenced UI resources exist on the server", + severity: "error", + direction: "server", + kind: "static", + specSection: "Resource Discovery > Behavior", + specText: "Resource MUST exist on the server", + }, + { + id: "APP-006", + title: 'Deprecated flat _meta["ui/resourceUri"] key', + severity: "warning", + direction: "server", + kind: "static", + specSection: "Resource Discovery", + specText: + 'The flat `_meta["ui/resourceUri"]` format is deprecated. Use `_meta.ui.resourceUri` instead. The deprecated format will be removed before GA.', + }, + { + id: "APP-007", + title: "Tool _meta.ui matches the McpUiToolMeta schema", + severity: "error", + direction: "server", + kind: "static", + specSection: "Resource Discovery", + specText: + 'Tools are associated with UI resources through the `_meta.ui` field (`resourceUri`, `visibility` of "model" | "app")', + }, + { + id: "APP-008", + title: "Resource _meta.ui matches the UIResourceMeta schema", + severity: "error", + direction: "server", + kind: "static", + specSection: "UI Resource Format", + specText: + "Resource metadata for security and rendering configuration (`csp`, `permissions`, `domain`, `prefersBorder`) on `resources/list` entries and/or `resources/read` content items", + }, + { + id: "APP-009", + title: "Declared CSP domains are well-formed origins", + severity: "error", + direction: "server", + kind: "static", + specSection: "UI Resource Format > McpUiResourceCsp", + specText: + "Servers declare which external origins their UI needs to access (e.g. `https://api.weather.com`; wildcard subdomains supported: `https://*.example.com`)", + }, + { + id: "APP-010", + title: "UI-enabled tools return a meaningful content array", + severity: "warning", + direction: "server", + kind: "static", + specSection: "Client<>Server Capability Negotiation > Graceful Degradation", + specText: + "Tools MUST return meaningful content array even when UI is available", + }, + { + id: "APP-011", + title: 'visibility values are limited to "model" and "app"', + severity: "error", + direction: "server", + kind: "static", + specSection: "Resource Discovery > Visibility", + specText: + '`visibility` defaults to `["model", "app"]` if omitted; `"model"`: visible to and callable by the agent; `"app"`: callable by the app from the same server connection only', + }, + { + id: "APP-100", + title: "App sends ui/initialize on load", + severity: "error", + direction: "app", + kind: "behavioral", + specSection: "Lifecycle > UI Initialization", + specText: + "UI iframes act as MCP clients: the View sends `ui/initialize` and completes the MCP-like handshake with the host", + }, + { + id: "APP-101", + title: "ui/initialize includes appCapabilities", + severity: "error", + direction: "app", + kind: "behavioral", + specSection: "App Capabilities in ui/initialize", + specText: + "When the View sends an `ui/initialize` request to the Host, it MUST include its capabilities in the `appCapabilities` field", + }, + { + id: "APP-102", + title: "App sends ui/notifications/initialized after initialize", + severity: "error", + direction: "app", + kind: "behavioral", + specSection: "Sandbox proxy", + specText: + "Lifecycle messages, e.g., `ui/initialize` request & `ui/notifications/initialized` notification both sent by the View. The Host MUST NOT send any request or notification to the View before it receives an `initialized` notification.", + }, + { + id: "APP-103", + title: "View→Host messages are well-formed JSON-RPC with known methods", + severity: "error", + direction: "app", + kind: "behavioral", + specSection: "Communication Protocol", + specText: + "MCP Apps uses JSON-RPC 2.0 over `postMessage` for iframe-host communication; UI capabilities reuse MCP's existing protocol", + }, + { + id: "APP-104", + title: "App reports size changes", + severity: "warning", + direction: "app", + kind: "behavioral", + specSection: "Notifications (View → Host)", + specText: + "The View SHOULD send this notification when rendered content body size changes (e.g. using ResizeObserver API to report up to date size)", + }, + { + id: "APP-105", + title: "App with tools capability responds to tools/list", + severity: "error", + direction: "app", + kind: "behavioral", + specSection: "Requests (Host → App)", + specText: + "Apps MUST implement `onlisttools` handler if they declare `tools` capability", + }, + { + id: "APP-106", + title: "Display-mode requests stay within host-declared modes", + severity: "error", + direction: "app", + kind: "behavioral", + specSection: "Display Modes > Requirements", + specText: + "View MUST check if the requested mode is in `availableDisplayModes` from host context before requesting a mode change", + }, +] as const satisfies readonly Rule[]; + +export type RuleId = (typeof RULES)[number]["id"]; + +const RULES_BY_ID = new Map(RULES.map((rule) => [rule.id, rule])); + +/** Look up a rule by id. Throws on unknown ids so typos fail fast in tests. */ +export function getRule(id: RuleId): Rule { + const rule = RULES_BY_ID.get(id); + if (!rule) { + throw new Error(`Unknown rule id: ${id}`); + } + return rule; +} diff --git a/src/validator/static.ts b/src/validator/static.ts new file mode 100644 index 000000000..027e26b0a --- /dev/null +++ b/src/validator/static.ts @@ -0,0 +1,334 @@ +/** + * Static (no-rendering) validation of an MCP App server: inspects + * `tools/list`, `resources/list`, and `resources/read` responses against the + * app-side requirements catalogue in `rules.ts`. + * + * @module validator + */ + +import type { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import type { Resource, Tool } from "@modelcontextprotocol/sdk/types.js"; + +import { RESOURCE_MIME_TYPE, RESOURCE_URI_META_KEY } from "../app.js"; +import { McpUiResourceMetaSchema, McpUiToolMetaSchema } from "../types.js"; +import { makeFinding, type Finding } from "./report.js"; +import type { RuleId } from "./rules.js"; + +/** Rule ids exercised by {@link validateServerStatically}. */ +export const STATIC_RULE_IDS: RuleId[] = [ + "APP-001", + "APP-002", + "APP-003", + "APP-004", + "APP-005", + "APP-006", + "APP-007", + "APP-008", + "APP-009", + "APP-011", +]; + +interface UiToolRef { + tool: Tool; + resourceUri: string; +} + +/** A fetched UI resource with its findings-relevant fields. */ +export interface FetchedUiResource { + uri: string; + html?: string; +} + +function getUiMeta(tool: Tool): Record | undefined { + return tool._meta?.ui as Record | undefined; +} + +function getLegacyUri(tool: Tool): string | undefined { + return tool._meta?.[RESOURCE_URI_META_KEY] as string | undefined; +} + +/** Extract every tool that declares a UI resource, in either metadata format. */ +export function collectUiTools(tools: Tool[]): UiToolRef[] { + const refs: UiToolRef[] = []; + for (const tool of tools) { + const resourceUri = + (getUiMeta(tool)?.resourceUri as string | undefined) ?? + getLegacyUri(tool); + if (resourceUri !== undefined) { + refs.push({ tool, resourceUri }); + } + } + return refs; +} + +const ORIGIN_PATTERN = /^(https?|wss?):\/\/(\*\.)?[^\s/*]+$/; + +function checkCspDomains( + csp: Record | undefined, + subject: string, + findings: Finding[], +): void { + if (!csp) return; + for (const [key, value] of Object.entries(csp)) { + if (!Array.isArray(value)) continue; + for (const domain of value) { + if (typeof domain !== "string" || !ORIGIN_PATTERN.test(domain)) { + findings.push( + makeFinding( + "APP-009", + `csp.${key} entry ${JSON.stringify(domain)} is not a well-formed origin (expected e.g. "https://api.example.com" or "https://*.example.com")`, + subject, + ), + ); + } + } + } +} + +function checkResourceMeta( + meta: unknown, + subject: string, + findings: Finding[], +): void { + if (meta === undefined) return; + const parsed = McpUiResourceMetaSchema.safeParse(meta); + if (!parsed.success) { + findings.push( + makeFinding( + "APP-008", + `_meta.ui does not match the UIResourceMeta schema: ${parsed.error.issues + .map((issue) => `${issue.path.join(".")}: ${issue.message}`) + .join("; ")}`, + subject, + ), + ); + return; + } + checkCspDomains( + (meta as { csp?: Record }).csp, + subject, + findings, + ); +} + +/** + * Cheap structural HTML5 sanity check. This intentionally does not attempt + * full HTML validation — it catches the failure modes that break rendering + * outright (empty content, JSON or plain text served as HTML). + */ +export function looksLikeHtmlDocument(content: string): boolean { + const trimmed = content.trim(); + if (trimmed.length === 0) return false; + const head = trimmed.slice(0, 1024).toLowerCase(); + return head.startsWith("]/.test(head); +} + +export interface StaticValidationResult { + findings: Finding[]; + /** Successfully fetched UI resources, for downstream behavioral checks. */ + resources: FetchedUiResource[]; +} + +/** + * Run all static app-side checks against a connected MCP client. + * + * The client must already be initialized (with the MCP Apps extension + * capability advertised, so the server registers its UI-enabled tools). + */ +export async function validateServerStatically( + client: Pick, +): Promise { + const findings: Finding[] = []; + + const { tools } = await client.listTools(); + let listedResources: Resource[] = []; + try { + listedResources = (await client.listResources()).resources; + } catch { + // Servers MAY omit UI-only resources from resources/list entirely, and + // some don't implement resources/list at all. Not a finding by itself. + } + + // Listing-level resource checks (uri scheme + mimeType + _meta.ui shape). + for (const resource of listedResources) { + const isUiUri = resource.uri.startsWith("ui://"); + const uiMeta = (resource._meta as { ui?: unknown } | undefined)?.ui; + if ( + !isUiUri && + (uiMeta !== undefined || resource.mimeType === RESOURCE_MIME_TYPE) + ) { + findings.push( + makeFinding( + "APP-001", + `resource declares UI metadata or the MCP App MIME type but its URI does not use the ui:// scheme`, + resource.uri, + ), + ); + } + if (isUiUri) { + if (resource.mimeType !== RESOURCE_MIME_TYPE) { + findings.push( + makeFinding( + "APP-002", + `listed mimeType is ${JSON.stringify(resource.mimeType)}, expected "${RESOURCE_MIME_TYPE}"`, + resource.uri, + ), + ); + } + checkResourceMeta(uiMeta, resource.uri, findings); + } + } + + // Tool-level checks. + const uiTools = collectUiTools(tools); + const referencedUris = new Set(); + for (const { tool, resourceUri } of uiTools) { + referencedUris.add(resourceUri); + const subject = `tool ${tool.name}`; + + if (!resourceUri.startsWith("ui://")) { + findings.push( + makeFinding( + "APP-001", + `referenced UI resource URI ${JSON.stringify(resourceUri)} does not use the ui:// scheme`, + subject, + ), + ); + } + + const uiMeta = getUiMeta(tool); + // The SDK's registerAppTool mirrors the modern key into the legacy one + // for host compatibility, so only a tool with no modern key is actually + // relying on the deprecated format. + if (getLegacyUri(tool) !== undefined && uiMeta?.resourceUri === undefined) { + findings.push( + makeFinding( + "APP-006", + `uses the deprecated flat _meta["${RESOURCE_URI_META_KEY}"] key; use _meta.ui.resourceUri (the flat key will be removed before GA)`, + subject, + ), + ); + } + + if (uiMeta !== undefined) { + // Report bad visibility entries under the dedicated rule (APP-011), + // then everything else the schema catches under APP-007. + if (Array.isArray(uiMeta.visibility)) { + for (const entry of uiMeta.visibility) { + if (entry !== "model" && entry !== "app") { + findings.push( + makeFinding( + "APP-011", + `visibility entry ${JSON.stringify(entry)} is not one of "model" | "app"`, + subject, + ), + ); + } + } + } + const parsed = McpUiToolMetaSchema.safeParse(uiMeta); + if (!parsed.success) { + const issues = parsed.error.issues.filter( + (issue) => issue.path[0] !== "visibility", + ); + if (issues.length > 0) { + findings.push( + makeFinding( + "APP-007", + `_meta.ui does not match the McpUiToolMeta schema: ${issues + .map((issue) => `${issue.path.join(".")}: ${issue.message}`) + .join("; ")}`, + subject, + ), + ); + } + } + } + } + + // Read every referenced UI resource: existence + content checks. + const fetched: FetchedUiResource[] = []; + for (const uri of referencedUris) { + let contents; + try { + contents = (await client.readResource({ uri })).contents; + } catch (error) { + // The SDK client validates resources/read responses, so a content item + // that violates the text-or-blob requirement surfaces here as a schema + // error rather than as inspectable data. + const issues = JSON.stringify( + (error as { issues?: unknown }).issues ?? "", + ); + if (issues.includes('["text"]') && issues.includes('["blob"]')) { + findings.push( + makeFinding( + "APP-003", + `resources/read returned content providing neither text nor blob`, + uri, + ), + ); + } else { + findings.push( + makeFinding( + "APP-005", + `resources/read failed for tool-referenced resource: ${error instanceof Error ? error.message : String(error)}`, + uri, + ), + ); + } + continue; + } + + if (contents.length === 0) { + findings.push( + makeFinding("APP-005", `resources/read returned no contents`, uri), + ); + continue; + } + + for (const content of contents) { + if (content.mimeType !== RESOURCE_MIME_TYPE) { + findings.push( + makeFinding( + "APP-002", + `content mimeType is ${JSON.stringify(content.mimeType)}, expected "${RESOURCE_MIME_TYPE}"`, + uri, + ), + ); + } + + const { text, blob } = content as { + text?: string; + blob?: string; + }; + if (text === undefined && blob === undefined) { + findings.push( + makeFinding("APP-003", `content provides neither text nor blob`, uri), + ); + continue; + } + + const html = + text ?? Buffer.from(blob as string, "base64").toString("utf-8"); + if (!looksLikeHtmlDocument(html)) { + findings.push( + makeFinding( + "APP-004", + `content does not look like an HTML5 document (no or in the first 1024 bytes)`, + uri, + ), + ); + } else { + fetched.push({ uri, html }); + } + + checkResourceMeta( + (content._meta as { ui?: unknown } | undefined)?.ui, + uri, + findings, + ); + } + } + + return { findings, resources: fetched }; +} diff --git a/src/validator/validator.test.ts b/src/validator/validator.test.ts new file mode 100644 index 000000000..4898dd3bd --- /dev/null +++ b/src/validator/validator.test.ts @@ -0,0 +1,347 @@ +import { describe, it, expect } from "bun:test"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; + +import { + registerAppResource, + registerAppTool, + RESOURCE_MIME_TYPE, + RESOURCE_URI_META_KEY, +} from "../server/index.js"; +import { getRule, RULES } from "./rules.js"; +import { + collectUiTools, + looksLikeHtmlDocument, + validateServerStatically, +} from "./static.js"; +import { formatJson, formatPretty, makeFinding } from "./report.js"; + +const VALID_HTML = "hello"; + +async function connectedClient(server: McpServer): Promise { + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + const client = new Client({ name: "validator-test", version: "0.0.0" }); + await Promise.all([ + server.connect(serverTransport), + client.connect(clientTransport), + ]); + return client; +} + +function makeServer(): McpServer { + return new McpServer({ name: "test-server", version: "0.0.0" }); +} + +describe("rules catalogue", () => { + it("has unique, stable ids", () => { + const ids = RULES.map((rule) => rule.id); + expect(new Set(ids).size).toBe(ids.length); + }); + + it("getRule throws on unknown ids", () => { + expect(() => getRule("APP-999" as never)).toThrow(); + }); +}); + +describe("validateServerStatically", () => { + it("passes a compliant server with no findings", async () => { + const server = makeServer(); + registerAppResource( + server, + "View", + "ui://test/view.html", + {}, + async () => ({ + contents: [ + { + uri: "ui://test/view.html", + mimeType: RESOURCE_MIME_TYPE, + text: VALID_HTML, + }, + ], + }), + ); + registerAppTool( + server, + "show-view", + { + description: "Show the view", + _meta: { ui: { resourceUri: "ui://test/view.html" } }, + }, + async () => ({ content: [{ type: "text", text: "ok" }] }), + ); + + const { findings, resources } = await validateServerStatically( + await connectedClient(server), + ); + expect(findings).toEqual([]); + expect(resources).toHaveLength(1); + expect(resources[0].uri).toBe("ui://test/view.html"); + }); + + it("flags a tool referencing a missing resource (APP-005)", async () => { + const server = makeServer(); + registerAppTool( + server, + "broken", + { + description: "References nothing", + _meta: { ui: { resourceUri: "ui://test/missing.html" } }, + }, + async () => ({ content: [{ type: "text", text: "ok" }] }), + ); + + const { findings } = await validateServerStatically( + await connectedClient(server), + ); + expect(findings.some((f) => f.rule.id === "APP-005")).toBe(true); + }); + + it("flags wrong content mimeType (APP-002)", async () => { + const server = makeServer(); + registerAppResource( + server, + "View", + "ui://test/view.html", + {}, + async () => ({ + contents: [ + { + uri: "ui://test/view.html", + mimeType: "text/html", + text: VALID_HTML, + }, + ], + }), + ); + registerAppTool( + server, + "show-view", + { + _meta: { ui: { resourceUri: "ui://test/view.html" } }, + }, + async () => ({ content: [{ type: "text", text: "ok" }] }), + ); + + const { findings } = await validateServerStatically( + await connectedClient(server), + ); + expect(findings.some((f) => f.rule.id === "APP-002")).toBe(true); + }); + + it("flags non-HTML content (APP-004)", async () => { + const server = makeServer(); + registerAppResource( + server, + "View", + "ui://test/view.html", + {}, + async () => ({ + contents: [ + { + uri: "ui://test/view.html", + mimeType: RESOURCE_MIME_TYPE, + text: JSON.stringify({ not: "html" }), + }, + ], + }), + ); + registerAppTool( + server, + "show-view", + { _meta: { ui: { resourceUri: "ui://test/view.html" } } }, + async () => ({ content: [{ type: "text", text: "ok" }] }), + ); + + const { findings } = await validateServerStatically( + await connectedClient(server), + ); + expect(findings.some((f) => f.rule.id === "APP-004")).toBe(true); + }); + + it("flags content missing text and blob (APP-003)", async () => { + const server = makeServer(); + registerAppResource( + server, + "View", + "ui://test/view.html", + {}, + // Cast: deliberately violates the text-or-blob requirement under test. + async () => + ({ + contents: [ + { uri: "ui://test/view.html", mimeType: RESOURCE_MIME_TYPE }, + ], + }) as never, + ); + registerAppTool( + server, + "show-view", + { _meta: { ui: { resourceUri: "ui://test/view.html" } } }, + async () => ({ content: [{ type: "text", text: "ok" }] }), + ); + + const { findings } = await validateServerStatically( + await connectedClient(server), + ); + expect(findings.some((f) => f.rule.id === "APP-003")).toBe(true); + }); + + it("warns on legacy-only metadata (APP-006) and still resolves the resource", async () => { + const server = makeServer(); + registerAppResource( + server, + "View", + "ui://test/view.html", + {}, + async () => ({ + contents: [ + { + uri: "ui://test/view.html", + mimeType: RESOURCE_MIME_TYPE, + text: VALID_HTML, + }, + ], + }), + ); + // Raw registerTool: registerAppTool would normalize the legacy key away. + server.registerTool( + "legacy-tool", + { _meta: { [RESOURCE_URI_META_KEY]: "ui://test/view.html" } }, + async () => ({ content: [{ type: "text", text: "ok" }] }), + ); + + const { findings } = await validateServerStatically( + await connectedClient(server), + ); + const deprecations = findings.filter((f) => f.rule.id === "APP-006"); + expect(deprecations).toHaveLength(1); + expect(deprecations[0].rule.severity).toBe("warning"); + expect(findings.some((f) => f.rule.id === "APP-005")).toBe(false); + }); + + it("flags invalid visibility entries (APP-011)", async () => { + const server = makeServer(); + registerAppResource( + server, + "View", + "ui://test/view.html", + {}, + async () => ({ + contents: [ + { + uri: "ui://test/view.html", + mimeType: RESOURCE_MIME_TYPE, + text: VALID_HTML, + }, + ], + }), + ); + server.registerTool( + "bad-visibility", + { + description: "Bad visibility", + _meta: { + ui: { + resourceUri: "ui://test/view.html", + visibility: ["model", "everyone"], + }, + }, + }, + async () => ({ content: [{ type: "text", text: "ok" }] }), + ); + + const { findings } = await validateServerStatically( + await connectedClient(server), + ); + expect(findings.some((f) => f.rule.id === "APP-011")).toBe(true); + expect(findings.some((f) => f.rule.id === "APP-007")).toBe(false); + }); + + it("flags malformed CSP domains (APP-009)", async () => { + const server = makeServer(); + registerAppResource( + server, + "View", + "ui://test/view.html", + { + _meta: { + ui: { csp: { connectDomains: ["not a domain"] } }, + }, + }, + async () => ({ + contents: [ + { + uri: "ui://test/view.html", + mimeType: RESOURCE_MIME_TYPE, + text: VALID_HTML, + }, + ], + }), + ); + registerAppTool( + server, + "show-view", + { _meta: { ui: { resourceUri: "ui://test/view.html" } } }, + async () => ({ content: [{ type: "text", text: "ok" }] }), + ); + + const { findings } = await validateServerStatically( + await connectedClient(server), + ); + expect(findings.some((f) => f.rule.id === "APP-009")).toBe(true); + }); +}); + +describe("collectUiTools", () => { + it("collects tools in both metadata formats", () => { + const tools = [ + { + name: "modern", + inputSchema: { type: "object" as const }, + _meta: { ui: { resourceUri: "ui://a" } }, + }, + { + name: "legacy", + inputSchema: { type: "object" as const }, + _meta: { [RESOURCE_URI_META_KEY]: "ui://b" }, + }, + { name: "plain", inputSchema: { type: "object" as const } }, + ]; + const refs = collectUiTools(tools); + expect(refs.map((r) => r.resourceUri)).toEqual(["ui://a", "ui://b"]); + }); +}); + +describe("looksLikeHtmlDocument", () => { + it("accepts doctype and bare ", () => { + expect(looksLikeHtmlDocument(VALID_HTML)).toBe(true); + expect(looksLikeHtmlDocument('')).toBe(true); + }); + + it("rejects JSON, empty, and plain text", () => { + expect(looksLikeHtmlDocument("{}")).toBe(false); + expect(looksLikeHtmlDocument("")).toBe(false); + expect(looksLikeHtmlDocument("hello world")).toBe(false); + }); +}); + +describe("report formatting", () => { + it("renders pretty and JSON output with counts", () => { + const report = { + target: "test", + findings: [makeFinding("APP-002", "wrong mimeType", "ui://x")], + checkedRules: ["APP-002" as const], + skippedRules: [{ id: "APP-100" as const, reason: "disabled" }], + }; + const pretty = formatPretty(report); + expect(pretty).toContain("APP-002"); + expect(pretty).toContain("1 error(s), 0 warning(s)"); + + const json = JSON.parse(formatJson(report)); + expect(json.errors).toBe(1); + expect(json.findings[0].ruleId).toBe("APP-002"); + }); +}); From e68bb84a0a9652b1de18df24b517314e4b9a03c2 Mon Sep 17 00:00:00 2001 From: Ryan Alberts <25306145+RyanAlberts@users.noreply.github.com> Date: Mon, 20 Jul 2026 20:40:32 -0500 Subject: [PATCH 2/2] docs: register validator in typedoc and document it in the testing guide - Add src/validator/index.ts to typedoc entryPoints; export the option and result types it references so the JSDoc validation check passes clean - Add a "Validate against the specification" section to testing-mcp-apps.md Refs #673 Co-Authored-By: Claude Fable 5 --- docs/testing-mcp-apps.md | 18 ++++++++++++++++++ src/validator/index.ts | 2 ++ typedoc.config.mjs | 1 + 3 files changed, 21 insertions(+) diff --git a/docs/testing-mcp-apps.md b/docs/testing-mcp-apps.md index df1396c5f..51ef2ede4 100644 --- a/docs/testing-mcp-apps.md +++ b/docs/testing-mcp-apps.md @@ -8,6 +8,24 @@ description: Test MCP Apps locally with the basic-host reference implementation This guide covers two approaches for testing your MCP App: using the `basic-host` reference implementation for local development, or using an MCP Apps-compatible host like Claude\.ai or VS Code. +## Validate against the specification + +The SDK ships a validator that checks your server and app against the app-side requirements of the MCP Apps specification — resource format, tool metadata, CSP declarations, and (with [Playwright](https://playwright.dev) installed) the app's observable protocol behavior under a mock host: + +```bash +# Validate a running server (streamable HTTP) +npx mcp-app-validator http://localhost:3001/mcp + +# Validate a server over stdio, or a built HTML document directly +npx mcp-app-validator --stdio node dist/server.js +npx mcp-app-validator --html dist/mcp-app.html + +# CI usage: JSON report, exit code 1 on any MUST-level violation +npx mcp-app-validator http://localhost:3001/mcp --json +``` + +Every finding cites the rule it violates and the spec section the rule derives from. See the {@link validator! validator} API documentation to run it programmatically. + ## Test with basic-host The [`basic-host`](https://github.com/modelcontextprotocol/ext-apps/tree/main/examples/basic-host) example in this repository is a reference host implementation that lets you select a tool, call it, and see your App UI rendered in a sandboxed iframe. diff --git a/src/validator/index.ts b/src/validator/index.ts index 7144aef1b..d2abbf6ff 100644 --- a/src/validator/index.ts +++ b/src/validator/index.ts @@ -43,6 +43,8 @@ export { validateServerStatically, validateAppBehavior, }; +export type { BehavioralOptions } from "./harness.js"; +export type { FetchedUiResource, StaticValidationResult } from "./static.js"; /** What to validate. Exactly one of the members must be provided. */ export type ValidationTarget = diff --git a/typedoc.config.mjs b/typedoc.config.mjs index 4c5baa320..664883b14 100644 --- a/typedoc.config.mjs +++ b/typedoc.config.mjs @@ -25,6 +25,7 @@ const config = { "src/app-bridge.ts", "src/message-transport.ts", "src/types.ts", + "src/validator/index.ts", ], excludePrivate: true, excludeInternal: false,