diff --git a/README.md b/README.md index 8a80fac30..92ec97b8a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ [![Stand With Ukraine](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner-direct-single.svg)](https://stand-with-ukraine.pp.ua) [](https://join.slack.com/t/codeceptjs/shared_invite/enQtMzA5OTM4NDM2MzA4LWE4MThhN2NmYTgxNTU5MTc4YzAyYWMwY2JkMmZlYWI5MWQ2MDM5MmRmYzZmYmNiNmY5NTAzM2EwMGIwOTNhOGQ) [](https://codecept.discourse.group) [![NPM version][npm-image]][npm-url] [](https://hub.docker.com/r/codeceptjs/codeceptjs) -[![AI features](https://img.shields.io/badge/AI-features?logo=openai&logoColor=white)](https://github.com/codeceptjs/CodeceptJS/edit/3.x/docs/ai.md) [![StandWithUkraine](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/badges/StandWithUkraine.svg)](https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md) +[![AI features](https://img.shields.io/badge/AI-features?logo=openai&logoColor=white)](https://github.com/codeceptjs/CodeceptJS/edit/3.x/docs/ai.md) [![MCP Server](https://img.shields.io/badge/MCP-server?logo=anthropic&logoColor=white)](https://github.com/codeceptjs/CodeceptJS/blob/main/docs/mcp.md) [![StandWithUkraine](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/badges/StandWithUkraine.svg)](https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md) ## Build Status @@ -276,6 +276,84 @@ Full support for Gherkin scenarios with proper feature formatting: The HTML reporter generates self-contained reports that can be easily shared with your team. Learn more about configuration and features in the [HTML Reporter documentation](https://codecept.io/plugins/#htmlreporter). +## MCP Server + +CodeceptJS includes a [Model Context Protocol (MCP)](https://modelcontextprotocol.io) server that enables AI agents (like Claude, ChatGPT) to interact with and control your tests programmatically. This allows AI to: + +- **List all tests** in your project +- **List all available actions** (I.* methods) from enabled helpers +- **Run arbitrary CodeceptJS code** and capture artifacts (ARIA snapshots, screenshots, HTML, logs) +- **Run specific tests** with AI-friendly trace files generated by the aiTrace plugin +- **Run tests step by step** for detailed debugging and analysis +- **Control browser sessions** (start/stop) + +### Features + +- **AI-Friendly Trace Files**: When enabled, the aiTrace plugin generates comprehensive trace files with screenshots, HTML, ARIA snapshots, browser logs, and HTTP requests +- **Real-time Monitoring**: Get trace file URLs at the start of test execution so AI can monitor tests in real-time +- **Artifact Capture**: Automatically capture ARIA snapshots, page source, console logs, and more +- **Browser Management**: Automatic or manual browser lifecycle control +- **Timeout Control**: Configurable timeouts for all operations + +### Quick Start + +1. Install the MCP SDK: +```sh +npm install @modelcontextprotocol/sdk +``` + +2. Configure your MCP client (e.g., Claude Desktop): +```json +{ + "mcpServers": { + "codeceptjs": { + "command": "node", + "args": ["path/to/codeceptjs/bin/mcp-server.js"] + } + } +} +``` + +3. Enable the aiTrace plugin in `codecept.conf.js`: +```javascript +plugins: { + aiTrace: { + enabled: true + } +} +``` + +### Usage Examples + +**List all tests:** +```json +{ + "name": "list_tests" +} +``` + +**Run CodeceptJS code:** +```json +{ + "name": "run_code", + "arguments": { + "code": "await I.amOnPage('/'); await I.see('Welcome')" + } +} +``` + +**Run a test with trace:** +```json +{ + "name": "run_test", + "arguments": { + "test": "tests/login_test.js" + } +} +``` + +Learn more about the MCP Server in the [MCP documentation](https://github.com/codeceptjs/CodeceptJS/blob/main/docs/mcp.md). + ## PageObjects CodeceptJS provides the most simple way to create and use page objects in your test. diff --git a/bin/mcp-server.js b/bin/mcp-server.js new file mode 100644 index 000000000..e03856234 --- /dev/null +++ b/bin/mcp-server.js @@ -0,0 +1,610 @@ +import { Server } from '@modelcontextprotocol/sdk/server/index.js' +import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js' +import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js' +import Codecept from '../lib/codecept.js' +import container from '../lib/container.js' +import { getParamsToString } from '../lib/parser.js' +import { methodsOfObject } from '../lib/utils.js' +import event from '../lib/event.js' +import { fileURLToPath } from 'url' +import { dirname, resolve as resolvePath } from 'path' +import path from 'path' +import crypto from 'crypto' +import { spawn } from 'child_process' +import { createRequire } from 'module' +import { existsSync, readdirSync } from 'fs' + +const require = createRequire(import.meta.url) + +const __filename = fileURLToPath(import.meta.url) +const __dirname = dirname(__filename) + +let codecept = null +let containerInitialized = false +let browserStarted = false + +let runLock = Promise.resolve() +async function withLock(fn) { + const prev = runLock + let release + runLock = new Promise(r => (release = r)) + await prev + try { return await fn() } + finally { release() } +} + +async function withSilencedIO(fn) { + const origOut = process.stdout.write.bind(process.stdout) + const origErr = process.stderr.write.bind(process.stderr) + + process.stdout.write = () => true + process.stderr.write = () => true + + try { + return await fn() + } finally { + process.stdout.write = origOut + process.stderr.write = origErr + } +} + +function runCmd(cmd, args, { cwd = process.cwd(), timeout = 60000 } = {}) { + return new Promise((resolve, reject) => { + const child = spawn(cmd, args, { + cwd, + env: { ...process.env, NODE_ENV: process.env.NODE_ENV || 'test' }, + stdio: ['ignore', 'pipe', 'pipe'], + }) + + let out = '' + let err = '' + + const t = setTimeout(() => { + child.kill('SIGKILL') + reject(new Error(`Timeout after ${timeout}ms`)) + }, timeout) + + child.stdout.on('data', d => (out += d.toString('utf8'))) + child.stderr.on('data', d => (err += d.toString('utf8'))) + + child.on('error', e => { + clearTimeout(t) + reject(e) + }) + + child.on('close', code => { + clearTimeout(t) + resolve({ code, out, err }) + }) + }) +} + +function resolveConfigPath(configPath) { + const cwd = process.cwd() + const envRoot = process.env.CODECEPTJS_PROJECT_DIR + + if (configPath && !path.isAbsolute(configPath)) { + const base = envRoot || cwd + configPath = path.resolve(base, configPath) + } + + if (!configPath) { + const base = envRoot || cwd + configPath = process.env.CODECEPTJS_CONFIG || path.resolve(base, 'codecept.conf.js') + if (!existsSync(configPath)) configPath = path.resolve(base, 'codecept.conf.cjs') + } + + if (!existsSync(configPath)) { + throw new Error( + `CodeceptJS config not found: ${configPath}\n` + + `CODECEPTJS_CONFIG=${process.env.CODECEPTJS_CONFIG || 'not set'}\n` + + `CODECEPTJS_PROJECT_DIR=${process.env.CODECEPTJS_PROJECT_DIR || 'not set'}\n` + + `cwd=${cwd}` + ) + } + + return { configPath, configDir: path.dirname(configPath) } +} + +function findCodeceptCliUpwards(startDir, { maxUp = 8 } = {}) { + let dir = startDir + + for (let i = 0; i <= maxUp; i++) { + const candidates = [ + path.resolve(dir, 'bin', 'codecept.js'), + path.resolve(dir, 'node_modules', 'codeceptjs', 'bin', 'codecept.js'), + path.resolve(dir, 'node_modules', '.bin', 'codeceptjs.cmd'), + path.resolve(dir, 'node_modules', '.bin', 'codeceptjs'), + ] + + for (const p of candidates) { + if (existsSync(p)) return { cli: p, root: dir } + } + + try { + const pkgJson = require.resolve('codeceptjs/package.json', { paths: [dir] }) + const pkgDir = path.dirname(pkgJson) + const jsCli = path.resolve(pkgDir, 'bin', 'codecept.js') + if (existsSync(jsCli)) return { cli: jsCli, root: dir } + } catch {} + + const parent = path.dirname(dir) + if (parent === dir) break + dir = parent + } + + throw new Error(`Cannot find CodeceptJS CLI walking up from: ${startDir}`) +} + +function looksLikePath(v) { + return typeof v === 'string' && ( + v.includes('/') || v.includes('\\') || + v.endsWith('.js') || v.endsWith('.ts') + ) +} + +function normalizePath(p) { + return String(p).replace(/\\/g, '/') +} + +function findFileByBasename(rootDir, baseNames, { maxDepth = 8 } = {}) { + const targets = new Set(baseNames.map(x => x.toLowerCase())) + + function walk(dir, depth) { + if (depth > maxDepth) return null + + let entries + try { entries = readdirSync(dir, { withFileTypes: true }) } catch { return null } + + for (const e of entries) { + const full = path.join(dir, e.name) + + if (e.isDirectory()) { + if (e.name === 'node_modules' || e.name === '.git' || e.name === 'output') continue + const res = walk(full, depth + 1) + if (res) return res + continue + } + + if (targets.has(e.name.toLowerCase())) return full + } + + return null + } + + return walk(rootDir, 0) +} + +async function listTestsJson({ cli, root, configPath }) { + const args = ['list', '--config', configPath, '--json'] + const isNodeScript = cli.endsWith('.js') + + const res = isNodeScript + ? await runCmd(process.execPath, [cli, ...args], { cwd: root, timeout: 60000 }) + : await runCmd(cli, args, { cwd: root, timeout: 60000 }) + + const out = (res.out || '').trim() + try { return JSON.parse(out) } catch { return null } +} + +function extractFilesFromListJson(json) { + if (!json) return [] + if (Array.isArray(json)) return json.map(String) + if (Array.isArray(json.tests)) return json.tests.map(String) + if (Array.isArray(json.files)) return json.files.map(String) + if (Array.isArray(json.testFiles)) return json.testFiles.map(String) + return [] +} + +async function resolveTestToFile({ cli, root, configPath, test }) { + if (looksLikePath(test)) return test + + const raw = String(test).trim() + const candidates = [ + raw, + `${raw}.js`, + `${raw}.ts`, + `${raw}_test.js`, + `${raw}.test.js`, + ].map(x => x.toLowerCase()) + + const json = await listTestsJson({ cli, root, configPath }) + const files = extractFilesFromListJson(json).map(normalizePath) + + if (files.length) { + const byName = files.find(f => candidates.some(c => path.basename(f).toLowerCase() === c)) + if (byName) return byName + + const byContains = files.find(f => f.toLowerCase().includes(raw.toLowerCase())) + if (byContains) return byContains + } + + const fsFound = findFileByBasename(root, candidates) + return fsFound ? normalizePath(fsFound) : null +} + +function clearString(str) { + return str.replace(/[^a-zA-Z0-9]/g, '_') +} + +function getTraceDir(testTitle, testFile) { + const hash = crypto.createHash('sha256').update(testFile + testTitle).digest('hex').slice(0, 8) + const cleanTitle = clearString(testTitle).slice(0, 200) + const outputDir = global.output_dir || resolvePath(process.cwd(), 'output') + return resolvePath(outputDir, `trace_${cleanTitle}_${hash}`) +} + +async function initCodecept(configPath) { + if (containerInitialized) return + + const testRoot = process.env.CODECEPTJS_PROJECT_DIR || process.cwd() + + if (!configPath) { + configPath = process.env.CODECEPTJS_CONFIG || resolvePath(testRoot, 'codecept.conf.js') + if (!existsSync(configPath)) configPath = resolvePath(testRoot, 'codecept.conf.cjs') + } + + if (!existsSync(configPath)) { + throw new Error( + `CodeceptJS config not found: ${configPath}\n` + + `CODECEPTJS_CONFIG=${process.env.CODECEPTJS_CONFIG || 'not set'}\n` + + `CODECEPTJS_PROJECT_DIR=${process.env.CODECEPTJS_PROJECT_DIR || 'not set'}\n` + + `cwd=${process.cwd()}` + ) + } + + console.log = () => {} + console.error = () => {} + console.warn = () => {} + + const { getConfig } = await import('../lib/command/utils.js') + const config = await getConfig(configPath) + + codecept = new Codecept(config, {}) + await codecept.init(testRoot) + await container.create(config, {}) + await container.started() + + containerInitialized = true + browserStarted = true +} + +const server = new Server( + { name: 'codeceptjs-mcp-server', version: '1.0.0' }, + { capabilities: { tools: {} } } +) + +server.setRequestHandler(ListToolsRequestSchema, async () => ({ + tools: [ + { + name: 'list_tests', + description: 'List all tests in the CodeceptJS project', + inputSchema: { type: 'object', properties: { config: { type: 'string' } } }, + }, + { + name: 'list_actions', + description: 'List all available CodeceptJS actions (I.* methods)', + inputSchema: { type: 'object', properties: { config: { type: 'string' } } }, + }, + { + name: 'run_code', + description: 'Run arbitrary CodeceptJS code.', + inputSchema: { + type: 'object', + properties: { + code: { type: 'string' }, + timeout: { type: 'number' }, + config: { type: 'string' }, + saveArtifacts: { type: 'boolean' }, + }, + required: ['code'], + }, + }, + { + name: 'run_test', + description: 'Run a specific test.', + inputSchema: { + type: 'object', + properties: { + test: { type: 'string' }, + timeout: { type: 'number' }, + config: { type: 'string' }, + }, + required: ['test'], + }, + }, + { + name: 'run_step_by_step', + description: 'Run a test step by step with pauses between steps.', + inputSchema: { + type: 'object', + properties: { + test: { type: 'string' }, + timeout: { type: 'number' }, + config: { type: 'string' }, + }, + required: ['test'], + }, + }, + { + name: 'start_browser', + description: 'Start the browser session.', + inputSchema: { type: 'object', properties: { config: { type: 'string' } } }, + }, + { + name: 'stop_browser', + description: 'Stop the browser session.', + inputSchema: { type: 'object', properties: {} }, + }, + ], +})) + +server.setRequestHandler(CallToolRequestSchema, async (request) => { + const { name, arguments: args } = request.params + + try { + switch (name) { + case 'list_tests': { + const configPath = args?.config + await initCodecept(configPath) + + codecept.loadTests() + const tests = codecept.testFiles.map(testFile => { + const relativePath = testFile.replace(process.cwd(), '').replace(/\\/g, '/') + return { + file: testFile, + relativePath: relativePath.startsWith('/') ? relativePath.slice(1) : relativePath, + } + }) + + return { content: [{ type: 'text', text: JSON.stringify({ count: tests.length, tests }, null, 2) }] } + } + + case 'list_actions': { + const configPath = args?.config + await initCodecept(configPath) + + const helpers = container.helpers() + const supportI = container.support('I') + const actions = [] + const actionDetails = [] + + for (const helperName in helpers) { + const helper = helpers[helperName] + methodsOfObject(helper).forEach(action => { + if (actions.includes(action)) return + actions.push(action) + const params = getParamsToString(helper[action]) + actionDetails.push({ helper: helperName, action, signature: `I.${action}(${params})` }) + }) + } + + for (const n in supportI) { + if (actions.includes(n)) continue + const actor = supportI[n] + const params = getParamsToString(actor) + actionDetails.push({ helper: 'SupportObject', action: n, signature: `I.${n}(${params})` }) + } + + return { content: [{ type: 'text', text: JSON.stringify({ count: actionDetails.length, actions: actionDetails }, null, 2) }] } + } + + case 'start_browser': { + const configPath = args?.config + if (browserStarted) { + return { content: [{ type: 'text', text: JSON.stringify({ status: 'Browser already started' }, null, 2) }] } + } + await initCodecept(configPath) + return { content: [{ type: 'text', text: JSON.stringify({ status: 'Browser started successfully' }, null, 2) }] } + } + + case 'stop_browser': { + if (!containerInitialized) { + return { content: [{ type: 'text', text: JSON.stringify({ status: 'Browser not initialized' }, null, 2) }] } + } + + const helpers = container.helpers() + for (const helperName in helpers) { + const helper = helpers[helperName] + try { if (helper._finish) await helper._finish() } catch {} + } + + browserStarted = false + containerInitialized = false + + return { content: [{ type: 'text', text: JSON.stringify({ status: 'Browser stopped successfully' }, null, 2) }] } + } + + case 'run_code': { + const { code, timeout = 60000, config: configPath, saveArtifacts = true } = args + await initCodecept(configPath) + + const I = container.support('I') + if (!I) throw new Error('I object not available. Make sure helpers are configured.') + + const result = { status: 'unknown', output: '', error: null, artifacts: {} } + + try { + const asyncFn = new Function('I', `return (async () => { ${code} })()`) + await Promise.race([ + asyncFn(I), + new Promise((_, reject) => setTimeout(() => reject(new Error(`Timeout after ${timeout}ms`)), timeout)), + ]) + + result.status = 'success' + result.output = 'Code executed successfully' + + if (saveArtifacts) { + const helpers = container.helpers() + const helper = Object.values(helpers)[0] + if (helper) { + try { + if (helper.grabAriaSnapshot) result.artifacts.aria = await helper.grabAriaSnapshot() + if (helper.grabCurrentUrl) result.artifacts.url = await helper.grabCurrentUrl() + if (helper.grabBrowserLogs) result.artifacts.consoleLogs = (await helper.grabBrowserLogs()) || [] + if (helper.grabSource) { + const html = await helper.grabSource() + result.artifacts.html = html.substring(0, 10000) + '...' + } + } catch (e) { + result.output += ` (Warning: ${e.message})` + } + } + } + } catch (error) { + result.status = 'failed' + result.error = error.message + result.output = error.stack || error.message + } + + return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] } + } + + case 'run_test': { + return await withLock(async () => { + const { test, timeout = 60000, config: configPathArg } = args || {} + const { configPath, configDir } = resolveConfigPath(configPathArg) + + const { cli, root } = findCodeceptCliUpwards(configDir) + const isNodeScript = cli.endsWith('.js') + + const resolvedFile = await resolveTestToFile({ cli, root, configPath, test }) + const runArgs = ['run', '--config', configPath, '--reporter', 'json'] + + if (resolvedFile) runArgs.push(resolvedFile) + else if (looksLikePath(test)) runArgs.push(test) + else runArgs.push('--grep', String(test)) + + const res = isNodeScript + ? await runCmd(process.execPath, [cli, ...runArgs], { cwd: root, timeout }) + : await runCmd(cli, runArgs, { cwd: root, timeout }) + + const { code, out, err } = res + + let parsed = null + const jsonStart = out.indexOf('{') + const jsonEnd = out.lastIndexOf('}') + if (jsonStart !== -1 && jsonEnd !== -1 && jsonEnd > jsonStart) { + try { parsed = JSON.parse(out.slice(jsonStart, jsonEnd + 1)) } catch {} + } + + return { + content: [{ + type: 'text', + text: JSON.stringify({ + meta: { exitCode: code, cli, root, configPath, args: runArgs, resolvedFile: resolvedFile || null }, + reporterJson: parsed, + stderr: err ? err.slice(0, 20000) : '', + rawStdout: parsed ? '' : out.slice(0, 20000), + }, null, 2), + }], + } + }) + } + + case 'run_step_by_step': { + const { test, timeout = 60000, config: configPath } = args + await initCodecept(configPath) + + return await withSilencedIO(async () => { + codecept.loadTests() + + let testFiles = codecept.testFiles + if (test) { + const testName = normalizePath(test).toLowerCase() + testFiles = codecept.testFiles.filter(f => { + const filePath = normalizePath(f).toLowerCase() + return filePath.includes(testName) || filePath.endsWith(testName) + }) + } + + if (!testFiles.length) throw new Error(`No tests found matching: ${test}`) + + const results = [] + const currentSteps = {} + let currentTestTitle = null + const testFile = testFiles[0] + + const onBefore = (t) => { + const traceDir = getTraceDir(t.title, t.file) + currentTestTitle = t.title + currentSteps[t.title] = [] + results.push({ + test: t.title, + file: t.file, + traceFile: `file://${resolvePath(traceDir, 'trace.md')}`, + status: 'running', + steps: [], + }) + } + + const onAfter = (t) => { + const r = results.find(x => x.test === t.title) + if (r) { + r.status = t.err ? 'failed' : 'completed' + if (t.err) r.error = t.err.message + } + currentTestTitle = null + } + + const onStepAfter = (step) => { + if (!currentTestTitle || !currentSteps[currentTestTitle]) return + currentSteps[currentTestTitle].push({ + step: step.toString(), + status: step.status, + time: step.endTime - step.startTime, + }) + const r = results.find(x => x.test === currentTestTitle) + if (r) r.steps = [...currentSteps[currentTestTitle]] + } + + event.dispatcher.on(event.test.before, onBefore) + event.dispatcher.on(event.test.after, onAfter) + event.dispatcher.on(event.step.after, onStepAfter) + + try { + await Promise.race([ + (async () => { + await codecept.bootstrap() + await codecept.run(testFile) + })(), + new Promise((_, reject) => setTimeout(() => reject(new Error(`Timeout after ${timeout}ms`)), timeout)), + ]) + } catch (error) { + const lastRunning = results.filter(r => r.status === 'running').pop() + if (lastRunning) { + lastRunning.status = 'failed' + lastRunning.error = error.message + } + } finally { + try { event.dispatcher.removeListener(event.test.before, onBefore) } catch {} + try { event.dispatcher.removeListener(event.test.after, onAfter) } catch {} + try { event.dispatcher.removeListener(event.step.after, onStepAfter) } catch {} + } + + return { content: [{ type: 'text', text: JSON.stringify({ results, stepByStep: true }, null, 2) }] } + }) + } + + default: + throw new Error(`Unknown tool: ${name}`) + } + } catch (error) { + return { + content: [{ type: 'text', text: JSON.stringify({ error: error.message, stack: error.stack }, null, 2) }], + isError: true, + } + } +}) + +async function main() { + const transport = new StdioServerTransport() + await server.connect(transport) +} + +main().catch((error) => { + import('fs').then(fs => { + const logFile = path.resolve(process.cwd(), 'mcp-server-error.log') + fs.appendFileSync(logFile, `${new Date().toISOString()} - ${error.stack}\n`) + }) +}) diff --git a/docs/mcp.md b/docs/mcp.md new file mode 100644 index 000000000..70e914e3e --- /dev/null +++ b/docs/mcp.md @@ -0,0 +1,545 @@ +# CodeceptJS MCP Server + +Model Context Protocol (MCP) server for CodeceptJS enables AI agents (like Claude) to interact with and control CodeceptJS tests programmatically. + +## Overview + +The MCP server provides AI agents with tools to: +- List all tests in a CodeceptJS project +- List all available CodeceptJS actions (I.* methods) +- Run arbitrary CodeceptJS code with artifacts capture +- Run specific tests with detailed output +- Run tests step by step for detailed analysis +- Start and stop browser sessions +- Capture screenshots, ARIA snapshots, HTML, and console logs + +## Installation + +Install the MCP SDK dependency: + +```bash +npm install @modelcontextprotocol/sdk +``` + +The MCP server binary is available at `bin/mcp-server.js`. + +## Configuration + +Configure the MCP server in your Claude Desktop or MCP-compatible client configuration: + +### Basic Configuration + +```json +{ + "mcpServers": { + "codeceptjs": { + "command": "node", + "args": ["path/to/codeceptjs/bin/mcp-server.js"] + } + } +} +``` + +With basic configuration, the server looks for `codecept.conf.js` in the current working directory. + +### Configuration with Environment Variables + +Use environment variables to specify the CodeceptJS project directory and config file: + +```json +{ + "mcpServers": { + "codeceptjs": { + "command": "node", + "args": ["path/to/codeceptjs/bin/mcp-server.js"], + "env": { + "CODECEPTJS_CONFIG": "/path/to/your/codecept.conf.js", + "CODECEPTJS_PROJECT_DIR": "/path/to/your/project" + } + } + } +} +``` + +**Environment Variables:** + +| Variable | Description | +|----------|-------------| +| `CODECEPTJS_CONFIG` | Absolute path to the CodeceptJS configuration file | +| `CODECEPTJS_PROJECT_DIR` | Absolute path to the project root directory | + +### Example: Full Claude Desktop Configuration + +```json +{ + "mcpServers": { + "codeceptjs-mcp": { + "command": "node", + "args": ["D:/projects/my-project/node_modules/codeceptjs/bin/mcp-server.js"], + "env": { + "CODECEPTJS_CONFIG": "D:/projects/my-project/codecept.conf.js", + "CODECEPTJS_PROJECT_DIR": "D:/projects/my-project" + } + } + } +} +``` + +## Available Tools + +### list_tests + +List all tests in the CodeceptJS project. + +**Parameters:** +- `config` (optional): Path to codecept.conf.js (default: codecept.conf.js) + +**Returns:** +```json +{ + "count": 5, + "tests": [ + { + "file": "/full/path/to/test/file.js", + "relativePath": "tests/example_test.js" + } + ] +} +``` + +**Example:** +```json +{ + "name": "list_tests", + "arguments": { + "config": "/path/to/codecept.conf.js" + } +} +``` + +### list_actions + +List all available CodeceptJS actions (I.* methods) from enabled helpers and support objects. + +**Parameters:** +- `config` (optional): Path to codecept.conf.js + +**Returns:** +```json +{ + "count": 120, + "actions": [ + { + "helper": "Playwright", + "action": "amOnPage", + "signature": "I.amOnPage(url)" + }, + { + "helper": "Playwright", + "action": "click", + "signature": "I.click(locator, context)" + } + ] +} +``` + +### run_code + +Run arbitrary CodeceptJS code. Returns status, ARIA snapshot, URL, console logs, and HTML. + +**Parameters:** +- `code` (required): CodeceptJS code to execute +- `timeout` (optional): Timeout in milliseconds (default: 60000) +- `config` (optional): Path to codecept.conf.js +- `saveArtifacts` (optional): Save artifacts like ARIA, URL, console logs, HTML (default: true) + +**Returns:** +```json +{ + "status": "success", + "output": "Code executed successfully", + "error": null, + "artifacts": { + "aria": "main -> \"Welcome\"...", + "url": "http://localhost:8000/", + "consoleLogs": [], + "html": "..." + } +} +``` + +**Example:** +```json +{ + "name": "run_code", + "arguments": { + "code": "await I.amOnPage('/'); await I.see('Welcome');", + "timeout": 30000, + "saveArtifacts": true + } +} +``` + +### run_test + +Run a specific test by name or file path. Uses subprocess to run tests with isolation. + +**Parameters:** +- `test` (required): Test name or file path +- `timeout` (optional): Timeout in milliseconds (default: 60000) +- `config` (optional): Path to codecept.conf.js + +**Returns:** +```json +{ + "meta": { + "exitCode": 0, + "cli": "/path/to/codecept.js", + "root": "/project/root", + "configPath": "/path/to/codecept.conf.js", + "args": ["run", "--config", "...", "--reporter", "json", "test_file.js"], + "resolvedFile": "/full/path/to/test_file.js" + }, + "reporterJson": { + "stats": { + "tests": 3, + "passes": 2, + "failures": 1 + } + }, + "stderr": "", + "rawStdout": "" +} +``` + +**Features:** +- Automatically resolves test names to file paths +- Supports partial test name matching +- Uses json reporter for structured output +- Executes in subprocess for isolation +- Includes stderr for debugging + +**Example:** +```json +{ + "name": "run_test", + "arguments": { + "test": "basic_navigation_test", + "timeout": 60000 + } +} +``` + +### run_step_by_step + +Run a test step by step with detailed step information including timing and status. Generates AI-friendly trace files. + +**Parameters:** +- `test` (required): Test name or file path +- `timeout` (optional): Timeout in milliseconds (default: 60000) +- `config` (optional): Path to codecept.conf.js + +**Returns:** +```json +{ + "stepByStep": true, + "results": [ + { + "test": "Navigate to homepage", + "file": "/path/to/test.js", + "traceFile": "file:///output/trace_Test_Name_abc123/trace.md", + "status": "completed", + "steps": [ + { + "step": "I.amOnPage(\"/\")", + "status": "passed", + "time": 150 + }, + { + "step": "I.seeInTitle(\"Test App\")", + "status": "passed", + "time": 50 + } + ] + } + ] +} +``` + +**Trace Files:** +- Generated in `{output_dir}/trace_{TestName}_{hash}/` +- Includes screenshots (PNG), page HTML, ARIA snapshots, console logs +- `trace.md` file provides structured summary for AI analysis +- Named with test title and hash for uniqueness + +**Example:** +```json +{ + "name": "run_step_by_step", + "arguments": { + "test": "authentication_test", + "timeout": 90000 + } +} +``` + +### start_browser + +Start the browser session (initializes CodeceptJS container). + +**Parameters:** +- `config` (optional): Path to codecept.conf.js + +**Returns:** +```json +{ + "status": "Browser started successfully" +} +``` + +**Note:** Browser is automatically started on first code execution. This tool is useful for pre-initialization. + +### stop_browser + +Stop the browser session and cleanup resources. + +**Parameters:** +- None + +**Returns:** +```json +{ + "status": "Browser stopped successfully" +} +``` + +**Note:** Useful for releasing resources between long-running sessions. + +## Testing + +### Run MCP Server Tests + +The MCP server includes a comprehensive test suite: + +```bash +node test/mcp/mcp_server_test.js +``` + +Tests cover: +- Tool listing and schema validation +- Test enumeration +- Action listing +- Code execution with artifacts +- Test execution (run_test) +- Step-by-step execution +- Browser lifecycle +- Error handling + +### Run Demo Tests with MCP + +**Important: Start the test web server first!** + +The MCP test scenarios require a web server running on port 8000. Start it in a separate terminal: + +```bash +# Option 1: Using http-server (recommended) +cd test/mcp +npx http-server -p 8000 + +# Option 2: Using Python +cd test/mcp +python -m http.server 8000 + +# Option 3: Using PHP +cd test/mcp +php -S localhost:8000 +``` + +The server will start at http://127.0.0.1:8000 + +**Keep this terminal open** while running tests through MCP/Claude. + +Once the server is running, you can use Claude to run tests: + +``` +"List all tests" +"Run basic navigation test" +"Run form interaction test step by step" +``` + +**Note:** If tests fail with ERR_CONNECTION_REFUSED, make sure the web server is running on port 8000. + +## Trace Files for AI Debugging + +When using `run_step_by_step`, the server generates trace files that provide rich context for AI agents: + +### Trace File Structure + +``` +output/ +└── trace_Test_Name_abc123/ + ├── 0000_screenshot.png # Screenshot after step 0 + ├── 0000_page.html # HTML snapshot after step 0 + ├── 0000_aria.txt # ARIA snapshot after step 0 + ├── 0000_console.json # Console logs after step 0 + ├── 0001_screenshot.png # Screenshot after step 1 + ├── 0001_page.html + ├── 0001_aria.txt + ├── 0001_console.json + └── trace.md # AI-friendly summary +``` + +### Using Trace Files with AI + +The `trace.md` file provides structured information perfect for AI analysis: + +```markdown +# Test: Login functionality + +**Status**: failed +**File**: tests/login_test.js + +## Steps + +1. **I.amOnPage("/login")** - passed (150ms) +2. **I.fillField("#username", "user")** - passed (80ms) +3. **I.fillField("#password", "pass")** - passed (75ms) +4. **I.click("#login")** - passed (100ms) +5. **I.see("Welcome")** - failed (50ms) + +## Error + +Element "Welcome" not found + +## Artifacts + +- Screenshot: 0005_screenshot.png +- HTML: 0005_page.html +- ARIA: 0005_aria.txt +``` + +AI agents can use these artifacts to: +- Visualize what the test saw at each step +- Analyze page structure via ARIA +- Debug issues using HTML snapshots +- Identify errors from console logs + +## Architecture + +### Request Flow + +1. MCP Client sends JSON-RPC request via stdin/stdout +2. Server processes request and calls appropriate tool +3. Tool executes CodeceptJS code or runs tests +4. Results formatted as JSON and returned +5. MCP Client receives response + +### Session Management + +- **Initialization**: CodeceptJS container initialized on first request +- **Browser**: Started once and reused across requests +- **Locking**: `run_test` uses locking to prevent concurrent test runs +- **Cleanup**: `stop_browser` releases all resources + +### Error Handling + +- All errors returned as JSON with error message and stack +- Invalid tools return error response +- Test failures included in results (not thrown) +- Timeout protection on all long-running operations + +## Troubleshooting + +### MCP Server Not Starting + +- Ensure `@modelcontextprotocol/sdk` is installed +- Check Node.js version (requires Node.js 16+) +- Verify the path to mcp-server.js in your MCP client config +- Check file permissions + +### Configuration Not Found + +- Set `CODECEPTJS_CONFIG` environment variable to absolute path of your config file +- Set `CODECEPTJS_PROJECT_DIR` environment variable to your project directory +- Use absolute paths in environment variables (e.g., `D:/projects/my-project/codecept.conf.js`) +- Verify config file exists and is valid JavaScript + +### Tests Not Found + +- Verify you're in the correct working directory +- Check that `codecept.conf.js` exists +- Use absolute paths for tests if relative paths don't work +- Check test patterns in config file match your test files + +### Browser Launch Issues + +- Ensure browser dependencies are installed (Chromium for Playwright) +- Check if browser is already running +- Verify `show: false` in config (headless mode recommended) +- Check firewall/proxy settings + +### Tests Stuck or Timing Out + +- Increase timeout parameter (default 60s) +- Check if web server is running (for tests that need it) +- Disable video recording and other heavy features +- Use `run_test` instead of `run_step_by_step` for faster execution + +## Advanced Usage + +### Custom Test Patterns + +```javascript +// In codecept.conf.js +export const config = { + tests: './tests/**/*_test.js', + // ... rest of config +} +``` + +### AI-Friendly Trace Integration + +For best results with AI agents: + +1. **Enable aiTrace plugin** in config (automatically enabled for `run_step_by_step`) +2. **Use descriptive test names** for better trace file organization +3. **Keep tests focused** - one scenario per test for clearer traces +4. **Add assertions with clear messages** - better error reporting + +### Running Tests from Different Directories + +```json +{ + "mcpServers": { + "codeceptjs": { + "command": "node", + "args": ["node_modules/codeceptjs/bin/mcp-server.js"], + "env": { + "CODECEPTJS_CONFIG": "/absolute/path/to/codecept.conf.js", + "CODECEPTJS_PROJECT_DIR": "/absolute/path/to/project" + } + } + } +} +``` + +## Security Considerations + +- MCP server runs with same permissions as calling process +- `run_code` allows arbitrary CodeceptJS execution - use in trusted environments only +- Test files should validate input if exposed to external systems +- Environment variables may contain sensitive paths - secure accordingly + +## Contributing + +When contributing to MCP server: + +1. Add tests for new tools in `test/mcp/mcp_server_test.js` +2. Update this documentation with new tools/parameters +3. Ensure error handling is consistent +4. Test with both Playwright and Puppeteer helpers +5. Verify trace files are generated correctly for `run_step_by_step` + +## License + +MIT diff --git a/package.json b/package.json index 067cb5600..649333186 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,8 @@ "./store": "./lib/store.js" }, "bin": { - "codeceptjs": "./bin/codecept.js" + "codeceptjs": "./bin/codecept.js", + "codeceptjs-mcp": "./bin/mcp-server.js" }, "repository": "Codeception/codeceptjs", "scripts": { @@ -90,6 +91,7 @@ "@cucumber/cucumber-expressions": "18", "@cucumber/gherkin": "38.0.0", "@cucumber/messages": "32.0.1", + "@modelcontextprotocol/sdk": "^1.26.0", "@xmldom/xmldom": "0.9.8", "acorn": "8.15.0", "ai": "^6.0.43", diff --git a/test/unit/mcpServer_test.js b/test/unit/mcpServer_test.js new file mode 100644 index 000000000..3dba334a2 --- /dev/null +++ b/test/unit/mcpServer_test.js @@ -0,0 +1,405 @@ +import { expect } from 'chai' +import sinon from 'sinon' +import { fileURLToPath } from 'url' +import { dirname, resolve, join } from 'path' +import { existsSync, mkdirSync, rmSync } from 'fs' +import { createHash } from 'crypto' + +const __filename = fileURLToPath(import.meta.url) +const __dirname = dirname(__filename) + +function extractFilesFromListJson(json) { + if (!json) return [] + if (Array.isArray(json)) return json.map(String) + if (Array.isArray(json.tests)) return json.tests.map(String) + if (Array.isArray(json.files)) return json.files.map(String) + if (Array.isArray(json.testFiles)) return json.testFiles.map(String) + return [] +} + +function looksLikePath(v) { + return typeof v === 'string' && ( + v.includes('/') || v.includes('\\') || + v.endsWith('.js') || v.endsWith('.ts') + ) +} + +function normalizePath(p) { + return String(p).replace(/\\/g, '/') +} + +function clearString(str) { + return str.replace(/[^a-zA-Z0-9]/g, '_') +} + +function getTraceDir(testTitle, testFile) { + const hash = createHash('sha256').update(testFile + testTitle).digest('hex').slice(0, 8) + const cleanTitle = clearString(testTitle).slice(0, 200) + const outputDir = global.output_dir || resolve(process.cwd(), 'output') + return resolve(outputDir, `trace_${cleanTitle}_${hash}`) +} + +describe('MCP Server Utilities', () => { + describe('extractFilesFromListJson', () => { + it('should extract files from array', () => { + const json = ['/path/to/test1.js', '/path/to/test2.js'] + const result = extractFilesFromListJson(json) + expect(result).to.deep.equal(['/path/to/test1.js', '/path/to/test2.js']) + }) + + it('should extract files from object with tests property', () => { + const json = { tests: ['/path/to/test1.js', '/path/to/test2.js'] } + const result = extractFilesFromListJson(json) + expect(result).to.deep.equal(['/path/to/test1.js', '/path/to/test2.js']) + }) + + it('should extract files from object with files property', () => { + const json = { files: ['/path/to/test1.js'] } + const result = extractFilesFromListJson(json) + expect(result).to.deep.equal(['/path/to/test1.js']) + }) + + it('should extract files from object with testFiles property', () => { + const json = { testFiles: ['/path/to/test1.js'] } + const result = extractFilesFromListJson(json) + expect(result).to.deep.equal(['/path/to/test1.js']) + }) + + it('should return empty array for null input', () => { + const result = extractFilesFromListJson(null) + expect(result).to.deep.equal([]) + }) + + it('should return empty array for invalid input', () => { + const result = extractFilesFromListJson({ invalid: true }) + expect(result).to.deep.equal([]) + }) + }) + + describe('looksLikePath', () => { + it('should return true for path with forward slashes', () => { + expect(looksLikePath('tests/example_test.js')).to.be.true + }) + + it('should return true for path with backslashes', () => { + expect(looksLikePath('tests\\example_test.js')).to.be.true + }) + + it('should return true for .js file', () => { + expect(looksLikePath('example_test.js')).to.be.true + }) + + it('should return true for .ts file', () => { + expect(looksLikePath('example_test.ts')).to.be.true + }) + + it('should return false for test name', () => { + expect(looksLikePath('my test')).to.be.false + }) + + it('should return false for simple string', () => { + expect(looksLikePath('login')).to.be.false + }) + }) + + describe('normalizePath', () => { + it('should convert backslashes to forward slashes', () => { + const result = normalizePath('path\\to\\file.js') + expect(result).to.equal('path/to/file.js') + }) + + it('should handle mixed slashes', () => { + const result = normalizePath('path/to\\file.js') + expect(result).to.equal('path/to/file.js') + }) + + it('should handle paths with no slashes', () => { + const result = normalizePath('file.js') + expect(result).to.equal('file.js') + }) + + it('should convert string to string', () => { + const result = normalizePath(123) + expect(result).to.equal('123') + }) + }) + + describe('clearString', () => { + it('should replace special characters with underscore', () => { + expect(clearString('test-name')).to.equal('test_name') + }) + + it('should handle multiple special characters', () => { + expect(clearString('test@#$%name')).to.equal('test____name') + }) + + it('should preserve alphanumeric characters', () => { + expect(clearString('test123Name')).to.equal('test123Name') + }) + + it('should replace spaces with underscores', () => { + expect(clearString('test name')).to.equal('test_name') + }) + + it('should handle empty string', () => { + expect(clearString('')).to.equal('') + }) + }) + + describe('getTraceDir', () => { + it('should generate unique trace directory name', () => { + const testFile = '/path/to/test.js' + const testTitle = 'My Test' + + const result = getTraceDir(testTitle, testFile) + + expect(result).to.be.a('string') + expect(result).to.include('trace_') + }) + + it('should use hash for uniqueness', () => { + const testFile = '/path/to/test.js' + const testTitle = 'My Test' + + const result1 = getTraceDir(testTitle, testFile) + const result2 = getTraceDir(testTitle, testFile) + + expect(result1).to.equal(result2) + }) + + it('should sanitize test title in directory name', () => { + const testFile = '/path/to/test.js' + const testTitle = 'Test: Special@#$ Characters' + + const result = getTraceDir(testTitle, testFile) + + expect(result).to.not.include('@') + expect(result).to.not.include('#') + expect(result).to.not.include('$') + expect(result).to.not.include(':') + }) + }) +}) + +describe('MCP Server Integration', () => { + let testOutputDir + + beforeEach(() => { + testOutputDir = resolve(__dirname, '../output/mcp-test') + if (existsSync(testOutputDir)) { + rmSync(testOutputDir, { recursive: true, force: true }) + } + mkdirSync(testOutputDir, { recursive: true }) + + sinon.stub(process, 'stdout').value({ + write: sinon.stub().returns(true), + }) + + sinon.stub(process, 'stderr').value({ + write: sinon.stub().returns(true), + }) + + sinon.stub(process, 'stdin').value({ + write: sinon.stub(), + }) + }) + + afterEach(() => { + sinon.restore() + + if (existsSync(testOutputDir)) { + try { + rmSync(testOutputDir, { recursive: true, force: true }) + } catch (e) { + } + } + }) + + describe('Tool Response Format', () => { + it('should return properly formatted tool response', () => { + const response = { + content: [ + { + type: 'text', + text: JSON.stringify({ status: 'success', output: 'test' }, null, 2), + }, + ], + } + + expect(response).to.have.property('content') + expect(response.content).to.be.an('array') + expect(response.content[0]).to.have.property('type', 'text') + expect(response.content[0]).to.have.property('text') + + const parsed = JSON.parse(response.content[0].text) + expect(parsed).to.have.property('status', 'success') + }) + + it('should return error response properly formatted', () => { + const errorResponse = { + content: [ + { + type: 'text', + text: JSON.stringify({ error: 'Test error', stack: 'error stack' }, null, 2), + }, + ], + isError: true, + } + + expect(errorResponse).to.have.property('isError', true) + expect(errorResponse.content[0].type).to.equal('text') + + const parsed = JSON.parse(errorResponse.content[0].text) + expect(parsed).to.have.property('error') + expect(parsed).to.have.property('stack') + }) + }) + + describe('Concurrency Control', () => { + it('should serialize execution using lock', async function () { + this.timeout(10000) + + let executionOrder = [] + let lock = Promise.resolve() + + async function withLock(fn) { + const prev = lock + let release + lock = new Promise(r => (release = r)) + + await prev + try { + return await fn() + } finally { + release() + } + } + + const task1 = withLock(async () => { + executionOrder.push('start1') + await new Promise(resolve => setTimeout(resolve, 100)) + executionOrder.push('end1') + }) + + const task2 = withLock(async () => { + executionOrder.push('start2') + await new Promise(resolve => setTimeout(resolve, 50)) + executionOrder.push('end2') + }) + + const task3 = withLock(async () => { + executionOrder.push('start3') + await new Promise(resolve => setTimeout(resolve, 50)) + executionOrder.push('end3') + }) + + await Promise.all([task1, task2, task3]) + + expect(executionOrder).to.deep.equal([ + 'start1', + 'end1', + 'start2', + 'end2', + 'start3', + 'end3', + ]) + }) + }) + + describe('Error Handling', () => { + it('should format error messages correctly', () => { + const error = new Error('Test error message') + const errorResponse = { + content: [ + { + type: 'text', + text: JSON.stringify({ error: error.message, stack: error.stack }, null, 2), + }, + ], + isError: true, + } + + expect(errorResponse.isError).to.be.true + const parsed = JSON.parse(errorResponse.content[0].text) + expect(parsed.error).to.equal('Test error message') + expect(parsed.stack).to.be.a('string') + }) + }) + + describe('Artifact Capture', () => { + it('should include all artifact types when saveArtifacts is true', () => { + const artifacts = { + aria: 'aria snapshot content', + url: 'http://localhost:8000/page', + consoleLogs: [{ level: 'info', text: 'log message' }], + html: 'Page content', + } + + expect(artifacts).to.have.property('aria') + expect(artifacts).to.have.property('url') + expect(artifacts).to.have.property('consoleLogs') + expect(artifacts).to.have.property('html') + }) + + it('should handle missing artifact methods gracefully', () => { + const partialArtifacts = { + url: 'http://localhost:8000/', + } + + expect(partialArtifacts).to.have.property('url') + expect(partialArtifacts).to.not.have.property('aria') + }) + }) + + describe('Test Result Formats', () => { + it('should format step-by-step results correctly', () => { + const results = [ + { + test: 'Login functionality', + file: '/path/to/login_test.js', + traceFile: 'file:///output/trace_Login_abc123/trace.md', + status: 'completed', + steps: [ + { step: 'I.amOnPage("/login")', status: 'passed', time: 150 }, + { step: 'I.fillField("#username", "user")', status: 'passed', time: 80 }, + ], + }, + ] + + expect(results).to.be.an('array') + expect(results[0]).to.have.property('test') + expect(results[0]).to.have.property('file') + expect(results[0]).to.have.property('traceFile') + expect(results[0]).to.have.property('status') + expect(results[0]).to.have.property('steps') + expect(results[0].steps).to.be.an('array') + expect(results[0].steps[0]).to.have.property('step') + expect(results[0].steps[0]).to.have.property('status') + expect(results[0].steps[0]).to.have.property('time') + }) + + it('should format run_test results correctly', () => { + const result = { + meta: { + exitCode: 0, + cli: '/path/to/codecept.js', + root: '/project/root', + configPath: '/path/to/codecept.conf.js', + args: ['run', '--config', '/path/to/codecept.conf.js', '--reporter', 'json', 'test.js'], + resolvedFile: '/full/path/to/test.js', + }, + reporterJson: { + stats: { tests: 3, passes: 2, failures: 1 }, + }, + stderr: '', + rawStdout: '', + } + + expect(result).to.have.property('meta') + expect(result.meta).to.have.property('exitCode', 0) + expect(result.meta).to.have.property('resolvedFile') + expect(result).to.have.property('reporterJson') + expect(result.reporterJson).to.have.property('stats') + }) + }) +})