From 39f1d8e179f98992a638f4b479a589d261372450 Mon Sep 17 00:00:00 2001 From: Ilya Topilskii Date: Mon, 27 Jul 2026 17:15:07 +0200 Subject: [PATCH 1/2] Add critical user flow test gate --- .github/workflows/ci.yml | 16 +- package.json | 1 + playwright.critical.config.ts | 56 ++++ tests/critical/README.md | 38 +++ .../critical/critical-user-flow-contracts.ts | 49 ++++ tests/critical/critical-user-flows.spec.ts | 277 ++++++++++++++++++ 6 files changed, 436 insertions(+), 1 deletion(-) create mode 100644 playwright.critical.config.ts create mode 100644 tests/critical/README.md create mode 100644 tests/critical/critical-user-flow-contracts.ts create mode 100644 tests/critical/critical-user-flows.spec.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6c8db6..dbec7dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,21 @@ jobs: run: pnpm build - name: Install Playwright browsers - run: pnpm exec playwright install --with-deps chromium + run: pnpm exec playwright install --with-deps chromium webkit + + - name: Run critical user flows + run: pnpm exec playwright test --config playwright.critical.config.ts - name: Run tests run: pnpm test + + - name: Upload critical Playwright failure artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: playwright-critical-${{ github.run_attempt }} + path: | + test-results/critical + playwright-report/critical + if-no-files-found: ignore + retention-days: 7 diff --git a/package.json b/package.json index fa09428..b9d40f9 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "test": "pnpm build && vitest run && pnpm exec playwright test", "test:smoke": "vitest run", "test:e2e": "pnpm exec playwright test", + "test:critical": "pnpm build && pnpm exec playwright test --config playwright.critical.config.ts", "test:docs-verify": "vitest run --config vitest.docs-verify.config.ts", "verify:images": "node scripts/verify-resource-images.mjs", "optimize:images": "bash scripts/optimize-images.sh", diff --git a/playwright.critical.config.ts b/playwright.critical.config.ts new file mode 100644 index 0000000..b0f5d6c --- /dev/null +++ b/playwright.critical.config.ts @@ -0,0 +1,56 @@ +import { defineConfig, devices } from "@playwright/test"; + +const PORT = 4173; +const isCI = Boolean(process.env.CI); + +export default defineConfig({ + testDir: "tests/critical", + outputDir: "test-results/critical", + fullyParallel: false, + forbidOnly: isCI, + retries: isCI ? 2 : 0, + workers: 1, + timeout: 45_000, + reporter: isCI + ? [ + ["list"], + [ + "html", + { + open: "never", + outputFolder: "playwright-report/critical", + }, + ], + ] + : "list", + use: { + baseURL: `http://localhost:${PORT}`, + screenshot: "only-on-failure", + trace: "retain-on-failure", + video: "retain-on-failure", + }, + projects: [ + { + name: "desktop-chromium", + use: { ...devices["Desktop Chrome"] }, + }, + { + name: "mobile-chromium", + use: { ...devices["Pixel 7"] }, + }, + { + name: "desktop-webkit", + use: { ...devices["Desktop Safari"] }, + }, + { + name: "mobile-webkit", + use: { ...devices["iPhone 15"] }, + }, + ], + webServer: { + command: `pnpm exec next start -p ${PORT}`, + port: PORT, + reuseExistingServer: true, + timeout: 60_000, + }, +}); diff --git a/tests/critical/README.md b/tests/critical/README.md new file mode 100644 index 0000000..0063fb7 --- /dev/null +++ b/tests/critical/README.md @@ -0,0 +1,38 @@ +# Critical user flow tests + +This suite is the release gate for the smallest set of DevHub journeys whose +failure prevents a developer or coding agent from getting useful content. It +runs against a local production build. It does not poll the production site, +send alerts, or run on a schedule. + +## Test plan + +| ID | Priority | User flow | Required outcome | +| ------------ | -------- | ------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| TC-WEB-001 | P0 | Home → Copy agent prompt | The real bootstrap API returns a complete prompt and the browser copies it. | +| TC-WEB-002 | P0 | Global search → Databricks CLI docs → Copy Markdown | Keyboard search navigates to the right doc and exports useful raw Markdown. | +| TC-WEB-003 | P0 | Templates → search → SaaS Tracker → agent handoff | The selected template copies a complete prompt and generates a valid Replit URL. | +| TC-WEB-004 | P1 | Template copy → clipboard error → retry | The failure is visible and a second attempt succeeds without reloading. | +| TC-AGENT-001 | P0 | Agent requests bootstrap, llms.txt, raw docs, and tools | Machine-readable entry points respond and the MCP server advertises its core tools. | + +The four browser journeys run in desktop Chromium, Pixel-sized Chromium, +desktop WebKit, and iPhone-sized WebKit. The browser-independent API contract +runs once in desktop Chromium. + +## Commands + +```bash +# Production build plus only the release-gating journeys +pnpm test:critical + +# Unit tests and the broad Chromium regression suite +pnpm test + +# The same two release gates CI runs +pnpm test:critical && pnpm test +``` + +On CI, a failed critical journey keeps its screenshot, trace, and video as a +short-lived workflow artifact. The broader Playwright and Vitest suites remain +the regression layer; tests should only move into this folder when a failure +blocks the core developer outcome described above. diff --git a/tests/critical/critical-user-flow-contracts.ts b/tests/critical/critical-user-flow-contracts.ts new file mode 100644 index 0000000..c4d7244 --- /dev/null +++ b/tests/critical/critical-user-flow-contracts.ts @@ -0,0 +1,49 @@ +export const criticalUserFlowContracts = { + bootstrapPrompt: { + id: "TC-WEB-001", + priority: "P0", + route: "/", + requiredPromptFragments: [ + "# About DevHub", + "# Working with DevHub prompts", + "# What the user just did", + "# Verify your local Databricks dev environment", + "Databricks CLI", + "llms.txt", + ], + }, + docsSearchAndExport: { + id: "TC-WEB-002", + priority: "P0", + route: "/docs/tools/databricks-cli", + query: "Databricks CLI", + requiredMarkdownFragments: [ + "# Databricks CLI", + "command-line interface", + "/templates", + ], + }, + templateDiscoveryAndHandoff: { + id: "TC-WEB-003", + priority: "P0", + route: "/templates/saas-tracker", + query: "SaaS Subscription Tracker", + title: "SaaS Subscription Tracker", + requiredPromptFragments: [ + "# About DevHub", + "# SaaS Subscription Tracker", + "Data Flow", + "llms.txt", + ], + }, + copyRecovery: { + id: "TC-WEB-004", + priority: "P1", + route: "/templates/saas-tracker", + }, + agentEntryPoints: { + id: "TC-AGENT-001", + priority: "P0", + mcpTools: ["get_doc_resource", "list_docs_resources"], + }, +} as const; diff --git a/tests/critical/critical-user-flows.spec.ts b/tests/critical/critical-user-flows.spec.ts new file mode 100644 index 0000000..c278149 --- /dev/null +++ b/tests/critical/critical-user-flows.spec.ts @@ -0,0 +1,277 @@ +import { expect, test, type Page } from "@playwright/test"; + +import { criticalUserFlowContracts as contracts } from "./critical-user-flow-contracts"; + +type TestWindow = Window & { + __clipboardWriteAttempts?: number; + __copiedText?: string; +}; + +function collectPageErrors(page: Page): string[] { + const errors: string[] = []; + page.on("pageerror", (error) => errors.push(error.message)); + return errors; +} + +function installClipboardMock(page: Page, failedWrites = 0): Promise { + return page.addInitScript((writesToFail) => { + const testWindow = window as TestWindow; + testWindow.__clipboardWriteAttempts = 0; + + Object.defineProperty(window.navigator, "clipboard", { + configurable: true, + value: { + writeText: async (value: string) => { + testWindow.__clipboardWriteAttempts = + (testWindow.__clipboardWriteAttempts ?? 0) + 1; + + if (testWindow.__clipboardWriteAttempts <= writesToFail) { + throw new Error("Simulated clipboard failure"); + } + + testWindow.__copiedText = value; + }, + }, + }); + }, failedWrites); +} + +function getCopiedText(page: Page): Promise { + return page.evaluate(() => (window as TestWindow).__copiedText ?? ""); +} + +function parseMcpResponse(contentType: string, body: string): unknown { + if (!contentType.includes("text/event-stream")) { + return JSON.parse(body); + } + + const dataLine = body.split("\n").find((line) => line.startsWith("data: ")); + + if (!dataLine) { + throw new Error(`Expected an SSE data line, received: ${body}`); + } + + return JSON.parse(dataLine.slice("data: ".length)); +} + +test.describe("critical user flows", () => { + test(`${contracts.bootstrapPrompt.id} copies a usable bootstrap prompt from the home page`, async ({ + page, + }) => { + const pageErrors = collectPageErrors(page); + await installClipboardMock(page); + + const response = await page.goto(contracts.bootstrapPrompt.route, { + waitUntil: "domcontentloaded", + }); + expect(response?.status()).toBe(200); + await expect( + page.getByRole("heading", { + level: 1, + name: /Build agentic applications/, + }), + ).toBeVisible(); + + const copyButton = page + .locator("main") + .getByRole("button", { name: "Copy agent prompt" }) + .first(); + await expect(copyButton).toBeEnabled(); + await copyButton.click(); + await expect( + page.locator("main").getByRole("button", { name: "Copied" }).first(), + ).toBeVisible(); + + const copiedPrompt = await getCopiedText(page); + for (const fragment of contracts.bootstrapPrompt.requiredPromptFragments) { + expect(copiedPrompt).toContain(fragment); + } + expect(pageErrors).toEqual([]); + }); + + test(`${contracts.docsSearchAndExport.id} finds docs with the keyboard and exports useful Markdown`, async ({ + page, + }, testInfo) => { + const pageErrors = collectPageErrors(page); + await installClipboardMock(page); + await page.goto("/"); + + if (testInfo.project.name.startsWith("mobile-")) { + await page.getByRole("button", { name: "Open menu" }).click(); + await expect(page.locator("#devhub-main-content")).toHaveJSProperty( + "inert", + true, + ); + } + + await page.getByRole("button", { name: "Search documentation" }).click(); + + const searchDialog = page.getByRole("dialog", { + name: "Search documentation", + }); + const searchInput = searchDialog.getByPlaceholder( + "What are you searching for?", + ); + await expect(searchInput).toBeFocused(); + await searchInput.fill(contracts.docsSearchAndExport.query); + await expect( + searchDialog.getByRole("option", { + name: /Databricks CLI/i, + }), + ).toBeVisible(); + await searchInput.press("Enter"); + + await expect(page).toHaveURL( + new RegExp(`${contracts.docsSearchAndExport.route}$`), + ); + await expect( + page.getByRole("heading", { + level: 1, + name: contracts.docsSearchAndExport.query, + }), + ).toBeVisible(); + + await page + .getByRole("button", { name: /copy (as|article)/i }) + .first() + .click(); + await page.getByRole("menuitem", { name: "Copy Markdown" }).click(); + await expect( + page.getByRole("button", { name: "Copied" }).first(), + ).toBeVisible(); + + const copiedMarkdown = await getCopiedText(page); + for (const fragment of contracts.docsSearchAndExport + .requiredMarkdownFragments) { + expect(copiedMarkdown).toContain(fragment); + } + expect(copiedMarkdown).not.toContain("# About DevHub"); + expect(pageErrors).toEqual([]); + }); + + test(`${contracts.templateDiscoveryAndHandoff.id} discovers a template and produces agent handoffs`, async ({ + page, + }) => { + const pageErrors = collectPageErrors(page); + await installClipboardMock(page); + await page.goto("/templates"); + + await page + .getByRole("searchbox") + .fill(contracts.templateDiscoveryAndHandoff.query); + await page + .getByRole("link", { + name: `Read ${contracts.templateDiscoveryAndHandoff.title}`, + }) + .click(); + + await expect(page).toHaveURL( + new RegExp(`${contracts.templateDiscoveryAndHandoff.route}$`), + ); + await expect( + page.getByRole("heading", { + level: 1, + name: contracts.templateDiscoveryAndHandoff.title, + }), + ).toBeVisible(); + + await page.getByRole("button", { name: "Copy prompt" }).first().click(); + await expect( + page.getByRole("button", { name: "Copied!" }).first(), + ).toBeVisible(); + + const copiedPrompt = await getCopiedText(page); + for (const fragment of contracts.templateDiscoveryAndHandoff + .requiredPromptFragments) { + expect(copiedPrompt).toContain(fragment); + } + + await page.getByRole("button", { name: "Open prompt in" }).click(); + const replitItem = page.getByRole("menuitem", { name: "Replit" }); + await expect(replitItem).toBeVisible(); + const replitHref = await replitItem.getAttribute("href"); + expect(replitHref).toBeTruthy(); + + const replitUrl = new URL(replitHref!); + expect(`${replitUrl.origin}${replitUrl.pathname}`).toBe( + "https://replit.com/", + ); + expect(replitUrl.searchParams.get("stack")).toBe("Build"); + expect(replitUrl.searchParams.get("prompt")?.length).toBeGreaterThan(50); + expect(pageErrors).toEqual([]); + }); + + test(`${contracts.copyRecovery.id} recovers after a clipboard failure`, async ({ + page, + }) => { + const pageErrors = collectPageErrors(page); + await installClipboardMock(page, 1); + await page.goto(contracts.copyRecovery.route); + + const copyButton = page + .getByRole("button", { name: "Copy prompt" }) + .first(); + await copyButton.click(); + await expect( + page.getByRole("button", { name: "Try again" }).first(), + ).toBeVisible(); + + await page.getByRole("button", { name: "Try again" }).first().click(); + await expect( + page.getByRole("button", { name: "Copied!" }).first(), + ).toBeVisible(); + await expect + .poll(() => getCopiedText(page)) + .toContain(contracts.templateDiscoveryAndHandoff.title); + expect(pageErrors).toEqual([]); + }); + + test(`${contracts.agentEntryPoints.id} serves the entry points used by coding agents`, async ({ + request, + }, testInfo) => { + test.skip( + testInfo.project.name !== "desktop-chromium", + "The browser-independent API contract only needs one execution.", + ); + + const bootstrapResponse = await request.get("/api/bootstrap-prompt"); + expect(bootstrapResponse.status()).toBe(200); + expect(bootstrapResponse.headers()["content-type"]).toContain( + "text/markdown", + ); + expect(await bootstrapResponse.text()).toContain("# About DevHub"); + + const llmsResponse = await request.get("/llms.txt"); + expect(llmsResponse.status()).toBe(200); + expect(await llmsResponse.text()).toContain("Databricks"); + + const rawDocsResponse = await request.get("/raw-docs/start-here.md"); + expect(rawDocsResponse.status()).toBe(200); + expect(rawDocsResponse.headers()["content-type"]).toContain( + "text/markdown", + ); + expect(await rawDocsResponse.text()).toContain("# Start here"); + + const mcpResponse = await request.post("/api/mcp", { + data: { + jsonrpc: "2.0", + id: 1, + method: "tools/list", + }, + headers: { + Accept: "application/json, text/event-stream", + }, + }); + expect(mcpResponse.status()).toBe(200); + + const mcpPayload = parseMcpResponse( + mcpResponse.headers()["content-type"] ?? "", + await mcpResponse.text(), + ) as { + result: { tools: Array<{ name: string }> }; + }; + expect(mcpPayload.result.tools.map(({ name }) => name).sort()).toEqual( + [...contracts.agentEntryPoints.mcpTools].sort(), + ); + }); +}); From 3f5c720d67abad9a563505ff318bcce4aa0fe0c5 Mon Sep 17 00:00:00 2001 From: Ilya Topilskii Date: Tue, 28 Jul 2026 14:41:09 +0200 Subject: [PATCH 2/2] Harden critical user flow tests --- playwright.critical.config.ts | 11 +- tests/critical/README.md | 31 ++-- .../critical/critical-user-flow-contracts.ts | 25 +-- tests/critical/critical-user-flows.spec.ts | 160 ++++++++++++++---- tests/mcp-e2e.test.ts | 3 +- tests/mcp.test.ts | 3 +- 6 files changed, 161 insertions(+), 72 deletions(-) diff --git a/playwright.critical.config.ts b/playwright.critical.config.ts index b0f5d6c..d173224 100644 --- a/playwright.critical.config.ts +++ b/playwright.critical.config.ts @@ -32,10 +32,13 @@ export default defineConfig({ projects: [ { name: "desktop-chromium", - use: { ...devices["Desktop Chrome"] }, + use: { + ...devices["Desktop Chrome"], + permissions: ["clipboard-read", "clipboard-write"], + }, }, { - name: "mobile-chromium", + name: "emulated-pixel-chromium", use: { ...devices["Pixel 7"] }, }, { @@ -43,14 +46,14 @@ export default defineConfig({ use: { ...devices["Desktop Safari"] }, }, { - name: "mobile-webkit", + name: "emulated-iphone-webkit", use: { ...devices["iPhone 15"] }, }, ], webServer: { command: `pnpm exec next start -p ${PORT}`, port: PORT, - reuseExistingServer: true, + reuseExistingServer: false, timeout: 60_000, }, }); diff --git a/tests/critical/README.md b/tests/critical/README.md index 0063fb7..70b9d38 100644 --- a/tests/critical/README.md +++ b/tests/critical/README.md @@ -7,17 +7,26 @@ send alerts, or run on a schedule. ## Test plan -| ID | Priority | User flow | Required outcome | -| ------------ | -------- | ------------------------------------------------------- | ----------------------------------------------------------------------------------- | -| TC-WEB-001 | P0 | Home → Copy agent prompt | The real bootstrap API returns a complete prompt and the browser copies it. | -| TC-WEB-002 | P0 | Global search → Databricks CLI docs → Copy Markdown | Keyboard search navigates to the right doc and exports useful raw Markdown. | -| TC-WEB-003 | P0 | Templates → search → SaaS Tracker → agent handoff | The selected template copies a complete prompt and generates a valid Replit URL. | -| TC-WEB-004 | P1 | Template copy → clipboard error → retry | The failure is visible and a second attempt succeeds without reloading. | -| TC-AGENT-001 | P0 | Agent requests bootstrap, llms.txt, raw docs, and tools | Machine-readable entry points respond and the MCP server advertises its core tools. | - -The four browser journeys run in desktop Chromium, Pixel-sized Chromium, -desktop WebKit, and iPhone-sized WebKit. The browser-independent API contract -runs once in desktop Chromium. +| ID | Priority | User flow | Required outcome | +| ------------ | -------- | ------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| TC-WEB-001 | P0 | Home → Copy agent prompt | The real bootstrap API returns a complete prompt and Chromium copies it through the browser Clipboard API. | +| TC-WEB-002 | P0 | Global search → Databricks CLI docs → Copy Markdown | Cmd/Ctrl-K on desktop or the mobile-menu trigger opens search, navigates to the right doc, and exports useful Markdown. | +| TC-WEB-003 | P0 | Templates → search → SaaS Tracker → agent handoff | The selected template copies a complete prompt and generates a valid Replit URL. | +| TC-WEB-004 | P1 | Template copy → simulated clipboard error → retry | The failure is visible and a second attempt succeeds without reloading. | +| TC-AGENT-001 | P0 | Agent requests bootstrap, llms.txt, raw docs, and tools | Machine-readable entry points respond, and the MCP server advertises and executes its required tools. | + +The four browser journeys run in desktop Chromium, Playwright's emulated Pixel +7 Chromium profile, desktop WebKit, and Playwright's emulated iPhone 15 WebKit +profile. Device profiles emulate browser settings such as viewport, user agent, +touch input, and device scale factor; they are not physical devices or installed +mobile Safari/Chrome. The browser-independent API contract runs once in desktop +Chromium. + +Successful copy flows use Chromium's browser Clipboard API with explicit +permissions in the desktop Chromium project. Other browser profiles use a +deterministic clipboard implementation because headless clipboard permission +support differs by engine. The clipboard failure and retry flow always uses a +controlled failure so the error state is reproducible. ## Commands diff --git a/tests/critical/critical-user-flow-contracts.ts b/tests/critical/critical-user-flow-contracts.ts index c4d7244..3f1b4fb 100644 --- a/tests/critical/critical-user-flow-contracts.ts +++ b/tests/critical/critical-user-flow-contracts.ts @@ -3,25 +3,16 @@ export const criticalUserFlowContracts = { id: "TC-WEB-001", priority: "P0", route: "/", - requiredPromptFragments: [ - "# About DevHub", - "# Working with DevHub prompts", - "# What the user just did", - "# Verify your local Databricks dev environment", - "Databricks CLI", - "llms.txt", - ], + minimumPromptLength: 500, + requiredPromptFragments: ["# About DevHub", "Databricks CLI", "llms.txt"], }, docsSearchAndExport: { id: "TC-WEB-002", priority: "P0", route: "/docs/tools/databricks-cli", query: "Databricks CLI", - requiredMarkdownFragments: [ - "# Databricks CLI", - "command-line interface", - "/templates", - ], + minimumMarkdownLength: 500, + requiredMarkdownFragments: ["# Databricks CLI"], }, templateDiscoveryAndHandoff: { id: "TC-WEB-003", @@ -29,12 +20,8 @@ export const criticalUserFlowContracts = { route: "/templates/saas-tracker", query: "SaaS Subscription Tracker", title: "SaaS Subscription Tracker", - requiredPromptFragments: [ - "# About DevHub", - "# SaaS Subscription Tracker", - "Data Flow", - "llms.txt", - ], + minimumPromptLength: 1_000, + requiredPromptFragments: ["# About DevHub", "# SaaS Subscription Tracker"], }, copyRecovery: { id: "TC-WEB-004", diff --git a/tests/critical/critical-user-flows.spec.ts b/tests/critical/critical-user-flows.spec.ts index c278149..d9bdda5 100644 --- a/tests/critical/critical-user-flows.spec.ts +++ b/tests/critical/critical-user-flows.spec.ts @@ -1,4 +1,10 @@ -import { expect, test, type Page } from "@playwright/test"; +import { + expect, + test, + type APIRequestContext, + type Page, + type TestInfo, +} from "@playwright/test"; import { criticalUserFlowContracts as contracts } from "./critical-user-flow-contracts"; @@ -36,10 +42,41 @@ function installClipboardMock(page: Page, failedWrites = 0): Promise { }, failedWrites); } -function getCopiedText(page: Page): Promise { +function usesBrowserClipboard(testInfo: TestInfo): boolean { + return testInfo.project.name === "desktop-chromium"; +} + +function prepareSuccessfulClipboard( + page: Page, + testInfo: TestInfo, +): Promise { + if (usesBrowserClipboard(testInfo)) { + return Promise.resolve(); + } + + return installClipboardMock(page); +} + +function getMockClipboardText(page: Page): Promise { return page.evaluate(() => (window as TestWindow).__copiedText ?? ""); } +function getCopiedText(page: Page, testInfo: TestInfo): Promise { + if (usesBrowserClipboard(testInfo)) { + return page.evaluate(() => navigator.clipboard.readText()); + } + + return getMockClipboardText(page); +} + +function clearBrowserClipboard(page: Page, testInfo: TestInfo): Promise { + if (!usesBrowserClipboard(testInfo)) { + return Promise.resolve(); + } + + return page.evaluate(() => navigator.clipboard.writeText("")); +} + function parseMcpResponse(contentType: string, body: string): unknown { if (!contentType.includes("text/event-stream")) { return JSON.parse(body); @@ -54,16 +91,41 @@ function parseMcpResponse(contentType: string, body: string): unknown { return JSON.parse(dataLine.slice("data: ".length)); } +async function callMcp( + request: APIRequestContext, + method: string, + params?: unknown, +): Promise { + const response = await request.post("/api/mcp", { + data: { + jsonrpc: "2.0", + id: 1, + method, + params, + }, + headers: { + Accept: "application/json, text/event-stream", + }, + }); + expect(response.status()).toBe(200); + + return parseMcpResponse( + response.headers()["content-type"] ?? "", + await response.text(), + ); +} + test.describe("critical user flows", () => { test(`${contracts.bootstrapPrompt.id} copies a usable bootstrap prompt from the home page`, async ({ page, - }) => { + }, testInfo) => { const pageErrors = collectPageErrors(page); - await installClipboardMock(page); + await prepareSuccessfulClipboard(page, testInfo); const response = await page.goto(contracts.bootstrapPrompt.route, { waitUntil: "domcontentloaded", }); + await clearBrowserClipboard(page, testInfo); expect(response?.status()).toBe(200); await expect( page.getByRole("heading", { @@ -82,30 +144,35 @@ test.describe("critical user flows", () => { page.locator("main").getByRole("button", { name: "Copied" }).first(), ).toBeVisible(); - const copiedPrompt = await getCopiedText(page); + const copiedPrompt = await getCopiedText(page, testInfo); + expect(copiedPrompt.length).toBeGreaterThanOrEqual( + contracts.bootstrapPrompt.minimumPromptLength, + ); for (const fragment of contracts.bootstrapPrompt.requiredPromptFragments) { expect(copiedPrompt).toContain(fragment); } expect(pageErrors).toEqual([]); }); - test(`${contracts.docsSearchAndExport.id} finds docs with the keyboard and exports useful Markdown`, async ({ + test(`${contracts.docsSearchAndExport.id} opens global search and exports useful Markdown`, async ({ page, }, testInfo) => { const pageErrors = collectPageErrors(page); - await installClipboardMock(page); + await prepareSuccessfulClipboard(page, testInfo); await page.goto("/"); + await clearBrowserClipboard(page, testInfo); - if (testInfo.project.name.startsWith("mobile-")) { + if (testInfo.project.name.startsWith("emulated-")) { await page.getByRole("button", { name: "Open menu" }).click(); await expect(page.locator("#devhub-main-content")).toHaveJSProperty( "inert", true, ); + await page.getByRole("button", { name: "Search documentation" }).click(); + } else { + await page.keyboard.press("ControlOrMeta+K"); } - await page.getByRole("button", { name: "Search documentation" }).click(); - const searchDialog = page.getByRole("dialog", { name: "Search documentation", }); @@ -140,7 +207,10 @@ test.describe("critical user flows", () => { page.getByRole("button", { name: "Copied" }).first(), ).toBeVisible(); - const copiedMarkdown = await getCopiedText(page); + const copiedMarkdown = await getCopiedText(page, testInfo); + expect(copiedMarkdown.length).toBeGreaterThanOrEqual( + contracts.docsSearchAndExport.minimumMarkdownLength, + ); for (const fragment of contracts.docsSearchAndExport .requiredMarkdownFragments) { expect(copiedMarkdown).toContain(fragment); @@ -151,10 +221,11 @@ test.describe("critical user flows", () => { test(`${contracts.templateDiscoveryAndHandoff.id} discovers a template and produces agent handoffs`, async ({ page, - }) => { + }, testInfo) => { const pageErrors = collectPageErrors(page); - await installClipboardMock(page); + await prepareSuccessfulClipboard(page, testInfo); await page.goto("/templates"); + await clearBrowserClipboard(page, testInfo); await page .getByRole("searchbox") @@ -180,7 +251,10 @@ test.describe("critical user flows", () => { page.getByRole("button", { name: "Copied!" }).first(), ).toBeVisible(); - const copiedPrompt = await getCopiedText(page); + const copiedPrompt = await getCopiedText(page, testInfo); + expect(copiedPrompt.length).toBeGreaterThanOrEqual( + contracts.templateDiscoveryAndHandoff.minimumPromptLength, + ); for (const fragment of contracts.templateDiscoveryAndHandoff .requiredPromptFragments) { expect(copiedPrompt).toContain(fragment); @@ -221,7 +295,7 @@ test.describe("critical user flows", () => { page.getByRole("button", { name: "Copied!" }).first(), ).toBeVisible(); await expect - .poll(() => getCopiedText(page)) + .poll(() => getMockClipboardText(page)) .toContain(contracts.templateDiscoveryAndHandoff.title); expect(pageErrors).toEqual([]); }); @@ -252,26 +326,44 @@ test.describe("critical user flows", () => { ); expect(await rawDocsResponse.text()).toContain("# Start here"); - const mcpResponse = await request.post("/api/mcp", { - data: { - jsonrpc: "2.0", - id: 1, - method: "tools/list", - }, - headers: { - Accept: "application/json, text/event-stream", - }, - }); - expect(mcpResponse.status()).toBe(200); - - const mcpPayload = parseMcpResponse( - mcpResponse.headers()["content-type"] ?? "", - await mcpResponse.text(), - ) as { + const mcpPayload = (await callMcp(request, "tools/list")) as { result: { tools: Array<{ name: string }> }; }; - expect(mcpPayload.result.tools.map(({ name }) => name).sort()).toEqual( - [...contracts.agentEntryPoints.mcpTools].sort(), - ); + const toolNames = mcpPayload.result.tools.map(({ name }) => name); + for (const requiredTool of contracts.agentEntryPoints.mcpTools) { + expect(toolNames).toContain(requiredTool); + } + + const resourcesPayload = (await callMcp(request, "tools/call", { + name: "list_docs_resources", + arguments: {}, + })) as { + result: { + content: Array<{ text: string }>; + isError?: boolean; + }; + }; + expect(resourcesPayload.result.isError).not.toBe(true); + const resourcesText = resourcesPayload.result.content + .map(({ text }) => text) + .join("\n"); + expect(resourcesText).toContain("# Databricks Developer"); + expect(resourcesText).toContain("/docs/start-here"); + + const docPayload = (await callMcp(request, "tools/call", { + name: "get_doc_resource", + arguments: { slug: "start-here" }, + })) as { + result: { + content: Array<{ text: string }>; + isError?: boolean; + }; + }; + expect(docPayload.result.isError).not.toBe(true); + const docText = docPayload.result.content + .map(({ text }) => text) + .join("\n"); + expect(docText).toContain("# Start here"); + expect(docText.length).toBeGreaterThan(100); }); }); diff --git a/tests/mcp-e2e.test.ts b/tests/mcp-e2e.test.ts index c0aeab8..17daee7 100644 --- a/tests/mcp-e2e.test.ts +++ b/tests/mcp-e2e.test.ts @@ -133,11 +133,10 @@ describe("MCP server e2e (mcporter)", () => { siteServer?.kill(); }); - test("list discovers both tools", async () => { + test("list discovers the required tools", async () => { const output = await mcporter("list"); expect(output).toContain("list_docs_resources"); expect(output).toContain("get_doc_resource"); - expect(output).toContain("2 tools"); }, 30_000); test("list_docs_resources returns the docs index", async () => { diff --git a/tests/mcp.test.ts b/tests/mcp.test.ts index 223e261..3af42c2 100644 --- a/tests/mcp.test.ts +++ b/tests/mcp.test.ts @@ -51,11 +51,10 @@ describe("MCP server handler", () => { expect(result.result.serverInfo.name).toBe("devhub-docs"); }); - test("tools/list returns both tools", async () => { + test("tools/list returns the required tools", async () => { const result = (await callMcp(rpc("tools/list"))) as { result: { tools: Array<{ name: string }> }; }; - expect(result.result.tools).toHaveLength(2); const names = result.result.tools.map((t) => t.name); expect(names).toContain("list_docs_resources"); expect(names).toContain("get_doc_resource");