diff --git a/apps/vscode-e2e/fixtures/apply-diff.json b/apps/vscode-e2e/fixtures/apply-diff.json new file mode 100644 index 0000000000..24cfe27724 --- /dev/null +++ b/apps/vscode-e2e/fixtures/apply-diff.json @@ -0,0 +1,74 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "APPLY_DIFF_SIMPLE_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "apply_diff", + "arguments": "{\"path\":\"apply-diff-tool-fixture/simple-modify.txt\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:1\\n-------\\nHello World\\n=======\\nHello Universe\\n>>>>>>> REPLACE\"}", + "id": "call_apply_diff_simple_001" + } + ] + } + }, + { + "match": { + "userMessage": "APPLY_DIFF_MULTI_REPLACE_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "apply_diff", + "arguments": "{\"path\":\"apply-diff-tool-fixture/multiple-replace.js\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:1\\n-------\\nfunction calculate(x, y) {\\n\\tconst sum = x + y\\n\\tconst product = x * y\\n\\treturn { sum: sum, product: product }\\n}\\n=======\\nfunction compute(a, b) {\\n\\tconst total = a + b\\n\\tconst result = a * b\\n\\treturn { total: total, result: result }\\n}\\n>>>>>>> REPLACE\"}", + "id": "call_apply_diff_multi_replace_001" + } + ] + } + }, + { + "match": { + "userMessage": "APPLY_DIFF_LINE_HINTS_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "apply_diff", + "arguments": "{\"path\":\"apply-diff-tool-fixture/line-hints.js\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:2\\n-------\\nfunction oldFunction() {\\n\\tconsole.log(\\\"Old implementation\\\")\\n}\\n=======\\nfunction newFunction() {\\n\\tconsole.log(\\\"New implementation\\\")\\n}\\n>>>>>>> REPLACE\"}", + "id": "call_apply_diff_line_hints_001" + } + ] + } + }, + { + "match": { + "userMessage": "APPLY_DIFF_ERROR_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "apply_diff", + "arguments": "{\"path\":\"apply-diff-tool-fixture/error-handling.txt\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:1\\n-------\\nThis content does not exist\\n=======\\nNew content\\n>>>>>>> REPLACE\"}", + "id": "call_apply_diff_error_001" + } + ] + } + }, + { + "match": { + "userMessage": "APPLY_DIFF_MULTI_BLOCK_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "apply_diff", + "arguments": "{\"path\":\"apply-diff-tool-fixture/multi-search-replace.js\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:1\\n-------\\nfunction processData(data) {\\n\\tconsole.log(\\\"Processing data\\\")\\n=======\\nfunction transformData(data) {\\n\\tconsole.log(\\\"Transforming data\\\")\\n>>>>>>> REPLACE\\n\\n<<<<<<< SEARCH\\n:start_line:12\\n-------\\nfunction validateInput(input) {\\n\\tconsole.log(\\\"Validating input\\\")\\n=======\\nfunction checkInput(input) {\\n\\tconsole.log(\\\"Checking input\\\")\\n>>>>>>> REPLACE\"}", + "id": "call_apply_diff_multi_block_001" + } + ] + } + } + ] +} diff --git a/apps/vscode-e2e/fixtures/execute-command.json b/apps/vscode-e2e/fixtures/execute-command.json new file mode 100644 index 0000000000..c7ebe5d7d4 --- /dev/null +++ b/apps/vscode-e2e/fixtures/execute-command.json @@ -0,0 +1,60 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "EXECUTE_COMMAND_SIMPLE_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "execute_command", + "arguments": "{\"command\":\"printf 'Hello from test\\\\n' > execute-command-tool-fixture/simple-echo.txt\"}", + "id": "call_execute_command_simple_001" + } + ] + } + }, + { + "match": { + "userMessage": "EXECUTE_COMMAND_CWD_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "execute_command", + "arguments": "{\"command\":\"printf 'Test in subdirectory\\\\n' > output.txt\",\"cwd\":\"execute-command-tool-fixture/custom-cwd\"}", + "id": "call_execute_command_cwd_001" + } + ] + } + }, + { + "match": { + "userMessage": "EXECUTE_COMMAND_MULTI_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "execute_command", + "arguments": "{\"command\":\"printf 'Line 1\\\\n' > execute-command-tool-fixture/multi-command.txt\"}", + "id": "call_execute_command_multi_001" + } + ] + } + }, + { + "match": { + "userMessage": "EXECUTE_COMMAND_LONG_RUNNING_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "execute_command", + "arguments": "{\"command\":\"node -e \\\"setTimeout(() => console.log('Command completed after delay'), 1000)\\\"\"}", + "id": "call_execute_command_long_running_001" + } + ] + } + } + ] +} diff --git a/apps/vscode-e2e/fixtures/write-to-file.json b/apps/vscode-e2e/fixtures/write-to-file.json new file mode 100644 index 0000000000..255fb77382 --- /dev/null +++ b/apps/vscode-e2e/fixtures/write-to-file.json @@ -0,0 +1,32 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "WRITE_TO_FILE_CREATE_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "write_to_file", + "arguments": "{\"path\":\"write-to-file-tool-fixture/write-to-file-smoke.txt\",\"content\":\"Hello, this is a test file!\"}", + "id": "call_write_to_file_create_001" + } + ] + } + }, + { + "match": { + "userMessage": "WRITE_TO_FILE_NESTED_SMOKE" + }, + "response": { + "toolCalls": [ + { + "name": "write_to_file", + "arguments": "{\"path\":\"write-to-file-tool-fixture/nested/deep/directory/write-to-file-nested-smoke.txt\",\"content\":\"File in nested directory\"}", + "id": "call_write_to_file_nested_001" + } + ] + } + } + ] +} diff --git a/apps/vscode-e2e/src/fixtures/apply-diff.ts b/apps/vscode-e2e/src/fixtures/apply-diff.ts new file mode 100644 index 0000000000..59114af238 --- /dev/null +++ b/apps/vscode-e2e/src/fixtures/apply-diff.ts @@ -0,0 +1,63 @@ +import { LLMock } from "@copilotkit/aimock" + +import { toolResultContains } from "./tool-result" + +type ApplyDiffFixture = { + toolCallId: string + expected: string[] + result: string + id: string +} + +export function addApplyDiffResultFixtures(mock: InstanceType) { + const fixtures: ApplyDiffFixture[] = [ + { + toolCallId: "call_apply_diff_simple_001", + expected: ['"path":"apply-diff-tool-fixture/simple-modify.txt"', '"operation":"modified"'], + result: "Updated `apply-diff-tool-fixture/simple-modify.txt` to say `Hello Universe`.", + id: "call_apply_diff_simple_002", + }, + { + toolCallId: "call_apply_diff_multi_replace_001", + expected: ['"path":"apply-diff-tool-fixture/multiple-replace.js"', '"operation":"modified"'], + result: "Updated `apply-diff-tool-fixture/multiple-replace.js` with the renamed function, parameters, and return fields.", + id: "call_apply_diff_multi_replace_002", + }, + { + toolCallId: "call_apply_diff_line_hints_001", + expected: ['"path":"apply-diff-tool-fixture/line-hints.js"', '"operation":"modified"'], + result: "Updated `apply-diff-tool-fixture/line-hints.js` so `oldFunction` became `newFunction` with the new log message.", + id: "call_apply_diff_line_hints_002", + }, + { + toolCallId: "call_apply_diff_error_001", + expected: ["No sufficiently similar match found at line: 1", "This content does not exist"], + result: "The apply_diff operation on `apply-diff-tool-fixture/error-handling.txt` was rejected - the search content did not match any content in the file, so it was not modified.", + id: "call_apply_diff_error_002", + }, + { + toolCallId: "call_apply_diff_multi_block_001", + expected: ['"path":"apply-diff-tool-fixture/multi-search-replace.js"', '"operation":"modified"'], + result: "Applied both search/replace blocks in `apply-diff-tool-fixture/multi-search-replace.js` to rename the two target functions.", + id: "call_apply_diff_multi_block_002", + }, + ] + + for (const fixture of fixtures) { + mock.addFixture({ + match: { + toolCallId: fixture.toolCallId, + predicate: (req) => toolResultContains(req, fixture.toolCallId, fixture.expected), + }, + response: { + toolCalls: [ + { + name: "attempt_completion", + arguments: JSON.stringify({ result: fixture.result }), + id: fixture.id, + }, + ], + }, + }) + } +} diff --git a/apps/vscode-e2e/src/fixtures/execute-command.ts b/apps/vscode-e2e/src/fixtures/execute-command.ts new file mode 100644 index 0000000000..dc0e44669a --- /dev/null +++ b/apps/vscode-e2e/src/fixtures/execute-command.ts @@ -0,0 +1,101 @@ +import { LLMock } from "@copilotkit/aimock" + +import { toolResultContains } from "./tool-result" + +type ExecuteCommandToolCall = { + name: "execute_command" | "attempt_completion" + params: Record + id: string +} + +type ExecuteCommandFixture = { + toolCallId: string + expected: string[] + toolCalls: ExecuteCommandToolCall[] +} + +export function addExecuteCommandResultFixtures(mock: InstanceType) { + const fixtures: ExecuteCommandFixture[] = [ + { + toolCallId: "call_execute_command_simple_001", + expected: ["Command executed in terminal within working directory '", "Exit code: 0\nOutput:\n"], + toolCalls: [ + { + name: "attempt_completion", + params: { + result: "Ran the echo command and created `execute-command-tool-fixture/simple-echo.txt`.", + }, + id: "call_execute_command_simple_002", + }, + ], + }, + { + toolCallId: "call_execute_command_cwd_001", + expected: ["execute-command-tool-fixture/custom-cwd'. Exit code: 0", "Output:\n"], + toolCalls: [ + { + name: "attempt_completion", + params: { + result: "Ran the command inside `execute-command-tool-fixture/custom-cwd` and created `output.txt`.", + }, + id: "call_execute_command_cwd_002", + }, + ], + }, + { + toolCallId: "call_execute_command_multi_001", + expected: ["Command executed in terminal within working directory '", "Exit code: 0\nOutput:\n"], + toolCalls: [ + { + name: "execute_command", + params: { + command: "printf 'Line 2\\n' >> execute-command-tool-fixture/multi-command.txt", + }, + id: "call_execute_command_multi_002", + }, + ], + }, + { + toolCallId: "call_execute_command_multi_002", + expected: ["Command executed in terminal within working directory '", "Exit code: 0\nOutput:\n"], + toolCalls: [ + { + name: "attempt_completion", + params: { + result: "Ran both commands and populated `execute-command-tool-fixture/multi-command.txt` with two lines.", + }, + id: "call_execute_command_multi_003", + }, + ], + }, + { + toolCallId: "call_execute_command_long_running_001", + expected: ["Exit code: 0", "Command completed after delay"], + toolCalls: [ + { + name: "attempt_completion", + params: { + result: "The delayed command completed and printed `Command completed after delay`.", + }, + id: "call_execute_command_long_running_002", + }, + ], + }, + ] + + for (const fixture of fixtures) { + mock.addFixture({ + match: { + toolCallId: fixture.toolCallId, + predicate: (req) => toolResultContains(req, fixture.toolCallId, fixture.expected), + }, + response: { + toolCalls: fixture.toolCalls.map((toolCall) => ({ + name: toolCall.name, + arguments: JSON.stringify(toolCall.params), + id: toolCall.id, + })), + }, + }) + } +} diff --git a/apps/vscode-e2e/src/fixtures/read-file.ts b/apps/vscode-e2e/src/fixtures/read-file.ts index 0740ada31e..dd9bb1c10b 100644 --- a/apps/vscode-e2e/src/fixtures/read-file.ts +++ b/apps/vscode-e2e/src/fixtures/read-file.ts @@ -1,7 +1,11 @@ import { LLMock } from "@copilotkit/aimock" -import type { ChatCompletionRequest, ChatMessage } from "@copilotkit/aimock" -type ToolResultExpectation = { toolCallId: string; expected: string[] } +import { + isToolResultExpectation, + toolResultContains, + toolResultsContain, + type ToolResultExpectation, +} from "./tool-result" type ReadFileResultFixture = { toolCallId: string @@ -10,28 +14,6 @@ type ReadFileResultFixture = { id: string } -function isToolResultExpectation(value: unknown): value is ToolResultExpectation { - return typeof value === "object" && value !== null && "toolCallId" in value && "expected" in value -} - -function toolResultContains(req: ChatCompletionRequest, toolCallId: string, expected: string[]) { - const messages = Array.isArray(req?.messages) ? req.messages : [] - const toolMessage = messages.find( - (message: ChatMessage) => message?.role === "tool" && message.tool_call_id === toolCallId, - ) - - const content = toolMessage?.content - if (typeof content !== "string") { - return false - } - - return expected.every((text) => content.includes(text)) -} - -function toolResultsContain(req: ChatCompletionRequest, expectations: ToolResultExpectation[]) { - return expectations.every(({ toolCallId, expected }) => toolResultContains(req, toolCallId, expected)) -} - export function addReadFileResultFixtures(mock: InstanceType) { const fixtures: ReadFileResultFixture[] = [ { diff --git a/apps/vscode-e2e/src/fixtures/tool-result.ts b/apps/vscode-e2e/src/fixtures/tool-result.ts new file mode 100644 index 0000000000..3deab7e410 --- /dev/null +++ b/apps/vscode-e2e/src/fixtures/tool-result.ts @@ -0,0 +1,25 @@ +import type { ChatCompletionRequest, ChatMessage } from "@copilotkit/aimock" + +export type ToolResultExpectation = { toolCallId: string; expected: string[] } + +export function isToolResultExpectation(value: unknown): value is ToolResultExpectation { + return typeof value === "object" && value !== null && "toolCallId" in value && "expected" in value +} + +export function toolResultContains(req: ChatCompletionRequest, toolCallId: string, expected: string[]) { + const messages = Array.isArray(req?.messages) ? req.messages : [] + const toolMessage = messages.find( + (message: ChatMessage) => message?.role === "tool" && message.tool_call_id === toolCallId, + ) + + const content = toolMessage?.content + if (typeof content !== "string") { + return false + } + + return expected.every((text) => content.includes(text)) +} + +export function toolResultsContain(req: ChatCompletionRequest, expectations: ToolResultExpectation[]) { + return expectations.every(({ toolCallId, expected }) => toolResultContains(req, toolCallId, expected)) +} diff --git a/apps/vscode-e2e/src/fixtures/write-to-file.ts b/apps/vscode-e2e/src/fixtures/write-to-file.ts new file mode 100644 index 0000000000..d0a31c4db7 --- /dev/null +++ b/apps/vscode-e2e/src/fixtures/write-to-file.ts @@ -0,0 +1,48 @@ +import { LLMock } from "@copilotkit/aimock" + +import { toolResultContains } from "./tool-result" + +type WriteToFileFixture = { + toolCallId: string + expected: string[] + result: string + id: string +} + +export function addWriteToFileResultFixtures(mock: InstanceType) { + const fixtures: WriteToFileFixture[] = [ + { + toolCallId: "call_write_to_file_create_001", + expected: ['"path":"write-to-file-tool-fixture/write-to-file-smoke.txt"', '"operation":"created"'], + result: "Created `write-to-file-tool-fixture/write-to-file-smoke.txt` with the requested content.", + id: "call_write_to_file_create_002", + }, + { + toolCallId: "call_write_to_file_nested_001", + expected: [ + '"path":"write-to-file-tool-fixture/nested/deep/directory/write-to-file-nested-smoke.txt"', + '"operation":"created"', + ], + result: "Created `write-to-file-tool-fixture/nested/deep/directory/write-to-file-nested-smoke.txt` with the requested content.", + id: "call_write_to_file_nested_002", + }, + ] + + for (const fixture of fixtures) { + mock.addFixture({ + match: { + toolCallId: fixture.toolCallId, + predicate: (req) => toolResultContains(req, fixture.toolCallId, fixture.expected), + }, + response: { + toolCalls: [ + { + name: "attempt_completion", + arguments: JSON.stringify({ result: fixture.result }), + id: fixture.id, + }, + ], + }, + }) + } +} diff --git a/apps/vscode-e2e/src/runTest.ts b/apps/vscode-e2e/src/runTest.ts index 1e53b885cb..36554dccd6 100644 --- a/apps/vscode-e2e/src/runTest.ts +++ b/apps/vscode-e2e/src/runTest.ts @@ -5,9 +5,12 @@ import * as fs from "fs/promises" import { runTests } from "@vscode/test-electron" import { LLMock } from "@copilotkit/aimock" +import { addApplyDiffResultFixtures } from "./fixtures/apply-diff" +import { addExecuteCommandResultFixtures } from "./fixtures/execute-command" import { addListFilesResultFixtures } from "./fixtures/list-files" import { addReadFileResultFixtures } from "./fixtures/read-file" import { addSearchFilesResultFixtures } from "./fixtures/search-files" +import { addWriteToFileResultFixtures } from "./fixtures/write-to-file" function getCliFlagValue(flag: string) { return process.argv.find((arg, index) => process.argv[index - 1] === flag) @@ -79,9 +82,12 @@ async function main() { mock.loadFixtureDir(fixturesDir) if (!isRecord) { + addApplyDiffResultFixtures(mock) + addExecuteCommandResultFixtures(mock) addListFilesResultFixtures(mock) addReadFileResultFixtures(mock) addSearchFilesResultFixtures(mock) + addWriteToFileResultFixtures(mock) // The modes test (switch_mode → ask) triggers a second API call whose last // user message starts with directly — no diff --git a/apps/vscode-e2e/src/suite/tools/apply-diff.test.ts b/apps/vscode-e2e/src/suite/tools/apply-diff.test.ts index c4f279f5f6..39488f8144 100644 --- a/apps/vscode-e2e/src/suite/tools/apply-diff.test.ts +++ b/apps/vscode-e2e/src/suite/tools/apply-diff.test.ts @@ -5,33 +5,33 @@ import * as vscode from "vscode" import { RooCodeEventName, type ClineMessage } from "@roo-code/types" -import { waitFor, sleep } from "../utils" +import { waitUntilCompleted, sleep } from "../utils" import { setDefaultSuiteTimeout } from "../test-utils" -suite.skip("Roo Code apply_diff Tool", function () { - setDefaultSuiteTimeout(this) - - let workspaceDir: string - - // Pre-created test files that will be used across tests - const testFiles = { - simpleModify: { - name: `test-file-simple-${Date.now()}.txt`, - content: "Hello World\nThis is a test file\nWith multiple lines", - path: "", - }, - multipleReplace: { - name: `test-func-multiple-${Date.now()}.js`, - content: `function calculate(x, y) { +const TEST_DIR_NAME = "apply-diff-tool-fixture" + +const testFiles = { + simpleModify: { + relativePath: `${TEST_DIR_NAME}/simple-modify.txt`, + originalContent: "Hello World\nThis is a test file\nWith multiple lines", + expectedContent: "Hello Universe\nThis is a test file\nWith multiple lines", + }, + multipleReplace: { + relativePath: `${TEST_DIR_NAME}/multiple-replace.js`, + originalContent: `function calculate(x, y) { const sum = x + y const product = x * y return { sum: sum, product: product } }`, - path: "", - }, - lineNumbers: { - name: `test-lines-${Date.now()}.js`, - content: `// Header comment + expectedContent: `function compute(a, b) { + const total = a + b + const result = a * b + return { total: total, result: result } +}`, + }, + lineHints: { + relativePath: `${TEST_DIR_NAME}/line-hints.js`, + originalContent: `// Header comment function oldFunction() { console.log("Old implementation") } @@ -42,16 +42,25 @@ function keepThis() { } // Footer comment`, - path: "", - }, - errorHandling: { - name: `test-error-${Date.now()}.txt`, - content: "Original content", - path: "", - }, - multiSearchReplace: { - name: `test-multi-search-${Date.now()}.js`, - content: `function processData(data) { + expectedContent: `// Header comment +function newFunction() { + console.log("New implementation") +} + +// Another function +function keepThis() { + console.log("Keep this") +} + +// Footer comment`, + }, + errorHandling: { + relativePath: `${TEST_DIR_NAME}/error-handling.txt`, + originalContent: "Original content", + }, + multiSearchReplace: { + relativePath: `${TEST_DIR_NAME}/multi-search-replace.js`, + originalContent: `function processData(data) { console.log("Processing data") return data.map(item => item * 2) } @@ -69,682 +78,363 @@ function validateInput(input) { } return true }`, - path: "", - }, + expectedContent: `function transformData(data) { + console.log("Transforming data") + return data.map(item => item * 2) +} + +// Some other code in between +const config = { + timeout: 5000, + retries: 3 +} + +function checkInput(input) { + console.log("Checking input") + if (!input) { + throw new Error("Invalid input") } + return true +}`, + }, +} + +suite("Roo Code apply_diff Tool", function () { + setDefaultSuiteTimeout(this) + + let workspaceDir: string + let testDir: string + + suiteSetup(async () => { + const aimockUrl = process.env.AIMOCK_URL + const isRecord = process.env.AIMOCK_RECORD === "true" + + await globalThis.api.setConfiguration({ + apiProvider: "openrouter" as const, + openRouterApiKey: aimockUrl && !isRecord ? "mock-key" : process.env.OPENROUTER_API_KEY!, + openRouterModelId: "anthropic/claude-sonnet-4.5", + ...(aimockUrl && { openRouterBaseUrl: `${aimockUrl}/v1` }), + }) - // Get the actual workspace directory that VSCode is using and create all test files - suiteSetup(async function () { - // Get the workspace folder from VSCode const workspaceFolders = vscode.workspace.workspaceFolders if (!workspaceFolders || workspaceFolders.length === 0) { throw new Error("No workspace folder found") } - workspaceDir = workspaceFolders[0]!.uri.fsPath - console.log("Using workspace directory:", workspaceDir) - - // Create all test files before any tests run - console.log("Creating test files in workspace...") - for (const [key, file] of Object.entries(testFiles)) { - file.path = path.join(workspaceDir, file.name) - await fs.writeFile(file.path, file.content) - console.log(`Created ${key} test file at:`, file.path) - } - // Verify all files exist - for (const [key, file] of Object.entries(testFiles)) { - const exists = await fs - .access(file.path) - .then(() => true) - .catch(() => false) - if (!exists) { - throw new Error(`Failed to create ${key} test file at ${file.path}`) - } - } + workspaceDir = workspaceFolders[0]!.uri.fsPath + testDir = path.join(workspaceDir, TEST_DIR_NAME) + await fs.rm(testDir, { recursive: true, force: true }) }) - // Clean up after all tests suiteTeardown(async () => { - // Cancel any running tasks before cleanup try { await globalThis.api.cancelCurrentTask() } catch { // Task might not be running } - // Clean up all test files - console.log("Cleaning up test files...") - for (const [key, file] of Object.entries(testFiles)) { - try { - await fs.unlink(file.path) - console.log(`Cleaned up ${key} test file`) - } catch (error) { - console.log(`Failed to clean up ${key} test file:`, error) - } - } + const aimockUrl = process.env.AIMOCK_URL + const isRecord = process.env.AIMOCK_RECORD === "true" + await globalThis.api.setConfiguration({ + apiProvider: "openrouter" as const, + openRouterApiKey: aimockUrl && !isRecord ? "mock-key" : process.env.OPENROUTER_API_KEY!, + openRouterModelId: "openai/gpt-4.1", + ...(aimockUrl && { openRouterBaseUrl: `${aimockUrl}/v1` }), + }) + + await fs.rm(testDir, { recursive: true, force: true }) }) - // Clean up before each test setup(async () => { - // Cancel any previous task try { await globalThis.api.cancelCurrentTask() } catch { // Task might not be running } - // Small delay to ensure clean state + await fs.rm(testDir, { recursive: true, force: true }) + await fs.mkdir(testDir, { recursive: true }) + + for (const fixture of Object.values(testFiles)) { + await fs.writeFile(path.join(workspaceDir, fixture.relativePath), fixture.originalContent) + } + await sleep(100) }) - // Clean up after each test teardown(async () => { - // Cancel the current task try { await globalThis.api.cancelCurrentTask() } catch { // Task might not be running } - // Small delay to ensure clean state await sleep(100) }) test("Should apply diff to modify existing file content", async function () { - // Increase timeout for this specific test - const api = globalThis.api const messages: ClineMessage[] = [] - const testFile = testFiles.simpleModify - const expectedContent = "Hello Universe\nThis is a test file\nWith multiple lines" - let taskStarted = false - let taskCompleted = false let errorOccurred: string | null = null - let applyDiffExecuted = false - // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { messages.push(message) - - // Log important messages for debugging if (message.type === "say" && message.say === "error") { errorOccurred = message.text || "Unknown error" - console.error("Error:", message.text) - } - if (message.type === "ask" && message.ask === "tool") { - console.log("Tool request:", message.text?.substring(0, 200)) - } - if (message.type === "say" && (message.say === "completion_result" || message.say === "text")) { - console.log("AI response:", message.text?.substring(0, 200)) - } - - // Check for tool execution - if (message.type === "say" && message.say === "api_req_started" && message.text) { - console.log("API request started:", message.text.substring(0, 200)) - try { - const requestData = JSON.parse(message.text) - if (requestData.request && requestData.request.includes("apply_diff")) { - applyDiffExecuted = true - console.log("apply_diff tool executed!") - } - } catch (e) { - console.log("Failed to parse api_req_started message:", e) - } } } api.on(RooCodeEventName.Message, messageHandler) - // Listen for task events - const taskStartedHandler = (id: string) => { - if (id === taskId) { - taskStarted = true - console.log("Task started:", id) - } - } - api.on(RooCodeEventName.TaskStarted, taskStartedHandler) - - const taskCompletedHandler = (id: string) => { - if (id === taskId) { - taskCompleted = true - console.log("Task completed:", id) - } - } - api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler) - - let taskId: string try { - // Start task with apply_diff instruction - file already exists - taskId = await api.startNewTask({ - configuration: { - mode: "code", - autoApprovalEnabled: true, - alwaysAllowWrite: true, - alwaysAllowReadOnly: true, - alwaysAllowReadOnlyOutsideWorkspace: true, - }, - text: `Use apply_diff on the file ${testFile.name} to change "Hello World" to "Hello Universe". The file already exists with this content: -${testFile.content}\nAssume the file exists and you can modify it directly.`, - }) //Temporary measure since list_files ignores all the files inside a tmp workspace - - console.log("Task ID:", taskId) - console.log("Test filename:", testFile.name) - - // Wait for task to start - await waitFor(() => taskStarted, { timeout: 60_000 }) - - // Check for early errors - if (errorOccurred) { - console.error("Early error detected:", errorOccurred) - } - - // Wait for task completion - await waitFor(() => taskCompleted, { timeout: 60_000 }) - - // Give extra time for file system operations - await sleep(2000) - - // Check if the file was modified correctly - const actualContent = await fs.readFile(testFile.path, "utf-8") - console.log("File content after modification:", actualContent) + await waitUntilCompleted({ + api, + start: () => + api.startNewTask({ + configuration: { + mode: "code", + autoApprovalEnabled: true, + alwaysAllowWrite: true, + alwaysAllowReadOnly: true, + alwaysAllowReadOnlyOutsideWorkspace: true, + }, + text: "APPLY_DIFF_SIMPLE_SMOKE", + }), + timeout: 60_000, + }) + await sleep(1_000) - // Verify tool was executed - assert.strictEqual(applyDiffExecuted, true, "apply_diff tool should have been executed") + assert.strictEqual(errorOccurred, null, `Error occurred: ${errorOccurred}`) - // Verify file content + const actualContent = await fs.readFile( + path.join(workspaceDir, testFiles.simpleModify.relativePath), + "utf-8", + ) assert.strictEqual( actualContent.trim(), - expectedContent.trim(), + testFiles.simpleModify.expectedContent.trim(), "File content should be modified correctly", ) - console.log("Test passed! apply_diff tool executed and file modified successfully") + const completionMessage = messages.find( + (message) => + message.type === "say" && + (message.say === "completion_result" || message.say === "text") && + message.text?.includes("Hello Universe"), + ) + assert.ok(completionMessage, "AI should have acknowledged the updated file content") } finally { - // Clean up api.off(RooCodeEventName.Message, messageHandler) - api.off(RooCodeEventName.TaskStarted, taskStartedHandler) - api.off(RooCodeEventName.TaskCompleted, taskCompletedHandler) } }) test("Should apply multiple search/replace blocks in single diff", async function () { - // Increase timeout for this specific test - const api = globalThis.api const messages: ClineMessage[] = [] - const testFile = testFiles.multipleReplace - const expectedContent = `function compute(a, b) { - const total = a + b - const result = a * b - return { total: total, result: result } -}` - let taskStarted = false - let taskCompleted = false - let applyDiffExecuted = false + let errorOccurred: string | null = null - // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { messages.push(message) - if (message.type === "ask" && message.ask === "tool") { - console.log("Tool request:", message.text?.substring(0, 200)) - } - if (message.type === "say" && message.text) { - console.log("AI response:", message.text.substring(0, 200)) - } - - // Check for tool execution - if (message.type === "say" && message.say === "api_req_started" && message.text) { - console.log("API request started:", message.text.substring(0, 200)) - try { - const requestData = JSON.parse(message.text) - if (requestData.request && requestData.request.includes("apply_diff")) { - applyDiffExecuted = true - console.log("apply_diff tool executed!") - } - } catch (e) { - console.log("Failed to parse api_req_started message:", e) - } + if (message.type === "say" && message.say === "error") { + errorOccurred = message.text || "Unknown error" } } api.on(RooCodeEventName.Message, messageHandler) - // Listen for task events - const taskStartedHandler = (id: string) => { - if (id === taskId) { - taskStarted = true - console.log("Task started:", id) - } - } - api.on(RooCodeEventName.TaskStarted, taskStartedHandler) - - const taskCompletedHandler = (id: string) => { - if (id === taskId) { - taskCompleted = true - console.log("Task completed:", id) - } - } - api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler) - - let taskId: string try { - // Start task with multiple replacements - file already exists - taskId = await api.startNewTask({ - configuration: { - mode: "code", - autoApprovalEnabled: true, - alwaysAllowWrite: true, - alwaysAllowReadOnly: true, - alwaysAllowReadOnlyOutsideWorkspace: true, - }, - text: `Use apply_diff on the file ${testFile.name} to make ALL of these changes: -1. Rename function "calculate" to "compute" -2. Rename parameters "x, y" to "a, b" -3. Rename variable "sum" to "total" (including in the return statement) -4. Rename variable "product" to "result" (including in the return statement) -5. In the return statement, change { sum: sum, product: product } to { total: total, result: result } - -The file already exists with this content: -${testFile.content}\nAssume the file exists and you can modify it directly.`, + await waitUntilCompleted({ + api, + start: () => + api.startNewTask({ + configuration: { + mode: "code", + autoApprovalEnabled: true, + alwaysAllowWrite: true, + alwaysAllowReadOnly: true, + alwaysAllowReadOnlyOutsideWorkspace: true, + }, + text: "APPLY_DIFF_MULTI_REPLACE_SMOKE", + }), + timeout: 60_000, }) + await sleep(1_000) - console.log("Task ID:", taskId) - console.log("Test filename:", testFile.name) - - // Wait for task to start - await waitFor(() => taskStarted, { timeout: 60_000 }) - - // Wait for task completion - await waitFor(() => taskCompleted, { timeout: 60_000 }) - - // Give extra time for file system operations - await sleep(2000) - - // Check the file was modified correctly - const actualContent = await fs.readFile(testFile.path, "utf-8") - console.log("File content after modification:", actualContent) - - // Verify tool was executed - assert.strictEqual(applyDiffExecuted, true, "apply_diff tool should have been executed") + assert.strictEqual(errorOccurred, null, `Error occurred: ${errorOccurred}`) - // Verify file content + const actualContent = await fs.readFile( + path.join(workspaceDir, testFiles.multipleReplace.relativePath), + "utf-8", + ) assert.strictEqual( actualContent.trim(), - expectedContent.trim(), + testFiles.multipleReplace.expectedContent.trim(), "All replacements should be applied correctly", ) - console.log("Test passed! apply_diff tool executed and multiple replacements applied successfully") + const completionMessage = messages.find( + (message) => + message.type === "say" && + (message.say === "completion_result" || message.say === "text") && + message.text?.includes("multiple-replace.js"), + ) + assert.ok(completionMessage, "AI should have acknowledged the multiple replacements") } finally { - // Clean up api.off(RooCodeEventName.Message, messageHandler) - api.off(RooCodeEventName.TaskStarted, taskStartedHandler) - api.off(RooCodeEventName.TaskCompleted, taskCompletedHandler) } }) test("Should handle apply_diff with line number hints", async function () { - // Increase timeout for this specific test - const api = globalThis.api const messages: ClineMessage[] = [] - const testFile = testFiles.lineNumbers - const expectedContent = `// Header comment -function newFunction() { - console.log("New implementation") -} - -// Another function -function keepThis() { - console.log("Keep this") -} - -// Footer comment` - - let taskStarted = false - let taskCompleted = false - let applyDiffExecuted = false + let errorOccurred: string | null = null - // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { messages.push(message) - if (message.type === "ask" && message.ask === "tool") { - console.log("Tool request:", message.text?.substring(0, 200)) - } - - // Check for tool execution - if (message.type === "say" && message.say === "api_req_started" && message.text) { - console.log("API request started:", message.text.substring(0, 200)) - try { - const requestData = JSON.parse(message.text) - if (requestData.request && requestData.request.includes("apply_diff")) { - applyDiffExecuted = true - console.log("apply_diff tool executed!") - } - } catch (e) { - console.log("Failed to parse api_req_started message:", e) - } + if (message.type === "say" && message.say === "error") { + errorOccurred = message.text || "Unknown error" } } api.on(RooCodeEventName.Message, messageHandler) - // Listen for task events - const taskStartedHandler = (id: string) => { - if (id === taskId) { - taskStarted = true - } - } - api.on(RooCodeEventName.TaskStarted, taskStartedHandler) - - const taskCompletedHandler = (id: string) => { - if (id === taskId) { - taskCompleted = true - } - } - api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler) - - let taskId: string try { - // Start task with line number context - file already exists - taskId = await api.startNewTask({ - configuration: { - mode: "code", - autoApprovalEnabled: true, - alwaysAllowWrite: true, - alwaysAllowReadOnly: true, - alwaysAllowReadOnlyOutsideWorkspace: true, - }, - text: `Use apply_diff on the file ${testFile.name} to change "oldFunction" to "newFunction" and update its console.log to "New implementation". Keep the rest of the file unchanged. - -The file already exists with this content: -${testFile.content}\nAssume the file exists and you can modify it directly.`, + await waitUntilCompleted({ + api, + start: () => + api.startNewTask({ + configuration: { + mode: "code", + autoApprovalEnabled: true, + alwaysAllowWrite: true, + alwaysAllowReadOnly: true, + alwaysAllowReadOnlyOutsideWorkspace: true, + }, + text: "APPLY_DIFF_LINE_HINTS_SMOKE", + }), + timeout: 60_000, }) + await sleep(1_000) - console.log("Task ID:", taskId) - console.log("Test filename:", testFile.name) - - // Wait for task to start - await waitFor(() => taskStarted, { timeout: 60_000 }) - - // Wait for task completion - await waitFor(() => taskCompleted, { timeout: 60_000 }) - - // Give extra time for file system operations - await sleep(2000) - - // Check the file was modified correctly - const actualContent = await fs.readFile(testFile.path, "utf-8") - console.log("File content after modification:", actualContent) - - // Verify tool was executed - assert.strictEqual(applyDiffExecuted, true, "apply_diff tool should have been executed") + assert.strictEqual(errorOccurred, null, `Error occurred: ${errorOccurred}`) - // Verify file content + const actualContent = await fs.readFile(path.join(workspaceDir, testFiles.lineHints.relativePath), "utf-8") assert.strictEqual( actualContent.trim(), - expectedContent.trim(), - "Only specified function should be modified", + testFiles.lineHints.expectedContent.trim(), + "Only the targeted function should be modified", ) - console.log("Test passed! apply_diff tool executed and targeted modification successful") + const completionMessage = messages.find( + (message) => + message.type === "say" && + (message.say === "completion_result" || message.say === "text") && + message.text?.includes("newFunction"), + ) + assert.ok(completionMessage, "AI should have acknowledged the targeted change") } finally { - // Clean up api.off(RooCodeEventName.Message, messageHandler) - api.off(RooCodeEventName.TaskStarted, taskStartedHandler) - api.off(RooCodeEventName.TaskCompleted, taskCompletedHandler) } }) test("Should handle apply_diff errors gracefully", async function () { const api = globalThis.api const messages: ClineMessage[] = [] - const testFile = testFiles.errorHandling - let taskStarted = false - let taskCompleted = false - let errorDetected = false - let applyDiffAttempted = false - // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { messages.push(message) - - // Check for error messages - if (message.type === "say" && message.say === "error") { - errorDetected = true - console.log("Error detected:", message.text) - } - - // Check if AI mentions it couldn't find the content - if (message.type === "say" && message.text?.toLowerCase().includes("could not find")) { - errorDetected = true - console.log("AI reported search failure:", message.text) - } - - // Check for tool execution attempt - if (message.type === "say" && message.say === "api_req_started" && message.text) { - console.log("API request started:", message.text.substring(0, 200)) - try { - const requestData = JSON.parse(message.text) - if (requestData.request && requestData.request.includes("apply_diff")) { - applyDiffAttempted = true - console.log("apply_diff tool attempted!") - } - } catch (e) { - console.log("Failed to parse api_req_started message:", e) - } - } } api.on(RooCodeEventName.Message, messageHandler) - // Listen for task events - const taskStartedHandler = (id: string) => { - if (id === taskId) { - taskStarted = true - } - } - api.on(RooCodeEventName.TaskStarted, taskStartedHandler) - - const taskCompletedHandler = (id: string) => { - if (id === taskId) { - taskCompleted = true - } - } - api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler) - - let taskId: string try { - // Start task with invalid search content - file already exists - taskId = await api.startNewTask({ - configuration: { - mode: "code", - autoApprovalEnabled: true, - alwaysAllowWrite: true, - alwaysAllowReadOnly: true, - alwaysAllowReadOnlyOutsideWorkspace: true, - }, - text: `Use apply_diff on the file ${testFile.name} to replace "This content does not exist" with "New content". - -The file already exists with this content: -${testFile.content} - -IMPORTANT: The search pattern "This content does not exist" is NOT in the file. When apply_diff cannot find the search pattern, it should fail gracefully and the file content should remain unchanged. Do NOT try to use write_to_file or any other tool to modify the file. Only use apply_diff, and if the search pattern is not found, report that it could not be found. - -Assume the file exists and you can modify it directly.`, + await waitUntilCompleted({ + api, + start: () => + api.startNewTask({ + configuration: { + mode: "code", + autoApprovalEnabled: true, + alwaysAllowWrite: true, + alwaysAllowReadOnly: true, + alwaysAllowReadOnlyOutsideWorkspace: true, + }, + text: "APPLY_DIFF_ERROR_SMOKE", + }), + timeout: 60_000, }) + await sleep(1_000) - console.log("Task ID:", taskId) - console.log("Test filename:", testFile.name) - // Wait for task to start - await waitFor(() => taskStarted, { timeout: 90_000 }) - - // Wait for task completion or error - await waitFor(() => taskCompleted || errorDetected, { timeout: 90_000 }) - - // Give time for any final operations - await sleep(2000) - - // The file content should remain unchanged since the search pattern wasn't found - const actualContent = await fs.readFile(testFile.path, "utf-8") - console.log("File content after task:", actualContent) - - // The AI should have attempted to use apply_diff - assert.strictEqual(applyDiffAttempted, true, "apply_diff tool should have been attempted") - - // The content should remain unchanged since the search pattern wasn't found + const actualContent = await fs.readFile( + path.join(workspaceDir, testFiles.errorHandling.relativePath), + "utf-8", + ) assert.strictEqual( actualContent.trim(), - testFile.content.trim(), - "File content should remain unchanged when search pattern not found", + testFiles.errorHandling.originalContent.trim(), + "File content should remain unchanged when the search pattern is not found", ) - console.log("Test passed! apply_diff attempted and error handled gracefully") + const completionMessage = messages.find( + (message) => + message.type === "say" && + (message.say === "completion_result" || message.say === "text") && + message.text?.includes("did not match"), + ) + assert.ok(completionMessage, "AI should have acknowledged the graceful apply_diff failure") } finally { - // Clean up api.off(RooCodeEventName.Message, messageHandler) - api.off(RooCodeEventName.TaskStarted, taskStartedHandler) - api.off(RooCodeEventName.TaskCompleted, taskCompletedHandler) } }) test("Should apply multiple search/replace blocks to edit two separate functions", async function () { const api = globalThis.api const messages: ClineMessage[] = [] - const testFile = testFiles.multiSearchReplace - const expectedContent = `function transformData(data) { - console.log("Transforming data") - return data.map(item => item * 2) -} - -// Some other code in between -const config = { - timeout: 5000, - retries: 3 -} - -function checkInput(input) { - console.log("Checking input") - if (!input) { - throw new Error("Invalid input") - } - return true -}` - let taskStarted = false - let taskCompleted = false let errorOccurred: string | null = null - let applyDiffExecuted = false - let applyDiffCount = 0 - // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { messages.push(message) - - // Log important messages for debugging if (message.type === "say" && message.say === "error") { errorOccurred = message.text || "Unknown error" - console.error("Error:", message.text) - } - if (message.type === "ask" && message.ask === "tool") { - console.log("Tool request:", message.text?.substring(0, 200)) - } - if (message.type === "say" && (message.say === "completion_result" || message.say === "text")) { - console.log("AI response:", message.text?.substring(0, 200)) - } - - // Check for tool execution - if (message.type === "say" && message.say === "api_req_started" && message.text) { - console.log("API request started:", message.text.substring(0, 200)) - try { - const requestData = JSON.parse(message.text) - if (requestData.request && requestData.request.includes("apply_diff")) { - applyDiffExecuted = true - applyDiffCount++ - console.log(`apply_diff tool executed! (count: ${applyDiffCount})`) - } - } catch (e) { - console.log("Failed to parse api_req_started message:", e) - } } } api.on(RooCodeEventName.Message, messageHandler) - // Listen for task events - const taskStartedHandler = (id: string) => { - if (id === taskId) { - taskStarted = true - console.log("Task started:", id) - } - } - api.on(RooCodeEventName.TaskStarted, taskStartedHandler) - - const taskCompletedHandler = (id: string) => { - if (id === taskId) { - taskCompleted = true - console.log("Task completed:", id) - } - } - api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler) - - let taskId: string try { - // Start task with instruction to edit two separate functions using multiple search/replace blocks - taskId = await api.startNewTask({ - configuration: { - mode: "code", - autoApprovalEnabled: true, - alwaysAllowWrite: true, - alwaysAllowReadOnly: true, - alwaysAllowReadOnlyOutsideWorkspace: true, - }, - text: `Use apply_diff on the file ${testFile.name} to make these changes. You MUST use TWO SEPARATE search/replace blocks within a SINGLE apply_diff call: - -FIRST search/replace block: Edit the processData function to rename it to "transformData" and change "Processing data" to "Transforming data" - -SECOND search/replace block: Edit the validateInput function to rename it to "checkInput" and change "Validating input" to "Checking input" - -Important: Use multiple SEARCH/REPLACE blocks in one apply_diff call, NOT multiple apply_diff calls. Each function should have its own search/replace block. - -The file already exists with this content: -${testFile.content} - -Assume the file exists and you can modify it directly.`, + await waitUntilCompleted({ + api, + start: () => + api.startNewTask({ + configuration: { + mode: "code", + autoApprovalEnabled: true, + alwaysAllowWrite: true, + alwaysAllowReadOnly: true, + alwaysAllowReadOnlyOutsideWorkspace: true, + }, + text: "APPLY_DIFF_MULTI_BLOCK_SMOKE", + }), + timeout: 60_000, }) + await sleep(1_000) - console.log("Task ID:", taskId) - console.log("Test filename:", testFile.name) - - // Wait for task to start - await waitFor(() => taskStarted, { timeout: 60_000 }) - - // Check for early errors - if (errorOccurred) { - console.error("Early error detected:", errorOccurred) - } - - // Wait for task completion - await waitFor(() => taskCompleted, { timeout: 60_000 }) + assert.strictEqual(errorOccurred, null, `Error occurred: ${errorOccurred}`) - // Give extra time for file system operations - await sleep(2000) - - // Check if the file was modified correctly - const actualContent = await fs.readFile(testFile.path, "utf-8") - console.log("File content after modification:", actualContent) - - // Verify tool was executed - assert.strictEqual(applyDiffExecuted, true, "apply_diff tool should have been executed") - console.log(`apply_diff was executed ${applyDiffCount} time(s)`) - - // Verify file content + const actualContent = await fs.readFile( + path.join(workspaceDir, testFiles.multiSearchReplace.relativePath), + "utf-8", + ) assert.strictEqual( actualContent.trim(), - expectedContent.trim(), + testFiles.multiSearchReplace.expectedContent.trim(), "Both functions should be modified with separate search/replace blocks", ) - console.log("Test passed! apply_diff tool executed and multiple search/replace blocks applied successfully") + const completionMessage = messages.find( + (message) => + message.type === "say" && + (message.say === "completion_result" || message.say === "text") && + message.text?.includes("multi-search-replace.js"), + ) + assert.ok(completionMessage, "AI should have acknowledged the multi-block apply_diff update") } finally { - // Clean up api.off(RooCodeEventName.Message, messageHandler) - api.off(RooCodeEventName.TaskStarted, taskStartedHandler) - api.off(RooCodeEventName.TaskCompleted, taskCompletedHandler) } }) }) diff --git a/apps/vscode-e2e/src/suite/tools/execute-command.test.ts b/apps/vscode-e2e/src/suite/tools/execute-command.test.ts index 3dbfb70934..586bb85c15 100644 --- a/apps/vscode-e2e/src/suite/tools/execute-command.test.ts +++ b/apps/vscode-e2e/src/suite/tools/execute-command.test.ts @@ -5,554 +5,271 @@ import * as vscode from "vscode" import { RooCodeEventName, type ClineMessage } from "@roo-code/types" -import { waitFor, sleep, waitUntilCompleted } from "../utils" +import { sleep, waitUntilCompleted } from "../utils" import { setDefaultSuiteTimeout } from "../test-utils" -suite.skip("Roo Code execute_command Tool", function () { +const TEST_DIR_NAME = "execute-command-tool-fixture" +const CUSTOM_CWD_RELATIVE_PATH = `${TEST_DIR_NAME}/custom-cwd` +const SIMPLE_FILE_RELATIVE_PATH = `${TEST_DIR_NAME}/simple-echo.txt` +const MULTI_COMMAND_FILE_RELATIVE_PATH = `${TEST_DIR_NAME}/multi-command.txt` +const CUSTOM_CWD_OUTPUT_RELATIVE_PATH = `${CUSTOM_CWD_RELATIVE_PATH}/output.txt` + +suite("Roo Code execute_command Tool", function () { setDefaultSuiteTimeout(this) let workspaceDir: string + let testDir: string - // Pre-created test files that will be used across tests - const testFiles = { - simpleEcho: { - name: `test-echo-${Date.now()}.txt`, - content: "", - path: "", - }, - multiCommand: { - name: `test-multi-${Date.now()}.txt`, - content: "", - path: "", - }, - cwdTest: { - name: `test-cwd-${Date.now()}.txt`, - content: "", - path: "", - }, - longRunning: { - name: `test-long-${Date.now()}.txt`, - content: "", - path: "", - }, - } - - // Create test files before all tests suiteSetup(async () => { - // Get workspace directory + const aimockUrl = process.env.AIMOCK_URL + const isRecord = process.env.AIMOCK_RECORD === "true" + + await globalThis.api.setConfiguration({ + apiProvider: "openrouter" as const, + openRouterApiKey: aimockUrl && !isRecord ? "mock-key" : process.env.OPENROUTER_API_KEY!, + openRouterModelId: "anthropic/claude-sonnet-4.5", + ...(aimockUrl && { openRouterBaseUrl: `${aimockUrl}/v1` }), + }) + const workspaceFolders = vscode.workspace.workspaceFolders if (!workspaceFolders || workspaceFolders.length === 0) { throw new Error("No workspace folder found") } + workspaceDir = workspaceFolders[0]!.uri.fsPath - console.log("Workspace directory:", workspaceDir) - - // Create test files - for (const [key, file] of Object.entries(testFiles)) { - file.path = path.join(workspaceDir, file.name) - if (file.content) { - await fs.writeFile(file.path, file.content) - console.log(`Created ${key} test file at:`, file.path) - } - } + testDir = path.join(workspaceDir, TEST_DIR_NAME) + await fs.rm(testDir, { recursive: true, force: true }) }) - // Clean up after all tests suiteTeardown(async () => { - // Cancel any running tasks before cleanup try { await globalThis.api.cancelCurrentTask() } catch { // Task might not be running } - // Clean up all test files - console.log("Cleaning up test files...") - for (const [key, file] of Object.entries(testFiles)) { - try { - await fs.unlink(file.path) - console.log(`Cleaned up ${key} test file`) - } catch (error) { - console.log(`Failed to clean up ${key} test file:`, error) - } - } + const aimockUrl = process.env.AIMOCK_URL + const isRecord = process.env.AIMOCK_RECORD === "true" + await globalThis.api.setConfiguration({ + apiProvider: "openrouter" as const, + openRouterApiKey: aimockUrl && !isRecord ? "mock-key" : process.env.OPENROUTER_API_KEY!, + openRouterModelId: "openai/gpt-4.1", + ...(aimockUrl && { openRouterBaseUrl: `${aimockUrl}/v1` }), + }) - // Clean up subdirectory if created - try { - const subDir = path.join(workspaceDir, "test-subdir") - await fs.rmdir(subDir) - } catch { - // Directory might not exist - } + await fs.rm(testDir, { recursive: true, force: true }) }) - // Clean up before each test setup(async () => { - // Cancel any previous task try { await globalThis.api.cancelCurrentTask() } catch { // Task might not be running } - // Small delay to ensure clean state + await fs.rm(testDir, { recursive: true, force: true }) + await fs.mkdir(path.join(workspaceDir, CUSTOM_CWD_RELATIVE_PATH), { recursive: true }) await sleep(100) }) - // Clean up after each test teardown(async () => { - // Cancel the current task try { await globalThis.api.cancelCurrentTask() } catch { // Task might not be running } - // Small delay to ensure clean state await sleep(100) }) test("Should execute simple echo command", async function () { const api = globalThis.api - const testFile = testFiles.simpleEcho - let taskStarted = false - let _taskCompleted = false + const messages: ClineMessage[] = [] let errorOccurred: string | null = null - let executeCommandToolCalled = false - let commandExecuted = "" - // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { - // Log important messages for debugging + messages.push(message) if (message.type === "say" && message.say === "error") { errorOccurred = message.text || "Unknown error" - console.error("Error:", message.text) - } - - // Check for tool execution - if (message.type === "say" && message.say === "api_req_started" && message.text) { - console.log("API request started:", message.text.substring(0, 200)) - try { - const requestData = JSON.parse(message.text) - if (requestData.request && requestData.request.includes("execute_command")) { - executeCommandToolCalled = true - // The request contains the actual tool execution result - commandExecuted = requestData.request - console.log("execute_command tool called, full request:", commandExecuted.substring(0, 300)) - } - } catch (e) { - console.log("Failed to parse api_req_started message:", e) - } } } api.on(RooCodeEventName.Message, messageHandler) - // Listen for task events - const taskStartedHandler = (id: string) => { - if (id === taskId) { - taskStarted = true - console.log("Task started:", id) - } - } - api.on(RooCodeEventName.TaskStarted, taskStartedHandler) - - const taskCompletedHandler = (id: string) => { - if (id === taskId) { - _taskCompleted = true - console.log("Task completed:", id) - } - } - api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler) - - let taskId: string try { - // Start task with execute_command instruction - taskId = await api.startNewTask({ - configuration: { - mode: "code", - autoApprovalEnabled: true, - alwaysAllowExecute: true, - allowedCommands: ["*"], - terminalShellIntegrationDisabled: true, - }, - text: `Use the execute_command tool to run this command: echo "Hello from test" > ${testFile.name} - -The file ${testFile.name} will be created in the current workspace directory. Assume you can execute this command directly. - -Then use the attempt_completion tool to complete the task. Do not suggest any commands in the attempt_completion.`, + await waitUntilCompleted({ + api, + start: () => + api.startNewTask({ + configuration: { + mode: "code", + autoApprovalEnabled: true, + alwaysAllowExecute: true, + allowedCommands: ["*"], + terminalShellIntegrationDisabled: true, + }, + text: "EXECUTE_COMMAND_SIMPLE_SMOKE", + }), + timeout: 60_000, }) - - console.log("Task ID:", taskId) - console.log("Test file:", testFile.name) - - // Wait for task to start - await waitFor(() => taskStarted, { timeout: 45_000 }) - - // Wait for task completion - await waitUntilCompleted({ api, taskId, timeout: 60_000 }) - - // Verify no errors occurred assert.strictEqual(errorOccurred, null, `Error occurred: ${errorOccurred}`) - // Verify tool was called - assert.ok(executeCommandToolCalled, "execute_command tool should have been called") - assert.ok( - commandExecuted.includes("echo") && commandExecuted.includes(testFile.name), - `Command should include 'echo' and test file name. Got: ${commandExecuted.substring(0, 200)}`, - ) - - // Verify file was created with correct content - const content = await fs.readFile(testFile.path, "utf-8") + const content = await fs.readFile(path.join(workspaceDir, SIMPLE_FILE_RELATIVE_PATH), "utf-8") assert.ok(content.includes("Hello from test"), "File should contain the echoed text") - console.log("Test passed! Command executed successfully") + const completionMessage = messages.find( + (message) => + message.type === "say" && + (message.say === "completion_result" || message.say === "text") && + message.text?.includes("simple-echo.txt"), + ) + assert.ok(completionMessage, "AI should have acknowledged the created file") } finally { - // Clean up event listeners api.off(RooCodeEventName.Message, messageHandler) - api.off(RooCodeEventName.TaskStarted, taskStartedHandler) - api.off(RooCodeEventName.TaskCompleted, taskCompletedHandler) } }) test("Should execute command with custom working directory", async function () { const api = globalThis.api - let taskStarted = false - let _taskCompleted = false + const messages: ClineMessage[] = [] let errorOccurred: string | null = null - let executeCommandToolCalled = false - let cwdUsed = "" - - // Create subdirectory - const subDir = path.join(workspaceDir, "test-subdir") - await fs.mkdir(subDir, { recursive: true }) - // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { + messages.push(message) if (message.type === "say" && message.say === "error") { errorOccurred = message.text || "Unknown error" - console.error("Error:", message.text) - } - - // Check for tool execution - if (message.type === "say" && message.say === "api_req_started" && message.text) { - console.log("API request started:", message.text.substring(0, 200)) - try { - const requestData = JSON.parse(message.text) - if (requestData.request && requestData.request.includes("execute_command")) { - executeCommandToolCalled = true - // Check if the request contains the cwd - if (requestData.request.includes(subDir) || requestData.request.includes("test-subdir")) { - cwdUsed = subDir - } - console.log("execute_command tool called, checking for cwd in request") - } - } catch (e) { - console.log("Failed to parse api_req_started message:", e) - } } } api.on(RooCodeEventName.Message, messageHandler) - // Listen for task events - const taskStartedHandler = (id: string) => { - if (id === taskId) { - taskStarted = true - console.log("Task started:", id) - } - } - api.on(RooCodeEventName.TaskStarted, taskStartedHandler) - - const taskCompletedHandler = (id: string) => { - if (id === taskId) { - _taskCompleted = true - console.log("Task completed:", id) - } - } - api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler) - - let taskId: string try { - // Start task with execute_command instruction using cwd parameter - taskId = await api.startNewTask({ - configuration: { - mode: "code", - autoApprovalEnabled: true, - alwaysAllowExecute: true, - allowedCommands: ["*"], - terminalShellIntegrationDisabled: true, - }, - text: `Use the execute_command tool with these exact parameters: -- command: echo "Test in subdirectory" > output.txt -- cwd: ${subDir} - -The subdirectory ${subDir} exists in the workspace. Assume you can execute this command directly with the specified working directory. - -Avoid at all costs suggesting a command when using the attempt_completion tool`, + await waitUntilCompleted({ + api, + start: () => + api.startNewTask({ + configuration: { + mode: "code", + autoApprovalEnabled: true, + alwaysAllowExecute: true, + allowedCommands: ["*"], + terminalShellIntegrationDisabled: true, + }, + text: "EXECUTE_COMMAND_CWD_SMOKE", + }), + timeout: 60_000, }) - - console.log("Task ID:", taskId) - console.log("Subdirectory:", subDir) - - // Wait for task to start - await waitFor(() => taskStarted, { timeout: 45_000 }) - - // Wait for task completion - await waitUntilCompleted({ api, taskId, timeout: 60_000 }) - - // Verify no errors occurred assert.strictEqual(errorOccurred, null, `Error occurred: ${errorOccurred}`) - // Verify tool was called with correct cwd - assert.ok(executeCommandToolCalled, "execute_command tool should have been called") - assert.ok( - cwdUsed.includes(subDir) || cwdUsed.includes("test-subdir"), - "Command should have used the subdirectory as cwd", - ) - - // Verify file was created in subdirectory - const outputPath = path.join(subDir, "output.txt") - const content = await fs.readFile(outputPath, "utf-8") + const content = await fs.readFile(path.join(workspaceDir, CUSTOM_CWD_OUTPUT_RELATIVE_PATH), "utf-8") assert.ok(content.includes("Test in subdirectory"), "File should contain the echoed text") - // Clean up created file - await fs.unlink(outputPath) - - console.log("Test passed! Command executed in custom directory") + const completionMessage = messages.find( + (message) => + message.type === "say" && + (message.say === "completion_result" || message.say === "text") && + message.text?.includes("custom-cwd"), + ) + assert.ok(completionMessage, "AI should have acknowledged the custom cwd execution") } finally { - // Clean up event listeners api.off(RooCodeEventName.Message, messageHandler) - api.off(RooCodeEventName.TaskStarted, taskStartedHandler) - api.off(RooCodeEventName.TaskCompleted, taskCompletedHandler) - - // Clean up subdirectory - try { - await fs.rmdir(subDir) - } catch { - // Directory might not be empty - } } }) test("Should execute multiple commands sequentially", async function () { const api = globalThis.api - const testFile = testFiles.multiCommand - let taskStarted = false - let _taskCompleted = false + const messages: ClineMessage[] = [] let errorOccurred: string | null = null - let executeCommandCallCount = 0 - const commandsExecuted: string[] = [] - // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { + messages.push(message) if (message.type === "say" && message.say === "error") { errorOccurred = message.text || "Unknown error" - console.error("Error:", message.text) - } - - // Check for tool execution - if (message.type === "say" && message.say === "api_req_started" && message.text) { - console.log("API request started:", message.text.substring(0, 200)) - try { - const requestData = JSON.parse(message.text) - if (requestData.request && requestData.request.includes("execute_command")) { - executeCommandCallCount++ - // Store the full request to check for command content - commandsExecuted.push(requestData.request) - console.log(`execute_command tool call #${executeCommandCallCount}`) - } - } catch (e) { - console.log("Failed to parse api_req_started message:", e) - } } } api.on(RooCodeEventName.Message, messageHandler) - // Listen for task events - const taskStartedHandler = (id: string) => { - if (id === taskId) { - taskStarted = true - console.log("Task started:", id) - } - } - api.on(RooCodeEventName.TaskStarted, taskStartedHandler) - - const taskCompletedHandler = (id: string) => { - if (id === taskId) { - _taskCompleted = true - console.log("Task completed:", id) - } - } - api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler) - - let taskId: string try { - // Start task with multiple commands - simplified to just 2 commands - taskId = await api.startNewTask({ - configuration: { - mode: "code", - autoApprovalEnabled: true, - alwaysAllowExecute: true, - allowedCommands: ["*"], - terminalShellIntegrationDisabled: true, - }, - text: `Use the execute_command tool to create a file with multiple lines. Execute these commands one by one: -1. echo "Line 1" > ${testFile.name} -2. echo "Line 2" >> ${testFile.name} - -The file ${testFile.name} will be created in the current workspace directory. Assume you can execute these commands directly. - -Important: Use only the echo command which is available on all Unix platforms. Execute each command separately using the execute_command tool. - -After both commands are executed, use the attempt_completion tool to complete the task.`, + await waitUntilCompleted({ + api, + start: () => + api.startNewTask({ + configuration: { + mode: "code", + autoApprovalEnabled: true, + alwaysAllowExecute: true, + allowedCommands: ["*"], + terminalShellIntegrationDisabled: true, + }, + text: "EXECUTE_COMMAND_MULTI_SMOKE", + }), + timeout: 90_000, }) - - console.log("Task ID:", taskId) - console.log("Test file:", testFile.name) - - // Wait for task to start - await waitFor(() => taskStarted, { timeout: 90_000 }) - - // Wait for task completion with increased timeout - await waitUntilCompleted({ api, taskId, timeout: 90_000 }) - - // Verify no errors occurred assert.strictEqual(errorOccurred, null, `Error occurred: ${errorOccurred}`) - // Verify tool was called multiple times (reduced to 2) - assert.ok( - executeCommandCallCount >= 2, - `execute_command tool should have been called at least 2 times, was called ${executeCommandCallCount} times`, - ) - assert.ok( - commandsExecuted.some((cmd) => cmd.includes("Line 1")), - `Should have executed first command. Commands: ${commandsExecuted.map((c) => c.substring(0, 100)).join(", ")}`, - ) - assert.ok( - commandsExecuted.some((cmd) => cmd.includes("Line 2")), - "Should have executed second command", - ) - - // Verify file contains outputs - const content = await fs.readFile(testFile.path, "utf-8") - assert.ok(content.includes("Line 1"), "Should contain first line") - assert.ok(content.includes("Line 2"), "Should contain second line") + const content = await fs.readFile(path.join(workspaceDir, MULTI_COMMAND_FILE_RELATIVE_PATH), "utf-8") + assert.ok(content.includes("Line 1"), "Should contain the first line") + assert.ok(content.includes("Line 2"), "Should contain the second line") - console.log("Test passed! Multiple commands executed successfully") + const completionMessage = messages.find( + (message) => + message.type === "say" && + (message.say === "completion_result" || message.say === "text") && + message.text?.includes("multi-command.txt"), + ) + assert.ok(completionMessage, "AI should have acknowledged the multi-command run") } finally { - // Clean up event listeners api.off(RooCodeEventName.Message, messageHandler) - api.off(RooCodeEventName.TaskStarted, taskStartedHandler) - api.off(RooCodeEventName.TaskCompleted, taskCompletedHandler) } }) test("Should handle long-running commands", async function () { const api = globalThis.api - let taskStarted = false - let _taskCompleted = false - let _commandCompleted = false + const messages: ClineMessage[] = [] let errorOccurred: string | null = null - let executeCommandToolCalled = false - let commandExecuted = "" + let commandCompleted = false - // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { + messages.push(message) if (message.type === "say" && message.say === "error") { errorOccurred = message.text || "Unknown error" - console.error("Error:", message.text) } if (message.type === "say" && message.say === "command_output") { - if (message.text?.includes("completed after delay")) { - _commandCompleted = true - } - console.log("Command output:", message.text?.substring(0, 200)) - } - - // Check for tool execution - if (message.type === "say" && message.say === "api_req_started" && message.text) { - console.log("API request started:", message.text.substring(0, 200)) - try { - const requestData = JSON.parse(message.text) - if (requestData.request && requestData.request.includes("execute_command")) { - executeCommandToolCalled = true - // The request contains the actual tool execution result - commandExecuted = requestData.request - console.log("execute_command tool called, full request:", commandExecuted.substring(0, 300)) - } - } catch (e) { - console.log("Failed to parse api_req_started message:", e) + if (message.text?.includes("Command completed after delay")) { + commandCompleted = true } } } api.on(RooCodeEventName.Message, messageHandler) - // Listen for task events - const taskStartedHandler = (id: string) => { - if (id === taskId) { - taskStarted = true - console.log("Task started:", id) - } - } - api.on(RooCodeEventName.TaskStarted, taskStartedHandler) - - const taskCompletedHandler = (id: string) => { - if (id === taskId) { - _taskCompleted = true - console.log("Task completed:", id) - } - } - api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler) - - let taskId: string try { - // Platform-specific sleep command - const sleepCommand = process.platform === "win32" ? "timeout /t 3 /nobreak" : "sleep 3" - - // Start task with long-running command - taskId = await api.startNewTask({ - configuration: { - mode: "code", - autoApprovalEnabled: true, - alwaysAllowExecute: true, - allowedCommands: ["*"], - terminalShellIntegrationDisabled: true, - }, - text: `Use the execute_command tool to run: ${sleepCommand} && echo "Command completed after delay" - -Assume you can execute this command directly in the current workspace directory. - -Avoid at all costs suggesting a command when using the attempt_completion tool`, + await waitUntilCompleted({ + api, + start: () => + api.startNewTask({ + configuration: { + mode: "code", + autoApprovalEnabled: true, + alwaysAllowExecute: true, + allowedCommands: ["*"], + terminalShellIntegrationDisabled: true, + }, + text: "EXECUTE_COMMAND_LONG_RUNNING_SMOKE", + }), + timeout: 60_000, }) + await sleep(500) - console.log("Task ID:", taskId) - - // Wait for task to start - await waitFor(() => taskStarted, { timeout: 45_000 }) - - // Wait for task completion (the command output check will verify execution) - await waitUntilCompleted({ api, taskId, timeout: 45_000 }) - - // Give a bit of time for final output processing - await sleep(1000) - - // Verify no errors occurred assert.strictEqual(errorOccurred, null, `Error occurred: ${errorOccurred}`) + assert.ok(commandCompleted, "Command output should include the delayed completion text") - // Verify tool was called - assert.ok(executeCommandToolCalled, "execute_command tool should have been called") - assert.ok( - commandExecuted.includes("sleep") || commandExecuted.includes("timeout"), - `Command should include sleep or timeout command. Got: ${commandExecuted.substring(0, 200)}`, + const completionMessage = messages.find( + (message) => + message.type === "say" && + (message.say === "completion_result" || message.say === "text") && + message.text?.includes("Command completed after delay"), ) - - // The command output check in the message handler will verify execution - - console.log("Test passed! Long-running command handled successfully") + assert.ok(completionMessage, "AI should have acknowledged the long-running command result") } finally { - // Clean up event listeners api.off(RooCodeEventName.Message, messageHandler) - api.off(RooCodeEventName.TaskStarted, taskStartedHandler) - api.off(RooCodeEventName.TaskCompleted, taskCompletedHandler) } }) }) diff --git a/apps/vscode-e2e/src/suite/tools/list-files.test.ts b/apps/vscode-e2e/src/suite/tools/list-files.test.ts index 7cf5a8abca..d8afa52989 100644 --- a/apps/vscode-e2e/src/suite/tools/list-files.test.ts +++ b/apps/vscode-e2e/src/suite/tools/list-files.test.ts @@ -36,7 +36,6 @@ suite("Roo Code list_files Tool", function () { throw new Error("No workspace folder found") } workspaceDir = workspaceFolders[0]!.uri.fsPath - console.log("Workspace directory:", workspaceDir) // Create test directory structure const testDir = path.join(workspaceDir, TEST_DIR_NAME) @@ -126,9 +125,6 @@ This directory contains various files and subdirectories for testing the list_fi - Hidden file - Configuration files (yaml)`, ) - - console.log("Test directory structure created:", testDir) - console.log("Test files:", testFiles) }) // Clean up test files and directories after all tests @@ -146,9 +142,8 @@ This directory contains various files and subdirectories for testing the list_fi try { await fs.rm(testDir, { recursive: true, force: true }) - console.log("Cleaned up test directory:", testDir) - } catch (error) { - console.log("Failed to clean up test directory:", error) + } catch { + // cleanup failure is non-fatal } }) @@ -210,8 +205,6 @@ This directory contains various files and subdirectories for testing the list_fi text: "List the files in the list-files-tool-fixture directory without recursing into subdirectories, and report what you find.", }) - console.log("Task ID:", taskId) - // Wait for task completion await waitFor(() => taskCompleted, { timeout: 60_000 }) @@ -224,8 +217,6 @@ This directory contains various files and subdirectories for testing the list_fi m.text?.includes("nested/"), ) assert.ok(completionMessage, "AI should have summarized the non-recursive directory contents") - - console.log("Test passed! Directory listing (non-recursive) executed successfully") } finally { // Clean up api.off(RooCodeEventName.Message, messageHandler) @@ -265,8 +256,6 @@ This directory contains various files and subdirectories for testing the list_fi text: "List every file in the list-files-tool-fixture directory recursively and confirm that the nested path for deep-nested-file.ts is included.", }) - console.log("Task ID:", taskId) - // Wait for task completion await waitFor(() => taskCompleted, { timeout: 60_000 }) @@ -278,8 +267,6 @@ This directory contains various files and subdirectories for testing the list_fi m.text?.includes("deep-nested-file.ts"), ) assert.ok(completionMessage, "AI should have summarized the recursive directory contents") - - console.log("Test passed! Directory listing (recursive) executed successfully") } finally { // Clean up api.off(RooCodeEventName.Message, messageHandler) @@ -326,11 +313,7 @@ This directory contains various files and subdirectories for testing the list_fi try { await fs.symlink(sourceFile, symlinkFile) await fs.symlink(sourceDir, symlinkDir) - console.log("Created symlinks successfully") - } catch (symlinkError) { - console.log("Symlink creation failed (might be platform limitation):", symlinkError) - // Skip test if symlinks can't be created - console.log("Skipping symlink test - platform doesn't support symlinks") + } catch { return } @@ -345,8 +328,6 @@ This directory contains various files and subdirectories for testing the list_fi text: "Call list_files with path='list-files-symlink-fixture' and recursive=false. Report everything the tool returns.", }) - console.log("Symlink test Task ID:", taskId) - // 120s: real models may loop before finding the symlink fixture path. await waitFor(() => taskCompleted, { timeout: 120_000 }) @@ -363,8 +344,6 @@ This directory contains various files and subdirectories for testing the list_fi }) assert.ok(completionMessage, "AI should have summarized both the original and symlinked directory contents") - console.log("Test passed! Symlinked files and directories are now visible") - // Cleanup await fs.rm(testDir, { recursive: true, force: true }) } finally { @@ -406,8 +385,6 @@ This directory contains various files and subdirectories for testing the list_fi text: "List the files in the workspace root directory without recursing and confirm whether list-files-tool-fixture or list-files-symlink-fixture is present.", }) - console.log("Task ID:", taskId) - // Wait for task completion await waitFor(() => taskCompleted, { timeout: 60_000 }) @@ -419,8 +396,6 @@ This directory contains various files and subdirectories for testing the list_fi (m.text?.includes("list-files-tool-fixture") || m.text?.includes("list-files-symlink-fixture")), ) assert.ok(completionMessage, "AI should have mentioned workspace contents") - - console.log("Test passed! Workspace root directory listing executed successfully") } finally { // Clean up api.off(RooCodeEventName.Message, messageHandler) diff --git a/apps/vscode-e2e/src/suite/tools/write-to-file.test.ts b/apps/vscode-e2e/src/suite/tools/write-to-file.test.ts index fee15add17..9c73a94b4e 100644 --- a/apps/vscode-e2e/src/suite/tools/write-to-file.test.ts +++ b/apps/vscode-e2e/src/suite/tools/write-to-file.test.ts @@ -1,448 +1,180 @@ import * as assert from "assert" import * as fs from "fs/promises" import * as path from "path" -import * as os from "os" +import * as vscode from "vscode" import { RooCodeEventName, type ClineMessage } from "@roo-code/types" -import { waitFor, sleep } from "../utils" +import { waitUntilCompleted, sleep } from "../utils" import { setDefaultSuiteTimeout } from "../test-utils" -suite.skip("Roo Code write_to_file Tool", function () { +const TEST_DIR_NAME = "write-to-file-tool-fixture" +const SIMPLE_FILE_RELATIVE_PATH = `${TEST_DIR_NAME}/write-to-file-smoke.txt` +const NESTED_FILE_RELATIVE_PATH = `${TEST_DIR_NAME}/nested/deep/directory/write-to-file-nested-smoke.txt` +const SIMPLE_FILE_CONTENT = "Hello, this is a test file!" +const NESTED_FILE_CONTENT = "File in nested directory" + +suite("Roo Code write_to_file Tool", function () { setDefaultSuiteTimeout(this) - let tempDir: string - let testFilePath: string + let workspaceDir: string + let testDir: string - // Create a temporary directory for test files suiteSetup(async () => { - tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "roo-test-")) + const aimockUrl = process.env.AIMOCK_URL + const isRecord = process.env.AIMOCK_RECORD === "true" + + await globalThis.api.setConfiguration({ + apiProvider: "openrouter" as const, + openRouterApiKey: aimockUrl && !isRecord ? "mock-key" : process.env.OPENROUTER_API_KEY!, + openRouterModelId: "anthropic/claude-sonnet-4.5", + ...(aimockUrl && { openRouterBaseUrl: `${aimockUrl}/v1` }), + }) + + const workspaceFolders = vscode.workspace.workspaceFolders + if (!workspaceFolders || workspaceFolders.length === 0) { + throw new Error("No workspace folder found") + } + + workspaceDir = workspaceFolders[0]!.uri.fsPath + testDir = path.join(workspaceDir, TEST_DIR_NAME) + await fs.rm(testDir, { recursive: true, force: true }) }) - // Clean up temporary directory after tests suiteTeardown(async () => { - // Cancel any running tasks before cleanup try { await globalThis.api.cancelCurrentTask() } catch { // Task might not be running } - await fs.rm(tempDir, { recursive: true, force: true }) + + const aimockUrl = process.env.AIMOCK_URL + const isRecord = process.env.AIMOCK_RECORD === "true" + await globalThis.api.setConfiguration({ + apiProvider: "openrouter" as const, + openRouterApiKey: aimockUrl && !isRecord ? "mock-key" : process.env.OPENROUTER_API_KEY!, + openRouterModelId: "openai/gpt-4.1", + ...(aimockUrl && { openRouterBaseUrl: `${aimockUrl}/v1` }), + }) + + await fs.rm(testDir, { recursive: true, force: true }) }) - // Clean up test file before each test setup(async () => { - // Cancel any previous task try { await globalThis.api.cancelCurrentTask() } catch { // Task might not be running } - // Generate unique file name for each test to avoid conflicts - testFilePath = path.join(tempDir, `test-file-${Date.now()}.txt`) - - // Small delay to ensure clean state + await fs.rm(testDir, { recursive: true, force: true }) + await fs.mkdir(testDir, { recursive: true }) await sleep(100) }) - // Clean up after each test teardown(async () => { - // Cancel the current task try { await globalThis.api.cancelCurrentTask() } catch { // Task might not be running } - // Clean up the test file - try { - await fs.unlink(testFilePath) - } catch { - // File might not exist - } - - // Small delay to ensure clean state await sleep(100) }) test("Should create a new file with content", async function () { - // Increase timeout for this specific test - const api = globalThis.api const messages: ClineMessage[] = [] - const fileContent = "Hello, this is a test file!" - let taskStarted = false - let taskCompleted = false let errorOccurred: string | null = null - let writeToFileToolExecuted = false - let toolExecutionDetails = "" + const targetPath = path.join(workspaceDir, SIMPLE_FILE_RELATIVE_PATH) - // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { messages.push(message) - - // Check for tool execution - if (message.type === "say" && message.say === "api_req_started") { - console.log("Tool execution:", message.text?.substring(0, 200)) - if (message.text && message.text.includes("write_to_file")) { - writeToFileToolExecuted = true - toolExecutionDetails = message.text - // Try to parse the tool execution details - try { - const parsed = JSON.parse(message.text) - console.log("write_to_file tool called with request:", parsed.request?.substring(0, 300)) - } catch (_e) { - console.log("Could not parse tool execution details") - } - } - } - - // Log important messages for debugging if (message.type === "say" && message.say === "error") { errorOccurred = message.text || "Unknown error" - console.error("Error:", message.text) - } - if (message.type === "ask" && message.ask === "tool") { - console.log("Tool request:", message.text?.substring(0, 200)) - } - if (message.type === "say" && (message.say === "completion_result" || message.say === "text")) { - console.log("AI response:", message.text?.substring(0, 200)) } } api.on(RooCodeEventName.Message, messageHandler) - // Listen for task events - const taskStartedHandler = (id: string) => { - if (id === taskId) { - taskStarted = true - console.log("Task started:", id) - } - } - api.on(RooCodeEventName.TaskStarted, taskStartedHandler) - - const taskCompletedHandler = (id: string) => { - if (id === taskId) { - taskCompleted = true - console.log("Task completed:", id) - } - } - api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler) - - let taskId: string try { - // Start task with a very simple prompt - const baseFileName = path.basename(testFilePath) - taskId = await api.startNewTask({ - configuration: { - mode: "code", - autoApprovalEnabled: true, - alwaysAllowWrite: true, - alwaysAllowReadOnly: true, - alwaysAllowReadOnlyOutsideWorkspace: true, - }, - text: `Create a file named "${baseFileName}" with the following content:\n${fileContent}`, + await waitUntilCompleted({ + api, + start: () => + api.startNewTask({ + configuration: { + mode: "code", + autoApprovalEnabled: true, + alwaysAllowWrite: true, + alwaysAllowReadOnly: true, + alwaysAllowReadOnlyOutsideWorkspace: true, + }, + text: "WRITE_TO_FILE_CREATE_SMOKE", + }), + timeout: 45_000, }) - console.log("Task ID:", taskId) - console.log("Base filename:", baseFileName) - console.log("Expecting file at:", testFilePath) - - // Wait for task to start - await waitFor(() => taskStarted, { timeout: 45_000 }) - - // Check for early errors - if (errorOccurred) { - console.error("Early error detected:", errorOccurred) - } - - // Wait for task completion - await waitFor(() => taskCompleted, { timeout: 45_000 }) - - // Give extra time for file system operations - await sleep(2000) - - // The file might be created in different locations, let's check them all - const possibleLocations = [ - testFilePath, // Expected location - path.join(tempDir, baseFileName), // In temp directory - path.join(process.cwd(), baseFileName), // In current working directory - path.join("/tmp/roo-test-workspace-" + "*", baseFileName), // In workspace created by runTest.ts - ] - - let fileFound = false - let actualFilePath = "" - let actualContent = "" - - // First check the workspace directory that was created - const workspaceDirs = await fs - .readdir("/tmp") - .then((files) => files.filter((f) => f.startsWith("roo-test-workspace-"))) - .catch(() => []) - - for (const wsDir of workspaceDirs) { - const wsFilePath = path.join("/tmp", wsDir, baseFileName) - try { - await fs.access(wsFilePath) - fileFound = true - actualFilePath = wsFilePath - actualContent = await fs.readFile(wsFilePath, "utf-8") - console.log("File found in workspace directory:", wsFilePath) - break - } catch { - // Continue checking - } - } - - // If not found in workspace, check other locations - if (!fileFound) { - for (const location of possibleLocations) { - try { - await fs.access(location) - fileFound = true - actualFilePath = location - actualContent = await fs.readFile(location, "utf-8") - console.log("File found at:", location) - break - } catch { - // Continue checking - } - } - } + assert.strictEqual(errorOccurred, null, `Error occurred: ${errorOccurred}`) - // If still not found, list directories to help debug - if (!fileFound) { - console.log("File not found in expected locations. Debugging info:") + const actualContent = await fs.readFile(targetPath, "utf-8") + assert.strictEqual(actualContent.trim(), SIMPLE_FILE_CONTENT, "File content should match expected content") - // List temp directory - try { - const tempFiles = await fs.readdir(tempDir) - console.log("Files in temp directory:", tempFiles) - } catch (e) { - console.log("Could not list temp directory:", e) - } - - // List current working directory - try { - const cwdFiles = await fs.readdir(process.cwd()) - console.log( - "Files in CWD:", - cwdFiles.filter((f) => f.includes("test-file")), - ) - } catch (e) { - console.log("Could not list CWD:", e) - } - - // List /tmp for test files - try { - const tmpFiles = await fs.readdir("/tmp") - console.log( - "Test files in /tmp:", - tmpFiles.filter((f) => f.includes("test-file") || f.includes("roo-test")), - ) - } catch (e) { - console.log("Could not list /tmp:", e) - } - } - - assert.ok(fileFound, `File should have been created. Expected filename: ${baseFileName}`) - assert.strictEqual(actualContent.trim(), fileContent, "File content should match expected content") - - // Verify that write_to_file tool was actually executed - assert.ok(writeToFileToolExecuted, "write_to_file tool should have been executed") - assert.ok( - toolExecutionDetails.includes(baseFileName) || toolExecutionDetails.includes(fileContent), - "Tool execution should include the filename or content", + const toolApprovalMessage = messages.find( + (message) => + message.type === "ask" && + message.ask === "tool" && + message.text?.includes("write-to-file-smoke.txt"), ) - - console.log("Test passed! File created successfully at:", actualFilePath) - console.log("write_to_file tool was properly executed") + assert.ok(toolApprovalMessage, "Task should have requested approval for the file write") } finally { - // Clean up api.off(RooCodeEventName.Message, messageHandler) - api.off(RooCodeEventName.TaskStarted, taskStartedHandler) - api.off(RooCodeEventName.TaskCompleted, taskCompletedHandler) } }) test("Should create nested directories when writing file", async function () { - // Increase timeout for this specific test - const api = globalThis.api const messages: ClineMessage[] = [] - const content = "File in nested directory" - const fileName = `file-${Date.now()}.txt` - const nestedPath = path.join(tempDir, "nested", "deep", "directory", fileName) - let taskStarted = false - let taskCompleted = false - let writeToFileToolExecuted = false - let toolExecutionDetails = "" + let errorOccurred: string | null = null + const targetPath = path.join(workspaceDir, NESTED_FILE_RELATIVE_PATH) - // Listen for messages const messageHandler = ({ message }: { message: ClineMessage }) => { messages.push(message) - - // Check for tool execution - if (message.type === "say" && message.say === "api_req_started") { - console.log("Tool execution:", message.text?.substring(0, 200)) - if (message.text && message.text.includes("write_to_file")) { - writeToFileToolExecuted = true - toolExecutionDetails = message.text - // Try to parse the tool execution details - try { - const parsed = JSON.parse(message.text) - console.log("write_to_file tool called with request:", parsed.request?.substring(0, 300)) - } catch (_e) { - console.log("Could not parse tool execution details") - } - } - } - - if (message.type === "ask" && message.ask === "tool") { - console.log("Tool request:", message.text?.substring(0, 200)) + if (message.type === "say" && message.say === "error") { + errorOccurred = message.text || "Unknown error" } } api.on(RooCodeEventName.Message, messageHandler) - // Listen for task events - const taskStartedHandler = (id: string) => { - if (id === taskId) { - taskStarted = true - console.log("Task started:", id) - } - } - api.on(RooCodeEventName.TaskStarted, taskStartedHandler) - - const taskCompletedHandler = (id: string) => { - if (id === taskId) { - taskCompleted = true - console.log("Task completed:", id) - } - } - api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler) - - let taskId: string try { - // Start task to create file in nested directory - taskId = await api.startNewTask({ - configuration: { - mode: "code", - autoApprovalEnabled: true, - alwaysAllowWrite: true, - alwaysAllowReadOnly: true, - alwaysAllowReadOnlyOutsideWorkspace: true, - }, - text: `Create a file named "${fileName}" in a nested directory structure "nested/deep/directory/" with the following content:\n${content}`, + await waitUntilCompleted({ + api, + start: () => + api.startNewTask({ + configuration: { + mode: "code", + autoApprovalEnabled: true, + alwaysAllowWrite: true, + alwaysAllowReadOnly: true, + alwaysAllowReadOnlyOutsideWorkspace: true, + }, + text: "WRITE_TO_FILE_NESTED_SMOKE", + }), + timeout: 45_000, }) - console.log("Task ID:", taskId) - console.log("Expected nested path:", nestedPath) - - // Wait for task to start - await waitFor(() => taskStarted, { timeout: 45_000 }) - - // Wait for task completion - await waitFor(() => taskCompleted, { timeout: 45_000 }) - - // Give extra time for file system operations - await sleep(2000) - - // Check various possible locations - let fileFound = false - let actualFilePath = "" - let actualContent = "" - - // Check workspace directories - const workspaceDirs = await fs - .readdir("/tmp") - .then((files) => files.filter((f) => f.startsWith("roo-test-workspace-"))) - .catch(() => []) - - for (const wsDir of workspaceDirs) { - // Check in nested structure within workspace - const wsNestedPath = path.join("/tmp", wsDir, "nested", "deep", "directory", fileName) - try { - await fs.access(wsNestedPath) - fileFound = true - actualFilePath = wsNestedPath - actualContent = await fs.readFile(wsNestedPath, "utf-8") - console.log("File found in workspace nested directory:", wsNestedPath) - break - } catch { - // Also check if file was created directly in workspace root - const wsFilePath = path.join("/tmp", wsDir, fileName) - try { - await fs.access(wsFilePath) - fileFound = true - actualFilePath = wsFilePath - actualContent = await fs.readFile(wsFilePath, "utf-8") - console.log("File found in workspace root (nested dirs not created):", wsFilePath) - break - } catch { - // Continue checking - } - } - } - - // If not found in workspace, check the expected location - if (!fileFound) { - try { - await fs.access(nestedPath) - fileFound = true - actualFilePath = nestedPath - actualContent = await fs.readFile(nestedPath, "utf-8") - console.log("File found at expected nested path:", nestedPath) - } catch { - // File not found - } - } - - // Debug output if file not found - if (!fileFound) { - console.log("File not found. Debugging info:") + assert.strictEqual(errorOccurred, null, `Error occurred: ${errorOccurred}`) - // List workspace directories and their contents - for (const wsDir of workspaceDirs) { - const wsPath = path.join("/tmp", wsDir) - try { - const files = await fs.readdir(wsPath) - console.log(`Files in workspace ${wsDir}:`, files) + const actualContent = await fs.readFile(targetPath, "utf-8") + assert.strictEqual(actualContent.trim(), NESTED_FILE_CONTENT, "Nested file content should match") - // Check if nested directory was created - const nestedDir = path.join(wsPath, "nested") - try { - await fs.access(nestedDir) - console.log("Nested directory exists in workspace") - } catch { - console.log("Nested directory NOT created in workspace") - } - } catch (e) { - console.log(`Could not list workspace ${wsDir}:`, e) - } - } - } - - assert.ok(fileFound, `File should have been created. Expected filename: ${fileName}`) - assert.strictEqual(actualContent.trim(), content, "File content should match") - - // Verify that write_to_file tool was actually executed - assert.ok(writeToFileToolExecuted, "write_to_file tool should have been executed") - assert.ok( - toolExecutionDetails.includes(fileName) || - toolExecutionDetails.includes(content) || - toolExecutionDetails.includes("nested"), - "Tool execution should include the filename, content, or nested directory reference", + const toolApprovalMessage = messages.find( + (message) => + message.type === "ask" && + message.ask === "tool" && + message.text?.includes("write-to-file-nested-smoke.txt"), ) - - // Note: We're not checking if the nested directory structure was created, - // just that the file exists with the correct content - console.log("Test passed! File created successfully at:", actualFilePath) - console.log("write_to_file tool was properly executed") + assert.ok(toolApprovalMessage, "Task should have requested approval for the nested file write") } finally { - // Clean up api.off(RooCodeEventName.Message, messageHandler) - api.off(RooCodeEventName.TaskStarted, taskStartedHandler) - api.off(RooCodeEventName.TaskCompleted, taskCompletedHandler) } }) }) diff --git a/apps/vscode-e2e/src/suite/utils.ts b/apps/vscode-e2e/src/suite/utils.ts index 874ded9acc..7203f87681 100644 --- a/apps/vscode-e2e/src/suite/utils.ts +++ b/apps/vscode-e2e/src/suite/utils.ts @@ -52,13 +52,26 @@ export const waitUntilAborted = async ({ api, taskId, ...options }: WaitUntilAbo type WaitUntilCompletedOptions = WaitForOptions & { api: RooCodeAPI - taskId: string + taskId?: string + start?: () => Promise } -export const waitUntilCompleted = async ({ api, taskId, ...options }: WaitUntilCompletedOptions) => { - const set = new Set() - api.on(RooCodeEventName.TaskCompleted, (taskId) => set.add(taskId)) - await waitFor(() => set.has(taskId), options) +export const waitUntilCompleted = async ({ + api, + taskId: passedTaskId, + start, + ...options +}: WaitUntilCompletedOptions): Promise => { + const completed = new Set() + const handler = (id: string) => completed.add(id) + api.on(RooCodeEventName.TaskCompleted, handler) + try { + const taskId = passedTaskId ?? (await start!()) + await waitFor(() => completed.has(taskId), options) + return taskId + } finally { + api.off(RooCodeEventName.TaskCompleted, handler) + } } export const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))