[Chore] Unskip VS Code e2e replay for mutating tools

apps/vscode-e2e/fixtures/apply-diff.json

@@ -0,0 +1,74 @@
+{
+	"fixtures": [
+		{
+			"match": {
+				"userMessage": "APPLY_DIFF_SIMPLE_SMOKE"
+			},
+			"response": {
+				"toolCalls": [
+					{
+						"name": "apply_diff",
+						"arguments": "{\"path\":\"apply-diff-tool-fixture/simple-modify.txt\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:1\\n-------\\nHello World\\n=======\\nHello Universe\\n>>>>>>> REPLACE\"}",
+						"id": "call_apply_diff_simple_001"
+					}
+				]
+			}
+		},
+		{
+			"match": {
+				"userMessage": "APPLY_DIFF_MULTI_REPLACE_SMOKE"
+			},
+			"response": {
+				"toolCalls": [
+					{
+						"name": "apply_diff",
+						"arguments": "{\"path\":\"apply-diff-tool-fixture/multiple-replace.js\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:1\\n-------\\nfunction calculate(x, y) {\\n\\tconst sum = x + y\\n\\tconst product = x * y\\n\\treturn { sum: sum, product: product }\\n}\\n=======\\nfunction compute(a, b) {\\n\\tconst total = a + b\\n\\tconst result = a * b\\n\\treturn { total: total, result: result }\\n}\\n>>>>>>> REPLACE\"}",
+						"id": "call_apply_diff_multi_replace_001"
+					}
+				]
+			}
+		},
+		{
+			"match": {
+				"userMessage": "APPLY_DIFF_LINE_HINTS_SMOKE"
+			},
+			"response": {
+				"toolCalls": [
+					{
+						"name": "apply_diff",
+						"arguments": "{\"path\":\"apply-diff-tool-fixture/line-hints.js\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:2\\n-------\\nfunction oldFunction() {\\n\\tconsole.log(\\\"Old implementation\\\")\\n}\\n=======\\nfunction newFunction() {\\n\\tconsole.log(\\\"New implementation\\\")\\n}\\n>>>>>>> REPLACE\"}",
+						"id": "call_apply_diff_line_hints_001"
+					}
+				]
+			}
+		},
+		{
+			"match": {
+				"userMessage": "APPLY_DIFF_ERROR_SMOKE"
+			},
+			"response": {
+				"toolCalls": [
+					{
+						"name": "apply_diff",
+						"arguments": "{\"path\":\"apply-diff-tool-fixture/error-handling.txt\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:1\\n-------\\nThis content does not exist\\n=======\\nNew content\\n>>>>>>> REPLACE\"}",
+						"id": "call_apply_diff_error_001"
+					}
+				]
+			}
+		},
+		{
+			"match": {
+				"userMessage": "APPLY_DIFF_MULTI_BLOCK_SMOKE"
+			},
+			"response": {
+				"toolCalls": [
+					{
+						"name": "apply_diff",
+						"arguments": "{\"path\":\"apply-diff-tool-fixture/multi-search-replace.js\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:1\\n-------\\nfunction processData(data) {\\n\\tconsole.log(\\\"Processing data\\\")\\n=======\\nfunction transformData(data) {\\n\\tconsole.log(\\\"Transforming data\\\")\\n>>>>>>> REPLACE\\n\\n<<<<<<< SEARCH\\n:start_line:12\\n-------\\nfunction validateInput(input) {\\n\\tconsole.log(\\\"Validating input\\\")\\n=======\\nfunction checkInput(input) {\\n\\tconsole.log(\\\"Checking input\\\")\\n>>>>>>> REPLACE\"}",
+						"id": "call_apply_diff_multi_block_001"
+					}
+				]
+			}
+		}
+	]
+}

apps/vscode-e2e/fixtures/execute-command.json

@@ -0,0 +1,60 @@
+{
+	"fixtures": [
+		{
+			"match": {
+				"userMessage": "EXECUTE_COMMAND_SIMPLE_SMOKE"
+			},
+			"response": {
+				"toolCalls": [
+					{
+						"name": "execute_command",
+						"arguments": "{\"command\":\"printf 'Hello from test\\\\n' > execute-command-tool-fixture/simple-echo.txt\"}",
+						"id": "call_execute_command_simple_001"
+					}
+				]
+			}
+		},
+		{
+			"match": {
+				"userMessage": "EXECUTE_COMMAND_CWD_SMOKE"
+			},
+			"response": {
+				"toolCalls": [
+					{
+						"name": "execute_command",
+						"arguments": "{\"command\":\"printf 'Test in subdirectory\\\\n' > output.txt\",\"cwd\":\"execute-command-tool-fixture/custom-cwd\"}",
+						"id": "call_execute_command_cwd_001"
+					}
+				]
+			}
+		},
+		{
+			"match": {
+				"userMessage": "EXECUTE_COMMAND_MULTI_SMOKE"
+			},
+			"response": {
+				"toolCalls": [
+					{
+						"name": "execute_command",
+						"arguments": "{\"command\":\"printf 'Line 1\\\\n' > execute-command-tool-fixture/multi-command.txt\"}",
+						"id": "call_execute_command_multi_001"
+					}
+				]
+			}
+		},
+		{
+			"match": {
+				"userMessage": "EXECUTE_COMMAND_LONG_RUNNING_SMOKE"
+			},
+			"response": {
+				"toolCalls": [
+					{
+						"name": "execute_command",
+						"arguments": "{\"command\":\"node -e \\\"setTimeout(() => console.log('Command completed after delay'), 1000)\\\"\"}",
+						"id": "call_execute_command_long_running_001"
+					}
+				]
+			}
+		}
+	]
+}

apps/vscode-e2e/fixtures/write-to-file.json

@@ -0,0 +1,32 @@
+{
+	"fixtures": [
+		{
+			"match": {
+				"userMessage": "WRITE_TO_FILE_CREATE_SMOKE"
+			},
+			"response": {
+				"toolCalls": [
+					{
+						"name": "write_to_file",
+						"arguments": "{\"path\":\"write-to-file-tool-fixture/write-to-file-smoke.txt\",\"content\":\"Hello, this is a test file!\"}",
+						"id": "call_write_to_file_create_001"
+					}
+				]
+			}
+		},
+		{
+			"match": {
+				"userMessage": "WRITE_TO_FILE_NESTED_SMOKE"
+			},
+			"response": {
+				"toolCalls": [
+					{
+						"name": "write_to_file",
+						"arguments": "{\"path\":\"write-to-file-tool-fixture/nested/deep/directory/write-to-file-nested-smoke.txt\",\"content\":\"File in nested directory\"}",
+						"id": "call_write_to_file_nested_001"
+					}
+				]
+			}
+		}
+	]
+}

apps/vscode-e2e/src/fixtures/apply-diff.ts

@@ -0,0 +1,63 @@
+import { LLMock } from "@copilotkit/aimock"
+
+import { toolResultContains } from "./tool-result"
+
+type ApplyDiffFixture = {
+	toolCallId: string
+	expected: string[]
+	result: string
+	id: string
+}
+
+export function addApplyDiffResultFixtures(mock: InstanceType<typeof LLMock>) {
+	const fixtures: ApplyDiffFixture[] = [
+		{
+			toolCallId: "call_apply_diff_simple_001",
+			expected: ['"path":"apply-diff-tool-fixture/simple-modify.txt"', '"operation":"modified"'],
+			result: "Updated `apply-diff-tool-fixture/simple-modify.txt` to say `Hello Universe`.",
+			id: "call_apply_diff_simple_002",
+		},
+		{
+			toolCallId: "call_apply_diff_multi_replace_001",
+			expected: ['"path":"apply-diff-tool-fixture/multiple-replace.js"', '"operation":"modified"'],
+			result: "Updated `apply-diff-tool-fixture/multiple-replace.js` with the renamed function, parameters, and return fields.",
+			id: "call_apply_diff_multi_replace_002",
+		},
+		{
+			toolCallId: "call_apply_diff_line_hints_001",
+			expected: ['"path":"apply-diff-tool-fixture/line-hints.js"', '"operation":"modified"'],
+			result: "Updated `apply-diff-tool-fixture/line-hints.js` so `oldFunction` became `newFunction` with the new log message.",
+			id: "call_apply_diff_line_hints_002",
+		},
+		{
+			toolCallId: "call_apply_diff_error_001",
+			expected: ["No sufficiently similar match found at line: 1", "This content does not exist"],
+			result: "The apply_diff operation on `apply-diff-tool-fixture/error-handling.txt` was rejected - the search content did not match any content in the file, so it was not modified.",
+			id: "call_apply_diff_error_002",
+		},
+		{
+			toolCallId: "call_apply_diff_multi_block_001",
+			expected: ['"path":"apply-diff-tool-fixture/multi-search-replace.js"', '"operation":"modified"'],
+			result: "Applied both search/replace blocks in `apply-diff-tool-fixture/multi-search-replace.js` to rename the two target functions.",
+			id: "call_apply_diff_multi_block_002",
+		},
+	]
+
+	for (const fixture of fixtures) {
+		mock.addFixture({
+			match: {
+				toolCallId: fixture.toolCallId,
+				predicate: (req) => toolResultContains(req, fixture.toolCallId, fixture.expected),
+			},
+			response: {
+				toolCalls: [
+					{
+						name: "attempt_completion",
+						arguments: JSON.stringify({ result: fixture.result }),
+						id: fixture.id,
+					},
+				],
+			},
+		})
+	}
+}

apps/vscode-e2e/src/fixtures/execute-command.ts

@@ -0,0 +1,101 @@
+import { LLMock } from "@copilotkit/aimock"
+
+import { toolResultContains } from "./tool-result"
+
+type ExecuteCommandToolCall = {
+	name: "execute_command" | "attempt_completion"
+	params: Record<string, unknown>
+	id: string
+}
+
+type ExecuteCommandFixture = {
+	toolCallId: string
+	expected: string[]
+	toolCalls: ExecuteCommandToolCall[]
+}
+
+export function addExecuteCommandResultFixtures(mock: InstanceType<typeof LLMock>) {
+	const fixtures: ExecuteCommandFixture[] = [
+		{
+			toolCallId: "call_execute_command_simple_001",
+			expected: ["Command executed in terminal within working directory '", "Exit code: 0\nOutput:\n"],
+			toolCalls: [
+				{
+					name: "attempt_completion",
+					params: {
+						result: "Ran the echo command and created `execute-command-tool-fixture/simple-echo.txt`.",
+					},
+					id: "call_execute_command_simple_002",
+				},
+			],
+		},
+		{
+			toolCallId: "call_execute_command_cwd_001",
+			expected: ["execute-command-tool-fixture/custom-cwd'. Exit code: 0", "Output:\n"],
+			toolCalls: [
+				{
+					name: "attempt_completion",
+					params: {
+						result: "Ran the command inside `execute-command-tool-fixture/custom-cwd` and created `output.txt`.",
+					},
+					id: "call_execute_command_cwd_002",
+				},
+			],
+		},
+		{
+			toolCallId: "call_execute_command_multi_001",
+			expected: ["Command executed in terminal within working directory '", "Exit code: 0\nOutput:\n"],
+			toolCalls: [
+				{
+					name: "execute_command",
+					params: {
+						command: "printf 'Line 2\\n' >> execute-command-tool-fixture/multi-command.txt",
+					},
+					id: "call_execute_command_multi_002",
+				},
+			],
+		},
+		{
+			toolCallId: "call_execute_command_multi_002",
+			expected: ["Command executed in terminal within working directory '", "Exit code: 0\nOutput:\n"],
+			toolCalls: [
+				{
+					name: "attempt_completion",
+					params: {
+						result: "Ran both commands and populated `execute-command-tool-fixture/multi-command.txt` with two lines.",
+					},
+					id: "call_execute_command_multi_003",
+				},
+			],
+		},
+		{
+			toolCallId: "call_execute_command_long_running_001",
+			expected: ["Exit code: 0", "Command completed after delay"],
+			toolCalls: [
+				{
+					name: "attempt_completion",
+					params: {
+						result: "The delayed command completed and printed `Command completed after delay`.",
+					},
+					id: "call_execute_command_long_running_002",
+				},
+			],
+		},
+	]
+
+	for (const fixture of fixtures) {
+		mock.addFixture({
+			match: {
+				toolCallId: fixture.toolCallId,
+				predicate: (req) => toolResultContains(req, fixture.toolCallId, fixture.expected),
+			},
+			response: {
+				toolCalls: fixture.toolCalls.map((toolCall) => ({
+					name: toolCall.name,
+					arguments: JSON.stringify(toolCall.params),
+					id: toolCall.id,
+				})),
+			},
+		})
+	}
+}

apps/vscode-e2e/src/fixtures/read-file.ts

@@ -1,7 +1,11 @@
 import { LLMock } from "@copilotkit/aimock"
-import type { ChatCompletionRequest, ChatMessage } from "@copilotkit/aimock"
 
-type ToolResultExpectation = { toolCallId: string; expected: string[] }
+import {
+	isToolResultExpectation,
+	toolResultContains,
+	toolResultsContain,
+	type ToolResultExpectation,
+} from "./tool-result"
 
 type ReadFileResultFixture = {
 	toolCallId: string
@@ -10,28 +14,6 @@ type ReadFileResultFixture = {
 	id: string
 }
 
-function isToolResultExpectation(value: unknown): value is ToolResultExpectation {
-	return typeof value === "object" && value !== null && "toolCallId" in value && "expected" in value
-}
-
-function toolResultContains(req: ChatCompletionRequest, toolCallId: string, expected: string[]) {
-	const messages = Array.isArray(req?.messages) ? req.messages : []
-	const toolMessage = messages.find(
-		(message: ChatMessage) => message?.role === "tool" && message.tool_call_id === toolCallId,
-	)
-
-	const content = toolMessage?.content
-	if (typeof content !== "string") {
-		return false
-	}
-
-	return expected.every((text) => content.includes(text))
-}
-
-function toolResultsContain(req: ChatCompletionRequest, expectations: ToolResultExpectation[]) {
-	return expectations.every(({ toolCallId, expected }) => toolResultContains(req, toolCallId, expected))
-}
-
 export function addReadFileResultFixtures(mock: InstanceType<typeof LLMock>) {
 	const fixtures: ReadFileResultFixture[] = [
 		{