diff --git a/actions/setup/js/copilot_harness.cjs b/actions/setup/js/copilot_harness.cjs index 68b2cf057de..84658e5a6b1 100644 --- a/actions/setup/js/copilot_harness.cjs +++ b/actions/setup/js/copilot_harness.cjs @@ -49,7 +49,7 @@ "use strict"; -const { iterateJsonl } = require("./jsonl_helpers.cjs"); +const { iterateJsonl, readJsonlFile } = require("./jsonl_helpers.cjs"); require("./shim.cjs"); const { getErrorMessage } = require("./error_helpers.cjs"); @@ -105,7 +105,7 @@ const COPILOT_ROUTING_POLICY = getAWFModelRoutingPolicy("copilot"); const { loadModelsJson } = require("./model_costs.cjs"); const { applyCopilotWireAPI: configureCopilotWireAPI, warnCopilotSubagentEndpoints, recordModelEndpointMismatch, getSupportedEndpoints, MODEL_ENDPOINT_FIX } = require("./copilot_wire_api.cjs"); const { loadModelAliasMap, resolveEngineModel, normalizeForCopilotCLI, ModelAliasResolutionError, recordModelResolution, splitModelIdentifier } = require("./resolve_model_alias.cjs"); -const { COPILOT_MODEL_VERIFICATION_EXIT_CODE, createCopilotModelObserver, readCopilotModelSubstitution, recordCopilotModelSelection } = require("./copilot_model_selection.cjs"); +const { COPILOT_MODEL_VERIFICATION_EXIT_CODE, copilotSessionStateDirectories, createCopilotModelObserver, readCopilotModelSubstitution, recordCopilotModelSelection } = require("./copilot_model_selection.cjs"); const { getEngineSubAgentTarget } = require("./extract_inline_sub_agents.cjs"); const { inlineArtifactTarget } = require("./inline_artifact_target.cjs"); const MODEL_NOT_SUPPORTED_PATTERN = /The requested model is not supported|No model available\b[^\n]*policy enablement/i; @@ -1216,14 +1216,59 @@ function extractTokenCountFromOutput(output) { /** @param {string} output @returns {boolean} */ function isCopilotContentFilterError(output) { - for (const event of iterateJsonl(output)) { - if (event?.type !== "assistant.message") continue; + for (const event of iterateJsonl(output, { valueFilter: event => event?.type === "assistant.message" })) { const context = copilotSessionContext(event); if (context.agentId === undefined && !context.parentToolUseId && getCopilotMessageRefusal(event.data)?.reason === "content_filter") return true; } return false; } +/** @param {Iterable} events @returns {boolean} */ +function isCopilotToolSearchOnlyCompletion(events) { + let message; + for (const event of events) { + const context = copilotSessionContext(event); + if (context.agentId !== undefined || context.parentToolUseId) continue; + if (event?.type === "assistant.message") message = event.data; + if (event?.type === "assistant.refusal") message = undefined; + } + return ( + !!message && + (message.content === "" || message.content === null) && + Array.isArray(message.toolRequests) && + message.toolRequests.length === 0 && + !getCopilotMessageRefusal(message) && + message.serverTools?.provider === "openai-responses" && + Array.isArray(message.serverTools.items) && + message.serverTools.items.length > 0 && + message.serverTools.items.every(item => item?.type === "tool_search_call" && item.execution === "server" && item.status === "completed") + ); +} + +const copilotAssistantResponseLineFilter = line => /"type"\s*:\s*"assistant\.(?:message|refusal)"/.test(line); +const copilotAssistantResponseFilter = event => event?.type === "assistant.message" || event?.type === "assistant.refusal"; + +/** + * Native CLI stdout is plain text; inspect only this invocation's exact session. + * Never select another session by recency in a shared HOME. + * @param {string} output + * @param {string | undefined} sessionId + * @returns {boolean} + */ +function hasCopilotToolSearchOnlyCompletion(output, sessionId) { + if (sessionId && /^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i.test(sessionId)) { + for (const directory of copilotSessionStateDirectories()) { + try { + return isCopilotToolSearchOnlyCompletion(readJsonlFile(path.join(directory, sessionId, "events.jsonl"), { strict: true, valueFilter: copilotAssistantResponseFilter })); + } catch (error) { + const missing = error !== null && typeof error === "object" && "code" in error && error.code === "ENOENT"; + if (!missing) log("Could not inspect Copilot completion events; leaving missing-output validation to the collector"); + } + } + } + return isCopilotToolSearchOnlyCompletion(iterateJsonl(output, { lineFilter: copilotAssistantResponseLineFilter, valueFilter: copilotAssistantResponseFilter })); +} + /** * Classify a failed Copilot attempt into a short, named failure class. * @param {{ @@ -2108,6 +2153,13 @@ async function main() { result.runtimeGuardReason = "copilot_model_verification"; if (modelSubstitution) result.output += `\n${modelSubstitution}`; } + if (result.exitCode === 0 && (!safeOutputsPath || !hasTerminalSafeOutput(safeOutputsPath)) && hasCopilotToolSearchOnlyCompletion(result.output, sessionId)) { + result.exitCode = 1; + result.runtimeGuardReason = "copilot_tool_search_only_completion"; + log( + "Copilot stopped after server-side tool discovery without text, a local tool request, or a terminal safe output. Continuation belongs to Copilot CLI/upstream Responses handling; task completion is unverified. Not retrying to avoid duplicate side effects." + ); + } lastHasOutput = result.hasOutput; const attemptDetections = detectCopilotErrors(result.output); detectedCopilotErrors.inferenceAccessError ||= attemptDetections.inferenceAccessError; @@ -2118,6 +2170,10 @@ async function main() { return result; }, handleFailure: async ({ attempt, result }) => { + if (result.runtimeGuardReason === "copilot_tool_search_only_completion") { + result.failureClasses = ["copilot_tool_search_only_completion"]; + return { action: "stop" }; + } if (result.runtimeGuardReason === "copilot_model_verification") { result.failureClasses = ["copilot_model_verification"]; return { action: "stop" }; @@ -2566,6 +2622,8 @@ if (typeof module !== "undefined" && module.exports) { detectCopilotErrors, classifyCopilotFailure, isCopilotContentFilterError, + isCopilotToolSearchOnlyCompletion, + hasCopilotToolSearchOnlyCompletion, extractTokenCountFromOutput, shouldRetryFailedExecution, shouldContinueCopilotSessionAfterFailure, diff --git a/actions/setup/js/copilot_harness.test.cjs b/actions/setup/js/copilot_harness.test.cjs index 89f14ee03c4..ca0e3b2f90b 100644 --- a/actions/setup/js/copilot_harness.test.cjs +++ b/actions/setup/js/copilot_harness.test.cjs @@ -15,6 +15,8 @@ const { buildMissingToolPermissionIssuePayload, classifyCopilotFailure, isCopilotContentFilterError, + isCopilotToolSearchOnlyCompletion, + hasCopilotToolSearchOnlyCompletion, extractTokenCountFromOutput, buildMissingToolAlternatives, buildInfrastructureIncompletePayload, @@ -117,6 +119,115 @@ describe("native invocation model and session selection", () => { }); }); +describe("Copilot tool-search-only completion", () => { + // Shape observed in dotnet/android run 40700757014; no opaque IDs or prompt payloads. + const search = { + type: "assistant.message", + data: { + content: "", + toolRequests: [], + serverTools: { provider: "openai-responses", items: [{ type: "tool_search_call", execution: "server", status: "completed" }] }, + }, + }; + const tempDirs = []; + afterEach(() => { + vi.unstubAllEnvs(); + for (const dir of tempDirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); + }); + + it.each(["", null])("recognizes an empty final main response (%j)", content => { + expect(isCopilotToolSearchOnlyCompletion([{ ...search, data: { ...search.data, content } }, { type: "session.shutdown" }])).toBe(true); + }); + + it.each([ + { content: "Review complete." }, + { toolRequests: [{ name: "github.pull_request_read" }] }, + { refusal: "Cannot comply." }, + { serverTools: { provider: "openai-responses" } }, + { serverTools: { provider: "openai-responses", items: [{ type: "tool_search_call", execution: "server", status: "in_progress" }] } }, + { serverTools: { provider: "openai-responses", items: [{ type: "web_search_call", execution: "server", status: "completed" }] } }, + ])("does not mistake another response for discovery-only completion (%j)", overrides => { + expect(isCopilotToolSearchOnlyCompletion([{ ...search, data: { ...search.data, ...overrides } }])).toBe(false); + }); + + it("uses the last main assistant message and ignores subagents", () => { + const done = { type: "assistant.message", data: { content: "Done." } }; + expect(isCopilotToolSearchOnlyCompletion([search, done])).toBe(false); + expect(isCopilotToolSearchOnlyCompletion([done, { ...search, agentId: "child" }])).toBe(false); + expect(isCopilotToolSearchOnlyCompletion([done, { ...search, data: { ...search.data, parentToolCallId: "child-call" } }])).toBe(false); + expect(isCopilotToolSearchOnlyCompletion([search, { ...done, agentId: "child" }])).toBe(true); + expect(isCopilotToolSearchOnlyCompletion([search, { type: "assistant.refusal", data: { reason: "refusal" } }])).toBe(false); + expect(isCopilotToolSearchOnlyCompletion([])).toBe(false); + }); + + it.each(['{"type":"assistant.message"', '{"type":"assistant.mess', '{"type":"session.shut'])("reads only the exact native session and rejects truncated records (%s)", truncated => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "copilot-search-session-")); + tempDirs.push(root); + vi.stubEnv("COPILOT_HOME", root); + vi.stubEnv("GH_AW_TMP_DIR", root); + const sessionId = "aabbccdd-1122-3344-5566-778899aabbcc"; + const dir = path.join(root, "session-state", sessionId); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, "events.jsonl"), [{ type: "session.start" }, search, { type: "session.shutdown" }].map(event => JSON.stringify(event)).join("\n") + "\n"); + expect(hasCopilotToolSearchOnlyCompletion("plain CLI output", sessionId)).toBe(true); + expect(hasCopilotToolSearchOnlyCompletion("plain CLI output", "11223344-1122-3344-5566-778899aabbcc")).toBe(false); + expect(hasCopilotToolSearchOnlyCompletion("plain CLI output", "../other")).toBe(false); + expect(hasCopilotToolSearchOnlyCompletion(`${JSON.stringify({ type: "session.start" })}\n${JSON.stringify(search)}`, undefined)).toBe(true); + fs.appendFileSync(path.join(dir, "events.jsonl"), truncated); + expect(hasCopilotToolSearchOnlyCompletion("plain CLI output", sessionId)).toBe(false); + }); + + it.each([false, true])("native zero exit is rejected only without a terminal safe output (terminal=%s)", terminal => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "copilot-search-exit-")); + tempDirs.push(root); + const stub = path.join(root, "stub.cjs"); + const safeOutputs = path.join(root, "safe-outputs.jsonl"); + const configPath = path.join(root, "awf-config.json"); + fs.writeFileSync(configPath, "{}"); + fs.writeFileSync(safeOutputs, terminal ? '{"type":"submit_pull_request_review"}\n' : ""); + fs.writeFileSync( + stub, + `const fs = require("fs"), path = require("path"); +const args = process.argv.slice(2); +const id = args[args.indexOf("--session-id") + 1]; +const dir = path.join(process.env.COPILOT_HOME, "session-state", id); +fs.mkdirSync(dir, { recursive: true }); +fs.writeFileSync(path.join(dir, "events.jsonl"), JSON.stringify(${JSON.stringify({ ...search, data: { ...search.data, model: "gpt-6.1-sol" } })}) + "\\n"); +process.stdout.write("Skill loaded successfully.\\n");` + ); + const result = spawnSync(process.execPath, [require.resolve("./copilot_harness.cjs"), process.execPath, stub], { + cwd: root, + env: { + PATH: process.env.PATH, + HOME: root, + TMPDIR: root, + GITHUB_WORKSPACE: root, + GH_AW_ENGINE_CWD: root, + GH_AW_AWF_CONFIG_PATH: configPath, + GH_AW_HARNESS_INITIAL_DELAY_MS: "1", + GH_AW_HARNESS_MAX_DELAY_MS: "1", + COPILOT_HOME: root, + COPILOT_MODEL: "gpt-6.1-sol", + GH_AW_PHASE: "agent", + GH_AW_TMP_DIR: root, + GH_AW_SAFE_OUTPUTS: safeOutputs, + GH_AW_MODEL_ROUTING: "0", + AWF_REFLECT_ENABLED: "0", + }, + encoding: "utf8", + timeout: 10000, + }); + expect(result.status, result.stderr).toBe(terminal ? 0 : 1); + expect(result.stderr).toContain(terminal ? "success on attempt 1" : "Copilot stopped after server-side tool discovery"); + expect(result.stderr).not.toContain("retry 1/"); + expect(fs.readFileSync(safeOutputs, "utf8")).toBe(terminal ? '{"type":"submit_pull_request_review"}\n' : ""); + if (!terminal) { + const retry = JSON.parse(fs.readFileSync(path.join(root, "agent/harness_retry.json"), "utf8")); + expect(retry.attempt_results[0].failure_classes).toEqual(["copilot_tool_search_only_completion"]); + } + }); +}); + const { detectNonRetryableHarnessGuard, buildSoftTimeoutGuard } = require("./harness_retry_guard.cjs"); const { detectErrors } = require("./agent_error_patterns.cjs"); diff --git a/actions/setup/js/jsonl_helpers.cjs b/actions/setup/js/jsonl_helpers.cjs index 94516ced21f..e8ae48f6e04 100644 --- a/actions/setup/js/jsonl_helpers.cjs +++ b/actions/setup/js/jsonl_helpers.cjs @@ -8,6 +8,7 @@ const { prepareJsonSchema } = require("./json_schema_validation.cjs"); * @typedef {object} JsonlOptions * @property {(line: string, lineNumber: number) => boolean} [lineFilter] Receives trimmed text before transformation and decoding. * @property {(line: string, lineNumber: number) => string} [lineTransform] Transforms trimmed record text after filtering, retaining the original rawLine. + * @property {(value: any, lineNumber: number) => boolean} [valueFilter] Selects decoded, schema-validated values without skipping invalid records. * @property {(error: Error, lineNumber: number) => void} [onError] Reports rejected nonblank lines without exposing their contents. * @property {number} [maxLineLength] Optional limit in UTF-16 code units; unlimited by default. * @property {{field: string, value: string|number|boolean|null}} [fieldFilter] Match an own top-level field, prefiltering text before decoding. @@ -246,6 +247,7 @@ function parseEvent(rawLine, lineNumber, options, fieldFilter, schemaValidator) if (fieldFilter && (!value || typeof value !== "object" || Array.isArray(value) || !Object.hasOwn(value, fieldFilter.field) || value[fieldFilter.field] !== fieldFilter.value)) return; const validationError = schemaValidator?.(value, line); if (validationError) return invalidEvent(rawLine, lineNumber, "schema", options, validationError); + if (options.valueFilter && !options.valueFilter(value, lineNumber)) return; return { type: "json", value, rawLine, lineNumber }; } return invalidEvent(rawLine, lineNumber, tooLong ? "too-long" : "syntax", options); diff --git a/actions/setup/js/jsonl_helpers.test.cjs b/actions/setup/js/jsonl_helpers.test.cjs index c42f99e6fe6..82359b3f863 100644 --- a/actions/setup/js/jsonl_helpers.test.cjs +++ b/actions/setup/js/jsonl_helpers.test.cjs @@ -14,6 +14,64 @@ async function collect(source) { } describe("jsonl_helpers", () => { + describe("parsed-value filtering", () => { + const valueFilter = value => value?.type === "wanted"; + + it("filters decoded values consistently across text, files, and streams", async () => { + const text = 'null\nfalse\n0\n{"nested":{"type":"wanted"}}\n{"type":"other"}\n{"\\u0074ype":"wanted","id":7}\n'; + const expected = [{ type: "wanted", id: 7 }]; + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "gh-aw-jsonl-value-filter-")); + const file = path.join(directory, "records.jsonl"); + try { + fs.writeFileSync(file, text); + expect([...iterateJsonl(text, { valueFilter })]).toEqual(expected); + expect([...readJsonlFile(file, { valueFilter })]).toEqual(expected); + expect(await collect(streamJsonl([text], { valueFilter }))).toEqual(expected); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it("stops decoding and closes the file after the first parsed match", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "gh-aw-jsonl-value-search-")); + const file = path.join(directory, "records.jsonl"); + fs.writeFileSync(file, '{"type":"other"}\n{"type":"wanted"}\n{"broken":}\n'); + const filter = vi.fn(valueFilter); + const parse = vi.spyOn(JSON, "parse"); + const close = vi.spyOn(fs, "closeSync"); + try { + for (const record of readJsonlFile(file, { strict: true, valueFilter: filter })) { + expect(record).toEqual({ type: "wanted" }); + break; + } + expect(filter.mock.calls).toEqual([ + [{ type: "other" }, 1], + [{ type: "wanted" }, 2], + ]); + expect(parse).toHaveBeenCalledTimes(2); + expect(close).toHaveBeenCalledTimes(1); + } finally { + parse.mockRestore(); + close.mockRestore(); + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it("preserves strict validation before filtering and propagates predicate errors", () => { + const filter = vi.fn(valueFilter); + expect(() => [...iterateJsonl('{"type":"wanted"}\n{"type":"oth', { strict: true, valueFilter: filter })]).toThrow(SyntaxError); + expect(filter).toHaveBeenCalledTimes(1); + const error = new Error("predicate failed"); + expect(() => [ + ...iterateJsonl("null", { + valueFilter: () => { + throw error; + }, + }), + ]).toThrow(error); + }); + }); + describe("field-value search", () => { const options = { fieldFilter: { field: "type", value: "wanted" } };