From 8666e3b370bde61817eb124067b15744522055a1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:35:14 +0000 Subject: [PATCH 1/2] Initial plan From 0ffba25118f28f7f7c94b03f35cc41cc26d6d365 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:41:54 +0000 Subject: [PATCH 2/2] Filter Claude tool results from failure classifiers Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/claude_harness.cjs | 63 ++++++++++++++++-------- actions/setup/js/claude_harness.test.cjs | 31 +++++++++++- 2 files changed, 71 insertions(+), 23 deletions(-) diff --git a/actions/setup/js/claude_harness.cjs b/actions/setup/js/claude_harness.cjs index ea5ecab20d8..8e5d3b20ba8 100644 --- a/actions/setup/js/claude_harness.cjs +++ b/actions/setup/js/claude_harness.cjs @@ -169,6 +169,27 @@ function isRateLimitError(output) { return RATE_LIMIT_ERROR_PATTERN.test(output); } +/** + * Remove stream-JSON user events, which contain tool results, from output used + * by failure classifiers. + * + * @param {string} output - Collected stdout+stderr from the process + * @returns {string} + */ +function classifiableOutput(output) { + return output + .split("\n") + .filter(line => { + if (!line.startsWith("{")) return true; + try { + return JSON.parse(line).type !== "user"; + } catch { + return true; + } + }) + .join("\n"); +} + /** * Determines if the collected output signals a clean max-turns exit. * When Claude Code hits its turn limit it emits a result object with @@ -490,9 +511,10 @@ async function main() { return runProcess({ command, args: currentArgs, attempt, log, logArgs, env: childEnv }); }, handleFailure: ({ attempt, result }) => { - const isOverloaded = isOverloadedError(result.output); - const isRateLimit = isRateLimitError(result.output); - const isAuthenticationFailed = isAuthenticationFailedError(result.output); + const classifierOutput = classifiableOutput(result.output); + const isOverloaded = isOverloadedError(classifierOutput); + const isRateLimit = isRateLimitError(classifierOutput); + const isAuthenticationFailed = isAuthenticationFailedError(classifierOutput); const isMaxTurns = isMaxTurnsExit(result.output); const isNoDeferredMarker = isNoDeferredMarkerError(result.output); const isInvalidModel = isInvalidModelError(result.output); @@ -502,8 +524,8 @@ async function main() { // observed, it stays true for the remainder of this session's --continue attempts, even // if a later attempt's own output contains nothing but startup/transport errors. sessionHasProgress = sessionHasProgress || hasClaudeSessionProgress(result.output); - const permissionDeniedCount = countPermissionDeniedIssues(result.output); - const hasNumerousPermissionDenied = hasNumerousPermissionDeniedIssues(result.output); + const permissionDeniedCount = countPermissionDeniedIssues(classifierOutput); + const hasNumerousPermissionDenied = hasNumerousPermissionDeniedIssues(classifierOutput); const crashSignalName = crashSignalNameForExitCode(result.exitCode); log( `attempt ${attempt + 1} failed:` + @@ -528,7 +550,7 @@ async function main() { return { action: "stop", exitCode: 0 }; } - const nonRetryableGuard = detectNonRetryableHarnessGuard(result.output); + const nonRetryableGuard = detectNonRetryableHarnessGuard(classifierOutput); const proxyAICreditsRejection = parseAICreditsExceededProxyRejection(result.output); if (proxyAICreditsRejection) { log(`attempt ${attempt + 1}: AWF API proxy rejected the request with HTTP 403 max-AI-credits (${proxyAICreditsRejection.aiCredits}/${proxyAICreditsRejection.maxAICredits}) — trusted budget-abort evidence`); @@ -558,6 +580,18 @@ async function main() { return { action: "stop" }; } + const isSignalTermination = isSignalTerminationExitCode(result.exitCode); + const isCrashSignal = isCrashSignalExitCode(result.exitCode); + if (attempt < maxRetries && result.hasOutput && (isSignalTermination || isCrashSignal)) { + continueDisabledPermanently = true; + useContinueOnRetry = false; + const reason = isCrashSignal + ? `fatal-signal crash exitCode=${result.exitCode} (signal=${crashSignalName}, failure_reason=sandbox_runtime_crash)` + : `signal-style termination exitCode=${result.exitCode} (failure_reason=cancelled_or_timed_out)`; + log(`attempt ${attempt + 1}: ${reason} — will retry with fresh run (--continue disabled permanently) (attempt ${attempt + 2}/${maxRetries + 1})`); + return { action: "retry" }; + } + if (attempt === 0 && isAuthenticationFailed) { log(`attempt ${attempt + 1}: authentication failed — not retrying (first-attempt auth failure is non-retryable)`); return { action: "stop" }; @@ -627,9 +661,6 @@ async function main() { } if (attempt < maxRetries && result.hasOutput) { - const isSignalTermination = isSignalTerminationExitCode(result.exitCode); - const isCrashSignal = isCrashSignalExitCode(result.exitCode); - const crashSignalName = crashSignalNameForExitCode(result.exitCode); const retryWithContinue = shouldRetryWithContinue({ attempt, maxRetries, @@ -638,18 +669,7 @@ async function main() { isNoDeferredMarker, continueDisabledPermanently, }); - if (isSignalTermination || isCrashSignal) { - continueDisabledPermanently = true; - } - const reason = isCrashSignal - ? `fatal-signal crash exitCode=${result.exitCode} (signal=${crashSignalName}, failure_reason=sandbox_runtime_crash)` - : isSignalTermination - ? `signal-style termination exitCode=${result.exitCode} (failure_reason=cancelled_or_timed_out)` - : isOverloaded - ? "overloaded_error (transient)" - : isRateLimit - ? "rate_limit_error (transient)" - : "partial execution"; + const reason = isOverloaded ? "overloaded_error (transient)" : isRateLimit ? "rate_limit_error (transient)" : "partial execution"; useContinueOnRetry = retryWithContinue; const retryMode = retryWithContinue ? "--continue" : "fresh run (--continue disabled permanently)"; log(`attempt ${attempt + 1}: ${reason} — will retry with ${retryMode} (attempt ${attempt + 2}/${maxRetries + 1})`); @@ -688,6 +708,7 @@ if (typeof module !== "undefined" && module.exports) { module.exports = { resolveClaudePromptFileArgs, stripPromptFileArgs, + classifiableOutput, isRateLimitError, isAuthenticationFailedError, isMaxTurnsExit, diff --git a/actions/setup/js/claude_harness.test.cjs b/actions/setup/js/claude_harness.test.cjs index 4c9192a43ab..be6cc9fc79e 100644 --- a/actions/setup/js/claude_harness.test.cjs +++ b/actions/setup/js/claude_harness.test.cjs @@ -9,6 +9,7 @@ const require = createRequire(import.meta.url); const { resolveClaudePromptFileArgs, stripPromptFileArgs, + classifiableOutput, isRateLimitError, isAuthenticationFailedError, isMaxTurnsExit, @@ -196,6 +197,16 @@ describe("claude_harness.cjs", () => { }); }); + describe("classifiableOutput", () => { + it("drops user events while preserving result events and plain text", () => { + const userEvent = JSON.stringify({ type: "user", message: { content: [{ type: "tool_result", content: "not logged in" }] } }); + const resultEvent = JSON.stringify({ type: "result", result: "not logged in" }); + + expect(classifiableOutput(`${userEvent}\n${resultEvent}\nplain-text error`)).toBe(`${resultEvent}\nplain-text error`); + expect(isRateLimitError(classifiableOutput(JSON.stringify({ type: "user", result: "rate limiting" })))).toBe(false); + }); + }); + describe("isAuthenticationFailedError", () => { it("returns true for authentication failed with request id", () => { expect(isAuthenticationFailedError("Authentication failed (Request ID: C818:3ED713:19D401B:1C446B7:69D653CA)")).toBe(true); @@ -218,6 +229,15 @@ describe("claude_harness.cjs", () => { expect(isAuthenticationFailedError("NOT LOGGED IN")).toBe(true); }); + it("ignores authentication text in user tool results", () => { + const userEvent = JSON.stringify({ type: "user", message: { content: [{ type: "tool_result", content: "not logged in" }] } }); + const resultEvent = JSON.stringify({ type: "result", result: "not logged in" }); + + expect(isAuthenticationFailedError(classifiableOutput(userEvent))).toBe(false); + expect(isAuthenticationFailedError(classifiableOutput(resultEvent))).toBe(true); + expect(isAuthenticationFailedError(classifiableOutput("not logged in"))).toBe(true); + }); + describe("isInvalidModelError", () => { it("returns true for model-not-supported errors", () => { expect(isInvalidModelError("Execution failed: CAPIError: 400 The requested model is not supported.")).toBe(true); @@ -384,6 +404,11 @@ describe("claude_harness.cjs", () => { expect(hasNumerousPermissionDeniedIssues("permission denied")).toBe(false); }); + it("ignores repeated permission-denied signals in user tool results", () => { + const output = [1, 2, 3].map(index => JSON.stringify({ type: "user", tool_use_result: `upload ${index}: EACCES` })).join("\n"); + expect(hasNumerousPermissionDeniedIssues(classifiableOutput(output))).toBe(false); + }); + it("builds missing_tool payload for permission issues", () => { const payload = JSON.parse(buildMissingToolPermissionIssuePayload()); expect(payload.type).toBe("missing_tool"); @@ -577,7 +602,7 @@ process.exit(0); expect(calls.map(call => call.args.includes("--continue"))).toEqual([true, true, false]); }, 50000); - it("uses a fresh retry after signal-style termination instead of --continue", () => { + it("uses a fresh retry after signal-style termination even when output is classified as an auth failure", () => { const stubScript = ` const fs = require("fs"); const callsPath = process.env.CLAUDE_HARNESS_STUB_CALLS; @@ -586,7 +611,8 @@ const priorCalls = fs.existsSync(callsPath) ? fs.readFileSync(callsPath, "utf8") fs.appendFileSync(callsPath, JSON.stringify({ args }) + "\\n", "utf8"); if (priorCalls === 0) { - process.stdout.write("partial execution before SIGTERM-style exit\\n"); + process.stdout.write('{"type":"user","message":{"content":[{"type":"tool_result","content":"not logged in"}]}}\\n'); + process.stdout.write('{"type":"result","result":"not logged in"}\\n'); process.exit(143); } @@ -603,6 +629,7 @@ process.exit(0); expect(calls.map(call => call.args.includes("--continue"))).toEqual([false, false]); expect(calls[1].args).toContain("fix the bug"); expect(result.stderr).toContain("failure_reason=cancelled_or_timed_out"); + expect(result.stderr).toContain("isAuthenticationFailedError=true"); }, 30000); it("retries a connection-refused failure before the first assistant response as a fresh run", () => {