Skip to content

Prevent Claude tool results from triggering failure classifiers #60349

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pelikhan merged 3 commits into from
Sep 12, 2026
+71 −23
Merged

Prevent Claude tool results from triggering failure classifiers #60349

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8666e3b
Initial plan
Copilot Sep 11, 2026
0ffba25
Filter Claude tool results from failure classifiers
Copilot Sep 11, 2026
5773504
Merge branch 'main' into copilot/fix-authentication-failure-classifiers
pelikhan Sep 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 42 additions & 21 deletions actions/setup/js/claude_harness.cjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,27 @@ function isRateLimitError(output) {
return RATE_LIMIT_ERROR_PATTERN.test(output);
}

/**
* Remove stream-JSON user events, which contain tool results, from output used
* by failure classifiers.
*
* @param {string} output - Collected stdout+stderr from the process
* @returns {string}
*/
function classifiableOutput(output) {
return output
.split("\n")
.filter(line => {
if (!line.startsWith("{")) return true;
try {
return JSON.parse(line).type !== "user";
} catch {
return true;
}
})
.join("\n");
}

/**
* Determines if the collected output signals a clean max-turns exit.
* When Claude Code hits its turn limit it emits a result object with
Expand Down Expand Up @@ -490,9 +511,10 @@ async function main() {
return runProcess({ command, args: currentArgs, attempt, log, logArgs, env: childEnv });
},
handleFailure: ({ attempt, result }) => {
const isOverloaded = isOverloadedError(result.output);
const isRateLimit = isRateLimitError(result.output);
const isAuthenticationFailed = isAuthenticationFailedError(result.output);
const classifierOutput = classifiableOutput(result.output);
const isOverloaded = isOverloadedError(classifierOutput);
const isRateLimit = isRateLimitError(classifierOutput);
const isAuthenticationFailed = isAuthenticationFailedError(classifierOutput);
const isMaxTurns = isMaxTurnsExit(result.output);
const isNoDeferredMarker = isNoDeferredMarkerError(result.output);
const isInvalidModel = isInvalidModelError(result.output);
Expand All @@ -502,8 +524,8 @@ async function main() {
// observed, it stays true for the remainder of this session's --continue attempts, even
// if a later attempt's own output contains nothing but startup/transport errors.
sessionHasProgress = sessionHasProgress || hasClaudeSessionProgress(result.output);
const permissionDeniedCount = countPermissionDeniedIssues(result.output);
const hasNumerousPermissionDenied = hasNumerousPermissionDeniedIssues(result.output);
const permissionDeniedCount = countPermissionDeniedIssues(classifierOutput);
const hasNumerousPermissionDenied = hasNumerousPermissionDeniedIssues(classifierOutput);
const crashSignalName = crashSignalNameForExitCode(result.exitCode);
log(
`attempt ${attempt + 1} failed:` +
Expand All @@ -528,7 +550,7 @@ async function main() {
return { action: "stop", exitCode: 0 };
}

const nonRetryableGuard = detectNonRetryableHarnessGuard(result.output);
const nonRetryableGuard = detectNonRetryableHarnessGuard(classifierOutput);
const proxyAICreditsRejection = parseAICreditsExceededProxyRejection(result.output);
if (proxyAICreditsRejection) {
log(`attempt ${attempt + 1}: AWF API proxy rejected the request with HTTP 403 max-AI-credits (${proxyAICreditsRejection.aiCredits}/${proxyAICreditsRejection.maxAICredits}) — trusted budget-abort evidence`);
Expand Down Expand Up @@ -558,6 +580,18 @@ async function main() {
return { action: "stop" };
}

const isSignalTermination = isSignalTerminationExitCode(result.exitCode);
const isCrashSignal = isCrashSignalExitCode(result.exitCode);
if (attempt < maxRetries && result.hasOutput && (isSignalTermination || isCrashSignal)) {
continueDisabledPermanently = true;
useContinueOnRetry = false;
const reason = isCrashSignal
? `fatal-signal crash exitCode=${result.exitCode} (signal=${crashSignalName}, failure_reason=sandbox_runtime_crash)`
: `signal-style termination exitCode=${result.exitCode} (failure_reason=cancelled_or_timed_out)`;
log(`attempt ${attempt + 1}: ${reason} — will retry with fresh run (--continue disabled permanently) (attempt ${attempt + 2}/${maxRetries + 1})`);
return { action: "retry" };
}

if (attempt === 0 && isAuthenticationFailed) {
log(`attempt ${attempt + 1}: authentication failed — not retrying (first-attempt auth failure is non-retryable)`);
return { action: "stop" };
Expand Down Expand Up @@ -627,9 +661,6 @@ async function main() {
}

if (attempt < maxRetries && result.hasOutput) {
const isSignalTermination = isSignalTerminationExitCode(result.exitCode);
const isCrashSignal = isCrashSignalExitCode(result.exitCode);
const crashSignalName = crashSignalNameForExitCode(result.exitCode);
const retryWithContinue = shouldRetryWithContinue({
attempt,
maxRetries,
Expand All @@ -638,18 +669,7 @@ async function main() {
isNoDeferredMarker,
continueDisabledPermanently,
});
if (isSignalTermination || isCrashSignal) {
continueDisabledPermanently = true;
}
const reason = isCrashSignal
? `fatal-signal crash exitCode=${result.exitCode} (signal=${crashSignalName}, failure_reason=sandbox_runtime_crash)`
: isSignalTermination
? `signal-style termination exitCode=${result.exitCode} (failure_reason=cancelled_or_timed_out)`
: isOverloaded
? "overloaded_error (transient)"
: isRateLimit
? "rate_limit_error (transient)"
: "partial execution";
const reason = isOverloaded ? "overloaded_error (transient)" : isRateLimit ? "rate_limit_error (transient)" : "partial execution";
useContinueOnRetry = retryWithContinue;
const retryMode = retryWithContinue ? "--continue" : "fresh run (--continue disabled permanently)";
log(`attempt ${attempt + 1}: ${reason} — will retry with ${retryMode} (attempt ${attempt + 2}/${maxRetries + 1})`);
Expand Down Expand Up @@ -688,6 +708,7 @@ if (typeof module !== "undefined" && module.exports) {
module.exports = {
resolveClaudePromptFileArgs,
stripPromptFileArgs,
classifiableOutput,
isRateLimitError,
isAuthenticationFailedError,
isMaxTurnsExit,
Expand Down
31 changes: 29 additions & 2 deletions actions/setup/js/claude_harness.test.cjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ const require = createRequire(import.meta.url);
const {
resolveClaudePromptFileArgs,
stripPromptFileArgs,
classifiableOutput,
isRateLimitError,
isAuthenticationFailedError,
isMaxTurnsExit,
Expand Down Expand Up @@ -196,6 +197,16 @@ describe("claude_harness.cjs", () => {
});
});

describe("classifiableOutput", () => {
it("drops user events while preserving result events and plain text", () => {
const userEvent = JSON.stringify({ type: "user", message: { content: [{ type: "tool_result", content: "not logged in" }] } });
const resultEvent = JSON.stringify({ type: "result", result: "not logged in" });

expect(classifiableOutput(`${userEvent}\n${resultEvent}\nplain-text error`)).toBe(`${resultEvent}\nplain-text error`);
expect(isRateLimitError(classifiableOutput(JSON.stringify({ type: "user", result: "rate limiting" })))).toBe(false);
});
});

describe("isAuthenticationFailedError", () => {
it("returns true for authentication failed with request id", () => {
expect(isAuthenticationFailedError("Authentication failed (Request ID: C818:3ED713:19D401B:1C446B7:69D653CA)")).toBe(true);
Expand All @@ -218,6 +229,15 @@ describe("claude_harness.cjs", () => {
expect(isAuthenticationFailedError("NOT LOGGED IN")).toBe(true);
});

it("ignores authentication text in user tool results", () => {
const userEvent = JSON.stringify({ type: "user", message: { content: [{ type: "tool_result", content: "not logged in" }] } });
const resultEvent = JSON.stringify({ type: "result", result: "not logged in" });

expect(isAuthenticationFailedError(classifiableOutput(userEvent))).toBe(false);
expect(isAuthenticationFailedError(classifiableOutput(resultEvent))).toBe(true);
expect(isAuthenticationFailedError(classifiableOutput("not logged in"))).toBe(true);
});

describe("isInvalidModelError", () => {
it("returns true for model-not-supported errors", () => {
expect(isInvalidModelError("Execution failed: CAPIError: 400 The requested model is not supported.")).toBe(true);
Expand Down Expand Up @@ -384,6 +404,11 @@ describe("claude_harness.cjs", () => {
expect(hasNumerousPermissionDeniedIssues("permission denied")).toBe(false);
});

it("ignores repeated permission-denied signals in user tool results", () => {
const output = [1, 2, 3].map(index => JSON.stringify({ type: "user", tool_use_result: `upload ${index}: EACCES` })).join("\n");
expect(hasNumerousPermissionDeniedIssues(classifiableOutput(output))).toBe(false);
});

it("builds missing_tool payload for permission issues", () => {
const payload = JSON.parse(buildMissingToolPermissionIssuePayload());
expect(payload.type).toBe("missing_tool");
Expand Down Expand Up @@ -577,7 +602,7 @@ process.exit(0);
expect(calls.map(call => call.args.includes("--continue"))).toEqual([true, true, false]);
}, 50000);

it("uses a fresh retry after signal-style termination instead of --continue", () => {
it("uses a fresh retry after signal-style termination even when output is classified as an auth failure", () => {
const stubScript = `
const fs = require("fs");
const callsPath = process.env.CLAUDE_HARNESS_STUB_CALLS;
Expand All @@ -586,7 +611,8 @@ const priorCalls = fs.existsSync(callsPath) ? fs.readFileSync(callsPath, "utf8")
fs.appendFileSync(callsPath, JSON.stringify({ args }) + "\\n", "utf8");

if (priorCalls === 0) {
process.stdout.write("partial execution before SIGTERM-style exit\\n");
process.stdout.write('{"type":"user","message":{"content":[{"type":"tool_result","content":"not logged in"}]}}\\n');
process.stdout.write('{"type":"result","result":"not logged in"}\\n');
process.exit(143);
}

Expand All @@ -603,6 +629,7 @@ process.exit(0);
expect(calls.map(call => call.args.includes("--continue"))).toEqual([false, false]);
expect(calls[1].args).toContain("fix the bug");
expect(result.stderr).toContain("failure_reason=cancelled_or_timed_out");
expect(result.stderr).toContain("isAuthenticationFailedError=true");
}, 30000);

it("retries a connection-refused failure before the first assistant response as a fresh run", () => {
Expand Down
Loading