From 60ed0941881e5fb5913768df5572f88783cc45c4 Mon Sep 17 00:00:00 2001 From: dovvnloading <157447210+dovvnloading@users.noreply.github.com> Date: Fri, 4 Sep 2026 10:31:46 -0400 Subject: [PATCH] Stop an interrupted tool call wedging a harness node forever backend/harness/loop.py appends the assistant turn - tool_calls and all - and persists it to the transcript BEFORE invoking any tool. An interruption in that window (a Stop, a timeout, a provider fault, the process dying) leaves the transcript ending on an assistant turn whose calls were never answered. Measured against the real transcript writer and loader: after a Stop before either tool ran, load_messages returns user("go"), assistant(tool_calls=[c1, c2]) with both calls unanswered. loop.py appends the follow-up user message after that, and the provider is handed assistant(tool_calls) -> user(...), which every major provider rejects: OpenAI requires a tool message per tool_call, Anthropic a tool_result per tool_use. The malformed turn is on disk, so every subsequent follow-up fails identically. The node is wedged permanently, and the only visible symptom is a provider error the user cannot act on. drop_leading_orphan_tools already handled the mirror case - a history that OPENS mid tool-sequence, from a tail cut. This is the other end, and it was missing. close_unanswered_tool_calls synthesizes a result for every requested call that has none, so the reloaded history always satisfies the one-result-per-call contract. Repaired on LOAD rather than when the run lands, deliberately: that fixes transcripts already on disk from before this existed, and covers interruption paths that never reach a landing handler at all. A synthetic result rather than dropping the assistant turn, because the turn is a real record of what the agent decided to do - dropping it would make the transcript lie about the run. Test plan: - 5 new tests: a fully interrupted turn gets synthetic results for both calls; a partially answered turn keeps the real result and fills only the gap; every requested call ends up answered (the invariant providers actually enforce); a healthy transcript is byte-for-byte untouched; and an assistant turn with no tool_calls is not disturbed. - Full suite: 3205 passed, 19 skipped. ruff clean. Co-Authored-By: Claude Opus 5 --- backend/harness/transcript.py | 66 +++++++++++++++++++++++++++- backend/tests/test_harness.py | 81 +++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 1 deletion(-) diff --git a/backend/harness/transcript.py b/backend/harness/transcript.py index 0f04f7bd..35455079 100644 --- a/backend/harness/transcript.py +++ b/backend/harness/transcript.py @@ -312,6 +312,70 @@ def drop_leading_orphan_tools(messages: list[dict]) -> list[dict]: return messages[index:] +_INTERRUPTED_TOOL_RESULT = ( + "Interrupted - the run stopped before this tool produced a result." +) + + +def close_unanswered_tool_calls(messages: list[dict]) -> list[dict]: + """Give every requested tool_call a result, synthesizing one where the run + never produced it. + + drop_leading_orphan_tools above handles a history that OPENS mid + tool-sequence. This is the other end, and it was missing. + + backend/harness/loop.py appends the assistant turn - tool_calls and all - + and persists it BEFORE invoking any tool, so an interruption between those + two points leaves the transcript ending on an assistant turn whose calls + were never answered. A Stop lands there, so does a timeout, a provider + fault, or the process dying. On the next follow-up load_messages returns + that turn and loop.py appends the new user message after it, producing: + + ... assistant(tool_calls=[c1, c2]), user("try again") + + which every major provider rejects - OpenAI requires a tool message per + tool_call, Anthropic requires a tool_result block per tool_use. The + follow-up fails, the bad turn is still on disk, and the node is wedged + permanently: every subsequent attempt fails the same way. + + Repairing on LOAD rather than when the run lands is deliberate. It fixes + transcripts already sitting on disk from before this existed, and it + covers interruption paths that never reach a landing handler at all (a + crash, a kill). A synthetic result rather than dropping the assistant + turn, because the turn is a real record of what the agent decided to do - + losing it would make the transcript lie about the run. + + Handles the partially-answered case too: a Stop between two calls leaves + the first answered and the second not. + """ + repaired: list[dict] = [] + index = 0 + while index < len(messages): + message = messages[index] + repaired.append(message) + index += 1 + calls = message.get("tool_calls") if message.get("role") == "assistant" else None + if not isinstance(calls, list) or not calls: + continue + answered: set[str] = set() + while index < len(messages) and messages[index].get("role") == "tool": + answered.add(str(messages[index].get("tool_call_id", ""))) + repaired.append(messages[index]) + index += 1 + for call in calls: + if not isinstance(call, dict): + continue + call_id = str(call.get("id", "")) + if call_id and call_id not in answered: + repaired.append({ + "role": "tool", + "tool_call_id": call_id, + "name": str(call.get("name", "")), + "content": _INTERRUPTED_TOOL_RESULT, + }) + return repaired + + def load_messages(workspace: Path) -> list[dict]: """The reload path: the last MAX_RELOADED_MESSAGES message lines, each content-capped, in file order. A missing file is an empty history (a @@ -359,4 +423,4 @@ def load_messages(workspace: Path) -> list[dict]: return [] if len(messages) > MAX_RELOADED_MESSAGES: messages = messages[-MAX_RELOADED_MESSAGES:] - return drop_leading_orphan_tools(messages) + return close_unanswered_tool_calls(drop_leading_orphan_tools(messages)) diff --git a/backend/tests/test_harness.py b/backend/tests/test_harness.py index 2d82e91f..7820f4b5 100644 --- a/backend/tests/test_harness.py +++ b/backend/tests/test_harness.py @@ -663,3 +663,84 @@ def test_blank_workspace_id_is_self_healed_on_load(self): payload["workspace_id"] = "" restored = _restore_harness_payload(payload) assert restored.state.harness_workspace_id + + +class TestInterruptedToolCallsAreClosedOnReload: + """A run interrupted between "the model asked for tools" and "the tools + ran" must not wedge the node. + + backend/harness/loop.py appends the assistant turn - tool_calls and all - + and persists it BEFORE invoking any tool. A Stop, timeout, provider fault + or crash in that window leaves the transcript ending on an assistant turn + whose calls were never answered. The follow-up then sends + assistant(tool_calls) immediately followed by user(...), which every major + provider rejects: OpenAI wants a tool message per tool_call, Anthropic a + tool_result per tool_use. The bad turn is on disk, so every subsequent + follow-up fails identically and the node never recovers. + + drop_leading_orphan_tools already handled the other end of this (a history + that OPENS mid tool-sequence); this is the missing half. + """ + + @staticmethod + def _assistant_with_calls(): + return { + "role": "assistant", "content": "", + "tool_calls": [ + {"id": "c1", "name": "fs.read", "arguments": "{}"}, + {"id": "c2", "name": "shell.exec", "arguments": "{}"}, + ], + } + + def test_a_fully_interrupted_turn_gets_synthetic_results(self, tmp_path): + append_message(tmp_path, {"role": "user", "content": "go"}) + append_message(tmp_path, self._assistant_with_calls()) + + loaded = load_messages(tmp_path) + + assert [m["role"] for m in loaded] == ["user", "assistant", "tool", "tool"] + assert [m["tool_call_id"] for m in loaded if m["role"] == "tool"] == ["c1", "c2"] + assert all("Interrupted" in m["content"] for m in loaded if m["role"] == "tool") + + def test_a_partially_answered_turn_only_fills_the_gap(self, tmp_path): + """A Stop between two calls leaves the first answered.""" + append_message(tmp_path, {"role": "user", "content": "go"}) + append_message(tmp_path, self._assistant_with_calls()) + append_message(tmp_path, {"role": "tool", "tool_call_id": "c1", "name": "fs.read", "content": "real result"}) + + loaded = load_messages(tmp_path) + + results = [m for m in loaded if m["role"] == "tool"] + assert [m["tool_call_id"] for m in results] == ["c1", "c2"] + assert results[0]["content"] == "real result" + assert "Interrupted" in results[1]["content"] + + def test_every_requested_call_ends_up_answered(self, tmp_path): + """The invariant the providers actually enforce, stated directly.""" + append_message(tmp_path, {"role": "user", "content": "go"}) + append_message(tmp_path, self._assistant_with_calls()) + + loaded = load_messages(tmp_path) + + requested = {call["id"] for m in loaded for call in m.get("tool_calls", [])} + answered = {m["tool_call_id"] for m in loaded if m["role"] == "tool"} + assert requested == answered + assert loaded[-1]["role"] != "assistant" or not loaded[-1].get("tool_calls") + + def test_a_healthy_transcript_is_left_alone(self, tmp_path): + append_message(tmp_path, {"role": "user", "content": "go"}) + append_message(tmp_path, self._assistant_with_calls()) + append_message(tmp_path, {"role": "tool", "tool_call_id": "c1", "name": "fs.read", "content": "a"}) + append_message(tmp_path, {"role": "tool", "tool_call_id": "c2", "name": "shell.exec", "content": "b"}) + append_message(tmp_path, {"role": "assistant", "content": "done"}) + + loaded = load_messages(tmp_path) + + assert [m["role"] for m in loaded] == ["user", "assistant", "tool", "tool", "assistant"] + assert [m["content"] for m in loaded if m["role"] == "tool"] == ["a", "b"] + + def test_an_assistant_turn_with_no_tool_calls_is_untouched(self, tmp_path): + append_message(tmp_path, {"role": "user", "content": "go"}) + append_message(tmp_path, {"role": "assistant", "content": "just an answer"}) + + assert [m["role"] for m in load_messages(tmp_path)] == ["user", "assistant"]