From 362d2dc437080d93a22aaa1374b383972ced9ff8 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 12 Oct 2026 18:40:17 +0200 Subject: [PATCH 1/3] feat(ai): never run the cut tool calls of an answer at the output limit, add chat({ truncatedToolResult }) - Assistant messages keep metadata.tanstack.finishReason. - A tool call of an answer that stopped at the output limit (finish reason length) never runs and asks for nothing. A stored cut call gets an error tool result, and a history call of a length message is skipped. The client processor runs no client tool after a length finish. - chat({ truncatedToolResult }) gives each cut call that text and calls the model again. Split out of #1555. --- .changeset/truncated-tool-result.md | 12 + docs/api/ai.md | 1 + docs/chat/agentic-cycle.md | 37 ++ docs/config.json | 4 +- packages/ai/src/activities/chat/index.ts | 136 ++++++- .../src/activities/chat/stream/processor.ts | 6 +- packages/ai/src/types.ts | 15 + ...t-provider-executed-thinking-order.test.ts | 114 +++++- packages/ai/tests/chat.test.ts | 356 ++++++++++++++++++ packages/ai/tests/replay-metadata.test.ts | 9 +- .../fixtures/truncated-tool-call/basic.json | 28 ++ testing/e2e/src/routeTree.gen.ts | 23 ++ ...pi.bedrock-converse-truncated-tool-call.ts | 90 +++++ ...drock-converse-truncated-tool-call.spec.ts | 29 ++ 14 files changed, 842 insertions(+), 18 deletions(-) create mode 100644 .changeset/truncated-tool-result.md create mode 100644 testing/e2e/fixtures/truncated-tool-call/basic.json create mode 100644 testing/e2e/src/routes/api.bedrock-converse-truncated-tool-call.ts create mode 100644 testing/e2e/tests/bedrock-converse-truncated-tool-call.spec.ts diff --git a/.changeset/truncated-tool-result.md b/.changeset/truncated-tool-result.md new file mode 100644 index 0000000000..f1d7df4880 --- /dev/null +++ b/.changeset/truncated-tool-result.md @@ -0,0 +1,12 @@ +--- +'@tanstack/ai': minor +--- + +A model answer with tool calls that stops at the output limit (finish reason `length`) can continue, and its cut tool calls never run. + +- Behavior change: `chat()` never runs a tool call of an answer that stopped at the output limit. The input of such a call can be incomplete JSON. The call also asks for no approval, and the client does not run a client tool for it. Before, a call of a cut answer could run, for example when a provider-executed tool kept the call in the transcript, or when a later `chat()` call got that history. +- Each assistant message that `chat()` stores keeps the finish reason of its model call in `metadata.tanstack.finishReason`. +- When `chat()` stores a cut answer that keeps its tool calls, each call gets an error tool result with a default text. A history that has such a call with no result does not run it. +- New `chat({ truncatedToolResult })` option: a string, or a function that gets `{ toolCallId, toolName }`. When it is set, the answer keeps its tool calls, each call gets that text as its error result, and the model is called again. This counts as one iteration of the loop. The `RUN_FINISHED` event of that model call has the finish reason `tool_calls`, so the client keeps the stream open. + +Migration: if your app needs the model to continue after a cut tool call, set `truncatedToolResult` to a text that tells the model to send the call again with the complete input. To find a cut answer on the server, read `metadata.tanstack.finishReason === 'length'` on the stored assistant message. diff --git a/docs/api/ai.md b/docs/api/ai.md index a350722534..ddca8d968f 100644 --- a/docs/api/ai.md +++ b/docs/api/ai.md @@ -57,6 +57,7 @@ const stream = chat({ - `context?` - Typed runtime context passed to server tools and middleware. If a tool or middleware declares a concrete context type, `chat()` requires a compatible value here - `systemPrompts?` - System prompts to prepend to messages - `agentLoopStrategy?` - Strategy for agent loops (default: `maxIterations(5)`). Strategies receive `{ iterationCount, finishReason, messages, toolCallCount, lastTurnToolCallCount }` and run between model turns. Iterations are model turns, not tool calls — for tool-call budgets use middleware (`onBeforeToolCall` + `onShouldContinue`); see [Tool-call budgets](../chat/agentic-cycle#tool-call-budgets-middleware-recipe). +- `truncatedToolResult?` - The error result of a tool call in an answer that stopped at the output limit (finish reason `length`). A string, or a function that gets `{ toolCallId, toolName }`. Such a call never runs. When set, the loop calls the model again. See [When the output limit cuts a tool call](../chat/agentic-cycle#when-the-output-limit-cuts-a-tool-call). - `middleware?` - Array of chat middleware. Use `onShouldContinue` / `onBeforeToolCall` for app-owned tool budgets. - `abortController?` - AbortController for cancellation - `modelOptions?` - Provider-native model options. This is where sampling parameters live — `temperature`, `top_p`/`topP`, and the provider's token-limit key (`max_output_tokens`, `max_tokens`, `maxOutputTokens`, …) — under each provider's canonical name, rather than as generic root-level props. See [Moving Sampling Options into modelOptions](../migration/sampling-options-to-model-options). (Renamed from `providerOptions`.) diff --git a/docs/chat/agentic-cycle.md b/docs/chat/agentic-cycle.md index 14e52b84ff..35d1cbef67 100644 --- a/docs/chat/agentic-cycle.md +++ b/docs/chat/agentic-cycle.md @@ -189,6 +189,43 @@ export async function POST(request: Request) { } ``` +### When the output limit cuts a tool call + +A model can run out of output tokens in the middle of a tool call, for example while it writes a long file. The provider stops with the finish reason `length`, and the input of the call is incomplete. `chat()` never runs such a call. By default, the run ends. + +Set `truncatedToolResult`, and the model can try again: + +```typescript +import { chat, toServerSentEventsResponse } from "@tanstack/ai"; +import { openaiText } from "@tanstack/ai-openai"; +import { writeFile } from "./tools"; + +export async function POST(request: Request) { + const { messages } = await request.json(); + const stream = chat({ + adapter: openaiText("gpt-6.1-sol"), + messages, + tools: [writeFile], + truncatedToolResult: (call) => + `Tool call "${call.toolName}" did not run: your answer hit the output limit, so its input can be incomplete. Call it again with the complete input.`, + }); + return toServerSentEventsResponse(stream); +} +``` + +What happens when an answer with tool calls stops at the output limit: + +1. The answer keeps its tool calls in the transcript. +2. Each call gets an error tool result with your text. The call does not run, and it asks for no approval and no client tool. +3. The loop calls the model again. This counts as one iteration, so `maxIterations` still stops a model that hits the limit every time. + +Some details: + +- `truncatedToolResult` is a string, or a function that gets the call (`{ toolCallId, toolName }`). +- The `RUN_FINISHED` event of that model call has the finish reason `tool_calls`, so the client keeps the stream open. +- Each assistant message that `chat()` stores keeps the finish reason of its model call in `metadata.tanstack.finishReason`. A cut message keeps `length`. +- If a history has a call of a `length` message with no result, `chat()` does not run that call. + ### Tool-call budgets (middleware recipe) > **Iterations ≠ tool calls.** One model turn can emit many parallel tool calls. `maxIterations` only bounds **model turns**. Strategies run *between* turns, so without a per-turn cap a single runaway turn can still fan out unbounded. diff --git a/docs/config.json b/docs/config.json index 47007a89ba..65bd321050 100644 --- a/docs/config.json +++ b/docs/config.json @@ -166,7 +166,7 @@ "label": "Agentic Cycle", "to": "chat/agentic-cycle", "addedAt": "2026-04-15", - "updatedAt": "2026-07-21" + "updatedAt": "2026-10-12" }, { "label": "Thinking & Reasoning", @@ -1478,7 +1478,7 @@ "label": "@tanstack/ai", "to": "api/ai", "addedAt": "2026-04-15", - "updatedAt": "2026-10-08" + "updatedAt": "2026-10-12" }, { "label": "@tanstack/ai-client", diff --git a/packages/ai/src/activities/chat/index.ts b/packages/ai/src/activities/chat/index.ts index a8e0980553..b298f9ffb8 100644 --- a/packages/ai/src/activities/chat/index.ts +++ b/packages/ai/src/activities/chat/index.ts @@ -233,6 +233,10 @@ function turnBlockOrder( return buildBlockOrder(entries) } +/** The default tool result of a call in an answer cut at the output limit. */ +const TRUNCATED_TOOL_RESULT = + 'The answer was cut off at the output limit before this tool call was complete. The call did not run.' + // =========================== // Activity Kind // =========================== @@ -590,6 +594,25 @@ export interface TextActivityOptions< agentLoopStrategy?: TextOptions['agentLoopStrategy'] /** How the server tools of one model turn run. Default `'parallel'`. */ toolExecution?: TextOptions['toolExecution'] + /** + * Continue after a model answer with tool calls that stopped at the output + * limit (finish reason `length`). Each call that the provider did not run + * gets this text as an error result and does not run. Then the model is + * called again, so it can send the calls again with complete arguments. + * The function form gets the call. Not set: the answer ends the run. + * + * @example + * ```ts + * chat({ + * adapter, + * messages, + * tools, + * truncatedToolResult: (call) => + * `Tool call "${call.toolName}" was cut off. Send it again.`, + * }) + * ``` + */ + truncatedToolResult?: TextOptions['truncatedToolResult'] /** * Retry a rate-limited model call after the wait the provider asks for. * @@ -2146,8 +2169,10 @@ class TextEngine< if (chunk.type === EventType.RUN_FINISHED) { const model = chunk.model ?? incoming?.model const responseId = chunk.responseId ?? incoming?.responseId + const finishReason = chunk.finishReason ?? incoming?.finishReason if (model !== undefined) metadata.model = model if (responseId !== undefined) metadata.responseId = responseId + if (finishReason) metadata.finishReason = finishReason } if (chunk.type === EventType.RUN_ERROR) { metadata.stopReason ??= 'error' @@ -2635,6 +2660,47 @@ class TextEngine< this.addAssistantToolCallMessage(toolCalls) + // The output limit cut this answer. Each call that the provider did not + // run has its error result now (closeTruncatedToolCalls). No call runs, + // and the loop goes on, so the model can call again. + if (this.lastFinishReason === 'length') { + // A client reads it as a tool turn, so it waits for the next call. + this.deferredToolCallRunFinishedChunks = + this.deferredToolCallRunFinishedChunks.map((chunk) => + chunk.type === EventType.RUN_FINISHED + ? { ...chunk, finishReason: 'tool_calls' } + : chunk, + ) + yield* this.flushDeferredToolCallRunFinishedChunks() + const results = toolCalls.flatMap((call): Array => { + const result = this.messages.find( + (message) => + message.role === 'tool' && message.toolCallId === call.id, + ) + return result + ? [ + { + toolCallId: call.id, + toolName: call.function.name, + result: result.content, + state: 'output-error', + }, + ] + : [] + }) + for (const chunk of this.buildToolResultChunks( + results, + finishEvent, + undefined, + true, + )) { + yield* this.pipeThroughMiddleware(chunk) + } + this.toolCallManager.clear() + this.setToolPhase('continue') + return + } + // Handle undiscovered lazy tool calls with self-correcting error messages const undiscoveredLazyResults: Array = [] const executableToolCalls = toolCalls.filter((tc) => { @@ -2902,7 +2968,14 @@ class TextEngine< private shouldExecuteToolPhase(): boolean { return ( - this.lastFinishReason === 'tool_calls' && + (this.lastFinishReason === 'tool_calls' || + (this.lastFinishReason === 'length' && + this.params.truncatedToolResult !== undefined && + // Provider-executed calls have their results. With no other call, + // the cut answer ends the run like a text answer. + this.toolCallManager + .getToolCalls() + .some((call) => !isProviderExecutedToolCall(call)))) && this.tools.length > 0 && this.toolCallManager.hasToolCalls() ) @@ -3051,10 +3124,40 @@ class TextEngine< }, ]), ] + this.closeTruncatedToolCalls(from) this.saveMidConversationChange(from) this.middlewareCtx.messages = this.messages } + /** + * Give each call of the assistant messages from index `from` an error result + * when the output limit cut their answer. Such a call is not complete, so it + * never runs and asks for nothing. A provider-executed call has its result. + */ + private closeTruncatedToolCalls(from: number): void { + const cut = this.messages + .slice(from) + .flatMap((message) => + tanstackMetadata(message)?.finishReason === 'length' + ? (message.toolCalls ?? []).filter( + (call) => !isProviderExecutedToolCall(call), + ) + : [], + ) + if (cut.length === 0) return + const option = this.params.truncatedToolResult + this.messages = [ + ...this.messages, + ...cut.map((call): ModelMessage => { + const text = + typeof option === 'function' + ? option({ toolCallId: call.id, toolName: call.function.name }) + : (option ?? TRUNCATED_TOOL_RESULT) + return { role: 'tool', toolCallId: call.id, content: text, error: text } + }), + ] + } + private addTerminalAssistantMessages(): void { this.finalizeCurrentThinkingStep() @@ -3179,6 +3282,7 @@ class TextEngine< } this.messages = messages + this.closeTruncatedToolCalls(startedLength) // An in-place update does not grow the array. The assistant message is // already at existingStructuredIndex, so a search from the old length // misses it and leaves the record pending. @@ -3831,6 +3935,7 @@ class TextEngine< results: Array, _finishEvent: RunFinishedEvent, argsMap?: Map, + replaceExisting = false, ): Array { const chunks: Array = [] @@ -3912,17 +4017,19 @@ class TextEngine< } }) const isDeniedResult = result.outcome === 'denied' - const existingToolResultIdx = isDeniedResult - ? this.messages.findIndex( - (message) => - message.role === 'tool' && - message.toolCallId === result.toolCallId, - ) - : -1 + const existingToolResultIdx = + isDeniedResult || replaceExisting + ? this.messages.findIndex( + (message) => + message.role === 'tool' && + message.toolCallId === result.toolCallId, + ) + : -1 const resultMessageIdx = existingToolResultIdx >= 0 ? existingToolResultIdx : placeholderIdx - // Only a denied result keeps the old message's fields. A replaced + // Only a denied result or `replaceExisting` (the cut calls of an answer + // at the output limit) keeps the old message's fields. A replaced // pendingExecution placeholder must not leak into the real result. const existingToolMessage = existingToolResultIdx >= 0 @@ -4041,8 +4148,15 @@ class TextEngine< for (const message of this.messages) { if (message.role === 'assistant' && message.toolCalls) { - const stopReason = tanstackMetadata(message)?.stopReason - if (stopReason === 'error' || stopReason === 'aborted') continue + const metadata = tanstackMetadata(message) + // The calls of a failed answer, or of an answer that the output limit + // cut, are not complete. They never run. + if ( + metadata?.stopReason === 'error' || + metadata?.stopReason === 'aborted' || + metadata?.finishReason === 'length' + ) + continue for (const toolCall of message.toolCalls) { // Provider-executed tool calls (e.g. Anthropic `web_search`) were // already run by the provider; they carry no client result, so they diff --git a/packages/ai/src/activities/chat/stream/processor.ts b/packages/ai/src/activities/chat/stream/processor.ts index 05faf7b5a9..9be067d3d0 100644 --- a/packages/ai/src/activities/chat/stream/processor.ts +++ b/packages/ai/src/activities/chat/stream/processor.ts @@ -2687,7 +2687,11 @@ export class StreamProcessor { if (isIntermediateToolTurn) { return } - if (!chunk.outcome || chunk.outcome.type === 'success') { + // The calls of an answer cut at the output limit never run. + if ( + (!chunk.outcome || chunk.outcome.type === 'success') && + this.finishReason !== 'length' + ) { this.handlePendingToolCalls(chunk.outcome?.pendingToolCallIds) } this.isDone = true diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 5fa8acf9d1..6e9f45c84b 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -702,6 +702,11 @@ export interface TanStackMessageMetadata { source?: MessageSource stopReason?: 'error' | 'aborted' responseId?: string + /** + * Why the model call of this assistant message ended. A `'length'` message + * stopped at the output limit, so its tool calls are not complete. + */ + finishReason?: TanStackRunMetadata['finishReason'] /** Parent chat run that produced this assistant message. */ runId?: string /** @@ -1277,6 +1282,16 @@ export interface TextOptions< * starts them together, and `'sequential'` runs them one at a time. */ toolExecution?: 'parallel' | 'sequential' + /** + * The tool result of a call in a model answer that stopped at the output + * limit (finish reason `length`). The function form gets the call. Such a + * call never runs. When set, each call gets this text as an error result, + * and the loop calls the model again, as after a tool phase. Not set: the + * answer ends the run, and a kept call gets a default text. + */ + truncatedToolResult?: + | string + | ((call: { toolCallId: string; toolName: string }) => string) /** * Retry a model call when the provider error has a wait time (usually a * 429). `chat()` waits for the `retryAfterMs` time of the `RUN_ERROR`, then diff --git a/packages/ai/tests/chat-provider-executed-thinking-order.test.ts b/packages/ai/tests/chat-provider-executed-thinking-order.test.ts index 084c33c1d5..5e08867f8a 100644 --- a/packages/ai/tests/chat-provider-executed-thinking-order.test.ts +++ b/packages/ai/tests/chat-provider-executed-thinking-order.test.ts @@ -1,5 +1,6 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { chat } from '../src/activities/chat' +import { defineChatMiddleware } from '../src/activities/chat/middleware/define' import { convertMessagesToModelMessages, modelMessagesToUIMessages, @@ -16,7 +17,7 @@ import { serverTool, } from './test-utils' import { EventType } from '../src/types' -import type { StreamChunk, UIMessage } from '../src/types' +import type { ModelMessage, StreamChunk, UIMessage } from '../src/types' /** * Anthropic runs web_search / web_fetch inside one provider response and @@ -439,5 +440,114 @@ describe('provider-executed tools interleaved with signed thinking', () => { 1, ) }) + + it('gives the calls of an answer cut at the output limit an error result, so a later run does not run them', async () => { + const lookup = vi.fn(() => ({ found: true })) + const { adapter, calls } = createMockAdapter({ + iterations: [ + [ + ev.runStarted(), + ...reasoning('step-1', 'plan', 'sig-a'), + providerToolStart('srvtoolu_1', 'one'), + ev.toolEnd('srvtoolu_1'), + ...reasoning('step-2', 'refine', 'sig-b'), + ev.toolStart('call_1', 'lookup'), + ev.toolArgs('call_1', '{"id":'), + ev.runFinished('length'), + ], + [ + ev.runStarted(), + ev.textStart(), + ev.textContent('Done.'), + ev.textEnd(), + ev.runFinished('stop'), + ], + ], + }) + let transcript: Array = [] + const keep = defineChatMiddleware({ + onFinish(ctx) { + transcript = [...ctx.messages] + }, + }) + const tools = [serverTool('lookup', lookup)] + + await collectChunks( + chat({ + adapter, + messages: [{ role: 'user', content: 'Research' }], + tools, + middleware: [keep], + }) as AsyncIterable, + ) + + const text = + 'The answer was cut off at the output limit before this tool call was complete. The call did not run.' + expect(transcript.filter((message) => message.role === 'tool')).toEqual([ + { role: 'tool', toolCallId: 'call_1', content: text, error: text }, + ]) + + // The next run from this transcript. + await collectChunks( + chat({ + adapter, + messages: transcript, + tools, + }) as AsyncIterable, + ) + expect(lookup).not.toHaveBeenCalled() + expect(calls).toHaveLength(2) + }) + + it('with truncatedToolResult, gives the cut call that text and calls the model again', async () => { + const lookup = vi.fn(() => ({ found: true })) + const { adapter, calls } = createMockAdapter({ + iterations: [ + [ + ev.runStarted(), + ...reasoning('step-1', 'plan', 'sig-a'), + providerToolStart('srvtoolu_1', 'one'), + ev.toolEnd('srvtoolu_1'), + ...reasoning('step-2', 'refine', 'sig-b'), + ev.toolStart('call_1', 'lookup'), + ev.toolArgs('call_1', '{"id":'), + ev.runFinished('length'), + ], + [ + ev.runStarted(), + ev.textStart(), + ev.textContent('Done.'), + ev.textEnd(), + ev.runFinished('stop'), + ], + ], + }) + + await collectChunks( + chat({ + adapter, + messages: [{ role: 'user', content: 'Research' }], + tools: [serverTool('lookup', lookup)], + truncatedToolResult: 'Cut off. Not run.', + }) as AsyncIterable, + ) + + expect(lookup).not.toHaveBeenCalled() + expect(calls).toHaveLength(2) + const history = calls[1]!.messages as Array + expect( + history.map((message) => + message.role === 'tool' + ? { tool: message.toolCallId, content: message.content } + : { [message.role]: message.toolCalls?.map((call) => call.id) }, + ), + ).toEqual([ + { user: undefined }, + { assistant: ['srvtoolu_1'] }, + { assistant: ['call_1'] }, + // The provider-executed call keeps its own result. + { tool: 'call_1', content: 'Cut off. Not run.' }, + ]) + }) }) }) diff --git a/packages/ai/tests/chat.test.ts b/packages/ai/tests/chat.test.ts index 995d607768..7b3b0f68df 100644 --- a/packages/ai/tests/chat.test.ts +++ b/packages/ai/tests/chat.test.ts @@ -3050,6 +3050,362 @@ describe('chat()', () => { }) }) + // ========================================================================== + // Answers cut at the output limit + // ========================================================================== + describe('answers cut at the output limit', () => { + const DEFAULT_TEXT = + 'The answer was cut off at the output limit before this tool call was complete. The call did not run.' + const truncatedToolResult = (call: { toolName: string }) => + `Tool call "${call.toolName}" was not executed: the response hit the output token limit, so its arguments may be truncated. Re-issue the tool call with complete arguments.` + const done = [ + ev.runStarted(), + ev.textStart('msg-2'), + ev.textContent('Done.', 'msg-2'), + ev.textEnd('msg-2'), + ev.runFinished('stop'), + ] + + it('gives a cut call the text, does not run it, and calls the model again', async () => { + const lookup = vi.fn(() => ({ found: true })) + const { adapter, calls } = createMockAdapter({ + iterations: [ + [ + ev.runStarted(), + ev.toolStart('call_1', 'lookup'), + ev.toolArgs('call_1', '{"q":'), + ev.runFinished('length'), + ], + done, + ], + }) + + const chunks = await collectChunks( + chat({ + adapter, + messages: [{ role: 'user', content: 'Look it up' }], + tools: [serverTool('lookup', lookup)], + truncatedToolResult, + }) as AsyncIterable, + ) + + const text = truncatedToolResult({ toolName: 'lookup' }) + expect(lookup).not.toHaveBeenCalled() + expect(calls).toHaveLength(2) + expect((calls[1]!.messages as Array).slice(1)).toEqual([ + expect.objectContaining({ + role: 'assistant', + toolCalls: [expect.objectContaining({ id: 'call_1' })], + metadata: { + tanstack: expect.objectContaining({ finishReason: 'length' }), + }, + }), + { role: 'tool', toolCallId: 'call_1', content: text, error: text }, + ]) + expect(chunks).toContainEqual( + expect.objectContaining({ + type: EventType.TOOL_CALL_RESULT, + toolCallId: 'call_1', + content: text, + }), + ) + }) + + it('gives each call of a cut answer its text, and asks nothing', async () => { + const remove = vi.fn(() => ({ ok: true })) + const { adapter, calls } = createMockAdapter({ + iterations: [ + [ + ev.runStarted(), + ev.toolStart('call_1', 'remove'), + ev.toolArgs('call_1', '{"path":"a.txt"}'), + ev.toolEnd('call_1'), + ev.toolStart('call_2', 'ask'), + ev.toolArgs('call_2', '{"q":'), + ev.runFinished('length'), + ], + done, + ], + }) + const onToolCall = vi.fn() + const onStreamEnd = vi.fn() + const processor = new StreamProcessor({ + events: { onToolCall, onStreamEnd }, + }) + + const chunks = await collectChunks( + chat({ + adapter, + messages: [{ role: 'user', content: 'Remove and ask' }], + tools: [ + { ...serverTool('remove', remove), needsApproval: true }, + clientTool('ask'), + ], + truncatedToolResult, + }) as AsyncIterable, + ) + for (const item of chunks) processor.processChunk(item) + + expect(remove).not.toHaveBeenCalled() + expect(calls).toHaveLength(2) + expect( + (calls[1]!.messages as Array).filter( + (message) => message.role === 'tool', + ), + ).toEqual( + ['remove', 'ask'].map((toolName, index) => { + const text = truncatedToolResult({ toolName }) + return { + role: 'tool', + toolCallId: `call_${index + 1}`, + content: text, + error: text, + } + }), + ) + expect( + chunks.some( + (item) => + item.type === EventType.RUN_FINISHED && + item.outcome?.type === 'interrupt', + ), + ).toBe(false) + // The client waits through the cut call, and runs no tool. + expect(onToolCall).not.toHaveBeenCalled() + expect(onStreamEnd).toHaveBeenCalledTimes(1) + }) + + it('counts each cut answer as an iteration of the loop strategy', async () => { + const cut = (id: string) => [ + ev.runStarted(), + ev.toolStart(id, 'lookup'), + ev.toolArgs(id, '{"q":'), + ev.runFinished('length'), + ] + const { adapter, calls } = createMockAdapter({ + iterations: [cut('call_1'), cut('call_2'), cut('call_3')], + }) + + await collectChunks( + chat({ + adapter, + messages: [{ role: 'user', content: 'Look it up' }], + tools: [serverTool('lookup', () => ({ found: true }))], + agentLoopStrategy: (state) => state.iterationCount < 2, + truncatedToolResult, + }) as AsyncIterable, + ) + + expect(calls).toHaveLength(2) + }) + + it('without the option, ends the run and keeps no calls of a plain cut answer', async () => { + const lookup = vi.fn(() => ({ found: true })) + const { adapter, calls } = createMockAdapter({ + iterations: [ + [ + ev.runStarted(), + ev.textStart(), + ev.textContent('Let me look.'), + ev.textEnd(), + ev.toolStart('call_1', 'lookup'), + ev.toolArgs('call_1', '{"q":'), + ev.runFinished('length'), + ], + done, + ], + }) + let transcript: Array = [] + const keep = defineChatMiddleware({ + onFinish(ctx) { + transcript = [...ctx.messages] + }, + }) + + await collectChunks( + chat({ + adapter, + messages: [{ role: 'user', content: 'Look it up' }], + tools: [serverTool('lookup', lookup)], + middleware: [keep], + }) as AsyncIterable, + ) + + expect(lookup).not.toHaveBeenCalled() + expect(calls).toHaveLength(1) + expect(transcript.slice(1)).toEqual([ + expect.objectContaining({ + role: 'assistant', + content: 'Let me look.', + metadata: { + tanstack: expect.objectContaining({ finishReason: 'length' }), + }, + }), + ]) + expect(transcript[1]).not.toHaveProperty('toolCalls') + }) + + it('without the option, the client runs no client tool of a cut answer', async () => { + const { adapter } = createMockAdapter({ + iterations: [ + [ + ev.runStarted(), + ev.toolStart('call_1', 'ask'), + ev.toolArgs('call_1', '{}'), + ev.toolEnd('call_1'), + ev.toolStart('call_2', 'ask'), + ev.toolArgs('call_2', '{"q":'), + ev.runFinished('length'), + ], + ], + }) + const onToolCall = vi.fn() + const processor = new StreamProcessor({ events: { onToolCall } }) + + const chunks = await collectChunks( + chat({ + adapter, + messages: [{ role: 'user', content: 'Ask' }], + tools: [clientTool('ask')], + }) as AsyncIterable, + ) + for (const item of chunks) processor.processChunk(item) + + expect(onToolCall).not.toHaveBeenCalled() + }) + + it('does not run or ask for the calls of a cut answer in the history', async () => { + const executeSpy = vi.fn().mockReturnValue({ ok: true }) + const { adapter, calls } = createMockAdapter({ + iterations: [ + [ + ev.runStarted(), + ev.textStart(), + ev.textContent('Going on.'), + ev.textEnd(), + ev.runFinished('stop'), + ], + ], + }) + const call = (id: string, name: string) => ({ + id, + type: 'function' as const, + function: { name, arguments: '{}' }, + }) + + // A client history that still has the cut calls without results. + const chunks = await collectChunks( + chat({ + adapter, + messages: [ + { role: 'user', content: 'Look up, remove, and ask' }, + { + role: 'assistant', + content: null, + metadata: { tanstack: { finishReason: 'length' } }, + toolCalls: [ + call('call_1', 'lookup'), + call('call_2', 'remove'), + call('call_3', 'ask'), + ], + }, + ], + tools: [ + serverTool('lookup', executeSpy), + { ...serverTool('remove', executeSpy), needsApproval: true }, + clientTool('ask'), + ], + }) as AsyncIterable, + ) + + expect(executeSpy).not.toHaveBeenCalled() + expect(expectSingleRunFinished(chunks).outcome?.type).not.toBe( + 'interrupt', + ) + expect(calls).toHaveLength(1) + }) + + it.each([ + ['the default text', undefined, DEFAULT_TEXT], + ['truncatedToolResult', 'Cut off. Not run.', 'Cut off. Not run.'], + ])( + 'gives the calls of a cut answer a result at an afterModel interrupt (%s)', + async (_label, option, text) => { + const review = defineInterrupt({ + id: 'after-model-cut', + responseSchema: z.object({ approved: z.boolean() }), + }) + const { adapter } = createMockAdapter({ + iterations: [ + [ + ev.runStarted(), + ev.toolStart('call-draft', 'saveDraft'), + ev.toolArgs('call-draft', '{}'), + ev.toolEnd('call-draft'), + ev.toolStart('call-remove', 'remove'), + ev.toolArgs('call-remove', '{"path":'), + ev.runFinished('length'), + ], + ], + }) + + const chunks = await collectChunks( + chat({ + adapter, + interrupts: [review], + middleware: [ + defineChatMiddleware({ + onInterruptBoundary(ctx) { + if (ctx.phase !== 'afterModel') return + return { + interrupts: [ + review.interrupt({ + key: 'after-model', + reason: 'review', + message: 'Review the answer', + }), + ], + } + }, + }), + ], + messages: [{ role: 'user', content: 'Save and remove' }], + tools: [ + clientTool('saveDraft'), + { + ...serverTool('remove', () => ({ ok: true })), + needsApproval: true, + }, + ], + ...(option ? { truncatedToolResult: option } : {}), + }) as AsyncIterable, + ) + + const terminal = expectSingleRunFinished(chunks) + if (terminal.outcome?.type !== 'interrupt') { + throw new Error('Expected afterModel interrupt') + } + expect(terminal.outcome.interrupts).toEqual([ + expect.objectContaining({ reason: 'review' }), + ]) + const snapshot = chunks.find( + (item) => item.type === EventType.MESSAGES_SNAPSHOT, + ) + if (!snapshot || snapshot.type !== EventType.MESSAGES_SNAPSHOT) { + throw new Error('Expected messages snapshot') + } + expect( + snapshot.messages + .filter((message) => message.role === 'tool') + .map((message) => [message.toolCallId, message.content]), + ).toEqual([ + ['call-draft', text], + ['call-remove', text], + ]) + }, + ) + }) + // ========================================================================== // Agent loop strategy // ========================================================================== diff --git a/packages/ai/tests/replay-metadata.test.ts b/packages/ai/tests/replay-metadata.test.ts index a30a968cc5..11c5e0a6ab 100644 --- a/packages/ai/tests/replay-metadata.test.ts +++ b/packages/ai/tests/replay-metadata.test.ts @@ -193,9 +193,14 @@ describe('replay metadata', () => { }) const assistants = saved.filter((message) => message.role === 'assistant') expect(assistants.length).toBeGreaterThan(1) + const tags = { + source, + responseId: 'response-segments', + finishReason: 'stop', + } expect(assistants.map((message) => message.metadata?.tanstack)).toEqual([ - { source, responseId: 'response-segments' }, - { source, responseId: 'response-segments' }, + tags, + tags, ]) }) diff --git a/testing/e2e/fixtures/truncated-tool-call/basic.json b/testing/e2e/fixtures/truncated-tool-call/basic.json new file mode 100644 index 0000000000..118b191a4e --- /dev/null +++ b/testing/e2e/fixtures/truncated-tool-call/basic.json @@ -0,0 +1,28 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "[truncated-tool-call] weather in Paris", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "lookup_weather", + "arguments": "{\"location\":\"Paris\"}" + } + ], + "finishReason": "length" + } + }, + { + "match": { + "userMessage": "[truncated-tool-call] weather in Paris", + "hasToolResult": true + }, + "response": { + "content": "Let me look again." + } + } + ] +} diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index 11dec53912..385e37fbc6 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -66,6 +66,7 @@ import { Route as ApiArktypeToolWireRouteImport } from './routes/api.arktype-too import { Route as ApiAudioRouteImport } from './routes/api.audio' import { Route as ApiAzureOpenaiWireRouteImport } from './routes/api.azure-openai-wire' import { Route as ApiBedrockConverseCacheRouteImport } from './routes/api.bedrock-converse-cache' +import { Route as ApiBedrockConverseTruncatedToolCallRouteImport } from './routes/api.bedrock-converse-truncated-tool-call' import { Route as ApiByokChatRouteImport } from './routes/api.byok-chat' import { Route as ApiByteplusEncryptedReasoningWireRouteImport } from './routes/api.byteplus-encrypted-reasoning-wire' import { Route as ApiByteplusSeedance1080pWireRouteImport } from './routes/api.byteplus-seedance-1080p-wire' @@ -464,6 +465,12 @@ const ApiBedrockConverseCacheRoute = ApiBedrockConverseCacheRouteImport.update({ path: '/api/bedrock-converse-cache', getParentRoute: () => rootRouteImport, } as any) +const ApiBedrockConverseTruncatedToolCallRoute = + ApiBedrockConverseTruncatedToolCallRouteImport.update({ + id: '/api/bedrock-converse-truncated-tool-call', + path: '/api/bedrock-converse-truncated-tool-call', + getParentRoute: () => rootRouteImport, + } as any) const ApiByokChatRoute = ApiByokChatRouteImport.update({ id: '/api/byok-chat', path: '/api/byok-chat', @@ -1047,6 +1054,7 @@ export interface FileRoutesByFullPath { '/api/audio': typeof ApiAudioRouteWithChildren '/api/azure-openai-wire': typeof ApiAzureOpenaiWireRoute '/api/bedrock-converse-cache': typeof ApiBedrockConverseCacheRoute + '/api/bedrock-converse-truncated-tool-call': typeof ApiBedrockConverseTruncatedToolCallRoute '/api/byok-chat': typeof ApiByokChatRoute '/api/byteplus-encrypted-reasoning-wire': typeof ApiByteplusEncryptedReasoningWireRoute '/api/byteplus-seedance-1080p-wire': typeof ApiByteplusSeedance1080pWireRoute @@ -1206,6 +1214,7 @@ export interface FileRoutesByTo { '/api/audio': typeof ApiAudioRouteWithChildren '/api/azure-openai-wire': typeof ApiAzureOpenaiWireRoute '/api/bedrock-converse-cache': typeof ApiBedrockConverseCacheRoute + '/api/bedrock-converse-truncated-tool-call': typeof ApiBedrockConverseTruncatedToolCallRoute '/api/byok-chat': typeof ApiByokChatRoute '/api/byteplus-encrypted-reasoning-wire': typeof ApiByteplusEncryptedReasoningWireRoute '/api/byteplus-seedance-1080p-wire': typeof ApiByteplusSeedance1080pWireRoute @@ -1366,6 +1375,7 @@ export interface FileRoutesById { '/api/audio': typeof ApiAudioRouteWithChildren '/api/azure-openai-wire': typeof ApiAzureOpenaiWireRoute '/api/bedrock-converse-cache': typeof ApiBedrockConverseCacheRoute + '/api/bedrock-converse-truncated-tool-call': typeof ApiBedrockConverseTruncatedToolCallRoute '/api/byok-chat': typeof ApiByokChatRoute '/api/byteplus-encrypted-reasoning-wire': typeof ApiByteplusEncryptedReasoningWireRoute '/api/byteplus-seedance-1080p-wire': typeof ApiByteplusSeedance1080pWireRoute @@ -1527,6 +1537,7 @@ export interface FileRouteTypes { | '/api/audio' | '/api/azure-openai-wire' | '/api/bedrock-converse-cache' + | '/api/bedrock-converse-truncated-tool-call' | '/api/byok-chat' | '/api/byteplus-encrypted-reasoning-wire' | '/api/byteplus-seedance-1080p-wire' @@ -1686,6 +1697,7 @@ export interface FileRouteTypes { | '/api/audio' | '/api/azure-openai-wire' | '/api/bedrock-converse-cache' + | '/api/bedrock-converse-truncated-tool-call' | '/api/byok-chat' | '/api/byteplus-encrypted-reasoning-wire' | '/api/byteplus-seedance-1080p-wire' @@ -1845,6 +1857,7 @@ export interface FileRouteTypes { | '/api/audio' | '/api/azure-openai-wire' | '/api/bedrock-converse-cache' + | '/api/bedrock-converse-truncated-tool-call' | '/api/byok-chat' | '/api/byteplus-encrypted-reasoning-wire' | '/api/byteplus-seedance-1080p-wire' @@ -2005,6 +2018,7 @@ export interface RootRouteChildren { ApiAudioRoute: typeof ApiAudioRouteWithChildren ApiAzureOpenaiWireRoute: typeof ApiAzureOpenaiWireRoute ApiBedrockConverseCacheRoute: typeof ApiBedrockConverseCacheRoute + ApiBedrockConverseTruncatedToolCallRoute: typeof ApiBedrockConverseTruncatedToolCallRoute ApiByokChatRoute: typeof ApiByokChatRoute ApiByteplusEncryptedReasoningWireRoute: typeof ApiByteplusEncryptedReasoningWireRoute ApiByteplusSeedance1080pWireRoute: typeof ApiByteplusSeedance1080pWireRoute @@ -2504,6 +2518,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiBedrockConverseCacheRouteImport parentRoute: typeof rootRouteImport } + '/api/bedrock-converse-truncated-tool-call': { + id: '/api/bedrock-converse-truncated-tool-call' + path: '/api/bedrock-converse-truncated-tool-call' + fullPath: '/api/bedrock-converse-truncated-tool-call' + preLoaderRoute: typeof ApiBedrockConverseTruncatedToolCallRouteImport + parentRoute: typeof rootRouteImport + } '/api/byok-chat': { id: '/api/byok-chat' path: '/api/byok-chat' @@ -3324,6 +3345,8 @@ const rootRouteChildren: RootRouteChildren = { ApiAudioRoute: ApiAudioRouteWithChildren, ApiAzureOpenaiWireRoute: ApiAzureOpenaiWireRoute, ApiBedrockConverseCacheRoute: ApiBedrockConverseCacheRoute, + ApiBedrockConverseTruncatedToolCallRoute: + ApiBedrockConverseTruncatedToolCallRoute, ApiByokChatRoute: ApiByokChatRoute, ApiByteplusEncryptedReasoningWireRoute: ApiByteplusEncryptedReasoningWireRoute, diff --git a/testing/e2e/src/routes/api.bedrock-converse-truncated-tool-call.ts b/testing/e2e/src/routes/api.bedrock-converse-truncated-tool-call.ts new file mode 100644 index 0000000000..eb4a7f1ee2 --- /dev/null +++ b/testing/e2e/src/routes/api.bedrock-converse-truncated-tool-call.ts @@ -0,0 +1,90 @@ +import { createFileRoute } from '@tanstack/react-router' +import { chat, createChatOptions, toolDefinition } from '@tanstack/ai' +import { BedrockConverseTextAdapter } from '@tanstack/ai-bedrock' +import { NodeHttpHandler } from '@smithy/node-http-handler' +import { z } from 'zod' +import type { ResolvedBedrockAuth } from '@tanstack/ai-bedrock' + +const LLMOCK_DEFAULT_BASE = process.env.LLMOCK_URL || 'http://127.0.0.1:4010' +const DUMMY_KEY = 'sk-e2e-test-dummy-key' +const MODEL = 'us.anthropic.claude-haiku-4-5-20251001-v1:0' + +/** HTTP/1.1, because aimock does not speak HTTP/2 (see the cache route). */ +class Http1ConverseAdapter extends BedrockConverseTextAdapter { + protected override buildClientConfig( + resolved: ResolvedBedrockAuth, + region: string, + endpoint: string | undefined, + ) { + return { + ...super.buildClientConfig(resolved, region, endpoint), + requestHandler: new NodeHttpHandler(), + } + } +} + +/** + * The answer of the first model call has one tool call and stops at + * `max_tokens` (the fixture in `fixtures/truncated-tool-call` sets + * `finishReason: 'length'`). Converse keeps the `length` finish with the + * call. With `truncatedToolResult`, the call does not run, it gets the text + * as its error result, and the model is called again. + */ +export const Route = createFileRoute( + '/api/bedrock-converse-truncated-tool-call', +)({ + server: { + handlers: { + POST: async () => { + const adapter = new Http1ConverseAdapter( + { + apiKey: DUMMY_KEY, + baseURL: LLMOCK_DEFAULT_BASE, + region: 'us-east-1', + }, + MODEL, + ) + let runs = 0 + const lookupWeather = toolDefinition({ + name: 'lookup_weather', + description: 'Look up the weather.', + inputSchema: z.object({ location: z.string() }), + }).server(async () => { + runs += 1 + return 'sunny' + }) + + let text = '' + const toolResults: Array = [] + let runError: string | undefined + try { + for await (const chunk of chat({ + ...createChatOptions({ adapter }), + tools: [lookupWeather], + messages: [ + { + role: 'user', + content: '[truncated-tool-call] weather in Paris', + }, + ], + truncatedToolResult: ({ toolName }) => + `Tool call "${toolName}" was cut off. Send it again.`, + })) { + if (chunk.type === 'TEXT_MESSAGE_CONTENT') text += chunk.delta + if (chunk.type === 'TOOL_CALL_RESULT') { + toolResults.push(chunk.content) + } + if (chunk.type === 'RUN_ERROR') runError = chunk.message + } + if (runError !== undefined) throw new Error(runError) + return Response.json({ ok: true, runs, toolResults, text }) + } catch (error) { + return Response.json({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }) + } + }, + }, + }, +}) diff --git a/testing/e2e/tests/bedrock-converse-truncated-tool-call.spec.ts b/testing/e2e/tests/bedrock-converse-truncated-tool-call.spec.ts new file mode 100644 index 0000000000..ce9c621376 --- /dev/null +++ b/testing/e2e/tests/bedrock-converse-truncated-tool-call.spec.ts @@ -0,0 +1,29 @@ +import { test, expect } from './fixtures' + +// The route and its aimock fixture are described at +// `src/routes/api.bedrock-converse-truncated-tool-call.ts`. +test.describe('bedrock-converse — tool call cut at the output limit', () => { + test('the cut call does not run, gets the text, and the model answers again', async ({ + request, + }) => { + const res = await request.post('/api/bedrock-converse-truncated-tool-call') + expect(res.ok()).toBe(true) + + const { ok, error, runs, toolResults, text } = (await res.json()) as { + ok: boolean + error?: string + runs?: number + toolResults?: Array + text?: string + } + + expect(error ?? null).toBeNull() + expect(ok).toBe(true) + expect(runs).toBe(0) + expect(toolResults).toEqual([ + 'Tool call "lookup_weather" was cut off. Send it again.', + ]) + // The second fixture answers only a request that has a tool result. + expect(text).toBe('Let me look again.') + }) +}) From dd65fa4efeeaa4e2924444139be39331e395dea5 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 12 Oct 2026 21:11:11 +0200 Subject: [PATCH 2/3] fix(ai): warn instead of running a client tool of a cut answer, and expect finishReason in the persistence E2E Move the length guard into handlePendingToolCalls so the processor still warns 'Did not run tool call'. The sandbox persistence E2E now expects metadata.tanstack.finishReason on the saved assistant message. --- .../ai/src/activities/chat/stream/processor.ts | 14 +++++++++----- testing/e2e/tests/sandbox-file-persistence.spec.ts | 3 ++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/ai/src/activities/chat/stream/processor.ts b/packages/ai/src/activities/chat/stream/processor.ts index 9be067d3d0..2e7a933116 100644 --- a/packages/ai/src/activities/chat/stream/processor.ts +++ b/packages/ai/src/activities/chat/stream/processor.ts @@ -2687,11 +2687,7 @@ export class StreamProcessor { if (isIntermediateToolTurn) { return } - // The calls of an answer cut at the output limit never run. - if ( - (!chunk.outcome || chunk.outcome.type === 'success') && - this.finishReason !== 'length' - ) { + if (!chunk.outcome || chunk.outcome.type === 'success') { this.handlePendingToolCalls(chunk.outcome?.pendingToolCallIds) } this.isDone = true @@ -2734,6 +2730,14 @@ export class StreamProcessor { ) { continue } + // A call of an answer cut at the output limit never runs, also when + // its arguments parse. + if (this.finishReason === 'length') { + console.warn( + `[StreamProcessor] Did not run tool call ${toolCallId}: the answer stopped at the output limit`, + ) + continue + } // A snapshot can carry a part with arguments but no input. Parse them // strictly, as completeToolCall() does. An empty argument string is a // call with no arguments. diff --git a/testing/e2e/tests/sandbox-file-persistence.spec.ts b/testing/e2e/tests/sandbox-file-persistence.spec.ts index 707a85241f..96988ff92d 100644 --- a/testing/e2e/tests/sandbox-file-persistence.spec.ts +++ b/testing/e2e/tests/sandbox-file-persistence.spec.ts @@ -47,9 +47,10 @@ test.describe('sandbox portable file snapshots', () => { createdAt: expect.any(String), id: 'automatic-message', // withPersistence records the run that produced the message, and - // chat() records the source of the answer. + // chat() records the source and the finish reason of the answer. metadata: { tanstack: { + finishReason: 'stop', run: { id: 'recover' }, source: { provider: 'fixed', api: 'text', model: 'test-model' }, }, From 6b99bb31433c741f08c067d3f92b60dc2ba22b73 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Tue, 13 Oct 2026 17:07:55 +0200 Subject: [PATCH 3/3] feat(ai)!: retry cut tool calls through agentLoopStrategy, not truncatedToolResult Remove the chat({ truncatedToolResult }) option. A tool call of an answer that stopped at the output limit (finish reason `length`) never runs and always gets an error tool result. - The default agentLoopStrategy is now combineStrategies([maxIterations(5), untilFinishReason(['length'])]), so by default a cut answer ends the run with `length`. A strategy that goes on after `length` makes the model send the cut calls again. - The loop decision for a cut answer is taken once in its tool phase (strategy and onShouldContinue), and shouldContinue() reuses it. The RUN_FINISHED says `tool_calls` only when the loop goes on. - onAfterToolCall gets each cut call (ok: false, error code 'max_tokens'), so middleware can replace the error text. - Replay sends the non-JSON input of a cut call as {}, so Bedrock does not throw on it. - Native combined structured output keeps a single terminal RUN_ERROR when the cut answer ends the run. --- .changeset/truncated-tool-result.md | 15 +- docs/advanced/middleware.md | 1 + docs/api/ai.md | 3 +- docs/chat/agentic-cycle.md | 49 ++- docs/config.json | 6 +- packages/ai/src/activities/chat/index.ts | 174 +++++---- .../src/activities/chat/middleware/types.ts | 4 + packages/ai/src/types.ts | 10 - packages/ai/src/utilities/replay-messages.ts | 28 ++ ...-native-combined-structured-output.test.ts | 37 +- ...t-provider-executed-thinking-order.test.ts | 13 +- packages/ai/tests/chat.test.ts | 348 +++++++++++++----- ...pi.bedrock-converse-truncated-tool-call.ts | 59 ++- ...drock-converse-truncated-tool-call.spec.ts | 43 ++- 14 files changed, 568 insertions(+), 222 deletions(-) diff --git a/.changeset/truncated-tool-result.md b/.changeset/truncated-tool-result.md index f1d7df4880..e59fa3ffa5 100644 --- a/.changeset/truncated-tool-result.md +++ b/.changeset/truncated-tool-result.md @@ -2,11 +2,14 @@ '@tanstack/ai': minor --- -A model answer with tool calls that stops at the output limit (finish reason `length`) can continue, and its cut tool calls never run. +A tool call in a model answer that stopped at the output limit (finish reason `length`) never runs. The model can send it again when the loop strategy goes on. -- Behavior change: `chat()` never runs a tool call of an answer that stopped at the output limit. The input of such a call can be incomplete JSON. The call also asks for no approval, and the client does not run a client tool for it. Before, a call of a cut answer could run, for example when a provider-executed tool kept the call in the transcript, or when a later `chat()` call got that history. -- Each assistant message that `chat()` stores keeps the finish reason of its model call in `metadata.tanstack.finishReason`. -- When `chat()` stores a cut answer that keeps its tool calls, each call gets an error tool result with a default text. A history that has such a call with no result does not run it. -- New `chat({ truncatedToolResult })` option: a string, or a function that gets `{ toolCallId, toolName }`. When it is set, the answer keeps its tool calls, each call gets that text as its error result, and the model is called again. This counts as one iteration of the loop. The `RUN_FINISHED` event of that model call has the finish reason `tool_calls`, so the client keeps the stream open. +- Behavior change: `chat()` never runs a tool call of an answer that stopped at the output limit. The input of such a call can be incomplete JSON. The call asks for no approval, and the client does not run a client tool for it. Before, such a call ran in some cases, for example as a client tool with `{}` input, or as a pending call of the next `chat()` request. +- Each cut call gets an error tool result. The default text tells the model to send the call again. +- The default `agentLoopStrategy` is now `combineStrategies([maxIterations(5), untilFinishReason(['length'])])`. By default, a cut answer ends the run with the finish reason `length`, as before. +- With a strategy that goes on after `length` (for example `maxIterations(5)`), the model is called again. The `RUN_FINISHED` event of the cut answer then has the finish reason `tool_calls`, so the client keeps the stream open. +- In a history message with `metadata.tanstack.finishReason: 'length'`, a call input that is not JSON goes to the provider as `{}`. Before, Bedrock threw on such a cut input. +- Middleware `onAfterToolCall` gets each cut call, with `ok: false` and an `Error` with the code `'max_tokens'`. Return `{ type: 'replaceResult', result }` to change the error text. +- Each assistant message that `chat()` stores on the server keeps the finish reason of its model call in `metadata.tanstack.finishReason`. -Migration: if your app needs the model to continue after a cut tool call, set `truncatedToolResult` to a text that tells the model to send the call again with the complete input. To find a cut answer on the server, read `metadata.tanstack.finishReason === 'length'` on the stored assistant message. +Migration: if you pass your own `agentLoopStrategy` and want a cut answer to end the run, add `untilFinishReason(['length'])` to it with `combineStrategies`. diff --git a/docs/advanced/middleware.md b/docs/advanced/middleware.md index 943feecee6..5052b66dfc 100644 --- a/docs/advanced/middleware.md +++ b/docs/advanced/middleware.md @@ -737,6 +737,7 @@ Good to know: - Each middleware gets the result of the one before it as `info.result`. The last replacement wins. - A failed call stays failed. Your replacement becomes the error that the model reads. - A `skip` result from `onBeforeToolCall` goes through this hook too. `info.result` is the parsed skip result, the same value the model gets. +- A call that the output limit cut goes through this hook too. It did not run, so `ok` is false, and `error` has the code `max_tokens`. See [When the output limit cuts a tool call](../chat/agentic-cycle#when-the-output-limit-cuts-a-tool-call). ### Tool hook order in one turn diff --git a/docs/api/ai.md b/docs/api/ai.md index ddca8d968f..4e125f4ee4 100644 --- a/docs/api/ai.md +++ b/docs/api/ai.md @@ -56,8 +56,7 @@ const stream = chat({ - `tools?` - Array of tools for function calling - `context?` - Typed runtime context passed to server tools and middleware. If a tool or middleware declares a concrete context type, `chat()` requires a compatible value here - `systemPrompts?` - System prompts to prepend to messages -- `agentLoopStrategy?` - Strategy for agent loops (default: `maxIterations(5)`). Strategies receive `{ iterationCount, finishReason, messages, toolCallCount, lastTurnToolCallCount }` and run between model turns. Iterations are model turns, not tool calls — for tool-call budgets use middleware (`onBeforeToolCall` + `onShouldContinue`); see [Tool-call budgets](../chat/agentic-cycle#tool-call-budgets-middleware-recipe). -- `truncatedToolResult?` - The error result of a tool call in an answer that stopped at the output limit (finish reason `length`). A string, or a function that gets `{ toolCallId, toolName }`. Such a call never runs. When set, the loop calls the model again. See [When the output limit cuts a tool call](../chat/agentic-cycle#when-the-output-limit-cuts-a-tool-call). +- `agentLoopStrategy?` - Strategy for agent loops (default: `combineStrategies([maxIterations(5), untilFinishReason(["length"])])`). A strategy that goes on after a `length` answer lets the model send a [cut tool call](../chat/agentic-cycle#when-the-output-limit-cuts-a-tool-call) again. Strategies receive `{ iterationCount, finishReason, messages, toolCallCount, lastTurnToolCallCount }` and run between model turns. Iterations are model turns, not tool calls. For tool-call budgets, use middleware (`onBeforeToolCall` + `onShouldContinue`). See [Tool-call budgets](../chat/agentic-cycle#tool-call-budgets-middleware-recipe). - `middleware?` - Array of chat middleware. Use `onShouldContinue` / `onBeforeToolCall` for app-owned tool budgets. - `abortController?` - AbortController for cancellation - `modelOptions?` - Provider-native model options. This is where sampling parameters live — `temperature`, `top_p`/`topP`, and the provider's token-limit key (`max_output_tokens`, `max_tokens`, `maxOutputTokens`, …) — under each provider's canonical name, rather than as generic root-level props. See [Moving Sampling Options into modelOptions](../migration/sampling-options-to-model-options). (Renamed from `providerOptions`.) diff --git a/docs/chat/agentic-cycle.md b/docs/chat/agentic-cycle.md index 35d1cbef67..f01effc7c3 100644 --- a/docs/chat/agentic-cycle.md +++ b/docs/chat/agentic-cycle.md @@ -155,11 +155,11 @@ export async function POST(request: Request) { 2. LLM calls `get_clothing_advice({temperature: 62, conditions: "cloudy"})` → Returns `{recommendation: "Light jacket recommended"}` 3. LLM generates: "The weather in San Francisco is 62°F and cloudy. I recommend wearing a light jacket." -The loop continues only while the model's finish reason is `tool_calls` (with pending tool calls) **and** the agent loop strategy permits another iteration; it ends as soon as the model returns a normal `stop` finish reason. +The loop continues only while the model's finish reason is `tool_calls` (with pending tool calls) **and** the agent loop strategy permits another iteration. It ends as soon as the model returns a normal `stop` finish reason. An answer that [hits the output limit](#when-the-output-limit-cuts-a-tool-call) is a special case. ### Controlling the loop -By default the loop is bounded by `maxIterations(5)` — after five **model turns** it stops even if the model would keep calling tools. Override this with the `agentLoopStrategy` option. +By default, the loop stops after five **model turns** (`maxIterations(5)`), even if the model calls more tools. It also stops after an answer that [hits the output limit](#when-the-output-limit-cuts-a-tool-call) (`untilFinishReason(["length"])`). Override this with the `agentLoopStrategy` option. Other built-in strategies: @@ -191,12 +191,12 @@ export async function POST(request: Request) { ### When the output limit cuts a tool call -A model can run out of output tokens in the middle of a tool call, for example while it writes a long file. The provider stops with the finish reason `length`, and the input of the call is incomplete. `chat()` never runs such a call. By default, the run ends. +A model can run out of output tokens in the middle of a tool call, for example while it writes a long file. The provider stops with the finish reason `length`, and the input of the call is not complete. `chat()` never runs such a call. It asks for no approval and starts no client tool. Each cut call gets an error tool result instead. -Set `truncatedToolResult`, and the model can try again: +By default, the run then ends with the finish reason `length`. To let the model send the cut call again, use a strategy that does not stop on `length`: ```typescript -import { chat, toServerSentEventsResponse } from "@tanstack/ai"; +import { chat, maxIterations, toServerSentEventsResponse } from "@tanstack/ai"; import { openaiText } from "@tanstack/ai-openai"; import { writeFile } from "./tools"; @@ -206,25 +206,42 @@ export async function POST(request: Request) { adapter: openaiText("gpt-6.1-sol"), messages, tools: [writeFile], - truncatedToolResult: (call) => - `Tool call "${call.toolName}" did not run: your answer hit the output limit, so its input can be incomplete. Call it again with the complete input.`, + agentLoopStrategy: maxIterations(5), }); return toServerSentEventsResponse(stream); } ``` -What happens when an answer with tool calls stops at the output limit: +Then this happens after a cut answer: -1. The answer keeps its tool calls in the transcript. -2. Each call gets an error tool result with your text. The call does not run, and it asks for no approval and no client tool. -3. The loop calls the model again. This counts as one iteration, so `maxIterations` still stops a model that hits the limit every time. +1. The loop calls the model again. The model reads the error result and sends the call again. +2. `maxIterations` counts the cut answer as one model turn. A model that hits the limit every time stops at the iteration limit. -Some details: +The client needs no change. When the loop goes on, the `RUN_FINISHED` event of the cut answer has the finish reason `tool_calls`, so `useChat` keeps the stream open for the next model turn. -- `truncatedToolResult` is a string, or a function that gets the call (`{ toolCallId, toolName }`). -- The `RUN_FINISHED` event of that model call has the finish reason `tool_calls`, so the client keeps the stream open. -- Each assistant message that `chat()` stores keeps the finish reason of its model call in `metadata.tanstack.finishReason`. A cut message keeps `length`. -- If a history has a call of a `length` message with no result, `chat()` does not run that call. +The default error text tells the model to send the calls again and to split a large input. To use your own text, return a new result from the `onAfterToolCall` middleware hook. The error of a cut call has the code `max_tokens`: + +```typescript +import { defineChatMiddleware } from "@tanstack/ai"; + +export const cutToolCallText = defineChatMiddleware({ + onAfterToolCall(_ctx, info) { + const cut = + info.error instanceof Error && + "code" in info.error && + info.error.code === "max_tokens"; + if (!cut) return; + return { + type: "replaceResult", + result: `Your call to ${info.toolName} was cut off. Write the file in parts of 200 lines or less.`, + }; + }, +}); +``` + +Add it with `middleware: [cutToolCallText]` next to `agentLoopStrategy`. + +On the server, each assistant message that `chat()` stores keeps `metadata.tanstack.finishReason`. A cut answer has `length`. If a transcript has a cut call with no result, `chat()` does not run that call. ### Tool-call budgets (middleware recipe) diff --git a/docs/config.json b/docs/config.json index 0210b28b21..f4226298a9 100644 --- a/docs/config.json +++ b/docs/config.json @@ -166,7 +166,7 @@ "label": "Agentic Cycle", "to": "chat/agentic-cycle", "addedAt": "2026-04-15", - "updatedAt": "2026-10-12" + "updatedAt": "2026-10-13" }, { "label": "Thinking & Reasoning", @@ -353,7 +353,7 @@ "label": "Middleware", "to": "advanced/middleware", "addedAt": "2026-04-15", - "updatedAt": "2026-10-12" + "updatedAt": "2026-10-13" }, { "label": "Built-in Middleware", @@ -1479,7 +1479,7 @@ "label": "@tanstack/ai", "to": "api/ai", "addedAt": "2026-04-15", - "updatedAt": "2026-10-12" + "updatedAt": "2026-10-13" }, { "label": "@tanstack/ai-client", diff --git a/packages/ai/src/activities/chat/index.ts b/packages/ai/src/activities/chat/index.ts index 13de8f697d..a5737e9677 100644 --- a/packages/ai/src/activities/chat/index.ts +++ b/packages/ai/src/activities/chat/index.ts @@ -96,7 +96,11 @@ import { hashSchemaInput, normalizeApprovalSchema, } from './tools/approval-schema' -import { maxIterations as maxIterationsStrategy } from './agent-loop-strategies' +import { + combineStrategies, + maxIterations as maxIterationsStrategy, + untilFinishReason, +} from './agent-loop-strategies' import { isCancelRequestedReason } from './cancel' import { applyActivityDeltaToRecords, @@ -234,9 +238,13 @@ function turnBlockOrder( return buildBlockOrder(entries) } -/** The default tool result of a call in an answer cut at the output limit. */ +/** + * The tool result of a call in an answer cut at the output limit. Middleware + * `onAfterToolCall` can replace it: the error of such a call has the code + * `'max_tokens'`. + */ const TRUNCATED_TOOL_RESULT = - 'The answer was cut off at the output limit before this tool call was complete. The call did not run.' + 'The answer stopped at the output limit, so no tool call in it ran. Send the tool calls again. Do not repeat text you already wrote. If one call has a large input, split the work into smaller calls.' // =========================== // Activity Kind @@ -591,29 +599,15 @@ export interface TextActivityOptions< reasoning?: ReasoningOptionFor> /** AbortController for cancellation */ abortController?: TextOptions['abortController'] - /** Strategy for controlling the agent loop */ + /** + * Strategy for controlling the agent loop. Default: + * `combineStrategies([maxIterations(5), untilFinishReason(['length'])])`. + * A strategy that goes on after a `length` answer makes the model send the + * cut tool calls of that answer again. + */ agentLoopStrategy?: TextOptions['agentLoopStrategy'] /** How the server tools of one model turn run. Default `'parallel'`. */ toolExecution?: TextOptions['toolExecution'] - /** - * Continue after a model answer with tool calls that stopped at the output - * limit (finish reason `length`). Each call that the provider did not run - * gets this text as an error result and does not run. Then the model is - * called again, so it can send the calls again with complete arguments. - * The function form gets the call. Not set: the answer ends the run. - * - * @example - * ```ts - * chat({ - * adapter, - * messages, - * tools, - * truncatedToolResult: (call) => - * `Tool call "${call.toolName}" was cut off. Send it again.`, - * }) - * ``` - */ - truncatedToolResult?: TextOptions['truncatedToolResult'] /** * Retry a rate-limited model call after the wait the provider asks for. * @@ -1007,6 +1001,8 @@ class TextEngine< /** Tool call IDs already counted toward `toolCallCount` (avoids double-count on resume). */ private readonly countedToolCallIds = new Set() private lastFinishReason: string | null = null + /** The loop decision of a cut answer, from its tool phase (processToolCalls). */ + private cutAnswerContinues: boolean | undefined private streamStartTime = 0 private totalChunkCount = 0 private currentMessageId: string | null = null @@ -1165,7 +1161,11 @@ class TextEngine< } this.systemPrompts = config.params.systemPrompts || [] this.loopStrategy = - config.params.agentLoopStrategy || maxIterationsStrategy(5) + config.params.agentLoopStrategy || + combineStrategies([ + maxIterationsStrategy(5), + untilFinishReason(['length']), + ]) this.initialMessageCount = config.params.messages.length // Extract client state (approvals, client tool results) from original messages BEFORE conversion @@ -2662,43 +2662,79 @@ class TextEngine< this.addAssistantToolCallMessage(toolCalls) // The output limit cut this answer. Each call that the provider did not - // run has its error result now (closeTruncatedToolCalls). No call runs, - // and the loop goes on, so the model can call again. + // run has its error result now (closeTruncatedToolCalls). No call runs. if (this.lastFinishReason === 'length') { - // A client reads it as a tool turn, so it waits for the next call. - this.deferredToolCallRunFinishedChunks = - this.deferredToolCallRunFinishedChunks.map((chunk) => - chunk.type === EventType.RUN_FINISHED - ? { ...chunk, finishReason: 'tool_calls' } - : chunk, - ) - yield* this.flushDeferredToolCallRunFinishedChunks() - const results = toolCalls.flatMap((call): Array => { - const result = this.messages.find( + const results: Array = [] + for (const call of toolCalls) { + const stored = this.messages.find( (message) => message.role === 'tool' && message.toolCallId === call.id, ) - return result - ? [ - { - toolCallId: call.id, - toolName: call.function.name, - result: result.content, - state: 'output-error', - }, - ] - : [] - }) - for (const chunk of this.buildToolResultChunks( + if (!stored) continue + // onAfterToolCall can replace the text. The code tells this error + // apart from a failed run. + const decision = yield* this.runWhileYielding( + this.middlewareRunner.runOnAfterToolCall(this.middlewareCtx, { + toolCall: call, + tool: this.tools.find((tool) => tool.name === call.function.name), + toolName: call.function.name, + toolCallId: call.id, + ok: false, + duration: 0, + error: Object.assign(new Error(TRUNCATED_TOOL_RESULT), { + code: 'max_tokens', + }), + }), + ) + results.push({ + toolCallId: call.id, + toolName: call.function.name, + result: decision ? decision.result : stored.content, + state: 'output-error', + }) + } + // This also stores the results, so the loop decision reads them. + const resultChunks = this.buildToolResultChunks( results, finishEvent, undefined, true, - )) { + ) + // Ask the strategy and middleware now, with the state that + // shouldContinue() gives them after this cycle. shouldContinue() then + // uses this answer. The RUN_FINISHED that the client sees depends on it. + const continues = yield* this.runWhileYielding( + this.loopContinues(this.iterationCount + 1), + ) + this.cutAnswerContinues = continues + if (continues) { + // A client reads it as a tool turn, so it waits for the next call. + // The chunks are already public, so the finish reason is in the + // metadata. + this.deferredToolCallRunFinishedChunks = + this.deferredToolCallRunFinishedChunks.map((chunk) => + chunk.type === EventType.RUN_FINISHED + ? withTanstackMetadata(chunk, { finishReason: 'tool_calls' }) + : chunk, + ) + yield* this.flushDeferredToolCallRunFinishedChunks() + } + for (const chunk of resultChunks) { yield* this.pipeThroughMiddleware(chunk) } + // A stopped run ends on the `length` RUN_FINISHED, after the results. + // Native combined structured output sends it after its harvest, and + // drops it when the harvest fails. + if (!continues && this.finalStructuredOutput?.nativeCombined) { + this.deferredModelRunFinishedChunks.push( + ...this.deferredToolCallRunFinishedChunks, + ) + this.deferredToolCallRunFinishedChunks = [] + } else if (!continues) { + yield* this.flushDeferredToolCallRunFinishedChunks() + } this.toolCallManager.clear() - this.setToolPhase('continue') + this.setToolPhase(continues ? 'continue' : 'stop') return } @@ -2971,7 +3007,6 @@ class TextEngine< return ( (this.lastFinishReason === 'tool_calls' || (this.lastFinishReason === 'length' && - this.params.truncatedToolResult !== undefined && // Provider-executed calls have their results. With no other call, // the cut answer ends the run like a text answer. this.toolCallManager @@ -3146,16 +3181,16 @@ class TextEngine< : [], ) if (cut.length === 0) return - const option = this.params.truncatedToolResult this.messages = [ ...this.messages, - ...cut.map((call): ModelMessage => { - const text = - typeof option === 'function' - ? option({ toolCallId: call.id, toolName: call.function.name }) - : (option ?? TRUNCATED_TOOL_RESULT) - return { role: 'tool', toolCallId: call.id, content: text, error: text } - }), + ...cut.map( + (call): ModelMessage => ({ + role: 'tool', + toolCallId: call.id, + content: TRUNCATED_TOOL_RESULT, + error: TRUNCATED_TOOL_RESULT, + }), + ), ] } @@ -4253,8 +4288,20 @@ class TextEngine< return true } + // The tool phase of a cut answer asked already. + const cutAnswerContinues = this.cutAnswerContinues + this.cutAnswerContinues = undefined + + return ( + (cutAnswerContinues ?? (await this.loopContinues(this.iterationCount))) && + this.toolPhase === 'continue' + ) + } + + /** Ask the loop strategy and middleware `onShouldContinue`. */ + private async loopContinues(iterationCount: number): Promise { const state = { - iterationCount: this.iterationCount, + iterationCount, messages: this.messages, finishReason: this.lastFinishReason, toolCallCount: this.toolCallCount, @@ -4263,16 +4310,13 @@ class TextEngine< // Evaluate strategy and middleware unconditionally (even when the // strategy already says stop) so every onShouldContinue observer still - // sees the final counters; AND all three at the end. + // sees the final counters. const strategyContinues = this.loopStrategy(state) const middlewareContinues = await this.middlewareRunner.runOnShouldContinue( this.middlewareCtx, state, ) - - return ( - strategyContinues && middlewareContinues && this.toolPhase === 'continue' - ) + return strategyContinues && middlewareContinues } /** diff --git a/packages/ai/src/activities/chat/middleware/types.ts b/packages/ai/src/activities/chat/middleware/types.ts index b46aff8c8f..40afa4c922 100644 --- a/packages/ai/src/activities/chat/middleware/types.ts +++ b/packages/ai/src/activities/chat/middleware/types.ts @@ -832,6 +832,10 @@ export interface ChatMiddleware< * Return `{ type: 'replaceResult', result }` to change the result that the * model and the stream see. Middleware run in order. Each one sees the * result of the one before it. + * + * Also called for a tool call that did not run because the output limit + * cut its answer. Then `ok` is false, and `error` is an `Error` with the + * code `'max_tokens'`. */ onAfterToolCall?: ( ctx: ChatMiddlewareContext, diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 141ae1a082..d803952d65 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -1306,16 +1306,6 @@ export interface TextOptions< * starts them together, and `'sequential'` runs them one at a time. */ toolExecution?: 'parallel' | 'sequential' - /** - * The tool result of a call in a model answer that stopped at the output - * limit (finish reason `length`). The function form gets the call. Such a - * call never runs. When set, each call gets this text as an error result, - * and the loop calls the model again, as after a tool phase. Not set: the - * answer ends the run, and a kept call gets a default text. - */ - truncatedToolResult?: - | string - | ((call: { toolCallId: string; toolName: string }) => string) /** * Retry a model call when the provider error has a wait time (usually a * 429). `chat()` waits for the `retryAfterMs` time of the `RUN_ERROR`, then diff --git a/packages/ai/src/utilities/replay-messages.ts b/packages/ai/src/utilities/replay-messages.ts index 044f0ed289..f291591630 100644 --- a/packages/ai/src/utilities/replay-messages.ts +++ b/packages/ai/src/utilities/replay-messages.ts @@ -50,6 +50,17 @@ function isForeign( ) } +/** A string input that does not parse. Some adapters keep an object here. */ +function isCutInput(input: unknown): boolean { + if (typeof input !== 'string') return false + try { + JSON.parse(input || '{}') + return false + } catch { + return true + } +} + function isFailed(message: ModelMessage): boolean { const stop = tanstackMetadata(message.metadata)?.stopReason return ( @@ -221,6 +232,23 @@ export function transformMessagesForReplay( } else { for (const call of message.toolCalls ?? []) idMap.set(call.id, call.id) } + // The output limit can cut a call in the middle of its input. Such a + // call did not run, and some providers (Bedrock) reject input that is not + // JSON. Other input that is not JSON stays, so such a provider still fails + // loudly on it. + if ( + tanstackMetadata(message.metadata)?.finishReason === 'length' && + mapped.toolCalls?.some((call) => isCutInput(call.function.arguments)) + ) { + mapped = { + ...mapped, + toolCalls: mapped.toolCalls.map((call) => + isCutInput(call.function.arguments) + ? { ...call, function: { ...call.function, arguments: '{}' } } + : call, + ), + } + } pending = [...pending, ...(mapped.toolCalls ?? [])] turnId ??= message.id messages.push(mapped) diff --git a/packages/ai/tests/chat-native-combined-structured-output.test.ts b/packages/ai/tests/chat-native-combined-structured-output.test.ts index 42d4db4ff3..4a850cd2dd 100644 --- a/packages/ai/tests/chat-native-combined-structured-output.test.ts +++ b/packages/ai/tests/chat-native-combined-structured-output.test.ts @@ -19,7 +19,7 @@ import { describe, expect, it } from 'vitest' import { z } from 'zod' import { chat } from '../src/activities/chat/index' import { EventType } from '../src/types' -import { collectChunks, createMockAdapter } from './test-utils' +import { collectChunks, createMockAdapter, ev, serverTool } from './test-utils' import type { AdapterYieldChunk } from '../src/utilities/adapter-yield-chunk' const PersonSchema = z.object({ @@ -244,6 +244,41 @@ describe('chat({ outputSchema, stream: true }) — native combined mode (#605)', }, ) + it('ends with one RUN_ERROR when the output limit cuts a tool call', async () => { + const { adapter } = createMockAdapter({ + iterations: [ + [ + ev.runStarted(), + ev.toolStart('call_1', 'lookup'), + ev.toolArgs('call_1', '{"q":'), + ev.runFinished('length'), + ], + ], + supportsCombinedToolsAndSchema: true, + }) + + const chunks = await collectChunks( + chat({ + adapter, + messages: [{ role: 'user', content: 'extract' }], + tools: [serverTool('lookup', () => ({ found: true }))], + outputSchema: PersonSchema, + stream: true, + }), + ) + + const terminals = chunks.filter( + (c) => + c.type === EventType.RUN_FINISHED || c.type === EventType.RUN_ERROR, + ) + expect(terminals).toEqual([ + expect.objectContaining({ + type: EventType.RUN_ERROR, + code: 'max_tokens', + }), + ]) + }) + it('Promise path puts the full raw text on the parse error (#1485)', async () => { const one = JSON.stringify({ ...validPerson, name: 'x'.repeat(150) }) const raw = `${one}\n\n${one}` diff --git a/packages/ai/tests/chat-provider-executed-thinking-order.test.ts b/packages/ai/tests/chat-provider-executed-thinking-order.test.ts index 5e08867f8a..4f67415356 100644 --- a/packages/ai/tests/chat-provider-executed-thinking-order.test.ts +++ b/packages/ai/tests/chat-provider-executed-thinking-order.test.ts @@ -17,6 +17,7 @@ import { serverTool, } from './test-utils' import { EventType } from '../src/types' +import { maxIterations } from '../src/activities/chat/agent-loop-strategies' import type { ModelMessage, StreamChunk, UIMessage } from '../src/types' /** @@ -441,6 +442,9 @@ describe('provider-executed tools interleaved with signed thinking', () => { ) }) + const DEFAULT_TEXT = + 'The answer stopped at the output limit, so no tool call in it ran. Send the tool calls again. Do not repeat text you already wrote. If one call has a large input, split the work into smaller calls.' + it('gives the calls of an answer cut at the output limit an error result, so a later run does not run them', async () => { const lookup = vi.fn(() => ({ found: true })) const { adapter, calls } = createMockAdapter({ @@ -481,8 +485,7 @@ describe('provider-executed tools interleaved with signed thinking', () => { }) as AsyncIterable, ) - const text = - 'The answer was cut off at the output limit before this tool call was complete. The call did not run.' + const text = DEFAULT_TEXT expect(transcript.filter((message) => message.role === 'tool')).toEqual([ { role: 'tool', toolCallId: 'call_1', content: text, error: text }, ]) @@ -499,7 +502,7 @@ describe('provider-executed tools interleaved with signed thinking', () => { expect(calls).toHaveLength(2) }) - it('with truncatedToolResult, gives the cut call that text and calls the model again', async () => { + it('with a strategy that goes on, gives the cut call the text and calls the model again', async () => { const lookup = vi.fn(() => ({ found: true })) const { adapter, calls } = createMockAdapter({ iterations: [ @@ -528,7 +531,7 @@ describe('provider-executed tools interleaved with signed thinking', () => { adapter, messages: [{ role: 'user', content: 'Research' }], tools: [serverTool('lookup', lookup)], - truncatedToolResult: 'Cut off. Not run.', + agentLoopStrategy: maxIterations(5), }) as AsyncIterable, ) @@ -546,7 +549,7 @@ describe('provider-executed tools interleaved with signed thinking', () => { { assistant: ['srvtoolu_1'] }, { assistant: ['call_1'] }, // The provider-executed call keeps its own result. - { tool: 'call_1', content: 'Cut off. Not run.' }, + { tool: 'call_1', content: DEFAULT_TEXT }, ]) }) }) diff --git a/packages/ai/tests/chat.test.ts b/packages/ai/tests/chat.test.ts index 7b3b0f68df..eac41978be 100644 --- a/packages/ai/tests/chat.test.ts +++ b/packages/ai/tests/chat.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest' import { z } from 'zod' import { chat, createChatOptions } from '../src/activities/chat/index' +import { maxIterations } from '../src/activities/chat/agent-loop-strategies' import { defineInterrupt } from '../src/interrupt-definition' import { genericInterruptContinuationFromDescriptor, @@ -3055,9 +3056,9 @@ describe('chat()', () => { // ========================================================================== describe('answers cut at the output limit', () => { const DEFAULT_TEXT = - 'The answer was cut off at the output limit before this tool call was complete. The call did not run.' - const truncatedToolResult = (call: { toolName: string }) => - `Tool call "${call.toolName}" was not executed: the response hit the output token limit, so its arguments may be truncated. Re-issue the tool call with complete arguments.` + 'The answer stopped at the output limit, so no tool call in it ran. Send the tool calls again. Do not repeat text you already wrote. If one call has a large input, split the work into smaller calls.' + // A strategy with no `length` stop: the model sends the cut calls again. + const retry = maxIterations(5) const done = [ ev.runStarted(), ev.textStart('msg-2'), @@ -3066,7 +3067,7 @@ describe('chat()', () => { ev.runFinished('stop'), ] - it('gives a cut call the text, does not run it, and calls the model again', async () => { + it('with a strategy that goes on, gives a cut call the text, does not run it, and calls the model again', async () => { const lookup = vi.fn(() => ({ found: true })) const { adapter, calls } = createMockAdapter({ iterations: [ @@ -3085,11 +3086,10 @@ describe('chat()', () => { adapter, messages: [{ role: 'user', content: 'Look it up' }], tools: [serverTool('lookup', lookup)], - truncatedToolResult, + agentLoopStrategy: retry, }) as AsyncIterable, ) - const text = truncatedToolResult({ toolName: 'lookup' }) expect(lookup).not.toHaveBeenCalled() expect(calls).toHaveLength(2) expect((calls[1]!.messages as Array).slice(1)).toEqual([ @@ -3100,8 +3100,81 @@ describe('chat()', () => { tanstack: expect.objectContaining({ finishReason: 'length' }), }, }), - { role: 'tool', toolCallId: 'call_1', content: text, error: text }, + { + role: 'tool', + toolCallId: 'call_1', + content: DEFAULT_TEXT, + error: DEFAULT_TEXT, + }, ]) + expect(chunks).toContainEqual( + expect.objectContaining({ + type: EventType.TOOL_CALL_RESULT, + toolCallId: 'call_1', + content: DEFAULT_TEXT, + }), + ) + // The client reads the cut answer as a tool turn. + expect( + chunks + .filter((item) => item.type === EventType.RUN_FINISHED) + .map((item) => tanstackMetadata(item)?.finishReason), + ).toEqual(['tool_calls', 'stop']) + }) + + it('lets onAfterToolCall replace the text of a cut call', async () => { + const seen: Array = [] + const { adapter, calls } = createMockAdapter({ + iterations: [ + [ + ev.runStarted(), + ev.toolStart('call_1', 'lookup'), + ev.toolArgs('call_1', '{"q":'), + ev.runFinished('length'), + ], + done, + ], + }) + const rewrite = defineChatMiddleware({ + onAfterToolCall(_ctx, info) { + seen.push(info.error) + const cut = + !info.ok && + info.error instanceof Error && + 'code' in info.error && + info.error.code === 'max_tokens' + if (!cut) return + return { + type: 'replaceResult', + result: `The ${info.toolName} call was cut off. Send it again.`, + } + }, + }) + + const chunks = await collectChunks( + chat({ + adapter, + messages: [{ role: 'user', content: 'Look it up' }], + tools: [serverTool('lookup', () => ({ found: true }))], + agentLoopStrategy: retry, + middleware: [rewrite], + }) as AsyncIterable, + ) + + const text = 'The lookup call was cut off. Send it again.' + expect(seen).toEqual([ + expect.objectContaining({ message: DEFAULT_TEXT, code: 'max_tokens' }), + ]) + expect( + (calls[1]!.messages as Array).find( + (message) => message.role === 'tool', + ), + ).toEqual({ + role: 'tool', + toolCallId: 'call_1', + content: text, + error: text, + }) expect(chunks).toContainEqual( expect.objectContaining({ type: EventType.TOOL_CALL_RESULT, @@ -3111,7 +3184,7 @@ describe('chat()', () => { ) }) - it('gives each call of a cut answer its text, and asks nothing', async () => { + it('gives each call of a cut answer the text, and asks nothing', async () => { const remove = vi.fn(() => ({ ok: true })) const { adapter, calls } = createMockAdapter({ iterations: [ @@ -3141,7 +3214,7 @@ describe('chat()', () => { { ...serverTool('remove', remove), needsApproval: true }, clientTool('ask'), ], - truncatedToolResult, + agentLoopStrategy: retry, }) as AsyncIterable, ) for (const item of chunks) processor.processChunk(item) @@ -3153,15 +3226,12 @@ describe('chat()', () => { (message) => message.role === 'tool', ), ).toEqual( - ['remove', 'ask'].map((toolName, index) => { - const text = truncatedToolResult({ toolName }) - return { - role: 'tool', - toolCallId: `call_${index + 1}`, - content: text, - error: text, - } - }), + ['call_1', 'call_2'].map((toolCallId) => ({ + role: 'tool', + toolCallId, + content: DEFAULT_TEXT, + error: DEFAULT_TEXT, + })), ) expect( chunks.some( @@ -3186,20 +3256,25 @@ describe('chat()', () => { iterations: [cut('call_1'), cut('call_2'), cut('call_3')], }) - await collectChunks( + const chunks = await collectChunks( chat({ adapter, messages: [{ role: 'user', content: 'Look it up' }], tools: [serverTool('lookup', () => ({ found: true }))], agentLoopStrategy: (state) => state.iterationCount < 2, - truncatedToolResult, }) as AsyncIterable, ) expect(calls).toHaveLength(2) + // The second cut answer stops the loop, so it keeps `length`. + expect( + chunks + .filter((item) => item.type === EventType.RUN_FINISHED) + .map((item) => tanstackMetadata(item)?.finishReason), + ).toEqual(['tool_calls', 'length']) }) - it('without the option, ends the run and keeps no calls of a plain cut answer', async () => { + it('by default, gives the cut call its result and ends the run with length', async () => { const lookup = vi.fn(() => ({ found: true })) const { adapter, calls } = createMockAdapter({ iterations: [ @@ -3216,13 +3291,15 @@ describe('chat()', () => { ], }) let transcript: Array = [] + const afterToolCall = vi.fn() const keep = defineChatMiddleware({ onFinish(ctx) { transcript = [...ctx.messages] }, + onAfterToolCall: afterToolCall, }) - await collectChunks( + const chunks = await collectChunks( chat({ adapter, messages: [{ role: 'user', content: 'Look it up' }], @@ -3232,20 +3309,100 @@ describe('chat()', () => { ) expect(lookup).not.toHaveBeenCalled() + expect(afterToolCall).toHaveBeenCalledTimes(1) expect(calls).toHaveLength(1) + expect( + tanstackMetadata(expectSingleRunFinished(chunks))?.finishReason, + ).toBe('length') + // The result comes before the RUN_FINISHED that ends the run. + expect( + chunks + .filter( + (item) => + item.type === EventType.TOOL_CALL_RESULT || + item.type === EventType.RUN_FINISHED, + ) + .map((item) => item.type), + ).toEqual([EventType.TOOL_CALL_RESULT, EventType.RUN_FINISHED]) expect(transcript.slice(1)).toEqual([ expect.objectContaining({ role: 'assistant', content: 'Let me look.', + toolCalls: [expect.objectContaining({ id: 'call_1' })], metadata: { tanstack: expect.objectContaining({ finishReason: 'length' }), }, }), + { + role: 'tool', + toolCallId: 'call_1', + content: DEFAULT_TEXT, + error: DEFAULT_TEXT, + }, ]) - expect(transcript[1]).not.toHaveProperty('toolCalls') }) - it('without the option, the client runs no client tool of a cut answer', async () => { + it('ends the run with length when onShouldContinue stops after a cut answer', async () => { + const { adapter, calls } = createMockAdapter({ + iterations: [ + [ + ev.runStarted(), + ev.toolStart('call_1', 'lookup'), + ev.toolArgs('call_1', '{"q":'), + ev.runFinished('length'), + ], + done, + ], + }) + const onShouldContinue = vi.fn(() => false) + + const chunks = await collectChunks( + chat({ + adapter, + messages: [{ role: 'user', content: 'Look it up' }], + tools: [serverTool('lookup', () => ({ found: true }))], + agentLoopStrategy: retry, + middleware: [defineChatMiddleware({ onShouldContinue })], + }) as AsyncIterable, + ) + + expect(calls).toHaveLength(1) + // One loop decision for the cut answer. + expect(onShouldContinue).toHaveBeenCalledTimes(1) + expect( + tanstackMetadata(expectSingleRunFinished(chunks))?.finishReason, + ).toBe('length') + }) + + it('sends the input of a cut call that is not JSON as an empty object', async () => { + const { adapter, calls } = createMockAdapter({ + iterations: [ + [ + ev.runStarted(), + ev.toolStart('call_1', 'lookup'), + ev.toolArgs('call_1', '{"q":'), + ev.runFinished('length'), + ], + done, + ], + }) + + await collectChunks( + chat({ + adapter, + messages: [{ role: 'user', content: 'Look it up' }], + tools: [serverTool('lookup', () => ({ found: true }))], + agentLoopStrategy: retry, + }) as AsyncIterable, + ) + + const assistant = (calls[1]!.messages as Array).find( + (message) => message.role === 'assistant', + ) + expect(assistant?.toolCalls?.[0]?.function.arguments).toBe('{}') + }) + + it('by default, the client runs no client tool of a cut answer', async () => { const { adapter } = createMockAdapter({ iterations: [ [ @@ -3325,85 +3482,78 @@ describe('chat()', () => { expect(calls).toHaveLength(1) }) - it.each([ - ['the default text', undefined, DEFAULT_TEXT], - ['truncatedToolResult', 'Cut off. Not run.', 'Cut off. Not run.'], - ])( - 'gives the calls of a cut answer a result at an afterModel interrupt (%s)', - async (_label, option, text) => { - const review = defineInterrupt({ - id: 'after-model-cut', - responseSchema: z.object({ approved: z.boolean() }), - }) - const { adapter } = createMockAdapter({ - iterations: [ - [ - ev.runStarted(), - ev.toolStart('call-draft', 'saveDraft'), - ev.toolArgs('call-draft', '{}'), - ev.toolEnd('call-draft'), - ev.toolStart('call-remove', 'remove'), - ev.toolArgs('call-remove', '{"path":'), - ev.runFinished('length'), - ], + it('gives the calls of a cut answer a result at an afterModel interrupt', async () => { + const review = defineInterrupt({ + id: 'after-model-cut', + responseSchema: z.object({ approved: z.boolean() }), + }) + const { adapter } = createMockAdapter({ + iterations: [ + [ + ev.runStarted(), + ev.toolStart('call-draft', 'saveDraft'), + ev.toolArgs('call-draft', '{}'), + ev.toolEnd('call-draft'), + ev.toolStart('call-remove', 'remove'), + ev.toolArgs('call-remove', '{"path":'), + ev.runFinished('length'), ], - }) + ], + }) - const chunks = await collectChunks( - chat({ - adapter, - interrupts: [review], - middleware: [ - defineChatMiddleware({ - onInterruptBoundary(ctx) { - if (ctx.phase !== 'afterModel') return - return { - interrupts: [ - review.interrupt({ - key: 'after-model', - reason: 'review', - message: 'Review the answer', - }), - ], - } - }, - }), - ], - messages: [{ role: 'user', content: 'Save and remove' }], - tools: [ - clientTool('saveDraft'), - { - ...serverTool('remove', () => ({ ok: true })), - needsApproval: true, + const chunks = await collectChunks( + chat({ + adapter, + interrupts: [review], + middleware: [ + defineChatMiddleware({ + onInterruptBoundary(ctx) { + if (ctx.phase !== 'afterModel') return + return { + interrupts: [ + review.interrupt({ + key: 'after-model', + reason: 'review', + message: 'Review the answer', + }), + ], + } }, - ], - ...(option ? { truncatedToolResult: option } : {}), - }) as AsyncIterable, - ) + }), + ], + messages: [{ role: 'user', content: 'Save and remove' }], + tools: [ + clientTool('saveDraft'), + { + ...serverTool('remove', () => ({ ok: true })), + needsApproval: true, + }, + ], + }) as AsyncIterable, + ) - const terminal = expectSingleRunFinished(chunks) - if (terminal.outcome?.type !== 'interrupt') { - throw new Error('Expected afterModel interrupt') - } - expect(terminal.outcome.interrupts).toEqual([ - expect.objectContaining({ reason: 'review' }), - ]) - const snapshot = chunks.find( - (item) => item.type === EventType.MESSAGES_SNAPSHOT, - ) - if (!snapshot || snapshot.type !== EventType.MESSAGES_SNAPSHOT) { - throw new Error('Expected messages snapshot') - } - expect( - snapshot.messages - .filter((message) => message.role === 'tool') - .map((message) => [message.toolCallId, message.content]), - ).toEqual([ - ['call-draft', text], - ['call-remove', text], - ]) - }, - ) + const terminal = expectSingleRunFinished(chunks) + if (terminal.outcome?.type !== 'interrupt') { + throw new Error('Expected afterModel interrupt') + } + expect(terminal.outcome.interrupts).toEqual([ + expect.objectContaining({ reason: 'review' }), + ]) + const snapshot = chunks.find( + (item) => item.type === EventType.MESSAGES_SNAPSHOT, + ) + if (!snapshot || snapshot.type !== EventType.MESSAGES_SNAPSHOT) { + throw new Error('Expected messages snapshot') + } + expect( + snapshot.messages + .filter((message) => message.role === 'tool') + .map((message) => [message.toolCallId, message.content]), + ).toEqual([ + ['call-draft', DEFAULT_TEXT], + ['call-remove', DEFAULT_TEXT], + ]) + }) }) // ========================================================================== diff --git a/testing/e2e/src/routes/api.bedrock-converse-truncated-tool-call.ts b/testing/e2e/src/routes/api.bedrock-converse-truncated-tool-call.ts index eb4a7f1ee2..61b66914a2 100644 --- a/testing/e2e/src/routes/api.bedrock-converse-truncated-tool-call.ts +++ b/testing/e2e/src/routes/api.bedrock-converse-truncated-tool-call.ts @@ -1,5 +1,11 @@ import { createFileRoute } from '@tanstack/react-router' -import { chat, createChatOptions, toolDefinition } from '@tanstack/ai' +import { + chat, + createChatOptions, + defineChatMiddleware, + maxIterations, + toolDefinition, +} from '@tanstack/ai' import { BedrockConverseTextAdapter } from '@tanstack/ai-bedrock' import { NodeHttpHandler } from '@smithy/node-http-handler' import { z } from 'zod' @@ -27,15 +33,20 @@ class Http1ConverseAdapter extends BedrockConverseTextAdapter { * The answer of the first model call has one tool call and stops at * `max_tokens` (the fixture in `fixtures/truncated-tool-call` sets * `finishReason: 'length'`). Converse keeps the `length` finish with the - * call. With `truncatedToolResult`, the call does not run, it gets the text - * as its error result, and the model is called again. + * call. The call never runs, and it gets an error tool result. aimock sends + * fixture input that is not JSON as `{}`, so the input here is complete. + * + * - Default strategy: the run ends with `length`. + * - `?retry`: `maxIterations(5)` goes on after `length`. A middleware + * replaces the error text of the cut call, and the model is called again. */ export const Route = createFileRoute( '/api/bedrock-converse-truncated-tool-call', )({ server: { handlers: { - POST: async () => { + POST: async ({ request }) => { + const retry = new URL(request.url).searchParams.has('retry') const adapter = new Http1ConverseAdapter( { apiKey: DUMMY_KEY, @@ -56,6 +67,7 @@ export const Route = createFileRoute( let text = '' const toolResults: Array = [] + const finishReasons: Array = [] let runError: string | undefined try { for await (const chunk of chat({ @@ -67,17 +79,50 @@ export const Route = createFileRoute( content: '[truncated-tool-call] weather in Paris', }, ], - truncatedToolResult: ({ toolName }) => - `Tool call "${toolName}" was cut off. Send it again.`, + ...(retry && { + agentLoopStrategy: maxIterations(5), + middleware: [ + defineChatMiddleware({ + onAfterToolCall(_ctx, info) { + const cut = + info.error instanceof Error && + 'code' in info.error && + info.error.code === 'max_tokens' + if (!cut) return + return { + type: 'replaceResult', + result: `Tool call "${info.toolName}" was cut off. Send it again.`, + } + }, + }), + ], + }), })) { if (chunk.type === 'TEXT_MESSAGE_CONTENT') text += chunk.delta if (chunk.type === 'TOOL_CALL_RESULT') { toolResults.push(chunk.content) } + if (chunk.type === 'RUN_FINISHED') { + const tanstack = chunk.metadata?.tanstack + finishReasons.push( + tanstack && + typeof tanstack === 'object' && + 'finishReason' in tanstack && + typeof tanstack.finishReason === 'string' + ? tanstack.finishReason + : undefined, + ) + } if (chunk.type === 'RUN_ERROR') runError = chunk.message } if (runError !== undefined) throw new Error(runError) - return Response.json({ ok: true, runs, toolResults, text }) + return Response.json({ + ok: true, + runs, + toolResults, + text, + finishReasons, + }) } catch (error) { return Response.json({ ok: false, diff --git a/testing/e2e/tests/bedrock-converse-truncated-tool-call.spec.ts b/testing/e2e/tests/bedrock-converse-truncated-tool-call.spec.ts index ce9c621376..dbbb849aef 100644 --- a/testing/e2e/tests/bedrock-converse-truncated-tool-call.spec.ts +++ b/testing/e2e/tests/bedrock-converse-truncated-tool-call.spec.ts @@ -1,21 +1,47 @@ import { test, expect } from './fixtures' +const DEFAULT_TEXT = + 'The answer stopped at the output limit, so no tool call in it ran. Send the tool calls again. Do not repeat text you already wrote. If one call has a large input, split the work into smaller calls.' + +type Result = { + ok: boolean + error?: string + runs?: number + toolResults?: Array + text?: string + finishReasons?: Array +} + // The route and its aimock fixture are described at // `src/routes/api.bedrock-converse-truncated-tool-call.ts`. test.describe('bedrock-converse — tool call cut at the output limit', () => { - test('the cut call does not run, gets the text, and the model answers again', async ({ + test('by default, the cut call does not run, gets the default text, and the run ends with length', async ({ request, }) => { const res = await request.post('/api/bedrock-converse-truncated-tool-call') expect(res.ok()).toBe(true) - const { ok, error, runs, toolResults, text } = (await res.json()) as { - ok: boolean - error?: string - runs?: number - toolResults?: Array - text?: string - } + const { ok, error, runs, toolResults, text, finishReasons } = + (await res.json()) as Result + + expect(error ?? null).toBeNull() + expect(ok).toBe(true) + expect(runs).toBe(0) + expect(toolResults).toEqual([DEFAULT_TEXT]) + expect(text).toBe('') + expect(finishReasons).toEqual(['length']) + }) + + test('with a strategy that goes on, the cut call gets the middleware text and the model answers again', async ({ + request, + }) => { + const res = await request.post( + '/api/bedrock-converse-truncated-tool-call?retry', + ) + expect(res.ok()).toBe(true) + + const { ok, error, runs, toolResults, text, finishReasons } = + (await res.json()) as Result expect(error ?? null).toBeNull() expect(ok).toBe(true) @@ -25,5 +51,6 @@ test.describe('bedrock-converse — tool call cut at the output limit', () => { ]) // The second fixture answers only a request that has a tool result. expect(text).toBe('Let me look again.') + expect(finishReasons).toEqual(['tool_calls', 'stop']) }) })