From 460d4fdd113bc0bb1599ce0777f4c19375c83317 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:58:28 +0000 Subject: [PATCH 1/2] fix(providers): keep tool batches ahead of media messages Defer synthetic user media until all contiguous tool-result messages have been emitted. OpenAI rejects media inserted between replies to parallel tool calls. Preserve accompanying text and the order of media payloads. Cover both Chat Completions serializers with separate and grouped tool results, two media results, and following conversation messages. > Xum acted on Mike's behalf. --- providers/openai/language_model_hooks.go | 11 +- .../openai/tool_result_media_order_test.go | 104 ++++++++++++++++++ .../openaicompat/language_model_hooks.go | 11 +- 3 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 providers/openai/tool_result_media_order_test.go diff --git a/providers/openai/language_model_hooks.go b/providers/openai/language_model_hooks.go index f1c851570..ce55cc9f7 100644 --- a/providers/openai/language_model_hooks.go +++ b/providers/openai/language_model_hooks.go @@ -293,7 +293,13 @@ func DefaultStreamProviderMetadataFunc(choice openai.ChatCompletionChoice, metad func DefaultToPrompt(prompt fantasy.Prompt, _, _ string) ([]openai.ChatCompletionMessageParamUnion, []fantasy.CallWarning) { var messages []openai.ChatCompletionMessageParamUnion var warnings []fantasy.CallWarning + // All tool replies in a batch must precede synthetic user media. + var pendingToolMedia []openai.ChatCompletionMessageParamUnion for _, msg := range prompt { + if msg.Role != fantasy.MessageRoleTool { + messages = append(messages, pendingToolMedia...) + pendingToolMedia = nil + } switch msg.Role { case fantasy.MessageRoleSystem: var systemPromptParts []string @@ -581,7 +587,8 @@ func DefaultToPrompt(prompt fantasy.Prompt, _, _ string) ([]openai.ChatCompletio // OpenAI Chat Completions tool messages cannot carry image // or audio content directly; see ToolResultMediaMessages. mediaMessages, mediaWarnings := ToolResultMediaMessages(output, toolResultPart.ToolCallID) - messages = append(messages, mediaMessages...) + messages = append(messages, mediaMessages[0]) + pendingToolMedia = append(pendingToolMedia, mediaMessages[1:]...) warnings = append(warnings, mediaWarnings...) default: warnings = append(warnings, fantasy.CallWarning{ @@ -592,7 +599,7 @@ func DefaultToPrompt(prompt fantasy.Prompt, _, _ string) ([]openai.ChatCompletio } } } - return messages, warnings + return append(messages, pendingToolMedia...), warnings } // ToolResultMediaMessages maps a tool-result media output to the chat diff --git a/providers/openai/tool_result_media_order_test.go b/providers/openai/tool_result_media_order_test.go new file mode 100644 index 000000000..ebd501eeb --- /dev/null +++ b/providers/openai/tool_result_media_order_test.go @@ -0,0 +1,104 @@ +package openai_test + +import ( + "testing" + + "charm.land/fantasy" + "charm.land/fantasy/providers/openai" + "charm.land/fantasy/providers/openaicompat" + openaisdk "github.com/openai/openai-go/v3" + "github.com/stretchr/testify/require" +) + +func TestToolMediaFollowsToolResultBatch(t *testing.T) { + t.Parallel() + + for _, provider := range []struct { + name string + toPrompt func(fantasy.Prompt, string, string) ([]openaisdk.ChatCompletionMessageParamUnion, []fantasy.CallWarning) + }{ + {name: "OpenAI", toPrompt: openai.DefaultToPrompt}, + {name: "OpenAICompatible", toPrompt: openaicompat.ToPromptFunc}, + } { + t.Run(provider.name, func(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + separateMessages bool + secondMedia bool + followup bool + }{ + {name: "SeparateMessagesMixed", separateMessages: true}, + {name: "SeparateMessagesTwoMedia", separateMessages: true, secondMedia: true}, + {name: "SameMessageMixed"}, + {name: "SameMessageTwoMedia", secondMedia: true}, + {name: "BeforeFollowingMessages", separateMessages: true, secondMedia: true, followup: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + media := fantasy.ToolResultOutputContentMedia{Data: "AAEC", MediaType: "image/png", Text: "first result"} + results := []fantasy.MessagePart{ + fantasy.ToolResultPart{ToolCallID: "call-a", Output: media}, + fantasy.ToolResultPart{ToolCallID: "call-b", Output: fantasy.ToolResultOutputContentText{Text: "second result"}}, + } + imageURLs := []string{"data:image/png;base64,AAEC"} + if tc.secondMedia { + media.Data, media.Text = "AwQF", "second result" + results[1] = fantasy.ToolResultPart{ToolCallID: "call-b", Output: media} + imageURLs = append(imageURLs, "data:image/png;base64,AwQF") + } + prompt := fantasy.Prompt{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ToolCallID: "call-a", ToolName: "first", Input: "{}"}, + fantasy.ToolCallPart{ToolCallID: "call-b", ToolName: "second", Input: "{}"}, + }, + }} + if tc.separateMessages { + for _, result := range results { + prompt = append(prompt, fantasy.Message{Role: fantasy.MessageRoleTool, Content: []fantasy.MessagePart{result}}) + } + } else { + prompt = append(prompt, fantasy.Message{Role: fantasy.MessageRoleTool, Content: results}) + } + if tc.followup { + prompt = append(prompt, + fantasy.Message{Role: fantasy.MessageRoleAssistant, Content: []fantasy.MessagePart{fantasy.TextPart{Text: "done"}}}, + fantasy.Message{Role: fantasy.MessageRoleUser, Content: []fantasy.MessagePart{fantasy.TextPart{Text: "thanks"}}}, + ) + } + + messages, warnings := provider.toPrompt(prompt, "", "") + require.Empty(t, warnings) + wantLen := 3 + len(imageURLs) + if tc.followup { + wantLen += 2 + } + require.Len(t, messages, wantLen) + require.NotNil(t, messages[0].OfAssistant) + require.Len(t, messages[0].OfAssistant.ToolCalls, 2) + for i, want := range []struct{ id, text string }{{"call-a", "first result"}, {"call-b", "second result"}} { + tool := messages[i+1].OfTool + require.NotNil(t, tool, "all tool results must precede synthetic user media") + require.Equal(t, want.id, tool.ToolCallID) + require.Equal(t, want.text, tool.Content.OfString.Value) + } + for i, url := range imageURLs { + user := messages[3+i].OfUser + require.NotNil(t, user) + require.Len(t, user.Content.OfArrayOfContentParts, 1) + image := user.Content.OfArrayOfContentParts[0].OfImageURL + require.NotNil(t, image) + require.Equal(t, url, image.ImageURL.URL) + } + if tc.followup { + require.NotNil(t, messages[wantLen-2].OfAssistant) + require.Equal(t, "done", messages[wantLen-2].OfAssistant.Content.OfString.Value) + require.NotNil(t, messages[wantLen-1].OfUser) + require.Equal(t, "thanks", messages[wantLen-1].OfUser.Content.OfString.Value) + } + }) + } + }) + } +} diff --git a/providers/openaicompat/language_model_hooks.go b/providers/openaicompat/language_model_hooks.go index b7d7ff5d1..daa616d5a 100644 --- a/providers/openaicompat/language_model_hooks.go +++ b/providers/openaicompat/language_model_hooks.go @@ -160,8 +160,14 @@ func ToPromptFunc(prompt fantasy.Prompt, _, _ string) ([]openaisdk.ChatCompletio var messages []openaisdk.ChatCompletionMessageParamUnion var warnings []fantasy.CallWarning hasReasoning := false + // All tool replies in a batch must precede synthetic user media. + var pendingToolMedia []openaisdk.ChatCompletionMessageParamUnion for _, msg := range prompt { + if msg.Role != fantasy.MessageRoleTool { + messages = append(messages, pendingToolMedia...) + pendingToolMedia = nil + } switch msg.Role { case fantasy.MessageRoleSystem: var blocks []openaisdk.ChatCompletionContentPartTextParam @@ -502,7 +508,8 @@ func ToPromptFunc(prompt fantasy.Prompt, _, _ string) ([]openaisdk.ChatCompletio // helper, which emits a text tool message plus a synthetic // user message holding the media. mediaMessages, mediaWarnings := openai.ToolResultMediaMessages(output, toolResultPart.ToolCallID) - messages = append(messages, mediaMessages...) + messages = append(messages, mediaMessages[0]) + pendingToolMedia = append(pendingToolMedia, mediaMessages[1:]...) warnings = append(warnings, mediaWarnings...) default: warnings = append(warnings, fantasy.CallWarning{ @@ -513,7 +520,7 @@ func ToPromptFunc(prompt fantasy.Prompt, _, _ string) ([]openaisdk.ChatCompletio } } } - return messages, warnings + return append(messages, pendingToolMedia...), warnings } // toolResultMediaUserPart maps a tool-result media output to an OpenAI chat From c1408f8095a274c7689d59bfb18ea2910aa5ade4 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:03:01 +0000 Subject: [PATCH 2/2] fix(providers/anthropic): keep metadata on web searches with no results A web_search_tool_result whose content is an empty list carried no WebSearchResultMetadata in Generate or Stream, so consumers could not tell a search that found nothing from a result that cannot be replayed. Coder's chat sanitizer drops provider tool results without that metadata, so these searches vanished from saved history after clients had seen them stream. Attach empty metadata instead. It replays as an empty result list, as the encoder's missing-metadata fallback already did, without the warning. > Xum acted on Mike's behalf. --- providers/anthropic/anthropic.go | 10 ++ providers/anthropic/anthropic_test.go | 153 ++++++++++++++++++++++++++ 2 files changed, 163 insertions(+) diff --git a/providers/anthropic/anthropic.go b/providers/anthropic/anthropic.go index 509939628..47ca97b9c 100644 --- a/providers/anthropic/anthropic.go +++ b/providers/anthropic/anthropic.go @@ -1682,6 +1682,12 @@ func (a languageModel) Generate(ctx context.Context, call fantasy.Call) (*fantas ErrorCode: string(webSearchResult.Content.ErrorCode), }, } + } else { + // A search that found nothing still needs metadata, or it + // reads as a result that cannot be replayed. + toolResult.ProviderMetadata = fantasy.ProviderMetadata{ + Name: &WebSearchResultMetadata{}, + } } content = append(content, toolResult) } @@ -1878,6 +1884,10 @@ func (a languageModel) Stream(ctx context.Context, call fantasy.Call) (fantasy.S ErrorCode: string(contentBlock.Content.ErrorCode), }, } + } else { + providerMeta = fantasy.ProviderMetadata{ + Name: &WebSearchResultMetadata{}, + } } if !yield(fantasy.StreamPart{ Type: fantasy.StreamPartTypeToolResult, diff --git a/providers/anthropic/anthropic_test.go b/providers/anthropic/anthropic_test.go index 1becece23..457fe0f65 100644 --- a/providers/anthropic/anthropic_test.go +++ b/providers/anthropic/anthropic_test.go @@ -1726,6 +1726,49 @@ func TestToPrompt_WebSearchProviderExecutedErrorRoundTrip(t *testing.T) { require.Equal(t, "I was unable to search.", assistantMsg.Content[2].OfText.Text) } +func TestToPrompt_WebSearchNoResultsRoundTrip(t *testing.T) { + t.Parallel() + + prompt := fantasy.Prompt{ + { + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "Search for the latest AI news"}, + }, + }, + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ + ToolCallID: "srvtoolu_empty", + ToolName: "web_search", + Input: `{"query":"latest AI news"}`, + ProviderExecuted: true, + }, + fantasy.ToolResultPart{ + ToolCallID: "srvtoolu_empty", + ProviderExecuted: true, + ProviderOptions: fantasy.ProviderOptions{ + Name: &WebSearchResultMetadata{}, + }, + }, + fantasy.TextPart{Text: "The search found nothing."}, + }, + }, + } + + _, messages, warnings := toPrompt(prompt, true) + require.Empty(t, warnings) + require.Len(t, messages, 2) + + webResult := messages[1].Content[1].OfWebSearchToolResult + require.NotNil(t, webResult) + require.Equal(t, "srvtoolu_empty", webResult.ToolUseID) + require.Nil(t, webResult.Content.OfRequestWebSearchToolResultError) + require.NotNil(t, webResult.Content.OfWebSearchToolResultBlockItem) + require.Empty(t, webResult.Content.OfWebSearchToolResultBlockItem) +} + func TestToPrompt_WebSearchProviderExecutedToolResults(t *testing.T) { t.Parallel() @@ -2202,6 +2245,59 @@ func TestGenerate_WebSearchErrorPreservesErrorCode(t *testing.T) { require.Equal(t, "I was unable to search.", texts[0].Text) } +func TestGenerate_WebSearchNoResultsKeepsMetadata(t *testing.T) { + t.Parallel() + + response := mockAnthropicWebSearchErrorResponse() + response["content"] = []any{ + map[string]any{ + "type": "server_tool_use", + "id": "srvtoolu_empty", + "name": "web_search", + "input": map[string]any{"query": "latest AI news"}, + }, + map[string]any{ + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_empty", + "content": []any{}, + }, + map[string]any{"type": "text", "text": "The search found nothing."}, + } + server, calls := newAnthropicJSONServer(response) + defer server.Close() + + provider, err := New( + WithAPIKey("test-api-key"), + WithBaseURL(server.URL), + ) + require.NoError(t, err) + + model, err := provider.LanguageModel(context.Background(), "claude-sonnet-4-20250514") + require.NoError(t, err) + + resp, err := model.Generate(context.Background(), fantasy.Call{ + Prompt: testPrompt(), + Tools: []fantasy.Tool{ + WebSearchTool(nil), + }, + }) + require.NoError(t, err) + _ = awaitAnthropicCall(t, calls) + + var toolResults []fantasy.ToolResultContent + for _, c := range resp.Content { + require.NotEqual(t, fantasy.ContentTypeSource, c.GetType()) + if v, ok := c.(fantasy.ToolResultContent); ok { + toolResults = append(toolResults, v) + } + } + require.Len(t, toolResults, 1) + require.Equal(t, "srvtoolu_empty", toolResults[0].ToolCallID) + webMeta := requireWebSearchResultMetadata(t, toolResults[0].ProviderMetadata) + require.Empty(t, webMeta.ErrorCode) + require.Empty(t, webMeta.Results) +} + func TestGenerate_WebSearchToolInRequest(t *testing.T) { t.Parallel() @@ -2742,6 +2838,63 @@ func TestStream_WebSearchErrorPreservesErrorCode(t *testing.T) { require.Empty(t, webMeta.Results) } +// A search that finds nothing still carries metadata, so consumers can tell +// it apart from a result that cannot be replayed. +func TestStream_WebSearchNoResultsKeepsMetadata(t *testing.T) { + t.Parallel() + + chunks := []string{ + "event: message_start\n", + `data: {"type":"message_start","message":{"id":"msg_01WebSearchEmpty","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[],"stop_reason":null,"usage":{"input_tokens":100,"output_tokens":0}}}` + "\n\n", + "event: content_block_start\n", + `data: {"type":"content_block_start","index":0,"content_block":{"type":"server_tool_use","id":"srvtoolu_empty","name":"web_search","input":{}}}` + "\n\n", + "event: content_block_stop\n", + `data: {"type":"content_block_stop","index":0}` + "\n\n", + "event: content_block_start\n", + `data: {"type":"content_block_start","index":1,"content_block":{"type":"web_search_tool_result","tool_use_id":"srvtoolu_empty","content":[]}}` + "\n\n", + "event: content_block_stop\n", + `data: {"type":"content_block_stop","index":1}` + "\n\n", + "event: message_stop\n", + `data: {"type":"message_stop"}` + "\n\n", + } + + server, calls := newAnthropicStreamingServer(chunks) + defer server.Close() + + provider, err := New( + WithAPIKey("test-api-key"), + WithBaseURL(server.URL), + ) + require.NoError(t, err) + + model, err := provider.LanguageModel(context.Background(), "claude-sonnet-4-20250514") + require.NoError(t, err) + + stream, err := model.Stream(context.Background(), fantasy.Call{ + Prompt: testPrompt(), + Tools: []fantasy.Tool{ + WebSearchTool(nil), + }, + }) + require.NoError(t, err) + + var toolResults []fantasy.StreamPart + stream(func(part fantasy.StreamPart) bool { + require.NotEqual(t, fantasy.StreamPartTypeSource, part.Type) + if part.Type == fantasy.StreamPartTypeToolResult { + toolResults = append(toolResults, part) + } + return true + }) + _ = awaitAnthropicCall(t, calls) + + require.Len(t, toolResults, 1) + require.Equal(t, "srvtoolu_empty", toolResults[0].ID) + webMeta := requireWebSearchResultMetadata(t, toolResults[0].ProviderMetadata) + require.Empty(t, webMeta.ErrorCode) + require.Empty(t, webMeta.Results) +} + func TestGenerate_ToolChoiceNone(t *testing.T) { t.Parallel()