From 460d4fdd113bc0bb1599ce0777f4c19375c83317 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:58:28 +0000 Subject: [PATCH] fix(providers): keep tool batches ahead of media messages Defer synthetic user media until all contiguous tool-result messages have been emitted. OpenAI rejects media inserted between replies to parallel tool calls. Preserve accompanying text and the order of media payloads. Cover both Chat Completions serializers with separate and grouped tool results, two media results, and following conversation messages. > Xum acted on Mike's behalf. --- providers/openai/language_model_hooks.go | 11 +- .../openai/tool_result_media_order_test.go | 104 ++++++++++++++++++ .../openaicompat/language_model_hooks.go | 11 +- 3 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 providers/openai/tool_result_media_order_test.go diff --git a/providers/openai/language_model_hooks.go b/providers/openai/language_model_hooks.go index f1c851570..ce55cc9f7 100644 --- a/providers/openai/language_model_hooks.go +++ b/providers/openai/language_model_hooks.go @@ -293,7 +293,13 @@ func DefaultStreamProviderMetadataFunc(choice openai.ChatCompletionChoice, metad func DefaultToPrompt(prompt fantasy.Prompt, _, _ string) ([]openai.ChatCompletionMessageParamUnion, []fantasy.CallWarning) { var messages []openai.ChatCompletionMessageParamUnion var warnings []fantasy.CallWarning + // All tool replies in a batch must precede synthetic user media. + var pendingToolMedia []openai.ChatCompletionMessageParamUnion for _, msg := range prompt { + if msg.Role != fantasy.MessageRoleTool { + messages = append(messages, pendingToolMedia...) + pendingToolMedia = nil + } switch msg.Role { case fantasy.MessageRoleSystem: var systemPromptParts []string @@ -581,7 +587,8 @@ func DefaultToPrompt(prompt fantasy.Prompt, _, _ string) ([]openai.ChatCompletio // OpenAI Chat Completions tool messages cannot carry image // or audio content directly; see ToolResultMediaMessages. mediaMessages, mediaWarnings := ToolResultMediaMessages(output, toolResultPart.ToolCallID) - messages = append(messages, mediaMessages...) + messages = append(messages, mediaMessages[0]) + pendingToolMedia = append(pendingToolMedia, mediaMessages[1:]...) warnings = append(warnings, mediaWarnings...) default: warnings = append(warnings, fantasy.CallWarning{ @@ -592,7 +599,7 @@ func DefaultToPrompt(prompt fantasy.Prompt, _, _ string) ([]openai.ChatCompletio } } } - return messages, warnings + return append(messages, pendingToolMedia...), warnings } // ToolResultMediaMessages maps a tool-result media output to the chat diff --git a/providers/openai/tool_result_media_order_test.go b/providers/openai/tool_result_media_order_test.go new file mode 100644 index 000000000..ebd501eeb --- /dev/null +++ b/providers/openai/tool_result_media_order_test.go @@ -0,0 +1,104 @@ +package openai_test + +import ( + "testing" + + "charm.land/fantasy" + "charm.land/fantasy/providers/openai" + "charm.land/fantasy/providers/openaicompat" + openaisdk "github.com/openai/openai-go/v3" + "github.com/stretchr/testify/require" +) + +func TestToolMediaFollowsToolResultBatch(t *testing.T) { + t.Parallel() + + for _, provider := range []struct { + name string + toPrompt func(fantasy.Prompt, string, string) ([]openaisdk.ChatCompletionMessageParamUnion, []fantasy.CallWarning) + }{ + {name: "OpenAI", toPrompt: openai.DefaultToPrompt}, + {name: "OpenAICompatible", toPrompt: openaicompat.ToPromptFunc}, + } { + t.Run(provider.name, func(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + separateMessages bool + secondMedia bool + followup bool + }{ + {name: "SeparateMessagesMixed", separateMessages: true}, + {name: "SeparateMessagesTwoMedia", separateMessages: true, secondMedia: true}, + {name: "SameMessageMixed"}, + {name: "SameMessageTwoMedia", secondMedia: true}, + {name: "BeforeFollowingMessages", separateMessages: true, secondMedia: true, followup: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + media := fantasy.ToolResultOutputContentMedia{Data: "AAEC", MediaType: "image/png", Text: "first result"} + results := []fantasy.MessagePart{ + fantasy.ToolResultPart{ToolCallID: "call-a", Output: media}, + fantasy.ToolResultPart{ToolCallID: "call-b", Output: fantasy.ToolResultOutputContentText{Text: "second result"}}, + } + imageURLs := []string{"data:image/png;base64,AAEC"} + if tc.secondMedia { + media.Data, media.Text = "AwQF", "second result" + results[1] = fantasy.ToolResultPart{ToolCallID: "call-b", Output: media} + imageURLs = append(imageURLs, "data:image/png;base64,AwQF") + } + prompt := fantasy.Prompt{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ToolCallID: "call-a", ToolName: "first", Input: "{}"}, + fantasy.ToolCallPart{ToolCallID: "call-b", ToolName: "second", Input: "{}"}, + }, + }} + if tc.separateMessages { + for _, result := range results { + prompt = append(prompt, fantasy.Message{Role: fantasy.MessageRoleTool, Content: []fantasy.MessagePart{result}}) + } + } else { + prompt = append(prompt, fantasy.Message{Role: fantasy.MessageRoleTool, Content: results}) + } + if tc.followup { + prompt = append(prompt, + fantasy.Message{Role: fantasy.MessageRoleAssistant, Content: []fantasy.MessagePart{fantasy.TextPart{Text: "done"}}}, + fantasy.Message{Role: fantasy.MessageRoleUser, Content: []fantasy.MessagePart{fantasy.TextPart{Text: "thanks"}}}, + ) + } + + messages, warnings := provider.toPrompt(prompt, "", "") + require.Empty(t, warnings) + wantLen := 3 + len(imageURLs) + if tc.followup { + wantLen += 2 + } + require.Len(t, messages, wantLen) + require.NotNil(t, messages[0].OfAssistant) + require.Len(t, messages[0].OfAssistant.ToolCalls, 2) + for i, want := range []struct{ id, text string }{{"call-a", "first result"}, {"call-b", "second result"}} { + tool := messages[i+1].OfTool + require.NotNil(t, tool, "all tool results must precede synthetic user media") + require.Equal(t, want.id, tool.ToolCallID) + require.Equal(t, want.text, tool.Content.OfString.Value) + } + for i, url := range imageURLs { + user := messages[3+i].OfUser + require.NotNil(t, user) + require.Len(t, user.Content.OfArrayOfContentParts, 1) + image := user.Content.OfArrayOfContentParts[0].OfImageURL + require.NotNil(t, image) + require.Equal(t, url, image.ImageURL.URL) + } + if tc.followup { + require.NotNil(t, messages[wantLen-2].OfAssistant) + require.Equal(t, "done", messages[wantLen-2].OfAssistant.Content.OfString.Value) + require.NotNil(t, messages[wantLen-1].OfUser) + require.Equal(t, "thanks", messages[wantLen-1].OfUser.Content.OfString.Value) + } + }) + } + }) + } +} diff --git a/providers/openaicompat/language_model_hooks.go b/providers/openaicompat/language_model_hooks.go index b7d7ff5d1..daa616d5a 100644 --- a/providers/openaicompat/language_model_hooks.go +++ b/providers/openaicompat/language_model_hooks.go @@ -160,8 +160,14 @@ func ToPromptFunc(prompt fantasy.Prompt, _, _ string) ([]openaisdk.ChatCompletio var messages []openaisdk.ChatCompletionMessageParamUnion var warnings []fantasy.CallWarning hasReasoning := false + // All tool replies in a batch must precede synthetic user media. + var pendingToolMedia []openaisdk.ChatCompletionMessageParamUnion for _, msg := range prompt { + if msg.Role != fantasy.MessageRoleTool { + messages = append(messages, pendingToolMedia...) + pendingToolMedia = nil + } switch msg.Role { case fantasy.MessageRoleSystem: var blocks []openaisdk.ChatCompletionContentPartTextParam @@ -502,7 +508,8 @@ func ToPromptFunc(prompt fantasy.Prompt, _, _ string) ([]openaisdk.ChatCompletio // helper, which emits a text tool message plus a synthetic // user message holding the media. mediaMessages, mediaWarnings := openai.ToolResultMediaMessages(output, toolResultPart.ToolCallID) - messages = append(messages, mediaMessages...) + messages = append(messages, mediaMessages[0]) + pendingToolMedia = append(pendingToolMedia, mediaMessages[1:]...) warnings = append(warnings, mediaWarnings...) default: warnings = append(warnings, fantasy.CallWarning{ @@ -513,7 +520,7 @@ func ToPromptFunc(prompt fantasy.Prompt, _, _ string) ([]openaisdk.ChatCompletio } } } - return messages, warnings + return append(messages, pendingToolMedia...), warnings } // toolResultMediaUserPart maps a tool-result media output to an OpenAI chat