Skip to content

Replay OpenAI tool calls as plain, stable JSON #325

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gtokman wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Replay OpenAI tool calls as plain, stable JSON #325

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8afdd28
Replay OpenAI tool-call arguments as plain JSON
gtokman Oct 13, 2026
fca48fe
Send a tool round the same way in flight and in history
gtokman Oct 13, 2026
9956048
Replay OpenAI tool-call text the same as in flight
cursoragent Oct 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 83 additions & 49 deletions Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -550,9 +550,6 @@ public struct OpenAILanguageModel: LanguageModel {

let toolCallMessage = choice.message
if let toolCalls = toolCallMessage.toolCalls, !toolCalls.isEmpty {
if let value = try? JSONValue(toolCallMessage) {
inFlightMessages.append(OpenAIMessage(role: .raw(rawContent: value), content: .text("")))
}
try toolRounds.record(toolCalls.map(\.roundCall))
let resolution = try await resolveToolCalls(
toolCalls,
Expand All @@ -562,7 +559,14 @@ public struct OpenAILanguageModel: LanguageModel {
switch resolution {
case .stop(let calls):
if !calls.isEmpty {
entries.append(.toolCalls(Transcript.ToolCalls(calls)))
entries.append(
.toolCalls(
Transcript.ToolCalls(
calls,
providerMetadata: assistantContentMetadata(toolCallMessage.content)
)
)
)
}
let empty = try emptyResponseContent(for: type)
return LanguageModelSession.Response(
Expand All @@ -573,7 +577,12 @@ public struct OpenAILanguageModel: LanguageModel {
)
case .invocations(let invocations):
if !invocations.isEmpty {
entries.append(.toolCalls(Transcript.ToolCalls(invocations.map { $0.call })))
let calls = Transcript.ToolCalls(
invocations.map(\.call),
providerMetadata: assistantContentMetadata(toolCallMessage.content)
)
entries.append(.toolCalls(calls))
inFlightMessages.append(toolCallsMessage(calls))
for invocation in invocations {
let output = invocation.output
entries.append(.toolOutput(output))
Expand Down Expand Up @@ -879,15 +888,6 @@ public struct OpenAILanguageModel: LanguageModel {
}
}
toolCalls = calls.keys.sorted().compactMap { calls[$0]?.toolCall }
if !toolCalls.isEmpty {
let message: JSONValue = .object([
"role": .string("assistant"), "content": .string(state.text),
"tool_calls": try JSONValue(toolCalls),
])
inFlightMessages.append(
.init(role: .raw(rawContent: message), content: .text(""))
)
}
}

guard !toolCalls.isEmpty else { break }
Expand All @@ -899,7 +899,15 @@ public struct OpenAILanguageModel: LanguageModel {
session: session
) {
case .stop(let calls):
state.entries.append(.toolCalls(Transcript.ToolCalls(calls)))
state.entries.append(
.toolCalls(
Transcript.ToolCalls(
calls,
providerMetadata: apiVariant == .chatCompletions
? assistantContentMetadata(state.text) : nil
)
)
)
continuation.yield(try state.stoppedSnapshot())
continuation.finish()
return
Expand All @@ -908,7 +916,15 @@ public struct OpenAILanguageModel: LanguageModel {
continuation.finish()
return
}
state.entries.append(.toolCalls(Transcript.ToolCalls(invocations.map(\.call))))
let calls = Transcript.ToolCalls(
invocations.map(\.call),
providerMetadata: apiVariant == .chatCompletions
? assistantContentMetadata(state.text) : nil
)
state.entries.append(.toolCalls(calls))
if apiVariant == .chatCompletions {
inFlightMessages.append(toolCallsMessage(calls))
}
for invocation in invocations {
state.entries.append(.toolOutput(invocation.output))
inFlightMessages.append(
Expand Down Expand Up @@ -1338,6 +1354,56 @@ private enum Responses {

// MARK: - Supporting Types

/// Key under which the assistant's text for a Chat Completions tool-call round is kept in
/// ``Transcript/ToolCalls/providerMetadata``.
private let assistantContentMetadataKey = "openai.content"

private func assistantContentMetadata(_ text: String?) -> [String: String]? {
guard let text, !text.isEmpty else { return nil }
return [assistantContentMetadataKey: text]
}

/// The assistant message that carries a round of tool calls.
///
/// Built from the transcript's calls both for the round in flight and when the history is
/// replayed on the next turn, so the two are byte-for-byte the same and a provider's prompt
/// cache (Anthropic's is an exact prefix match) keeps matching past the first tool call.
/// The arguments are their JSON, with sorted keys (`stableJSON`), not `GeneratedContent`'s
/// `Codable` form (`{"kind":鈥`), which models copy into their next calls.
///
/// The model's text from this round is stored on the transcript's `ToolCalls`, so replay
/// sends the same `content` as the in-flight request, including when that text is empty
/// (both paths send `null`).
private func toolCallsMessage(_ calls: Transcript.ToolCalls) -> OpenAIMessage {
let text = calls.providerMetadata?[assistantContentMetadataKey]
let toolCalls: [JSONValue] = calls.map { call in
.object([
"id": .string(call.id),
"type": .string("function"),
"function": .object([
"name": .string(call.toolName),
"arguments": .string(stableJSON(call.arguments)),
]),
])
}
let message: JSONValue = .object([
"role": .string("assistant"),
"content": text.map { .string($0) } ?? .null,
"tool_calls": .array(toolCalls),
])
return OpenAIMessage(role: .raw(rawContent: message), content: .text(""))
}

/// The content's JSON with its keys sorted, so the same arguments are always the same bytes.
/// `jsonString` goes through an unordered dictionary, so its key order can change from one
/// call to the next.
private func stableJSON(_ content: GeneratedContent) -> String {
guard let object = try? JSONSerialization.jsonObject(with: content.jsonData, options: [.fragmentsAllowed]),
let data = try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys, .fragmentsAllowed])
else { return content.jsonString }
return String(decoding: data, as: UTF8.self)
}

extension Transcript {
fileprivate func toOpenAIMessages() -> [OpenAIMessage] {
var messages = [OpenAIMessage]()
Expand Down Expand Up @@ -1368,39 +1434,7 @@ extension Transcript {
)
)
case .toolCalls(let toolCalls):
// Add assistant message with tool calls
let openAIToolCalls: [JSONValue] = toolCalls.map { call in
let argumentsJSON: String
if let data = try? JSONEncoder().encode(call.arguments),
let jsonString = String(data: data, encoding: .utf8)
{
argumentsJSON = jsonString
} else {
argumentsJSON = "{}"
}

return .object([
"id": .string(call.id),
"type": .string("function"),
"function": .object([
"name": .string(call.toolName),
"arguments": .string(argumentsJSON),
]),
])
}

let rawMessage: JSONValue = .object([
"role": .string("assistant"),
"content": .null,
"tool_calls": .array(openAIToolCalls),
])

messages.append(
.init(
role: .raw(rawContent: rawMessage),
content: .text("")
)
)
messages.append(toolCallsMessage(toolCalls))
case .toolOutput(let toolOutput):
messages.append(
.init(
Expand Down
Loading