diff --git a/README.md b/README.md index 9cf4ef30..a4a871e3 100644 --- a/README.md +++ b/README.md @@ -384,19 +384,22 @@ pass the model's multimodal projector with `mmprojPath:`. ### Prompt attachments -Image attachments use the Foundation Models 27 prompt construction syntax: +Image attachments use the Foundation Models 27 prompt construction syntax, +including ordinary Tool outputs: ```swift -let prompt = Prompt { - "Describe this image" - Attachment(imageURL: fileURL) +func call(arguments: Arguments) async throws -> Prompt { + Prompt { + "Image read from the selected file" + Attachment(imageURL: fileURL) + } } ``` On Apple platforms, `Attachment(CGImage, orientation:)` retains typed image -content and orientation. Prompt builders preserve the order of text and images, -and the OS 26 compatibility transcript continues to store images using its -existing `Transcript.Segment.image` Codable representation. +content and orientation. Prompts, builders, and Tool outputs preserve the order +of text and images, and the OS 26 compatibility transcript continues to store +images using its existing `Transcript.Segment.image` Codable representation. `SystemLanguageModel` and `FoundationLanguageModel` bridge prompt attachments to Foundation Models only in apps built with Swift 6.4 or later @@ -406,9 +409,9 @@ they throw an error when a prompt has an attachment. The system bridge requires local file URLs; other provider adapters retain URL images in their existing transcript representation. -This first compatibility surface covers prompt `CGImage` and `imageURL` -attachments. Tool outputs, instruction attachments, labels, `CIImage`, pixel -buffers, and newer data attachments remain outside this change. +This compatibility surface covers `CGImage` and `imageURL` attachments in +prompts and ordinary Tool outputs. Instruction attachments, labels, `CIImage`, +pixel buffers, and newer data attachments remain outside this change. ### Tool Calling @@ -641,6 +644,10 @@ say which API they follow. [image inputs](#image-inputs). Foundation Models 27-style [prompt attachments](#prompt-attachments) are also available for ordered text and image prompt construction. +- `Tool.makeOutputSegments(from:)`, exposed only through the `Compatibility` SPI: + an integration extension for back-deployment provider adapters to lower normal + Tool outputs into the compatibility transcript. It is not an Apple Tool API; + OS 27 canonical executors use Foundation Models' own Tool execution instead. - `ToolExecutionDelegate`, `ToolExecutionDecision`, and `toolExecutionDelegate`: [observing and controlling tool calls](#tool-calling). - `transcriptErrorHandlingPolicy` and `waitForResponseCompletion()`: diff --git a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift index 6a618a79..2fae7d71 100644 --- a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift @@ -716,7 +716,7 @@ public struct AnthropicLanguageModel: LanguageModel { .toolResult( .init( toolUseId: invocation.call.id, - content: convertSegmentsToAnthropicContent(invocation.output.segments) + content: try convertSegmentsToAnthropicContent(invocation.output.segments) ) ) ) @@ -1029,14 +1029,14 @@ extension Transcript { messages.append( .init( role: .user, - content: convertSegmentsToAnthropicContent(instructions.segments) + content: try convertSegmentsToAnthropicContent(instructions.segments) ) ) case .prompt(let prompt): messages.append( .init( role: .user, - content: convertSegmentsToAnthropicContent(prompt.segments) + content: try convertSegmentsToAnthropicContent(prompt.segments) ) ) case .reasoning(let reasoning): @@ -1063,7 +1063,7 @@ extension Transcript { case .response(let response): // Anthropic rejects text blocks without non-whitespace text, // such as the empty response of a turn that only called tools. - let content = convertSegmentsToAnthropicContent(response.segments).filter { block in + let content = try convertSegmentsToAnthropicContent(response.segments).filter { block in guard case .text(let text) = block else { return true } return !text.text.allSatisfy(\.isWhitespace) } @@ -1091,7 +1091,7 @@ extension Transcript { .toolResult( AnthropicToolResult( toolUseId: toolOutput.id, - content: convertSegmentsToAnthropicContent(toolOutput.segments) + content: try convertSegmentsToAnthropicContent(toolOutput.segments) ) ) ] @@ -1244,7 +1244,7 @@ private extension AnthropicThinking { } } -private func convertSegmentsToAnthropicContent(_ segments: [Transcript.Segment]) -> [AnthropicContent] { +private func convertSegmentsToAnthropicContent(_ segments: [Transcript.Segment]) throws -> [AnthropicContent] { var blocks: [AnthropicContent] = [] blocks.reserveCapacity(segments.count) for segment in segments { @@ -1256,7 +1256,26 @@ private func convertSegmentsToAnthropicContent(_ segments: [Transcript.Segment]) case .image(let img): switch img.source { case .url(let url): - blocks.append(.image(AnthropicImage(url: url.absoluteString))) + if url.isFileURL { + let mimeType: String + switch url.pathExtension.lowercased() { + case "jpg", "jpeg": mimeType = "image/jpeg" + case "png": mimeType = "image/png" + case "gif": mimeType = "image/gif" + case "webp": mimeType = "image/webp" + default: throw AnthropicImageAttachmentError.unsupportedImageType(url) + } + blocks.append( + .image( + AnthropicImage( + base64Data: try Data(contentsOf: url).base64EncodedString(), + mimeType: mimeType + ) + ) + ) + } else { + blocks.append(.image(AnthropicImage(url: url.absoluteString))) + } case .data(let data, let mimeType): blocks.append(.image(AnthropicImage(base64Data: data.base64EncodedString(), mimeType: mimeType))) } @@ -1265,6 +1284,17 @@ private func convertSegmentsToAnthropicContent(_ segments: [Transcript.Segment]) return blocks } +private enum AnthropicImageAttachmentError: LocalizedError { + case unsupportedImageType(URL) + + var errorDescription: String? { + switch self { + case .unsupportedImageType(let url): + "Anthropic image attachments support PNG, JPEG, GIF, and WebP files; cannot determine a supported MIME type for \(url.lastPathComponent)." + } + } +} + private struct AnthropicToolUse: Codable, Sendable { let type: String let id: String diff --git a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift index 5be74601..271378c5 100644 --- a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift @@ -254,7 +254,7 @@ public struct GeminiLanguageModel: LanguageModel { requestTranscript.append(entry) } let params = try createGenerateContentParams( - contents: requestTranscript.toGeminiContent(), + contents: try requestTranscript.toGeminiContent(), tools: geminiTools, generating: type, schema: responseSchema.schema, @@ -432,7 +432,7 @@ public struct GeminiLanguageModel: LanguageModel { requestTranscript.append(entry) } let params = try createGenerateContentParams( - contents: requestTranscript.toGeminiContent(), + contents: try requestTranscript.toGeminiContent(), tools: geminiTools, generating: type, schema: responseSchema.schema, @@ -773,32 +773,85 @@ private func emptyResponseContent( } } -private func toJSONValue(_ toolOutput: Transcript.ToolOutput) throws -> [String: JSONValue] { - var result: [String: JSONValue] = [:] +private func toGeminiFunctionResponse( + _ toolOutput: Transcript.ToolOutput +) throws -> GeminiFunctionResponse { + var resultValues: [String] = [] + var parts: [GeminiPart] = [] for segment in toolOutput.segments { switch segment { case .text(let text): - result["result"] = .string(text.content) + resultValues.append(text.content) case .structure(let structured): - // For structured segments, encode the content let data = try JSONEncoder().encode(structured.content) if let jsonString = String(data: data, encoding: .utf8) { - result["result"] = .string(jsonString) + resultValues.append(jsonString) + } + case .image(let image): + switch image.source { + case .data(let data, let mimeType): + parts.append( + .inlineData( + GeminiInlineData( + mimeType: mimeType, + data: data.base64EncodedString() + ) + ) + ) + case .url(let url): + parts.append(.inlineData(try geminiFunctionResponseData(from: url))) } - case .image: - // Ignore images in tool outputs for Gemini conversion - break } } - return result + var response: [String: JSONValue] = [:] + if !resultValues.isEmpty { + response["result"] = .string(resultValues.joined(separator: "\n")) + } + return GeminiFunctionResponse( + name: toolOutput.toolName, + response: response, + parts: parts.isEmpty ? nil : parts + ) +} + +private enum GeminiFunctionResponseAttachmentError: LocalizedError { + case unsupportedURL(URL) + case unsupportedImageType(URL) + + var errorDescription: String? { + switch self { + case .unsupportedURL(let url): + "Gemini Tool output attachments require a local file URL; cannot inline \(url.absoluteString)." + case .unsupportedImageType(let url): + "Gemini Tool output attachments support PNG, JPEG, and WebP files; cannot determine a supported MIME type for \(url.lastPathComponent)." + } + } +} + +private func geminiFunctionResponseData(from url: URL) throws -> GeminiInlineData { + guard url.isFileURL else { + throw GeminiFunctionResponseAttachmentError.unsupportedURL(url) + } + + let mimeType = + switch url.pathExtension.lowercased() { + case "png": "image/png" + case "jpg", "jpeg": "image/jpeg" + case "webp": "image/webp" + default: throw GeminiFunctionResponseAttachmentError.unsupportedImageType(url) + } + return try GeminiInlineData( + mimeType: mimeType, + data: Data(contentsOf: url).base64EncodedString() + ) } // MARK: - Supporting Types extension Transcript { - fileprivate func toGeminiContent() -> [GeminiContent] { + fileprivate func toGeminiContent() throws -> [GeminiContent] { var messages = [GeminiContent]() for item in self { switch item { @@ -848,11 +901,7 @@ extension Transcript { ) case .toolOutput(let toolOutput): // Add function response as a user message (Gemini API expects function responses from user role) - let response = try? toJSONValue(toolOutput) - let functionResponse = GeminiFunctionResponse( - name: toolOutput.toolName, - response: response ?? [:] - ) + let functionResponse = try toGeminiFunctionResponse(toolOutput) messages.append( .init( role: .user, @@ -1120,6 +1169,7 @@ private struct GeminiFunctionCall: Codable, Sendable { private struct GeminiFunctionResponse: Codable, Sendable { let name: String let response: [String: JSONValue] + let parts: [GeminiPart]? } private struct GeminiGenerateContentResponse: Codable, Sendable { diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index 3c9e5b6e..502cb0d1 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -577,7 +577,7 @@ @available(macOS 26.0, iOS 26.0, watchOS 27.0, tvOS 26.0, visionOS 26.0, *) private struct AnyToolWrapper: FoundationModels.Tool { typealias Arguments = FoundationModels.GeneratedContent - typealias Output = String + typealias Output = FoundationModels.Prompt let name: String let description: String @@ -596,7 +596,7 @@ func call(arguments: FoundationModels.GeneratedContent) async throws -> Output { let output = try await wrappedTool.callFunction(arguments: arguments) - return output.promptRepresentation.description + return try output.promptRepresentation.toFoundationModels() } } diff --git a/Sources/AnyLanguageModel/Tool.swift b/Sources/AnyLanguageModel/Tool.swift index 368a9961..59f5b800 100644 --- a/Sources/AnyLanguageModel/Tool.swift +++ b/Sources/AnyLanguageModel/Tool.swift @@ -110,7 +110,10 @@ extension Tool where Self.Arguments: Generable { // MARK: - Helpers extension Tool { - internal func makeOutputSegments(from arguments: GeneratedContent) async throws -> [Transcript.Segment] { + /// Compatibility integration SPI for provider adapters outside this module. + /// This is not part of Apple's public Tool contract. + @_spi(Compatibility) + public func makeOutputSegments(from arguments: GeneratedContent) async throws -> [Transcript.Segment] { let parsedArguments = try Arguments(arguments) let output = try await call(arguments: parsedArguments) @@ -124,7 +127,6 @@ extension Tool { return [Transcript.Segment.text(.init(content: stringOutput))] } - let fallback = output.promptRepresentation.description - return [Transcript.Segment.text(.init(content: fallback))] + return try output.promptRepresentation.makeTranscriptSegments() } } diff --git a/Tests/AnyLanguageModelTests/DynamicInstructionsTests.swift b/Tests/AnyLanguageModelTests/DynamicInstructionsTests.swift index af43ad37..baba7aa1 100644 --- a/Tests/AnyLanguageModelTests/DynamicInstructionsTests.swift +++ b/Tests/AnyLanguageModelTests/DynamicInstructionsTests.swift @@ -1,7 +1,7 @@ import Foundation import Testing -@testable import AnyLanguageModel +@_spi(Compatibility) @testable import AnyLanguageModel @Suite("Dynamic instructions") struct DynamicInstructionsTests { diff --git a/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift b/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift index 5265718f..006490ab 100644 --- a/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift +++ b/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift @@ -4,11 +4,27 @@ import Testing @testable import AnyLanguageModel #if canImport(Darwin) && !canImport(AsyncHTTPClient) + import CoreGraphics @Suite("GeminiLanguageModel thought signatures", .serialized) struct GeminiThoughtSignatureTests { private static let signature = "CvsBAdHtim8n5xQK1pVX2H0lPQeXAMPLEsignature==" + @Generable + struct ImageArguments { + var request: String + } + + private struct ImageOutputTool: Tool { + let name = "image" + let description = "Return an image" + let output: Prompt + + func call(arguments: ImageArguments) async throws -> Prompt { + output + } + } + private func makeModel() -> GeminiLanguageModel { GeminiLanguageModel( apiKey: "test-key", @@ -51,6 +67,21 @@ import Testing """ } + private func image() throws -> CGImage { + let context = try #require( + CGContext( + data: nil, + width: 2, + height: 2, + bitsPerComponent: 8, + bytesPerRow: 8, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) + ) + return try #require(context.makeImage()) + } + /// Signatures of every `functionCall` part in a request body, in order. /// A part without a signature contributes `nil`. private func functionCallSignatures(in body: Data) throws -> [String?] { @@ -108,6 +139,108 @@ import Testing #expect(signatures.allSatisfy { $0 == Self.signature }) } + @Test("preserves all text and image data in function responses") + func preservesSegmentedToolOutput() async throws { + StubURLProtocol.reset() + StubURLProtocol.enqueue( + json: + #"{"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"image","args":{"request":"read"}}}]},"finishReason":"STOP"}]}"# + ) + StubURLProtocol.enqueue(json: textResponse("OK")) + + let output = try Prompt { + "before" + Attachment(try image()) + "after" + } + let session = LanguageModelSession( + model: makeModel(), + tools: [ImageOutputTool(output: output)] + ) + _ = try await session.respond(to: "read image") + + let bodies = StubURLProtocol.recordedBodies + try #require(bodies.count == 2) + let json = try #require( + try JSONSerialization.jsonObject(with: bodies[1]) as? [String: Any] + ) + let contents = try #require(json["contents"] as? [[String: Any]]) + let outputContent = try #require(contents.last) + let outputParts = try #require(outputContent["parts"] as? [[String: Any]]) + let functionResponse = try #require( + outputParts.first?["functionResponse"] as? [String: Any] + ) + let response = try #require(functionResponse["response"] as? [String: Any]) + #expect(response["result"] as? String == "before\nafter") + + let mediaParts = try #require(functionResponse["parts"] as? [[String: Any]]) + let inlineData = try #require(mediaParts.first?["inlineData"] as? [String: Any]) + #expect(inlineData["mime_type"] as? String == "image/png") + #expect(Data(base64Encoded: try #require(inlineData["data"] as? String)) != nil) + } + + @Test("inlines URL-backed images in function responses") + func inlinesURLBackedToolOutput() async throws { + StubURLProtocol.reset() + StubURLProtocol.enqueue( + json: + #"{"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"image","args":{"request":"read"}}}]},"finishReason":"STOP"}]}"# + ) + StubURLProtocol.enqueue(json: textResponse("OK")) + + let imageData = Data([0x89, 0x50, 0x4E, 0x47]) + let url = FileManager.default.temporaryDirectory + .appending(path: UUID().uuidString) + .appendingPathExtension("png") + try imageData.write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + + let output = Prompt { Attachment(imageURL: url) } + let session = LanguageModelSession( + model: makeModel(), + tools: [ImageOutputTool(output: output)] + ) + _ = try await session.respond(to: "read image") + + let body = try #require(StubURLProtocol.recordedBodies.last) + let json = try #require( + try JSONSerialization.jsonObject(with: body) as? [String: Any] + ) + let contents = try #require(json["contents"] as? [[String: Any]]) + let outputParts = try #require(contents.last?["parts"] as? [[String: Any]]) + let functionResponse = try #require( + outputParts.first?["functionResponse"] as? [String: Any] + ) + let mediaParts = try #require(functionResponse["parts"] as? [[String: Any]]) + let inlineData = try #require(mediaParts.first?["inlineData"] as? [String: Any]) + #expect(inlineData["mime_type"] as? String == "image/png") + #expect(Data(base64Encoded: try #require(inlineData["data"] as? String)) == imageData) + #expect(mediaParts.first?["fileData"] == nil) + } + + @Test("rejects remote URL-backed images with an actionable error") + func rejectsRemoteURLBackedToolOutput() async throws { + StubURLProtocol.reset() + StubURLProtocol.enqueue( + json: + #"{"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"image","args":{"request":"read"}}}]},"finishReason":"STOP"}]}"# + ) + + let url = try #require(URL(string: "https://example.com/image.png")) + let output = Prompt { Attachment(imageURL: url) } + let session = LanguageModelSession( + model: makeModel(), + tools: [ImageOutputTool(output: output)] + ) + + do { + _ = try await session.respond(to: "read image") + Issue.record("Expected a remote Tool output attachment to be rejected") + } catch { + #expect(error.localizedDescription.contains("local file URL")) + } + } + @Test("does not replay the conversation history on later turns") func doesNotReplayHistoryOnLaterTurns() async throws { StubURLProtocol.reset() diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index 34d6cdf4..589ce23f 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -1,7 +1,7 @@ import Foundation import Testing -@testable import AnyLanguageModel +@_spi(Compatibility) @testable import AnyLanguageModel #if canImport(CoreGraphics) && canImport(ImageIO) import CoreGraphics @@ -122,6 +122,29 @@ struct PromptAttachmentTests { } } + @Test func ordinaryToolReturnsOrderedImageSegments() async throws { + let segments = try await ImageTool(output: prompt()).makeOutputSegments( + from: AnyLanguageModel.GeneratedContent("read") + ) + #expect(segments.count == 3) + guard case .text(let before) = segments[0], + case .image(let image) = segments[1], + case .text(let after) = segments[2] + else { + Issue.record("Tool output lost its ordered image") + return + } + #expect(before.content == "before") + #expect(after.content == "after") + #expect(image.source == .url(imageURL)) + + let transcript = AnyLanguageModel.Transcript(entries: [ + .toolOutput(.init(id: "image-call", toolName: "image", segments: segments)) + ]) + let encoded = try JSONEncoder().encode(transcript) + #expect(try JSONDecoder().decode(AnyLanguageModel.Transcript.self, from: encoded) == transcript) + } + #if canImport(CoreGraphics) && canImport(ImageIO) private func image() throws -> CGImage { let context = try #require( @@ -267,11 +290,41 @@ struct PromptAttachmentTests { #expect(second.content == "prompts:2 images:1 right:1") } + @available(macOS 27, iOS 27, visionOS 27, *) + @Test func nativeToolWrapperRetainsOrdinaryImageOutput() async throws { + let output = try AnyLanguageModel.Prompt { + "tool note" + AnyLanguageModel.Attachment(try image(), orientation: .right) + } + let tools: [any AnyLanguageModel.Tool] = [ImageTool(output: output)] + let session = FoundationModels.LanguageModelSession( + model: BridgeModel(callsTools: true), + tools: tools.toFoundationModels() + ) + _ = try await session.respond(to: "read image") + let recorded = try #require( + session.transcript.compactMap { entry -> FoundationModels.Transcript.ToolOutput? in + guard case .toolOutput(let output) = entry else { return nil } + return output + }.first + ) + #expect(recorded.id == "image-call") + #expect(recorded.segments.count == 2) + guard case .attachment(let segment) = recorded.segments[1], + case .image(let image) = segment.content + else { + Issue.record("Native Tool wrapper flattened image output") + return + } + #expect(image.orientation == .right) + } + @available(macOS 27, iOS 27, visionOS 27, *) private struct BridgeModel: FoundationModels.LanguageModel { typealias Executor = BridgeExecutor + var callsTools = false var capabilities: FoundationModels.LanguageModelCapabilities { - .init([.vision]) + .init([.vision, .toolCalling]) } var executorConfiguration: String { "bridge-fixture" @@ -287,6 +340,27 @@ struct PromptAttachmentTests { model: BridgeModel, streamingInto channel: FoundationModels.LanguageModelExecutorGenerationChannel ) async throws { + if model.callsTools, + !request.transcript.contains(where: { + if case .toolOutput = $0 { return true } + return false + }) + { + await channel.send( + .toolCalls( + action: .toolCall( + id: "image-call", + name: "image", + action: .appendArguments(#""read""#, tokenCount: 1) + ) + ) + ) + return + } + if model.callsTools { + await channel.send(.response(action: .appendText("OK", tokenCount: 1))) + return + } let prompts = request.transcript.compactMap { entry -> FoundationModels.Transcript.Prompt? in guard case .prompt(let prompt) = entry else { return nil } return prompt @@ -313,3 +387,17 @@ struct PromptAttachmentTests { } #endif } + +private struct ImageTool: AnyLanguageModel.Tool { + let name = "image" + let description = "Return an image" + let output: AnyLanguageModel.Prompt + + var parameters: AnyLanguageModel.GenerationSchema { + String.generationSchema + } + + func call(arguments: String) async throws -> AnyLanguageModel.Prompt { + output + } +} diff --git a/Tests/AnyLanguageModelTests/ProviderUsageTests.swift b/Tests/AnyLanguageModelTests/ProviderUsageTests.swift index c4331da1..a2f9cc7b 100644 --- a/Tests/AnyLanguageModelTests/ProviderUsageTests.swift +++ b/Tests/AnyLanguageModelTests/ProviderUsageTests.swift @@ -482,6 +482,57 @@ import Testing #expect(UsageURLProtocol.recordedBodies.count == 2) } + @Test(arguments: [false, true]) + func anthropicInlinesLocalToolImagesAndReplaysThem(_ streaming: Bool) async throws { + let provider = Provider.anthropic + UsageURLProtocol.reset() + UsageURLProtocol.enqueue( + json: try streaming ? provider.toolStream() : Self.json(provider.response(text: "", tool: true)) + ) + if streaming { + UsageURLProtocol.enqueue(json: try provider.stream()) + } + UsageURLProtocol.enqueue(json: try Self.json(provider.response())) + let url = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString) + .appendingPathExtension("png") + try testImageData.write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + let session = provider.makeSession(tools: [LocalImageWeatherTool(url: url)]) + + if streaming { + _ = try await session.streamResponse(to: "Weather?").collect() + } else { + _ = try await session.respond(to: "Weather?") + } + _ = try await session.respond(to: "Review the image") + + #expect(UsageURLProtocol.recordedBodies.count == (streaming ? 3 : 2)) + for body in UsageURLProtocol.recordedBodies.dropFirst() { + let request = try JSONDecoder().decode([String: JSONValue].self, from: body) + let messages = try #require(request["messages"]?.arrayValue) + let results = messages.flatMap { $0.objectValue?["content"]?.arrayValue ?? [] } + .filter { $0.objectValue?["type"] == .string("tool_result") } + let content = try #require(results.first?.objectValue?["content"]?.arrayValue) + let image = try #require(content.first { $0.objectValue?["type"] == .string("image") }) + let source = try #require(image.objectValue?["source"]?.objectValue) + #expect(source["type"] == .string("base64")) + #expect(source["media_type"] == .string("image/png")) + #expect(source["data"] == .string(testImageData.base64EncodedString())) + #expect(source["url"] == nil) + } + } + + private struct LocalImageWeatherTool: Tool { + let name = "getWeather" + let description = "Get weather as an image" + let url: URL + + func call(arguments: WeatherTool.Arguments) async throws -> Prompt { + Prompt { Attachment(imageURL: url) } + } + } + @Test(arguments: Provider.allCases) func stoppedStreamedTools(_ provider: Provider) async throws { UsageURLProtocol.reset() diff --git a/Tests/AnyLanguageModelTests/ToolExecutionDelegateTests.swift b/Tests/AnyLanguageModelTests/ToolExecutionDelegateTests.swift index 1b258e32..383cca86 100644 --- a/Tests/AnyLanguageModelTests/ToolExecutionDelegateTests.swift +++ b/Tests/AnyLanguageModelTests/ToolExecutionDelegateTests.swift @@ -1,6 +1,6 @@ import Testing -@testable import AnyLanguageModel +@_spi(Compatibility) @testable import AnyLanguageModel private actor ToolExecutionDelegateSpy: ToolExecutionDelegate { private(set) var generatedToolCalls: [Transcript.ToolCall] = []