Document attachments in tool outputs

README.md

@@ -384,19 +384,22 @@ pass the model's multimodal projector with `mmprojPath:`.
 
 ### Prompt attachments
 
-Image attachments use the Foundation Models 27 prompt construction syntax:
+Image attachments use the Foundation Models 27 prompt construction syntax,
+including ordinary Tool outputs:
 
 ```swift
-let prompt = Prompt {
-    "Describe this image"
-    Attachment(imageURL: fileURL)
+func call(arguments: Arguments) async throws -> Prompt {
+    Prompt {
+        "Image read from the selected file"
+        Attachment(imageURL: fileURL)
+    }
 }
 ```
 
 On Apple platforms, `Attachment(CGImage, orientation:)` retains typed image
-content and orientation. Prompt builders preserve the order of text and images,
-and the OS 26 compatibility transcript continues to store images using its
-existing `Transcript.Segment.image` Codable representation.
+content and orientation. Prompts, builders, and Tool outputs preserve the order
+of text and images, and the OS 26 compatibility transcript continues to store
+images using its existing `Transcript.Segment.image` Codable representation.
 
 `SystemLanguageModel` and `FoundationLanguageModel` bridge prompt attachments
 to Foundation Models only in apps built with Swift 6.4 or later
@@ -406,9 +409,9 @@ they throw an error when a prompt has an attachment.
 The system bridge requires local file URLs; other provider adapters retain URL
 images in their existing transcript representation.
 
-This first compatibility surface covers prompt `CGImage` and `imageURL`
-attachments. Tool outputs, instruction attachments, labels, `CIImage`, pixel
-buffers, and newer data attachments remain outside this change.
+This compatibility surface covers `CGImage` and `imageURL` attachments in
+prompts and ordinary Tool outputs. Instruction attachments, labels, `CIImage`,
+pixel buffers, and newer data attachments remain outside this change.
 
 ### Tool Calling
 
@@ -641,6 +644,10 @@ say which API they follow.
   [image inputs](#image-inputs).
   Foundation Models 27-style [prompt attachments](#prompt-attachments) are also
   available for ordered text and image prompt construction.
+- `Tool.makeOutputSegments(from:)`, exposed only through the `Compatibility` SPI:
+  an integration extension for back-deployment provider adapters to lower normal
+  Tool outputs into the compatibility transcript. It is not an Apple Tool API;
+  OS 27 canonical executors use Foundation Models' own Tool execution instead.
 - `ToolExecutionDelegate`, `ToolExecutionDecision`, and `toolExecutionDelegate`:
   [observing and controlling tool calls](#tool-calling).
 - `transcriptErrorHandlingPolicy` and `waitForResponseCompletion()`:

Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift

@@ -254,7 +254,7 @@ public struct GeminiLanguageModel: LanguageModel {
                 requestTranscript.append(entry)
             }
             let params = try createGenerateContentParams(
-                contents: requestTranscript.toGeminiContent(),
+                contents: try requestTranscript.toGeminiContent(),
                 tools: geminiTools,
                 generating: type,
                 schema: responseSchema.schema,
@@ -432,7 +432,7 @@ public struct GeminiLanguageModel: LanguageModel {
                             requestTranscript.append(entry)
                         }
                         let params = try createGenerateContentParams(
-                            contents: requestTranscript.toGeminiContent(),
+                            contents: try requestTranscript.toGeminiContent(),
                             tools: geminiTools,
                             generating: type,
                             schema: responseSchema.schema,
@@ -773,32 +773,85 @@ private func emptyResponseContent<Content: Generable>(
     }
 }
 
-private func toJSONValue(_ toolOutput: Transcript.ToolOutput) throws -> [String: JSONValue] {
-    var result: [String: JSONValue] = [:]
+private func toGeminiFunctionResponse(
+    _ toolOutput: Transcript.ToolOutput
+) throws -> GeminiFunctionResponse {
+    var resultValues: [String] = []
+    var parts: [GeminiPart] = []
 
     for segment in toolOutput.segments {
         switch segment {
         case .text(let text):
-            result["result"] = .string(text.content)
+            resultValues.append(text.content)
         case .structure(let structured):
-            // For structured segments, encode the content
             let data = try JSONEncoder().encode(structured.content)
             if let jsonString = String(data: data, encoding: .utf8) {
-                result["result"] = .string(jsonString)
+                resultValues.append(jsonString)
+            }
+        case .image(let image):
+            switch image.source {
+            case .data(let data, let mimeType):
+                parts.append(
+                    .inlineData(
+                        GeminiInlineData(
+                            mimeType: mimeType,
+                            data: data.base64EncodedString()
+                        )
+                    )
+                )
+            case .url(let url):
+                parts.append(.inlineData(try geminiFunctionResponseData(from: url)))
             }
-        case .image:
-            // Ignore images in tool outputs for Gemini conversion
-            break
         }
     }
 
-    return result
+    var response: [String: JSONValue] = [:]
+    if !resultValues.isEmpty {
+        response["result"] = .string(resultValues.joined(separator: "\n"))
+    }
+    return GeminiFunctionResponse(
+        name: toolOutput.toolName,
+        response: response,
+        parts: parts.isEmpty ? nil : parts
+    )
+}
+
+private enum GeminiFunctionResponseAttachmentError: LocalizedError {
+    case unsupportedURL(URL)
+    case unsupportedImageType(URL)
+
+    var errorDescription: String? {
+        switch self {
+        case .unsupportedURL(let url):
+            "Gemini Tool output attachments require a local file URL; cannot inline \(url.absoluteString)."
+        case .unsupportedImageType(let url):
+            "Gemini Tool output attachments support PNG, JPEG, and WebP files; cannot determine a supported MIME type for \(url.lastPathComponent)."
+        }
+    }
+}
+
+private func geminiFunctionResponseData(from url: URL) throws -> GeminiInlineData {
+    guard url.isFileURL else {
+        throw GeminiFunctionResponseAttachmentError.unsupportedURL(url)
+    }
+
+    let mimeType =
+        switch url.pathExtension.lowercased() {
+        case "png": "image/png"
+        case "jpg", "jpeg": "image/jpeg"
+        case "webp": "image/webp"
+        default: throw GeminiFunctionResponseAttachmentError.unsupportedImageType(url)
+        }
+    return try GeminiInlineData(
+        mimeType: mimeType,
+        data: Data(contentsOf: url).base64EncodedString()
+    )
 }
 
 // MARK: - Supporting Types
 
 extension Transcript {
-    fileprivate func toGeminiContent() -> [GeminiContent] {
+    fileprivate func toGeminiContent() throws -> [GeminiContent] {
         var messages = [GeminiContent]()
         for item in self {
             switch item {
@@ -848,11 +901,7 @@ extension Transcript {
                 )
             case .toolOutput(let toolOutput):
                 // Add function response as a user message (Gemini API expects function responses from user role)
-                let response = try? toJSONValue(toolOutput)
-                let functionResponse = GeminiFunctionResponse(
-                    name: toolOutput.toolName,
-                    response: response ?? [:]
-                )
+                let functionResponse = try toGeminiFunctionResponse(toolOutput)
                 messages.append(
                     .init(
                         role: .user,
@@ -1120,6 +1169,7 @@ private struct GeminiFunctionCall: Codable, Sendable {
 private struct GeminiFunctionResponse: Codable, Sendable {
     let name: String
     let response: [String: JSONValue]
+    let parts: [GeminiPart]?
 }
 
 private struct GeminiGenerateContentResponse: Codable, Sendable {

Sources/AnyLanguageModel/Models/SystemLanguageModel.swift

@@ -577,7 +577,7 @@
     @available(macOS 26.0, iOS 26.0, watchOS 27.0, tvOS 26.0, visionOS 26.0, *)
     private struct AnyToolWrapper: FoundationModels.Tool {
         typealias Arguments = FoundationModels.GeneratedContent
-        typealias Output = String
+        typealias Output = FoundationModels.Prompt
 
         let name: String
         let description: String
@@ -596,7 +596,7 @@
 
         func call(arguments: FoundationModels.GeneratedContent) async throws -> Output {
             let output = try await wrappedTool.callFunction(arguments: arguments)
-            return output.promptRepresentation.description
+            return try output.promptRepresentation.toFoundationModels()
         }
     }
 

Sources/AnyLanguageModel/Tool.swift

@@ -110,7 +110,10 @@ extension Tool where Self.Arguments: Generable {
 // MARK: - Helpers
 
 extension Tool {
-    internal func makeOutputSegments(from arguments: GeneratedContent) async throws -> [Transcript.Segment] {
+    /// Compatibility integration SPI for provider adapters outside this module.
+    /// This is not part of Apple's public Tool contract.
+    @_spi(Compatibility)
+    public func makeOutputSegments(from arguments: GeneratedContent) async throws -> [Transcript.Segment] {
         let parsedArguments = try Arguments(arguments)
         let output = try await call(arguments: parsedArguments)
 
@@ -124,7 +127,6 @@ extension Tool {
             return [Transcript.Segment.text(.init(content: stringOutput))]
         }
 
-        let fallback = output.promptRepresentation.description
-        return [Transcript.Segment.text(.init(content: fallback))]
+        return try output.promptRepresentation.makeTranscriptSegments()
     }
 }

Tests/AnyLanguageModelTests/DynamicInstructionsTests.swift

@@ -1,7 +1,7 @@
 import Foundation
 import Testing
 
-@testable import AnyLanguageModel
+@_spi(Compatibility) @testable import AnyLanguageModel
 
 @Suite("Dynamic instructions")
 struct DynamicInstructionsTests {

Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift

@@ -4,11 +4,27 @@ import Testing
 @testable import AnyLanguageModel
 
 #if canImport(Darwin) && !canImport(AsyncHTTPClient)
+    import CoreGraphics
 
     @Suite("GeminiLanguageModel thought signatures", .serialized)
     struct GeminiThoughtSignatureTests {
         private static let signature = "CvsBAdHtim8n5xQK1pVX2H0lPQeXAMPLEsignature=="
 
+        @Generable
+        struct ImageArguments {
+            var request: String
+        }
+
+        private struct ImageOutputTool: Tool {
+            let name = "image"
+            let description = "Return an image"
+            let output: Prompt
+
+            func call(arguments: ImageArguments) async throws -> Prompt {
+                output
+            }
+        }
+
         private func makeModel() -> GeminiLanguageModel {
             GeminiLanguageModel(
                 apiKey: "test-key",
@@ -51,6 +67,21 @@ import Testing
             """
         }
 
+        private func image() throws -> CGImage {
+            let context = try #require(
+                CGContext(
+                    data: nil,
+                    width: 2,
+                    height: 2,
+                    bitsPerComponent: 8,
+                    bytesPerRow: 8,
+                    space: CGColorSpaceCreateDeviceRGB(),
+                    bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
+                )
+            )
+            return try #require(context.makeImage())
+        }
+
         /// Signatures of every `functionCall` part in a request body, in order.
         /// A part without a signature contributes `nil`.
         private func functionCallSignatures(in body: Data) throws -> [String?] {
@@ -108,6 +139,108 @@ import Testing
             #expect(signatures.allSatisfy { $0 == Self.signature })
         }
 
+        @Test("preserves all text and image data in function responses")
+        func preservesSegmentedToolOutput() async throws {
+            StubURLProtocol.reset()
+            StubURLProtocol.enqueue(
+                json:
+                    #"{"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"image","args":{"request":"read"}}}]},"finishReason":"STOP"}]}"#
+            )
+            StubURLProtocol.enqueue(json: textResponse("OK"))
+
+            let output = try Prompt {
+                "before"
+                Attachment(try image())
+                "after"
+            }
+            let session = LanguageModelSession(
+                model: makeModel(),
+                tools: [ImageOutputTool(output: output)]
+            )
+            _ = try await session.respond(to: "read image")
+
+            let bodies = StubURLProtocol.recordedBodies
+            try #require(bodies.count == 2)
+            let json = try #require(
+                try JSONSerialization.jsonObject(with: bodies[1]) as? [String: Any]
+            )
+            let contents = try #require(json["contents"] as? [[String: Any]])
+            let outputContent = try #require(contents.last)
+            let outputParts = try #require(outputContent["parts"] as? [[String: Any]])
+            let functionResponse = try #require(
+                outputParts.first?["functionResponse"] as? [String: Any]
+            )
+            let response = try #require(functionResponse["response"] as? [String: Any])
+            #expect(response["result"] as? String == "before\nafter")
+
+            let mediaParts = try #require(functionResponse["parts"] as? [[String: Any]])
+            let inlineData = try #require(mediaParts.first?["inlineData"] as? [String: Any])
+            #expect(inlineData["mime_type"] as? String == "image/png")
+            #expect(Data(base64Encoded: try #require(inlineData["data"] as? String)) != nil)
+        }
+
+        @Test("inlines URL-backed images in function responses")
+        func inlinesURLBackedToolOutput() async throws {
+            StubURLProtocol.reset()
+            StubURLProtocol.enqueue(
+                json:
+                    #"{"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"image","args":{"request":"read"}}}]},"finishReason":"STOP"}]}"#
+            )
+            StubURLProtocol.enqueue(json: textResponse("OK"))
+
+            let imageData = Data([0x89, 0x50, 0x4E, 0x47])
+            let url = FileManager.default.temporaryDirectory
+                .appending(path: UUID().uuidString)
+                .appendingPathExtension("png")
+            try imageData.write(to: url)
+            defer { try? FileManager.default.removeItem(at: url) }
+
+            let output = Prompt { Attachment(imageURL: url) }
+            let session = LanguageModelSession(
+                model: makeModel(),
+                tools: [ImageOutputTool(output: output)]
+            )
+            _ = try await session.respond(to: "read image")
+
+            let body = try #require(StubURLProtocol.recordedBodies.last)
+            let json = try #require(
+                try JSONSerialization.jsonObject(with: body) as? [String: Any]
+            )
+            let contents = try #require(json["contents"] as? [[String: Any]])
+            let outputParts = try #require(contents.last?["parts"] as? [[String: Any]])
+            let functionResponse = try #require(
+                outputParts.first?["functionResponse"] as? [String: Any]
+            )
+            let mediaParts = try #require(functionResponse["parts"] as? [[String: Any]])
+            let inlineData = try #require(mediaParts.first?["inlineData"] as? [String: Any])
+            #expect(inlineData["mime_type"] as? String == "image/png")
+            #expect(Data(base64Encoded: try #require(inlineData["data"] as? String)) == imageData)
+            #expect(mediaParts.first?["fileData"] == nil)
+        }
+
+        @Test("rejects remote URL-backed images with an actionable error")
+        func rejectsRemoteURLBackedToolOutput() async throws {
+            StubURLProtocol.reset()
+            StubURLProtocol.enqueue(
+                json:
+                    #"{"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"image","args":{"request":"read"}}}]},"finishReason":"STOP"}]}"#
+            )
+
+            let url = try #require(URL(string: "https://example.com/image.png"))
+            let output = Prompt { Attachment(imageURL: url) }
+            let session = LanguageModelSession(
+                model: makeModel(),
+                tools: [ImageOutputTool(output: output)]
+            )
+
+            do {
+                _ = try await session.respond(to: "read image")
+                Issue.record("Expected a remote Tool output attachment to be rejected")
+            } catch {
+                #expect(error.localizedDescription.contains("local file URL"))
+            }
+        }
+
         @Test("does not replay the conversation history on later turns")
         func doesNotReplayHistoryOnLaterTurns() async throws {
             StubURLProtocol.reset()