From 760fe11aeb3cb35903ec7f021e149089d861ed3c Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Fri, 9 Oct 2026 16:00:01 +0800 Subject: [PATCH 1/8] Add Foundation Models 27 prompt attachments --- README.md | 31 ++- Sources/AnyLanguageModel/Attachment.swift | 107 +++++++++++ .../LanguageModelSession.swift | 18 +- .../Models/FoundationLanguageModel.swift | 10 +- .../Models/SystemLanguageModel.swift | 52 ++++- Sources/AnyLanguageModel/Prompt.swift | 104 ++++++++-- .../PromptAttachmentTests.swift | 181 ++++++++++++++++++ 7 files changed, 476 insertions(+), 27 deletions(-) create mode 100644 Sources/AnyLanguageModel/Attachment.swift create mode 100644 Tests/AnyLanguageModelTests/PromptAttachmentTests.swift diff --git a/README.md b/README.md index 5a969f0b..6826b869 100644 --- a/README.md +++ b/README.md @@ -382,6 +382,31 @@ pass the model's multimodal projector with `mmprojPath:`. > Image inputs are an AnyLanguageModel extension. > See [Differences from Foundation Models](#differences-from-foundation-models). +### Prompt attachments + +Image attachments use the Foundation Models 27 prompt construction syntax: + +```swift +let prompt = Prompt { + "Describe this image" + Attachment(imageURL: fileURL) +} +``` + +On Apple platforms, `Attachment(CGImage, orientation:)` retains typed image +content and orientation. Prompt builders preserve the order of text and images, +and the OS 26 compatibility transcript continues to store images using its +existing `Transcript.Segment.image` Codable representation. + +`SystemLanguageModel` and `FoundationLanguageModel` bridge prompt attachments +to Foundation Models on OS 27. Earlier system runtimes reject them explicitly. +The system bridge requires local file URLs; other provider adapters retain URL +images in their existing transcript representation. + +This first compatibility surface covers prompt `CGImage` and `imageURL` +attachments. Tool outputs, instruction attachments, labels, `CIImage`, pixel +buffers, and newer data attachments remain outside this change. + ### Tool Calling Tool calling is supported by all providers. @@ -611,8 +636,8 @@ say which API they follow. described in that provider's section under [Providers](#providers). - `respond(to:images:)`, `Transcript.ImageSegment`, and `Transcript.Segment.image`: [image inputs](#image-inputs). - Foundation Models 27 adds prompt attachments, - and AnyLanguageModel 2.0 will change to match them. + Foundation Models 27-style [prompt attachments](#prompt-attachments) are also + available for ordered text and image prompt construction. - `ToolExecutionDelegate`, `ToolExecutionDecision`, and `toolExecutionDelegate`: [observing and controlling tool calls](#tool-calling). - `transcriptErrorHandlingPolicy` and `waitForResponseCompletion()`: @@ -645,7 +670,7 @@ say which API they follow. ### Behavior differences - A session adds the prompt to its transcript before the model responds, - as a single text segment. + preserving its text and image segment order. - A structured response is recorded in the transcript as JSON text, not as a structured segment. - A session created from a transcript restores `instructions` diff --git a/Sources/AnyLanguageModel/Attachment.swift b/Sources/AnyLanguageModel/Attachment.swift new file mode 100644 index 00000000..a0cc88c0 --- /dev/null +++ b/Sources/AnyLanguageModel/Attachment.swift @@ -0,0 +1,107 @@ +import Foundation + +#if canImport(CoreGraphics) && canImport(ImageIO) + import CoreGraphics + import ImageIO +#endif + +/// A typed prompt attachment, following Foundation Models 27's image surface. +/// +/// AnyLanguageModel lowers image attachments into its existing compatibility +/// transcript image segments on deployment targets before OS 27. +public struct Attachment { + let content: Content +} + +/// Image content retained until a prompt is lowered into the compatibility transcript. +public struct ImageAttachmentContent: Sendable, Equatable { + enum Source: Sendable { + case url(URL, orientation: UInt32?) + #if canImport(CoreGraphics) && canImport(ImageIO) + case image(CGImage, orientation: CGImagePropertyOrientation?) + #endif + } + + let source: Source + + public static func == (lhs: Self, rhs: Self) -> Bool { + switch (lhs.source, rhs.source) { + case (.url(let a, let ao), .url(let b, let bo)): + return a == b && ao == bo + #if canImport(CoreGraphics) && canImport(ImageIO) + case (.image(let a, let ao), .image(let b, let bo)): + return a === b && ao == bo + #endif + default: + return false + } + } + + func makeSegment(id: String) throws -> Transcript.ImageSegment { + switch source { + case .url(let url, let orientation): + #if canImport(CoreGraphics) && canImport(ImageIO) + if let orientation { + guard let source = CGImageSourceCreateWithURL(url as CFURL, nil), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil) + else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + return try Self( + source: .image( + image, + orientation: CGImagePropertyOrientation(rawValue: orientation) + ) + ).makeSegment(id: id) + } + #endif + return .init(id: id, url: url) + #if canImport(CoreGraphics) && canImport(ImageIO) + case .image(let image, let orientation): + let bytes = NSMutableData() + guard + let destination = CGImageDestinationCreateWithData( + bytes, + "public.png" as CFString, + 1, + nil + ) + else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + var properties: [CFString: Any] = [:] + if let orientation { + properties[kCGImagePropertyOrientation] = orientation.rawValue + } + CGImageDestinationAddImage(destination, image, properties as CFDictionary) + guard CGImageDestinationFinalize(destination) else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + return .init(id: id, data: bytes as Data, mimeType: "image/png") + #endif + } + } +} + +extension Attachment where Content == ImageAttachmentContent { + #if canImport(CoreGraphics) && canImport(ImageIO) + public init(_ image: CGImage, orientation: CGImagePropertyOrientation? = nil) { + content = .init(source: .image(image, orientation: orientation)) + } + + public init(imageURL: URL, orientation: CGImagePropertyOrientation? = nil) { + // Preserve the URL itself; its bytes are not read merely by constructing a prompt. + content = .init(source: .url(imageURL, orientation: orientation?.rawValue)) + } + #else + public init(imageURL: URL) { + content = .init(source: .url(imageURL, orientation: nil)) + } + #endif +} + +extension Attachment: PromptRepresentable where Content == ImageAttachmentContent { + public var promptRepresentation: Prompt { + Prompt(components: [.image(id: UUID().uuidString, content: content)]) + } +} diff --git a/Sources/AnyLanguageModel/LanguageModelSession.swift b/Sources/AnyLanguageModel/LanguageModelSession.swift index 4f08efc6..b7775ce3 100644 --- a/Sources/AnyLanguageModel/LanguageModelSession.swift +++ b/Sources/AnyLanguageModel/LanguageModelSession.swift @@ -728,7 +728,7 @@ public final class LanguageModelSession: @unchecked Sendable { let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( id: promptID, - segments: [.text(.init(content: prompt.description))], + segments: try prompt.makeTranscriptSegments(), options: options, responseFormat: responseFormat ) @@ -776,9 +776,15 @@ public final class LanguageModelSession: @unchecked Sendable { options: GenerationOptions = GenerationOptions() ) -> sending ResponseStream where Content: Generable { // Add prompt to transcript + let segments: [Transcript.Segment] + do { + segments = try prompt.makeTranscriptSegments() + } catch { + return ResponseStream(stream: AsyncThrowingStream { $0.finish(throwing: error) }) + } let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( - segments: [.text(.init(content: prompt.description))], + segments: segments, options: options, responseFormat: type == String.self ? nil : .init(type: type) ) @@ -924,9 +930,15 @@ extension LanguageModelSession { includeSchemaInPrompt: Bool = true, options: GenerationOptions = GenerationOptions() ) -> sending ResponseStream { + let segments: [Transcript.Segment] + do { + segments = try prompt.makeTranscriptSegments() + } catch { + return ResponseStream(stream: AsyncThrowingStream { $0.finish(throwing: error) }) + } let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( - segments: [.text(.init(content: prompt.description))], + segments: segments, options: options, responseFormat: .init(schema: schema) ) diff --git a/Sources/AnyLanguageModel/Models/FoundationLanguageModel.swift b/Sources/AnyLanguageModel/Models/FoundationLanguageModel.swift index b5bf8599..6c54fcec 100644 --- a/Sources/AnyLanguageModel/Models/FoundationLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/FoundationLanguageModel.swift @@ -159,7 +159,7 @@ ) async throws -> LanguageModelSession.Response where Content: Generable { return try await fmRespond( makeSession: { try await self.makeSession(for: session, prompt: prompt) }, - fmPrompt: prompt.toFoundationModels(), + fmPrompt: try prompt.toFoundationModels(), fmOptions: options.toFoundationModels(), type: type, schema: schema, @@ -209,9 +209,15 @@ includeSchemaInPrompt: Bool, options: GenerationOptions ) -> sending LanguageModelSession.ResponseStream where Content: Generable { + let fmPrompt: FoundationModels.Prompt + do { + fmPrompt = try prompt.toFoundationModels() + } catch { + return .init(stream: AsyncThrowingStream { $0.finish(throwing: error) }) + } return fmStreamResponse( makeSession: { try await self.makeSession(for: session, prompt: prompt) }, - fmPrompt: prompt.toFoundationModels(), + fmPrompt: fmPrompt, fmOptions: options.toFoundationModels(), type: type, schema: schema, diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index 87763ed6..d66ab692 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -194,7 +194,7 @@ includeSchemaInPrompt: Bool, options: GenerationOptions ) async throws -> LanguageModelSession.Response where Content: Generable { - let fmPrompt = prompt.toFoundationModels() + let fmPrompt = try prompt.toFoundationModels() let fmOptions = options.toFoundationModels() return try await fmRespond( @@ -249,7 +249,12 @@ includeSchemaInPrompt: Bool, options: GenerationOptions ) -> sending LanguageModelSession.ResponseStream where Content: Generable { - let fmPrompt = prompt.toFoundationModels() + let fmPrompt: FoundationModels.Prompt + do { + fmPrompt = try prompt.toFoundationModels() + } catch { + return .init(stream: AsyncThrowingStream { $0.finish(throwing: error) }) + } let fmOptions = options.toFoundationModels() return fmStreamResponse( @@ -430,8 +435,42 @@ @available(macOS 26.0, iOS 26.0, watchOS 27.0, tvOS 26.0, visionOS 26.0, *) extension Prompt { - func toFoundationModels() -> FoundationModels.Prompt { - FoundationModels.Prompt(self.description) + func toFoundationModels() throws -> FoundationModels.Prompt { + let prompts: [FoundationModels.Prompt] = try components.map { component in + switch component { + case .text(let text): + return FoundationModels.Prompt(text.content) + case .image(_, let image): + #if compiler(>=6.4) && !os(tvOS) && !os(watchOS) + if #available(macOS 27, iOS 27, visionOS 27, *) { + switch image.source { + case .url(let url, let orientation): + guard url.isFileURL else { + throw PromptBridgeError.nonFileImageURL + } + return FoundationModels.Prompt( + FoundationModels.Attachment( + imageURL: url, + orientation: orientation.flatMap( + CGImagePropertyOrientation.init(rawValue:) + ) + ) + ) + case .image(let value, let orientation): + return FoundationModels.Prompt( + FoundationModels.Attachment(value, orientation: orientation) + ) + } + } + #endif + throw PromptBridgeError.attachmentsUnavailable + } + } + return FoundationModels.Prompt { + for prompt in prompts { + prompt + } + } } } @@ -517,6 +556,11 @@ } } + enum PromptBridgeError: Error { + case attachmentsUnavailable + case nonFileImageURL + } + /// A type-erased wrapper that bridges any `Tool` to `FoundationModels.Tool`. @available(macOS 26.0, iOS 26.0, watchOS 27.0, tvOS 26.0, visionOS 26.0, *) private struct AnyToolWrapper: FoundationModels.Tool { diff --git a/Sources/AnyLanguageModel/Prompt.swift b/Sources/AnyLanguageModel/Prompt.swift index 7e099134..5d215051 100644 --- a/Sources/AnyLanguageModel/Prompt.swift +++ b/Sources/AnyLanguageModel/Prompt.swift @@ -1,3 +1,5 @@ +import Foundation + /// A prompt from a person to the model. /// /// Prompts can contain content written by you, an outside source, or input directly from people using @@ -24,7 +26,12 @@ /// own prompt to better steer the model's response. For more information on handling inputs in your /// prompts, see . public struct Prompt: Sendable { - private let content: String + enum Component: Sendable { + case text(Transcript.TextSegment) + case image(id: String, content: ImageAttachmentContent) + } + + let components: [Component] /// Creates an instance with the content you specify. public init(_ representable: some PromptRepresentable) { @@ -34,12 +41,76 @@ public struct Prompt: Sendable { case let string as String: self.init(content: string) default: - self.init(content: representable.promptRepresentation.content) + self = representable.promptRepresentation } } init(content: String) { - self.content = content + components = [.text(.init(content: content))] + } + + init(components: [Component]) { + self.components = components + } + + func makeTranscriptSegments() throws -> [Transcript.Segment] { + try components.map { component in + switch component { + case .text(let text): + return .text(text) + case .image(let id, let content): + return .image(try content.makeSegment(id: id)) + } + } + } + + static func joining(_ prompts: [Prompt], trim: Bool = false) -> Prompt { + var components: [Component] = [] + for prompt in prompts { + for component in prompt.components { + if case .text(let text) = component, let last = components.last, + case .text(let previous) = last + { + components[components.count - 1] = .text( + .init( + id: previous.id, + content: previous.content + "\n" + text.content + ) + ) + } else { + components.append(component) + } + } + } + + if trim { + if let first = components.first, case .text(let text) = first { + components[0] = .text( + .init( + id: text.id, + content: text.content.replacingOccurrences( + of: #"^\s+"#, + with: "", + options: .regularExpression + ) + ) + ) + } + if let last = components.last, case .text(let text) = last { + components[components.count - 1] = .text( + .init( + id: text.id, + content: text.content.replacingOccurrences( + of: #"\s+$"#, + with: "", + options: .regularExpression + ) + ) + ) + } + } + + return Prompt(components: components) } public init(@PromptBuilder _ content: () throws -> Prompt) rethrows { @@ -50,7 +121,16 @@ public struct Prompt: Sendable { // MARK: - CustomStringConvertible extension Prompt: CustomStringConvertible { - public var description: String { content } + public var description: String { + components.map { component in + switch component { + case .text(let text): + return text.content + case .image: + return "" + } + }.joined(separator: "\n") + } } // MARK: - PromptBuilder @@ -59,10 +139,9 @@ extension Prompt: CustomStringConvertible { public struct PromptBuilder { public static func buildBlock(_ components: repeat each P) -> Prompt where repeat each P: PromptRepresentable { - var parts: [String] = [] - repeat parts.append((each components).promptRepresentation.description) - let combinedText = parts.joined(separator: "\n") - return Prompt(content: combinedText.trimmingCharacters(in: .whitespacesAndNewlines)) + var parts: [Prompt] = [] + repeat parts.append((each components).promptRepresentation) + return Prompt.joining(parts, trim: true) } public static func buildExpression

(_ expression: P) -> P where P: PromptRepresentable { @@ -70,10 +149,7 @@ public struct PromptBuilder { } public static func buildArray(_ prompts: [some PromptRepresentable]) -> Prompt { - let combinedText = prompts.map { - $0.promptRepresentation.description - }.joined(separator: "\n") - return Prompt(content: combinedText) + Prompt.joining(prompts.map(\.promptRepresentation)) } public static func buildOptional(_ component: Prompt?) -> Prompt { @@ -120,8 +196,6 @@ extension String: PromptRepresentable { extension Array: PromptRepresentable where Element: PromptRepresentable { /// An instance that represents a prompt. public var promptRepresentation: Prompt { - let combined = self.map { $0.promptRepresentation.description } - .joined(separator: "\n") - return Prompt(content: combined) + Prompt.joining(map(\.promptRepresentation)) } } diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift new file mode 100644 index 00000000..df184ca5 --- /dev/null +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -0,0 +1,181 @@ +import Foundation +import Testing + +@testable import AnyLanguageModel + +#if canImport(CoreGraphics) && canImport(ImageIO) + import CoreGraphics + import ImageIO +#endif +#if canImport(FoundationModels) && compiler(>=6.4) && !os(tvOS) && !os(watchOS) + import FoundationModels +#endif + +@Suite("Prompt attachments") +struct PromptAttachmentTests { + private let imageURL = URL(fileURLWithPath: "/tmp/attachment-fixture.png") + + private func prompt() -> AnyLanguageModel.Prompt { + AnyLanguageModel.Prompt { + "before" + AnyLanguageModel.Attachment(imageURL: imageURL) + "after" + } + } + + @Test func buildersAndRepresentablesPreserveOrderedImages() throws { + let original = prompt() + struct Representable: AnyLanguageModel.PromptRepresentable { + let promptRepresentation: AnyLanguageModel.Prompt + } + let wrapped = AnyLanguageModel.Prompt( + Representable(promptRepresentation: original) + ) + let composed = AnyLanguageModel.Prompt { + if true { + wrapped + } + for item in [original] { + item + } + } + + let expected = try original.makeTranscriptSegments() + #expect(try wrapped.makeTranscriptSegments() == expected) + let combined = try composed.makeTranscriptSegments() + #expect(combined.count == 5) + let sources: [AnyLanguageModel.Transcript.ImageSegment.Source] = combined.compactMap { + if case .image(let value) = $0 { + return value.source + } + return nil + } + let texts: [String] = combined.compactMap { + if case .text(let value) = $0 { + return value.content + } + return nil + } + #expect(sources == [.url(imageURL), .url(imageURL)]) + #expect(texts == ["before", "after\nbefore", "after"]) + #expect(original.description == "before\n\nafter") + #expect( + try AnyLanguageModel.Prompt([original, original]).makeTranscriptSegments() + == combined + ) + } + + @Test func directSessionPromptsRetainImagesInBothModes() async throws { + for streaming in [false, true] { + let session = AnyLanguageModel.LanguageModelSession(model: MockLanguageModel()) + let input = prompt() + if streaming { + _ = try await session.streamResponse(to: input).collect() + } else { + _ = try await session.respond(to: input) + } + + let first = try #require(session.transcript.first) + guard case .prompt(let recorded) = first else { + Issue.record("Missing prompt") + return + } + #expect(recorded.segments == (try input.makeTranscriptSegments())) + } + } + + #if canImport(CoreGraphics) && canImport(ImageIO) + private func image() throws -> CGImage { + let context = try #require( + CGContext( + data: nil, + width: 2, + height: 3, + bitsPerComponent: 8, + bytesPerRow: 8, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) + ) + return try #require(context.makeImage()) + } + + @Test func imageEncodingRetainsPixelsAndOrientation() throws { + let value = try image() + let input = AnyLanguageModel.Prompt { + "image note" + AnyLanguageModel.Attachment(value, orientation: .right) + } + let segments = try input.makeTranscriptSegments() + guard case .image(let segment) = segments.last, + case .data(let bytes, let mime) = segment.source + else { + Issue.record("No encoded image") + return + } + #expect(mime == "image/png") + let source = try #require( + CGImageSourceCreateWithData(bytes as CFData, nil) + ) + let decoded = try #require(CGImageSourceCreateImageAtIndex(source, 0, nil)) + #expect(decoded.width == value.width) + #expect(decoded.height == value.height) + let properties = try #require( + CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any] + ) + #expect( + (properties[kCGImagePropertyOrientation] as? NSNumber)?.uint32Value + == CGImagePropertyOrientation.right.rawValue + ) + } + #endif + + #if canImport(FoundationModels) && compiler(>=6.4) && !os(tvOS) && !os(watchOS) + @available(macOS 27, iOS 27, visionOS 27, *) + @Test func nativeBridgeRetainsAttachment() async throws { + let input = try AnyLanguageModel.Prompt { + "before" + AnyLanguageModel.Attachment(try image(), orientation: .right) + "after" + } + let session = FoundationModels.LanguageModelSession(model: BridgeModel()) + _ = try await session.respond(to: input.toFoundationModels()) + guard case .prompt(let recorded) = try #require(session.transcript.first) else { + Issue.record("Missing prompt") + return + } + #expect(recorded.segments.count == 3) + guard case .attachment(let segment) = recorded.segments[1], + case .image(let image) = segment.content + else { + Issue.record("Native bridge lost attachment") + return + } + #expect(image.orientation == .right) + } + + @available(macOS 27, iOS 27, visionOS 27, *) + private struct BridgeModel: FoundationModels.LanguageModel { + typealias Executor = BridgeExecutor + var capabilities: FoundationModels.LanguageModelCapabilities { + .init([.vision]) + } + var executorConfiguration: String { + "bridge-fixture" + } + } + + @available(macOS 27, iOS 27, visionOS 27, *) + private struct BridgeExecutor: FoundationModels.LanguageModelExecutor { + init(configuration: String) {} + + func respond( + to request: FoundationModels.LanguageModelExecutorGenerationRequest, + model: BridgeModel, + streamingInto channel: FoundationModels.LanguageModelExecutorGenerationChannel + ) async throws { + await channel.send(.response(action: .appendText("OK", tokenCount: 1))) + } + } + #endif +} From 379ee3cfca6dd026159d08b507baaecf6453480b Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Sun, 11 Oct 2026 22:07:15 +0800 Subject: [PATCH 2/8] Fix attachment prompt replay --- .../Models/SystemLanguageModel.swift | 18 ++++++--- .../PromptAttachmentTests.swift | 39 ++++++++++++++++++- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index d66ab692..9e17d200 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -341,11 +341,9 @@ guard let lastEntry = transcript.last, case .prompt(let lastPrompt) = lastEntry else { return transcript } - let lastText = lastPrompt.segments.compactMap { segment -> String? in - if case .text(let textSegment) = segment { return textSegment.content } - return nil - }.joined() - guard lastText == prompt.description else { + guard let promptSegments = try? prompt.makeTranscriptSegments(), + lastPrompt.segments == promptSegments + else { return transcript } return Transcript(entries: transcript.dropLast()) @@ -968,7 +966,15 @@ else { return nil } - self.init(cgImage) + let properties = + CGImageSourceCopyPropertiesAtIndex( + imageSource, + 0, + nil + ) as? [CFString: Any] + let orientation = (properties?[kCGImagePropertyOrientation] as? NSNumber) + .flatMap { CGImagePropertyOrientation(rawValue: $0.uint32Value) } + self.init(cgImage, orientation: orientation) } } } diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index df184ca5..9d6c6a2a 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -154,6 +154,22 @@ struct PromptAttachmentTests { #expect(image.orientation == .right) } + @available(macOS 27, iOS 27, visionOS 27, *) + @Test func foundationWrapperDoesNotDuplicatePromptsAndReplaysOrientation() async throws { + let model = AnyLanguageModel.FoundationLanguageModel(BridgeModel()) + let session = AnyLanguageModel.LanguageModelSession(model: model) + let input = try AnyLanguageModel.Prompt { + "before" + AnyLanguageModel.Attachment(try image(), orientation: .right) + "after" + } + let first = try await session.respond(to: input) + #expect(first.content == "prompts:1 images:1 right:1") + + let second = try await session.respond(to: "follow-up") + #expect(second.content == "prompts:2 images:1 right:1") + } + @available(macOS 27, iOS 27, visionOS 27, *) private struct BridgeModel: FoundationModels.LanguageModel { typealias Executor = BridgeExecutor @@ -174,7 +190,28 @@ struct PromptAttachmentTests { model: BridgeModel, streamingInto channel: FoundationModels.LanguageModelExecutorGenerationChannel ) async throws { - await channel.send(.response(action: .appendText("OK", tokenCount: 1))) + let prompts = request.transcript.compactMap { entry -> FoundationModels.Transcript.Prompt? in + guard case .prompt(let prompt) = entry else { return nil } + return prompt + } + let images = prompts.flatMap { prompt in + prompt.segments.compactMap { + segment -> FoundationModels.Transcript.ImageAttachment? in + guard case .attachment(let attachment) = segment, + case .image(let image) = attachment.content + else { return nil } + return image + } + } + let right = images.count { $0.orientation == .right } + await channel.send( + .response( + action: .appendText( + "prompts:\(prompts.count) images:\(images.count) right:\(right)", + tokenCount: 1 + ) + ) + ) } } #endif From 2d1a32b7352d38c111005fc1537fc32282fe8c3f Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Sun, 11 Oct 2026 22:10:10 +0800 Subject: [PATCH 3/8] Preserve attachments in tool outputs --- .../Models/SystemLanguageModel.swift | 4 +- Sources/AnyLanguageModel/Tool.swift | 8 +- .../DynamicInstructionsTests.swift | 2 +- .../PromptAttachmentTests.swift | 92 ++++++++++++++++++- .../ToolExecutionDelegateTests.swift | 2 +- 5 files changed, 99 insertions(+), 9 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index 9e17d200..bb451d50 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -563,7 +563,7 @@ @available(macOS 26.0, iOS 26.0, watchOS 27.0, tvOS 26.0, visionOS 26.0, *) private struct AnyToolWrapper: FoundationModels.Tool { typealias Arguments = FoundationModels.GeneratedContent - typealias Output = String + typealias Output = FoundationModels.Prompt let name: String let description: String @@ -582,7 +582,7 @@ func call(arguments: FoundationModels.GeneratedContent) async throws -> Output { let output = try await wrappedTool.callFunction(arguments: arguments) - return output.promptRepresentation.description + return try output.promptRepresentation.toFoundationModels() } } diff --git a/Sources/AnyLanguageModel/Tool.swift b/Sources/AnyLanguageModel/Tool.swift index 368a9961..59f5b800 100644 --- a/Sources/AnyLanguageModel/Tool.swift +++ b/Sources/AnyLanguageModel/Tool.swift @@ -110,7 +110,10 @@ extension Tool where Self.Arguments: Generable { // MARK: - Helpers extension Tool { - internal func makeOutputSegments(from arguments: GeneratedContent) async throws -> [Transcript.Segment] { + /// Compatibility integration SPI for provider adapters outside this module. + /// This is not part of Apple's public Tool contract. + @_spi(Compatibility) + public func makeOutputSegments(from arguments: GeneratedContent) async throws -> [Transcript.Segment] { let parsedArguments = try Arguments(arguments) let output = try await call(arguments: parsedArguments) @@ -124,7 +127,6 @@ extension Tool { return [Transcript.Segment.text(.init(content: stringOutput))] } - let fallback = output.promptRepresentation.description - return [Transcript.Segment.text(.init(content: fallback))] + return try output.promptRepresentation.makeTranscriptSegments() } } diff --git a/Tests/AnyLanguageModelTests/DynamicInstructionsTests.swift b/Tests/AnyLanguageModelTests/DynamicInstructionsTests.swift index af43ad37..baba7aa1 100644 --- a/Tests/AnyLanguageModelTests/DynamicInstructionsTests.swift +++ b/Tests/AnyLanguageModelTests/DynamicInstructionsTests.swift @@ -1,7 +1,7 @@ import Foundation import Testing -@testable import AnyLanguageModel +@_spi(Compatibility) @testable import AnyLanguageModel @Suite("Dynamic instructions") struct DynamicInstructionsTests { diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index 9d6c6a2a..ea85a885 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -1,7 +1,7 @@ import Foundation import Testing -@testable import AnyLanguageModel +@_spi(Compatibility) @testable import AnyLanguageModel #if canImport(CoreGraphics) && canImport(ImageIO) import CoreGraphics @@ -84,6 +84,29 @@ struct PromptAttachmentTests { } } + @Test func ordinaryToolReturnsOrderedImageSegments() async throws { + let segments = try await ImageTool(output: prompt()).makeOutputSegments( + from: AnyLanguageModel.GeneratedContent("read") + ) + #expect(segments.count == 3) + guard case .text(let before) = segments[0], + case .image(let image) = segments[1], + case .text(let after) = segments[2] + else { + Issue.record("Tool output lost its ordered image") + return + } + #expect(before.content == "before") + #expect(after.content == "after") + #expect(image.source == .url(imageURL)) + + let transcript = AnyLanguageModel.Transcript(entries: [ + .toolOutput(.init(id: "image-call", toolName: "image", segments: segments)) + ]) + let encoded = try JSONEncoder().encode(transcript) + #expect(try JSONDecoder().decode(AnyLanguageModel.Transcript.self, from: encoded) == transcript) + } + #if canImport(CoreGraphics) && canImport(ImageIO) private func image() throws -> CGImage { let context = try #require( @@ -170,11 +193,41 @@ struct PromptAttachmentTests { #expect(second.content == "prompts:2 images:1 right:1") } + @available(macOS 27, iOS 27, visionOS 27, *) + @Test func nativeToolWrapperRetainsOrdinaryImageOutput() async throws { + let output = try AnyLanguageModel.Prompt { + "tool note" + AnyLanguageModel.Attachment(try image(), orientation: .right) + } + let tools: [any AnyLanguageModel.Tool] = [ImageTool(output: output)] + let session = FoundationModels.LanguageModelSession( + model: BridgeModel(callsTools: true), + tools: tools.toFoundationModels() + ) + _ = try await session.respond(to: "read image") + let recorded = try #require( + session.transcript.compactMap { entry -> FoundationModels.Transcript.ToolOutput? in + guard case .toolOutput(let output) = entry else { return nil } + return output + }.first + ) + #expect(recorded.id == "image-call") + #expect(recorded.segments.count == 2) + guard case .attachment(let segment) = recorded.segments[1], + case .image(let image) = segment.content + else { + Issue.record("Native Tool wrapper flattened image output") + return + } + #expect(image.orientation == .right) + } + @available(macOS 27, iOS 27, visionOS 27, *) private struct BridgeModel: FoundationModels.LanguageModel { typealias Executor = BridgeExecutor + var callsTools = false var capabilities: FoundationModels.LanguageModelCapabilities { - .init([.vision]) + .init([.vision, .toolCalling]) } var executorConfiguration: String { "bridge-fixture" @@ -190,6 +243,27 @@ struct PromptAttachmentTests { model: BridgeModel, streamingInto channel: FoundationModels.LanguageModelExecutorGenerationChannel ) async throws { + if model.callsTools, + !request.transcript.contains(where: { + if case .toolOutput = $0 { return true } + return false + }) + { + await channel.send( + .toolCalls( + action: .toolCall( + id: "image-call", + name: "image", + action: .appendArguments(#""read""#, tokenCount: 1) + ) + ) + ) + return + } + if model.callsTools { + await channel.send(.response(action: .appendText("OK", tokenCount: 1))) + return + } let prompts = request.transcript.compactMap { entry -> FoundationModels.Transcript.Prompt? in guard case .prompt(let prompt) = entry else { return nil } return prompt @@ -216,3 +290,17 @@ struct PromptAttachmentTests { } #endif } + +private struct ImageTool: AnyLanguageModel.Tool { + let name = "image" + let description = "Return an image" + let output: AnyLanguageModel.Prompt + + var parameters: AnyLanguageModel.GenerationSchema { + String.generationSchema + } + + func call(arguments: String) async throws -> AnyLanguageModel.Prompt { + output + } +} diff --git a/Tests/AnyLanguageModelTests/ToolExecutionDelegateTests.swift b/Tests/AnyLanguageModelTests/ToolExecutionDelegateTests.swift index 1b258e32..383cca86 100644 --- a/Tests/AnyLanguageModelTests/ToolExecutionDelegateTests.swift +++ b/Tests/AnyLanguageModelTests/ToolExecutionDelegateTests.swift @@ -1,6 +1,6 @@ import Testing -@testable import AnyLanguageModel +@_spi(Compatibility) @testable import AnyLanguageModel private actor ToolExecutionDelegateSpy: ToolExecutionDelegate { private(set) var generatedToolCalls: [Transcript.ToolCall] = [] From f96d6decb4be0eca99af2741e8f8d2b2ba9ee524 Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Mon, 12 Oct 2026 09:04:53 +0800 Subject: [PATCH 4/8] Generate fresh prompt segment identifiers --- .../Models/SystemLanguageModel.swift | 20 ++++++++++- Sources/AnyLanguageModel/Prompt.swift | 6 ++-- .../PromptAttachmentTests.swift | 35 +++++++++++++++---- 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index bb451d50..fb176823 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -342,13 +342,31 @@ return transcript } guard let promptSegments = try? prompt.makeTranscriptSegments(), - lastPrompt.segments == promptSegments + lastPrompt.segments.hasSamePayload(as: promptSegments) else { return transcript } return Transcript(entries: transcript.dropLast()) } + private extension Array where Element == Transcript.Segment { + func hasSamePayload(as other: [Transcript.Segment]) -> Bool { + guard count == other.count else { return false } + return zip(self, other).allSatisfy { lhs, rhs in + switch (lhs, rhs) { + case (.text(let lhs), .text(let rhs)): + lhs.content == rhs.content + case (.structure(let lhs), .structure(let rhs)): + lhs.source == rhs.source && lhs.content == rhs.content + case (.image(let lhs), .image(let rhs)): + lhs.source == rhs.source + default: + false + } + } + } + } + @available(macOS 26.0, iOS 26.0, tvOS 26.0, visionOS 26.0, *) @available(watchOS, unavailable) extension SystemLanguageModel { diff --git a/Sources/AnyLanguageModel/Prompt.swift b/Sources/AnyLanguageModel/Prompt.swift index 5d215051..3a829982 100644 --- a/Sources/AnyLanguageModel/Prompt.swift +++ b/Sources/AnyLanguageModel/Prompt.swift @@ -57,9 +57,9 @@ public struct Prompt: Sendable { try components.map { component in switch component { case .text(let text): - return .text(text) - case .image(let id, let content): - return .image(try content.makeSegment(id: id)) + return .text(.init(content: text.content)) + case .image(_, let content): + return .image(try content.makeSegment(id: UUID().uuidString)) } } } diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index ea85a885..8f91183c 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -13,8 +13,28 @@ import Testing @Suite("Prompt attachments") struct PromptAttachmentTests { + private enum SegmentPayload: Equatable { + case text(String) + case image(AnyLanguageModel.Transcript.ImageSegment.Source) + } + private let imageURL = URL(fileURLWithPath: "/tmp/attachment-fixture.png") + private func payloads( + _ segments: [AnyLanguageModel.Transcript.Segment] + ) -> [SegmentPayload] { + segments.compactMap { segment in + switch segment { + case .text(let text): + .text(text.content) + case .image(let image): + .image(image.source) + case .structure: + nil + } + } + } + private func prompt() -> AnyLanguageModel.Prompt { AnyLanguageModel.Prompt { "before" @@ -41,9 +61,12 @@ struct PromptAttachmentTests { } let expected = try original.makeTranscriptSegments() - #expect(try wrapped.makeTranscriptSegments() == expected) + let wrappedSegments = try wrapped.makeTranscriptSegments() + #expect(payloads(wrappedSegments) == payloads(expected)) + #expect(Set(wrappedSegments.map(\.id)).isDisjoint(with: expected.map(\.id))) let combined = try composed.makeTranscriptSegments() #expect(combined.count == 5) + #expect(Set(combined.map(\.id)).count == combined.count) let sources: [AnyLanguageModel.Transcript.ImageSegment.Source] = combined.compactMap { if case .image(let value) = $0 { return value.source @@ -59,10 +82,10 @@ struct PromptAttachmentTests { #expect(sources == [.url(imageURL), .url(imageURL)]) #expect(texts == ["before", "after\nbefore", "after"]) #expect(original.description == "before\n\nafter") - #expect( - try AnyLanguageModel.Prompt([original, original]).makeTranscriptSegments() - == combined - ) + let repeated = try AnyLanguageModel.Prompt([original, original]).makeTranscriptSegments() + #expect(payloads(repeated) == payloads(combined)) + #expect(Set(repeated.map(\.id)).count == repeated.count) + #expect(Set(repeated.map(\.id)).isDisjoint(with: combined.map(\.id))) } @Test func directSessionPromptsRetainImagesInBothModes() async throws { @@ -80,7 +103,7 @@ struct PromptAttachmentTests { Issue.record("Missing prompt") return } - #expect(recorded.segments == (try input.makeTranscriptSegments())) + #expect(payloads(recorded.segments) == payloads(try input.makeTranscriptSegments())) } } From c7aaa57799f619a0a623abf53bd54d00f8306931 Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Mon, 12 Oct 2026 09:10:14 +0800 Subject: [PATCH 5/8] Preserve Gemini tool output segments --- README.md | 7 +- .../Models/GeminiLanguageModel.swift | 52 ++++++++++---- .../Models/SystemLanguageModel.swift | 11 ++- .../GeminiThoughtSignatureTests.swift | 71 +++++++++++++++++++ .../PromptAttachmentTests.swift | 11 +++ 5 files changed, 134 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 6826b869..6c4eb428 100644 --- a/README.md +++ b/README.md @@ -403,9 +403,10 @@ to Foundation Models on OS 27. Earlier system runtimes reject them explicitly. The system bridge requires local file URLs; other provider adapters retain URL images in their existing transcript representation. -This first compatibility surface covers prompt `CGImage` and `imageURL` -attachments. Tool outputs, instruction attachments, labels, `CIImage`, pixel -buffers, and newer data attachments remain outside this change. +This compatibility surface covers prompt `CGImage` and `imageURL` attachments, +including attachments returned from ordinary `Tool` output values. Instruction +attachments, labels, `CIImage`, pixel buffers, and newer data attachments remain +outside this change. ### Tool Calling diff --git a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift index 5fc12db9..bdc5da7f 100644 --- a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift @@ -769,26 +769,47 @@ private func emptyResponseContent( } } -private func toJSONValue(_ toolOutput: Transcript.ToolOutput) throws -> [String: JSONValue] { - var result: [String: JSONValue] = [:] +private func toGeminiFunctionResponse( + _ toolOutput: Transcript.ToolOutput +) throws -> GeminiFunctionResponse { + var resultValues: [String] = [] + var parts: [GeminiPart] = [] for segment in toolOutput.segments { switch segment { case .text(let text): - result["result"] = .string(text.content) + resultValues.append(text.content) case .structure(let structured): - // For structured segments, encode the content let data = try JSONEncoder().encode(structured.content) if let jsonString = String(data: data, encoding: .utf8) { - result["result"] = .string(jsonString) + resultValues.append(jsonString) + } + case .image(let image): + switch image.source { + case .data(let data, let mimeType): + parts.append( + .inlineData( + GeminiInlineData( + mimeType: mimeType, + data: data.base64EncodedString() + ) + ) + ) + case .url(let url): + parts.append(.fileData(GeminiFileData(fileURI: url.absoluteString))) } - case .image: - // Ignore images in tool outputs for Gemini conversion - break } } - return result + var response: [String: JSONValue] = [:] + if !resultValues.isEmpty { + response["result"] = .string(resultValues.joined(separator: "\n")) + } + return GeminiFunctionResponse( + name: toolOutput.toolName, + response: response, + parts: parts.isEmpty ? nil : parts + ) } // MARK: - Supporting Types @@ -844,11 +865,13 @@ extension Transcript { ) case .toolOutput(let toolOutput): // Add function response as a user message (Gemini API expects function responses from user role) - let response = try? toJSONValue(toolOutput) - let functionResponse = GeminiFunctionResponse( - name: toolOutput.toolName, - response: response ?? [:] - ) + let functionResponse = + (try? toGeminiFunctionResponse(toolOutput)) + ?? GeminiFunctionResponse( + name: toolOutput.toolName, + response: [:], + parts: nil + ) messages.append( .init( role: .user, @@ -1116,6 +1139,7 @@ private struct GeminiFunctionCall: Codable, Sendable { private struct GeminiFunctionResponse: Codable, Sendable { let name: String let response: [String: JSONValue] + let parts: [GeminiPart]? } private struct GeminiGenerateContentResponse: Codable, Sendable { diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index fb176823..7354e824 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -572,9 +572,18 @@ } } - enum PromptBridgeError: Error { + enum PromptBridgeError: LocalizedError, Sendable, Equatable { case attachmentsUnavailable case nonFileImageURL + + var errorDescription: String? { + switch self { + case .attachmentsUnavailable: + "Prompt attachments require an OS 27 Foundation Models runtime." + case .nonFileImageURL: + "Foundation Models prompt attachments require a local file URL." + } + } } /// A type-erased wrapper that bridges any `Tool` to `FoundationModels.Tool`. diff --git a/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift b/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift index 5265718f..d1a93925 100644 --- a/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift +++ b/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift @@ -4,11 +4,27 @@ import Testing @testable import AnyLanguageModel #if canImport(Darwin) && !canImport(AsyncHTTPClient) + import CoreGraphics @Suite("GeminiLanguageModel thought signatures", .serialized) struct GeminiThoughtSignatureTests { private static let signature = "CvsBAdHtim8n5xQK1pVX2H0lPQeXAMPLEsignature==" + @Generable + struct ImageArguments { + var request: String + } + + private struct ImageOutputTool: Tool { + let name = "image" + let description = "Return an image" + let output: Prompt + + func call(arguments: ImageArguments) async throws -> Prompt { + output + } + } + private func makeModel() -> GeminiLanguageModel { GeminiLanguageModel( apiKey: "test-key", @@ -51,6 +67,21 @@ import Testing """ } + private func image() throws -> CGImage { + let context = try #require( + CGContext( + data: nil, + width: 2, + height: 2, + bitsPerComponent: 8, + bytesPerRow: 8, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) + ) + return try #require(context.makeImage()) + } + /// Signatures of every `functionCall` part in a request body, in order. /// A part without a signature contributes `nil`. private func functionCallSignatures(in body: Data) throws -> [String?] { @@ -108,6 +139,46 @@ import Testing #expect(signatures.allSatisfy { $0 == Self.signature }) } + @Test("preserves all text and image data in function responses") + func preservesSegmentedToolOutput() async throws { + StubURLProtocol.reset() + StubURLProtocol.enqueue( + json: + #"{"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"image","args":{"request":"read"}}}]},"finishReason":"STOP"}]}"# + ) + StubURLProtocol.enqueue(json: textResponse("OK")) + + let output = try Prompt { + "before" + Attachment(try image()) + "after" + } + let session = LanguageModelSession( + model: makeModel(), + tools: [ImageOutputTool(output: output)] + ) + _ = try await session.respond(to: "read image") + + let bodies = StubURLProtocol.recordedBodies + try #require(bodies.count == 2) + let json = try #require( + try JSONSerialization.jsonObject(with: bodies[1]) as? [String: Any] + ) + let contents = try #require(json["contents"] as? [[String: Any]]) + let outputContent = try #require(contents.last) + let outputParts = try #require(outputContent["parts"] as? [[String: Any]]) + let functionResponse = try #require( + outputParts.first?["functionResponse"] as? [String: Any] + ) + let response = try #require(functionResponse["response"] as? [String: Any]) + #expect(response["result"] as? String == "before\nafter") + + let mediaParts = try #require(functionResponse["parts"] as? [[String: Any]]) + let inlineData = try #require(mediaParts.first?["inlineData"] as? [String: Any]) + #expect(inlineData["mime_type"] as? String == "image/png") + #expect(Data(base64Encoded: try #require(inlineData["data"] as? String)) != nil) + } + @Test("does not replay the conversation history on later turns") func doesNotReplayHistoryOnLaterTurns() async throws { StubURLProtocol.reset() diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index 8f91183c..81a3b3f7 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -177,6 +177,17 @@ struct PromptAttachmentTests { #endif #if canImport(FoundationModels) && compiler(>=6.4) && !os(tvOS) && !os(watchOS) + @Test func promptBridgeErrorsHaveLocalizedDescriptions() { + #expect( + AnyLanguageModel.PromptBridgeError.attachmentsUnavailable.errorDescription + == "Prompt attachments require an OS 27 Foundation Models runtime." + ) + #expect( + AnyLanguageModel.PromptBridgeError.nonFileImageURL.errorDescription + == "Foundation Models prompt attachments require a local file URL." + ) + } + @available(macOS 27, iOS 27, visionOS 27, *) @Test func nativeBridgeRetainsAttachment() async throws { let input = try AnyLanguageModel.Prompt { From d9de68af09d1a21e44d5a095423f9c5fa347dac7 Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Tue, 13 Oct 2026 09:08:28 +0800 Subject: [PATCH 6/8] Avoid relowering recorded prompt attachments --- .../LanguageModelSession.swift | 10 ++-- .../Models/SystemLanguageModel.swift | 25 ++------- Sources/AnyLanguageModel/Prompt.swift | 28 ++++++++-- .../PromptAttachmentTests.swift | 51 +++++++++++++++++++ 4 files changed, 87 insertions(+), 27 deletions(-) diff --git a/Sources/AnyLanguageModel/LanguageModelSession.swift b/Sources/AnyLanguageModel/LanguageModelSession.swift index b7775ce3..552c3fe0 100644 --- a/Sources/AnyLanguageModel/LanguageModelSession.swift +++ b/Sources/AnyLanguageModel/LanguageModelSession.swift @@ -728,7 +728,7 @@ public final class LanguageModelSession: @unchecked Sendable { let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( id: promptID, - segments: try prompt.makeTranscriptSegments(), + segments: try prompt.makeTranscriptSegments(occurrenceID: promptID), options: options, responseFormat: responseFormat ) @@ -776,14 +776,16 @@ public final class LanguageModelSession: @unchecked Sendable { options: GenerationOptions = GenerationOptions() ) -> sending ResponseStream where Content: Generable { // Add prompt to transcript + let promptID = UUID().uuidString let segments: [Transcript.Segment] do { - segments = try prompt.makeTranscriptSegments() + segments = try prompt.makeTranscriptSegments(occurrenceID: promptID) } catch { return ResponseStream(stream: AsyncThrowingStream { $0.finish(throwing: error) }) } let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( + id: promptID, segments: segments, options: options, responseFormat: type == String.self ? nil : .init(type: type) @@ -930,14 +932,16 @@ extension LanguageModelSession { includeSchemaInPrompt: Bool = true, options: GenerationOptions = GenerationOptions() ) -> sending ResponseStream { + let promptID = UUID().uuidString let segments: [Transcript.Segment] do { - segments = try prompt.makeTranscriptSegments() + segments = try prompt.makeTranscriptSegments(occurrenceID: promptID) } catch { return ResponseStream(stream: AsyncThrowingStream { $0.finish(throwing: error) }) } let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( + id: promptID, segments: segments, options: options, responseFormat: .init(schema: schema) diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index 7354e824..2eb7e785 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -341,32 +341,17 @@ guard let lastEntry = transcript.last, case .prompt(let lastPrompt) = lastEntry else { return transcript } - guard let promptSegments = try? prompt.makeTranscriptSegments(), - lastPrompt.segments.hasSamePayload(as: promptSegments) + guard + prompt.matchesTranscriptOccurrence( + lastPrompt.segments, + occurrenceID: lastPrompt.id + ) else { return transcript } return Transcript(entries: transcript.dropLast()) } - private extension Array where Element == Transcript.Segment { - func hasSamePayload(as other: [Transcript.Segment]) -> Bool { - guard count == other.count else { return false } - return zip(self, other).allSatisfy { lhs, rhs in - switch (lhs, rhs) { - case (.text(let lhs), .text(let rhs)): - lhs.content == rhs.content - case (.structure(let lhs), .structure(let rhs)): - lhs.source == rhs.source && lhs.content == rhs.content - case (.image(let lhs), .image(let rhs)): - lhs.source == rhs.source - default: - false - } - } - } - } - @available(macOS 26.0, iOS 26.0, tvOS 26.0, visionOS 26.0, *) @available(watchOS, unavailable) extension SystemLanguageModel { diff --git a/Sources/AnyLanguageModel/Prompt.swift b/Sources/AnyLanguageModel/Prompt.swift index 3a829982..9365c47c 100644 --- a/Sources/AnyLanguageModel/Prompt.swift +++ b/Sources/AnyLanguageModel/Prompt.swift @@ -53,18 +53,38 @@ public struct Prompt: Sendable { self.components = components } - func makeTranscriptSegments() throws -> [Transcript.Segment] { - try components.map { component in + func makeTranscriptSegments(occurrenceID: String = UUID().uuidString) throws -> [Transcript.Segment] { + try components.enumerated().map { index, component in + let id = Self.transcriptSegmentID(occurrenceID: occurrenceID, index: index) switch component { case .text(let text): - return .text(.init(content: text.content)) + return .text(.init(id: id, content: text.content)) case .image(_, let content): - return .image(try content.makeSegment(id: UUID().uuidString)) + return .image(try content.makeSegment(id: id)) } } } + func matchesTranscriptOccurrence( + _ segments: [Transcript.Segment], + occurrenceID: String + ) -> Bool { + guard components.count == segments.count else { return false } + return segments.indices.allSatisfy { index in + segments[index].id + == Self.transcriptSegmentID( + occurrenceID: occurrenceID, + index: index + ) + } + } + + private static func transcriptSegmentID(occurrenceID: String, index: Int) -> String { + "\(occurrenceID).\(index)" + } + static func joining(_ prompts: [Prompt], trim: Bool = false) -> Prompt { + guard !prompts.isEmpty else { return Prompt(content: "") } var components: [Component] = [] for prompt in prompts { for component in prompt.components { diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index 81a3b3f7..3283aee6 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -43,6 +43,21 @@ struct PromptAttachmentTests { } } + @Test func emptyBuildersKeepOneEmptyTextSegment() throws { + let built = AnyLanguageModel.Prompt {} + let array = AnyLanguageModel.Prompt([AnyLanguageModel.Prompt]()) + + for prompt in [built, array] { + let segments = try prompt.makeTranscriptSegments() + #expect(segments.count == 1) + guard case .text(let text) = segments[0] else { + Issue.record("Empty prompt did not retain its text segment") + continue + } + #expect(text.content.isEmpty) + } + } + @Test func buildersAndRepresentablesPreserveOrderedImages() throws { let original = prompt() struct Representable: AnyLanguageModel.PromptRepresentable { @@ -174,6 +189,42 @@ struct PromptAttachmentTests { == CGImagePropertyOrientation.right.rawValue ) } + + #if canImport(FoundationModels) + @available(macOS 26, iOS 26, visionOS 26, *) + @Test func duplicateSuppressionDoesNotReencodeRecordedAttachments() throws { + let encoded = try AnyLanguageModel.Prompt { + AnyLanguageModel.Attachment(try image(), orientation: .right) + }.makeTranscriptSegments() + guard case .image(let image) = try #require(encoded.first), + case .data(let data, _) = image.source + else { + Issue.record("Failed to encode the image fixture") + return + } + + let url = FileManager.default.temporaryDirectory + .appending(path: UUID().uuidString) + .appendingPathExtension("png") + try data.write(to: url) + let prompt = AnyLanguageModel.Prompt { + AnyLanguageModel.Attachment(imageURL: url, orientation: .right) + } + let occurrenceID = UUID().uuidString + let segments = try prompt.makeTranscriptSegments(occurrenceID: occurrenceID) + try FileManager.default.removeItem(at: url) + let transcript = AnyLanguageModel.Transcript(entries: [ + .prompt(.init(id: occurrenceID, segments: segments)) + ]) + + #expect( + AnyLanguageModel.fmTranscriptDroppingDuplicatePrompt( + transcript, + prompt: prompt + ).isEmpty + ) + } + #endif #endif #if canImport(FoundationModels) && compiler(>=6.4) && !os(tvOS) && !os(watchOS) From 9ab55ea4fe3351a52d1c4d7aefe381058c9faeb4 Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Tue, 13 Oct 2026 09:10:00 +0800 Subject: [PATCH 7/8] Clarify supported attachment platforms --- Sources/AnyLanguageModel/Models/SystemLanguageModel.swift | 2 +- Tests/AnyLanguageModelTests/PromptAttachmentTests.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index 2eb7e785..7727bbbf 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -564,7 +564,7 @@ var errorDescription: String? { switch self { case .attachmentsUnavailable: - "Prompt attachments require an OS 27 Foundation Models runtime." + "Prompt attachments require macOS 27, iOS 27, or visionOS 27." case .nonFileImageURL: "Foundation Models prompt attachments require a local file URL." } diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index 3283aee6..02cf014e 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -231,7 +231,7 @@ struct PromptAttachmentTests { @Test func promptBridgeErrorsHaveLocalizedDescriptions() { #expect( AnyLanguageModel.PromptBridgeError.attachmentsUnavailable.errorDescription - == "Prompt attachments require an OS 27 Foundation Models runtime." + == "Prompt attachments require macOS 27, iOS 27, or visionOS 27." ) #expect( AnyLanguageModel.PromptBridgeError.nonFileImageURL.errorDescription From 1892a39bca496b413e11a7dc0d570381a68e2b1d Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Thu, 15 Oct 2026 09:01:38 +0800 Subject: [PATCH 8/8] Inline Gemini tool output file attachments --- .../Models/GeminiLanguageModel.swift | 47 ++++++++++---- .../GeminiThoughtSignatureTests.swift | 62 +++++++++++++++++++ 2 files changed, 98 insertions(+), 11 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift index d531c426..cbb4cf2d 100644 --- a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift @@ -254,7 +254,7 @@ public struct GeminiLanguageModel: LanguageModel { requestTranscript.append(entry) } let params = try createGenerateContentParams( - contents: requestTranscript.toGeminiContent(), + contents: try requestTranscript.toGeminiContent(), tools: geminiTools, generating: type, schema: responseSchema.schema, @@ -432,7 +432,7 @@ public struct GeminiLanguageModel: LanguageModel { requestTranscript.append(entry) } let params = try createGenerateContentParams( - contents: requestTranscript.toGeminiContent(), + contents: try requestTranscript.toGeminiContent(), tools: geminiTools, generating: type, schema: responseSchema.schema, @@ -800,7 +800,7 @@ private func toGeminiFunctionResponse( ) ) case .url(let url): - parts.append(.fileData(GeminiFileData(fileURI: url.absoluteString))) + parts.append(.inlineData(try geminiFunctionResponseData(from: url))) } } } @@ -816,10 +816,41 @@ private func toGeminiFunctionResponse( ) } +private enum GeminiFunctionResponseAttachmentError: LocalizedError { + case unsupportedURL(URL) + case unsupportedImageType(URL) + + var errorDescription: String? { + switch self { + case .unsupportedURL(let url): + "Gemini Tool output attachments require a local file URL; cannot inline \(url.absoluteString)." + case .unsupportedImageType(let url): + "Gemini Tool output attachments support PNG, JPEG, and WebP files; cannot determine a supported MIME type for \(url.lastPathComponent)." + } + } +} + +private func geminiFunctionResponseData(from url: URL) throws -> GeminiInlineData { + guard url.isFileURL else { + throw GeminiFunctionResponseAttachmentError.unsupportedURL(url) + } + + let mimeType = switch url.pathExtension.lowercased() { + case "png": "image/png" + case "jpg", "jpeg": "image/jpeg" + case "webp": "image/webp" + default: throw GeminiFunctionResponseAttachmentError.unsupportedImageType(url) + } + return try GeminiInlineData( + mimeType: mimeType, + data: Data(contentsOf: url).base64EncodedString() + ) +} + // MARK: - Supporting Types extension Transcript { - fileprivate func toGeminiContent() -> [GeminiContent] { + fileprivate func toGeminiContent() throws -> [GeminiContent] { var messages = [GeminiContent]() for item in self { switch item { @@ -869,13 +900,7 @@ extension Transcript { ) case .toolOutput(let toolOutput): // Add function response as a user message (Gemini API expects function responses from user role) - let functionResponse = - (try? toGeminiFunctionResponse(toolOutput)) - ?? GeminiFunctionResponse( - name: toolOutput.toolName, - response: [:], - parts: nil - ) + let functionResponse = try toGeminiFunctionResponse(toolOutput) messages.append( .init( role: .user, diff --git a/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift b/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift index d1a93925..006490ab 100644 --- a/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift +++ b/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift @@ -179,6 +179,68 @@ import Testing #expect(Data(base64Encoded: try #require(inlineData["data"] as? String)) != nil) } + @Test("inlines URL-backed images in function responses") + func inlinesURLBackedToolOutput() async throws { + StubURLProtocol.reset() + StubURLProtocol.enqueue( + json: + #"{"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"image","args":{"request":"read"}}}]},"finishReason":"STOP"}]}"# + ) + StubURLProtocol.enqueue(json: textResponse("OK")) + + let imageData = Data([0x89, 0x50, 0x4E, 0x47]) + let url = FileManager.default.temporaryDirectory + .appending(path: UUID().uuidString) + .appendingPathExtension("png") + try imageData.write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + + let output = Prompt { Attachment(imageURL: url) } + let session = LanguageModelSession( + model: makeModel(), + tools: [ImageOutputTool(output: output)] + ) + _ = try await session.respond(to: "read image") + + let body = try #require(StubURLProtocol.recordedBodies.last) + let json = try #require( + try JSONSerialization.jsonObject(with: body) as? [String: Any] + ) + let contents = try #require(json["contents"] as? [[String: Any]]) + let outputParts = try #require(contents.last?["parts"] as? [[String: Any]]) + let functionResponse = try #require( + outputParts.first?["functionResponse"] as? [String: Any] + ) + let mediaParts = try #require(functionResponse["parts"] as? [[String: Any]]) + let inlineData = try #require(mediaParts.first?["inlineData"] as? [String: Any]) + #expect(inlineData["mime_type"] as? String == "image/png") + #expect(Data(base64Encoded: try #require(inlineData["data"] as? String)) == imageData) + #expect(mediaParts.first?["fileData"] == nil) + } + + @Test("rejects remote URL-backed images with an actionable error") + func rejectsRemoteURLBackedToolOutput() async throws { + StubURLProtocol.reset() + StubURLProtocol.enqueue( + json: + #"{"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"image","args":{"request":"read"}}}]},"finishReason":"STOP"}]}"# + ) + + let url = try #require(URL(string: "https://example.com/image.png")) + let output = Prompt { Attachment(imageURL: url) } + let session = LanguageModelSession( + model: makeModel(), + tools: [ImageOutputTool(output: output)] + ) + + do { + _ = try await session.respond(to: "read image") + Issue.record("Expected a remote Tool output attachment to be rejected") + } catch { + #expect(error.localizedDescription.contains("local file URL")) + } + } + @Test("does not replay the conversation history on later turns") func doesNotReplayHistoryOnLaterTurns() async throws { StubURLProtocol.reset()