diff --git a/README.md b/README.md index 5a969f0b..6826b869 100644 --- a/README.md +++ b/README.md @@ -382,6 +382,31 @@ pass the model's multimodal projector with `mmprojPath:`. > Image inputs are an AnyLanguageModel extension. > See [Differences from Foundation Models](#differences-from-foundation-models). +### Prompt attachments + +Image attachments use the Foundation Models 27 prompt construction syntax: + +```swift +let prompt = Prompt { + "Describe this image" + Attachment(imageURL: fileURL) +} +``` + +On Apple platforms, `Attachment(CGImage, orientation:)` retains typed image +content and orientation. Prompt builders preserve the order of text and images, +and the OS 26 compatibility transcript continues to store images using its +existing `Transcript.Segment.image` Codable representation. + +`SystemLanguageModel` and `FoundationLanguageModel` bridge prompt attachments +to Foundation Models on OS 27. Earlier system runtimes reject them explicitly. +The system bridge requires local file URLs; other provider adapters retain URL +images in their existing transcript representation. + +This first compatibility surface covers prompt `CGImage` and `imageURL` +attachments. Tool outputs, instruction attachments, labels, `CIImage`, pixel +buffers, and newer data attachments remain outside this change. + ### Tool Calling Tool calling is supported by all providers. @@ -611,8 +636,8 @@ say which API they follow. described in that provider's section under [Providers](#providers). - `respond(to:images:)`, `Transcript.ImageSegment`, and `Transcript.Segment.image`: [image inputs](#image-inputs). - Foundation Models 27 adds prompt attachments, - and AnyLanguageModel 2.0 will change to match them. + Foundation Models 27-style [prompt attachments](#prompt-attachments) are also + available for ordered text and image prompt construction. - `ToolExecutionDelegate`, `ToolExecutionDecision`, and `toolExecutionDelegate`: [observing and controlling tool calls](#tool-calling). - `transcriptErrorHandlingPolicy` and `waitForResponseCompletion()`: @@ -645,7 +670,7 @@ say which API they follow. ### Behavior differences - A session adds the prompt to its transcript before the model responds, - as a single text segment. + preserving its text and image segment order. - A structured response is recorded in the transcript as JSON text, not as a structured segment. - A session created from a transcript restores `instructions` diff --git a/Sources/AnyLanguageModel/Attachment.swift b/Sources/AnyLanguageModel/Attachment.swift new file mode 100644 index 00000000..a0cc88c0 --- /dev/null +++ b/Sources/AnyLanguageModel/Attachment.swift @@ -0,0 +1,107 @@ +import Foundation + +#if canImport(CoreGraphics) && canImport(ImageIO) + import CoreGraphics + import ImageIO +#endif + +/// A typed prompt attachment, following Foundation Models 27's image surface. +/// +/// AnyLanguageModel lowers image attachments into its existing compatibility +/// transcript image segments on deployment targets before OS 27. +public struct Attachment { + let content: Content +} + +/// Image content retained until a prompt is lowered into the compatibility transcript. +public struct ImageAttachmentContent: Sendable, Equatable { + enum Source: Sendable { + case url(URL, orientation: UInt32?) + #if canImport(CoreGraphics) && canImport(ImageIO) + case image(CGImage, orientation: CGImagePropertyOrientation?) + #endif + } + + let source: Source + + public static func == (lhs: Self, rhs: Self) -> Bool { + switch (lhs.source, rhs.source) { + case (.url(let a, let ao), .url(let b, let bo)): + return a == b && ao == bo + #if canImport(CoreGraphics) && canImport(ImageIO) + case (.image(let a, let ao), .image(let b, let bo)): + return a === b && ao == bo + #endif + default: + return false + } + } + + func makeSegment(id: String) throws -> Transcript.ImageSegment { + switch source { + case .url(let url, let orientation): + #if canImport(CoreGraphics) && canImport(ImageIO) + if let orientation { + guard let source = CGImageSourceCreateWithURL(url as CFURL, nil), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil) + else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + return try Self( + source: .image( + image, + orientation: CGImagePropertyOrientation(rawValue: orientation) + ) + ).makeSegment(id: id) + } + #endif + return .init(id: id, url: url) + #if canImport(CoreGraphics) && canImport(ImageIO) + case .image(let image, let orientation): + let bytes = NSMutableData() + guard + let destination = CGImageDestinationCreateWithData( + bytes, + "public.png" as CFString, + 1, + nil + ) + else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + var properties: [CFString: Any] = [:] + if let orientation { + properties[kCGImagePropertyOrientation] = orientation.rawValue + } + CGImageDestinationAddImage(destination, image, properties as CFDictionary) + guard CGImageDestinationFinalize(destination) else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + return .init(id: id, data: bytes as Data, mimeType: "image/png") + #endif + } + } +} + +extension Attachment where Content == ImageAttachmentContent { + #if canImport(CoreGraphics) && canImport(ImageIO) + public init(_ image: CGImage, orientation: CGImagePropertyOrientation? = nil) { + content = .init(source: .image(image, orientation: orientation)) + } + + public init(imageURL: URL, orientation: CGImagePropertyOrientation? = nil) { + // Preserve the URL itself; its bytes are not read merely by constructing a prompt. + content = .init(source: .url(imageURL, orientation: orientation?.rawValue)) + } + #else + public init(imageURL: URL) { + content = .init(source: .url(imageURL, orientation: nil)) + } + #endif +} + +extension Attachment: PromptRepresentable where Content == ImageAttachmentContent { + public var promptRepresentation: Prompt { + Prompt(components: [.image(id: UUID().uuidString, content: content)]) + } +} diff --git a/Sources/AnyLanguageModel/LanguageModelSession.swift b/Sources/AnyLanguageModel/LanguageModelSession.swift index 4f08efc6..552c3fe0 100644 --- a/Sources/AnyLanguageModel/LanguageModelSession.swift +++ b/Sources/AnyLanguageModel/LanguageModelSession.swift @@ -728,7 +728,7 @@ public final class LanguageModelSession: @unchecked Sendable { let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( id: promptID, - segments: [.text(.init(content: prompt.description))], + segments: try prompt.makeTranscriptSegments(occurrenceID: promptID), options: options, responseFormat: responseFormat ) @@ -776,9 +776,17 @@ public final class LanguageModelSession: @unchecked Sendable { options: GenerationOptions = GenerationOptions() ) -> sending ResponseStream where Content: Generable { // Add prompt to transcript + let promptID = UUID().uuidString + let segments: [Transcript.Segment] + do { + segments = try prompt.makeTranscriptSegments(occurrenceID: promptID) + } catch { + return ResponseStream(stream: AsyncThrowingStream { $0.finish(throwing: error) }) + } let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( - segments: [.text(.init(content: prompt.description))], + id: promptID, + segments: segments, options: options, responseFormat: type == String.self ? nil : .init(type: type) ) @@ -924,9 +932,17 @@ extension LanguageModelSession { includeSchemaInPrompt: Bool = true, options: GenerationOptions = GenerationOptions() ) -> sending ResponseStream { + let promptID = UUID().uuidString + let segments: [Transcript.Segment] + do { + segments = try prompt.makeTranscriptSegments(occurrenceID: promptID) + } catch { + return ResponseStream(stream: AsyncThrowingStream { $0.finish(throwing: error) }) + } let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( - segments: [.text(.init(content: prompt.description))], + id: promptID, + segments: segments, options: options, responseFormat: .init(schema: schema) ) diff --git a/Sources/AnyLanguageModel/Models/FoundationLanguageModel.swift b/Sources/AnyLanguageModel/Models/FoundationLanguageModel.swift index b5bf8599..6c54fcec 100644 --- a/Sources/AnyLanguageModel/Models/FoundationLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/FoundationLanguageModel.swift @@ -159,7 +159,7 @@ ) async throws -> LanguageModelSession.Response where Content: Generable { return try await fmRespond( makeSession: { try await self.makeSession(for: session, prompt: prompt) }, - fmPrompt: prompt.toFoundationModels(), + fmPrompt: try prompt.toFoundationModels(), fmOptions: options.toFoundationModels(), type: type, schema: schema, @@ -209,9 +209,15 @@ includeSchemaInPrompt: Bool, options: GenerationOptions ) -> sending LanguageModelSession.ResponseStream where Content: Generable { + let fmPrompt: FoundationModels.Prompt + do { + fmPrompt = try prompt.toFoundationModels() + } catch { + return .init(stream: AsyncThrowingStream { $0.finish(throwing: error) }) + } return fmStreamResponse( makeSession: { try await self.makeSession(for: session, prompt: prompt) }, - fmPrompt: prompt.toFoundationModels(), + fmPrompt: fmPrompt, fmOptions: options.toFoundationModels(), type: type, schema: schema, diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index 87763ed6..f699f437 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -194,7 +194,7 @@ includeSchemaInPrompt: Bool, options: GenerationOptions ) async throws -> LanguageModelSession.Response where Content: Generable { - let fmPrompt = prompt.toFoundationModels() + let fmPrompt = try prompt.toFoundationModels() let fmOptions = options.toFoundationModels() return try await fmRespond( @@ -249,7 +249,12 @@ includeSchemaInPrompt: Bool, options: GenerationOptions ) -> sending LanguageModelSession.ResponseStream where Content: Generable { - let fmPrompt = prompt.toFoundationModels() + let fmPrompt: FoundationModels.Prompt + do { + fmPrompt = try prompt.toFoundationModels() + } catch { + return .init(stream: AsyncThrowingStream { $0.finish(throwing: error) }) + } let fmOptions = options.toFoundationModels() return fmStreamResponse( @@ -336,11 +341,12 @@ guard let lastEntry = transcript.last, case .prompt(let lastPrompt) = lastEntry else { return transcript } - let lastText = lastPrompt.segments.compactMap { segment -> String? in - if case .text(let textSegment) = segment { return textSegment.content } - return nil - }.joined() - guard lastText == prompt.description else { + guard + prompt.matchesTranscriptOccurrence( + lastPrompt.segments, + occurrenceID: lastPrompt.id + ) + else { return transcript } return Transcript(entries: transcript.dropLast()) @@ -430,8 +436,42 @@ @available(macOS 26.0, iOS 26.0, watchOS 27.0, tvOS 26.0, visionOS 26.0, *) extension Prompt { - func toFoundationModels() -> FoundationModels.Prompt { - FoundationModels.Prompt(self.description) + func toFoundationModels() throws -> FoundationModels.Prompt { + let prompts: [FoundationModels.Prompt] = try components.map { component in + switch component { + case .text(let text): + return FoundationModels.Prompt(text.content) + case .image(_, let image): + #if compiler(>=6.4) && !os(tvOS) && !os(watchOS) + if #available(macOS 27, iOS 27, visionOS 27, *) { + switch image.source { + case .url(let url, let orientation): + guard url.isFileURL else { + throw PromptBridgeError.nonFileImageURL + } + return FoundationModels.Prompt( + FoundationModels.Attachment( + imageURL: url, + orientation: orientation.flatMap( + CGImagePropertyOrientation.init(rawValue:) + ) + ) + ) + case .image(let value, let orientation): + return FoundationModels.Prompt( + FoundationModels.Attachment(value, orientation: orientation) + ) + } + } + #endif + throw PromptBridgeError.attachmentsUnavailable + } + } + return FoundationModels.Prompt { + for prompt in prompts { + prompt + } + } } } @@ -517,6 +557,11 @@ } } + enum PromptBridgeError: Error { + case attachmentsUnavailable + case nonFileImageURL + } + /// A type-erased wrapper that bridges any `Tool` to `FoundationModels.Tool`. @available(macOS 26.0, iOS 26.0, watchOS 27.0, tvOS 26.0, visionOS 26.0, *) private struct AnyToolWrapper: FoundationModels.Tool { @@ -924,7 +969,15 @@ else { return nil } - self.init(cgImage) + let properties = + CGImageSourceCopyPropertiesAtIndex( + imageSource, + 0, + nil + ) as? [CFString: Any] + let orientation = (properties?[kCGImagePropertyOrientation] as? NSNumber) + .flatMap { CGImagePropertyOrientation(rawValue: $0.uint32Value) } + self.init(cgImage, orientation: orientation) } } } diff --git a/Sources/AnyLanguageModel/Prompt.swift b/Sources/AnyLanguageModel/Prompt.swift index 7e099134..9365c47c 100644 --- a/Sources/AnyLanguageModel/Prompt.swift +++ b/Sources/AnyLanguageModel/Prompt.swift @@ -1,3 +1,5 @@ +import Foundation + /// A prompt from a person to the model. /// /// Prompts can contain content written by you, an outside source, or input directly from people using @@ -24,7 +26,12 @@ /// own prompt to better steer the model's response. For more information on handling inputs in your /// prompts, see . public struct Prompt: Sendable { - private let content: String + enum Component: Sendable { + case text(Transcript.TextSegment) + case image(id: String, content: ImageAttachmentContent) + } + + let components: [Component] /// Creates an instance with the content you specify. public init(_ representable: some PromptRepresentable) { @@ -34,12 +41,96 @@ public struct Prompt: Sendable { case let string as String: self.init(content: string) default: - self.init(content: representable.promptRepresentation.content) + self = representable.promptRepresentation } } init(content: String) { - self.content = content + components = [.text(.init(content: content))] + } + + init(components: [Component]) { + self.components = components + } + + func makeTranscriptSegments(occurrenceID: String = UUID().uuidString) throws -> [Transcript.Segment] { + try components.enumerated().map { index, component in + let id = Self.transcriptSegmentID(occurrenceID: occurrenceID, index: index) + switch component { + case .text(let text): + return .text(.init(id: id, content: text.content)) + case .image(_, let content): + return .image(try content.makeSegment(id: id)) + } + } + } + + func matchesTranscriptOccurrence( + _ segments: [Transcript.Segment], + occurrenceID: String + ) -> Bool { + guard components.count == segments.count else { return false } + return segments.indices.allSatisfy { index in + segments[index].id + == Self.transcriptSegmentID( + occurrenceID: occurrenceID, + index: index + ) + } + } + + private static func transcriptSegmentID(occurrenceID: String, index: Int) -> String { + "\(occurrenceID).\(index)" + } + + static func joining(_ prompts: [Prompt], trim: Bool = false) -> Prompt { + guard !prompts.isEmpty else { return Prompt(content: "") } + var components: [Component] = [] + for prompt in prompts { + for component in prompt.components { + if case .text(let text) = component, let last = components.last, + case .text(let previous) = last + { + components[components.count - 1] = .text( + .init( + id: previous.id, + content: previous.content + "\n" + text.content + ) + ) + } else { + components.append(component) + } + } + } + + if trim { + if let first = components.first, case .text(let text) = first { + components[0] = .text( + .init( + id: text.id, + content: text.content.replacingOccurrences( + of: #"^\s+"#, + with: "", + options: .regularExpression + ) + ) + ) + } + if let last = components.last, case .text(let text) = last { + components[components.count - 1] = .text( + .init( + id: text.id, + content: text.content.replacingOccurrences( + of: #"\s+$"#, + with: "", + options: .regularExpression + ) + ) + ) + } + } + + return Prompt(components: components) } public init(@PromptBuilder _ content: () throws -> Prompt) rethrows { @@ -50,7 +141,16 @@ public struct Prompt: Sendable { // MARK: - CustomStringConvertible extension Prompt: CustomStringConvertible { - public var description: String { content } + public var description: String { + components.map { component in + switch component { + case .text(let text): + return text.content + case .image: + return "" + } + }.joined(separator: "\n") + } } // MARK: - PromptBuilder @@ -59,10 +159,9 @@ extension Prompt: CustomStringConvertible { public struct PromptBuilder { public static func buildBlock(_ components: repeat each P) -> Prompt where repeat each P: PromptRepresentable { - var parts: [String] = [] - repeat parts.append((each components).promptRepresentation.description) - let combinedText = parts.joined(separator: "\n") - return Prompt(content: combinedText.trimmingCharacters(in: .whitespacesAndNewlines)) + var parts: [Prompt] = [] + repeat parts.append((each components).promptRepresentation) + return Prompt.joining(parts, trim: true) } public static func buildExpression

(_ expression: P) -> P where P: PromptRepresentable { @@ -70,10 +169,7 @@ public struct PromptBuilder { } public static func buildArray(_ prompts: [some PromptRepresentable]) -> Prompt { - let combinedText = prompts.map { - $0.promptRepresentation.description - }.joined(separator: "\n") - return Prompt(content: combinedText) + Prompt.joining(prompts.map(\.promptRepresentation)) } public static func buildOptional(_ component: Prompt?) -> Prompt { @@ -120,8 +216,6 @@ extension String: PromptRepresentable { extension Array: PromptRepresentable where Element: PromptRepresentable { /// An instance that represents a prompt. public var promptRepresentation: Prompt { - let combined = self.map { $0.promptRepresentation.description } - .joined(separator: "\n") - return Prompt(content: combined) + Prompt.joining(map(\.promptRepresentation)) } } diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift new file mode 100644 index 00000000..2563045c --- /dev/null +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -0,0 +1,292 @@ +import Foundation +import Testing + +@testable import AnyLanguageModel + +#if canImport(CoreGraphics) && canImport(ImageIO) + import CoreGraphics + import ImageIO +#endif +#if canImport(FoundationModels) && compiler(>=6.4) && !os(tvOS) && !os(watchOS) + import FoundationModels +#endif + +@Suite("Prompt attachments") +struct PromptAttachmentTests { + private enum SegmentPayload: Equatable { + case text(String) + case image(AnyLanguageModel.Transcript.ImageSegment.Source) + } + + private let imageURL = URL(fileURLWithPath: "/tmp/attachment-fixture.png") + + private func payloads( + _ segments: [AnyLanguageModel.Transcript.Segment] + ) -> [SegmentPayload] { + segments.compactMap { segment in + switch segment { + case .text(let text): + .text(text.content) + case .image(let image): + .image(image.source) + case .structure: + nil + } + } + } + + private func prompt() -> AnyLanguageModel.Prompt { + AnyLanguageModel.Prompt { + "before" + AnyLanguageModel.Attachment(imageURL: imageURL) + "after" + } + } + + @Test func emptyBuildersKeepOneEmptyTextSegment() throws { + let built = AnyLanguageModel.Prompt {} + let array = AnyLanguageModel.Prompt([AnyLanguageModel.Prompt]()) + + for prompt in [built, array] { + let segments = try prompt.makeTranscriptSegments() + #expect(segments.count == 1) + guard case .text(let text) = segments[0] else { + Issue.record("Empty prompt did not retain its text segment") + continue + } + #expect(text.content.isEmpty) + } + } + + @Test func buildersAndRepresentablesPreserveOrderedImages() throws { + let original = prompt() + struct Representable: AnyLanguageModel.PromptRepresentable { + let promptRepresentation: AnyLanguageModel.Prompt + } + let wrapped = AnyLanguageModel.Prompt( + Representable(promptRepresentation: original) + ) + let composed = AnyLanguageModel.Prompt { + if true { + wrapped + } + for item in [original] { + item + } + } + + let expected = try original.makeTranscriptSegments() + let wrappedSegments = try wrapped.makeTranscriptSegments() + #expect(payloads(wrappedSegments) == payloads(expected)) + #expect(Set(wrappedSegments.map(\.id)).isDisjoint(with: expected.map(\.id))) + let combined = try composed.makeTranscriptSegments() + #expect(combined.count == 5) + #expect(Set(combined.map(\.id)).count == combined.count) + let sources: [AnyLanguageModel.Transcript.ImageSegment.Source] = combined.compactMap { + if case .image(let value) = $0 { + return value.source + } + return nil + } + let texts: [String] = combined.compactMap { + if case .text(let value) = $0 { + return value.content + } + return nil + } + #expect(sources == [.url(imageURL), .url(imageURL)]) + #expect(texts == ["before", "after\nbefore", "after"]) + #expect(original.description == "before\n\nafter") + let repeated = try AnyLanguageModel.Prompt([original, original]).makeTranscriptSegments() + #expect(payloads(repeated) == payloads(combined)) + #expect(Set(repeated.map(\.id)).count == repeated.count) + #expect(Set(repeated.map(\.id)).isDisjoint(with: combined.map(\.id))) + } + + @Test func directSessionPromptsRetainImagesInBothModes() async throws { + for streaming in [false, true] { + let session = AnyLanguageModel.LanguageModelSession(model: MockLanguageModel()) + let input = prompt() + if streaming { + _ = try await session.streamResponse(to: input).collect() + } else { + _ = try await session.respond(to: input) + } + + let first = try #require(session.transcript.first) + guard case .prompt(let recorded) = first else { + Issue.record("Missing prompt") + return + } + #expect(payloads(recorded.segments) == payloads(try input.makeTranscriptSegments())) + } + } + + #if canImport(CoreGraphics) && canImport(ImageIO) + private func image() throws -> CGImage { + let context = try #require( + CGContext( + data: nil, + width: 2, + height: 3, + bitsPerComponent: 8, + bytesPerRow: 8, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) + ) + return try #require(context.makeImage()) + } + + @Test func imageEncodingRetainsPixelsAndOrientation() throws { + let value = try image() + let input = AnyLanguageModel.Prompt { + "image note" + AnyLanguageModel.Attachment(value, orientation: .right) + } + let segments = try input.makeTranscriptSegments() + guard case .image(let segment) = segments.last, + case .data(let bytes, let mime) = segment.source + else { + Issue.record("No encoded image") + return + } + #expect(mime == "image/png") + let source = try #require( + CGImageSourceCreateWithData(bytes as CFData, nil) + ) + let decoded = try #require(CGImageSourceCreateImageAtIndex(source, 0, nil)) + #expect(decoded.width == value.width) + #expect(decoded.height == value.height) + let properties = try #require( + CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any] + ) + #expect( + (properties[kCGImagePropertyOrientation] as? NSNumber)?.uint32Value + == CGImagePropertyOrientation.right.rawValue + ) + } + + #if canImport(FoundationModels) + @available(macOS 26, iOS 26, visionOS 26, *) + @Test func duplicateSuppressionDoesNotReencodeRecordedAttachments() throws { + let encoded = try AnyLanguageModel.Prompt { + AnyLanguageModel.Attachment(try image(), orientation: .right) + }.makeTranscriptSegments() + guard case .image(let image) = try #require(encoded.first), + case .data(let data, _) = image.source + else { + Issue.record("Failed to encode the image fixture") + return + } + + let url = FileManager.default.temporaryDirectory + .appending(path: UUID().uuidString) + .appendingPathExtension("png") + try data.write(to: url) + let prompt = AnyLanguageModel.Prompt { + AnyLanguageModel.Attachment(imageURL: url, orientation: .right) + } + let occurrenceID = UUID().uuidString + let segments = try prompt.makeTranscriptSegments(occurrenceID: occurrenceID) + try FileManager.default.removeItem(at: url) + let transcript = AnyLanguageModel.Transcript(entries: [ + .prompt(.init(id: occurrenceID, segments: segments)) + ]) + + #expect( + AnyLanguageModel.fmTranscriptDroppingDuplicatePrompt( + transcript, + prompt: prompt + ).isEmpty + ) + } + #endif + #endif + + #if canImport(FoundationModels) && compiler(>=6.4) && !os(tvOS) && !os(watchOS) + @available(macOS 27, iOS 27, visionOS 27, *) + @Test func nativeBridgeRetainsAttachment() async throws { + let input = try AnyLanguageModel.Prompt { + "before" + AnyLanguageModel.Attachment(try image(), orientation: .right) + "after" + } + let session = FoundationModels.LanguageModelSession(model: BridgeModel()) + _ = try await session.respond(to: input.toFoundationModels()) + guard case .prompt(let recorded) = try #require(session.transcript.first) else { + Issue.record("Missing prompt") + return + } + #expect(recorded.segments.count == 3) + guard case .attachment(let segment) = recorded.segments[1], + case .image(let image) = segment.content + else { + Issue.record("Native bridge lost attachment") + return + } + #expect(image.orientation == .right) + } + + @available(macOS 27, iOS 27, visionOS 27, *) + @Test func foundationWrapperDoesNotDuplicatePromptsAndReplaysOrientation() async throws { + let model = AnyLanguageModel.FoundationLanguageModel(BridgeModel()) + let session = AnyLanguageModel.LanguageModelSession(model: model) + let input = try AnyLanguageModel.Prompt { + "before" + AnyLanguageModel.Attachment(try image(), orientation: .right) + "after" + } + let first = try await session.respond(to: input) + #expect(first.content == "prompts:1 images:1 right:1") + + let second = try await session.respond(to: "follow-up") + #expect(second.content == "prompts:2 images:1 right:1") + } + + @available(macOS 27, iOS 27, visionOS 27, *) + private struct BridgeModel: FoundationModels.LanguageModel { + typealias Executor = BridgeExecutor + var capabilities: FoundationModels.LanguageModelCapabilities { + .init([.vision]) + } + var executorConfiguration: String { + "bridge-fixture" + } + } + + @available(macOS 27, iOS 27, visionOS 27, *) + private struct BridgeExecutor: FoundationModels.LanguageModelExecutor { + init(configuration: String) {} + + func respond( + to request: FoundationModels.LanguageModelExecutorGenerationRequest, + model: BridgeModel, + streamingInto channel: FoundationModels.LanguageModelExecutorGenerationChannel + ) async throws { + let prompts = request.transcript.compactMap { entry -> FoundationModels.Transcript.Prompt? in + guard case .prompt(let prompt) = entry else { return nil } + return prompt + } + let images = prompts.flatMap { prompt in + prompt.segments.compactMap { + segment -> FoundationModels.Transcript.ImageAttachment? in + guard case .attachment(let attachment) = segment, + case .image(let image) = attachment.content + else { return nil } + return image + } + } + let right = images.count { $0.orientation == .right } + await channel.send( + .response( + action: .appendText( + "prompts:\(prompts.count) images:\(images.count) right:\(right)", + tokenCount: 1 + ) + ) + ) + } + } + #endif +}