diff --git a/README.md b/README.md index 9cf4ef30..e208a841 100644 --- a/README.md +++ b/README.md @@ -515,6 +515,86 @@ but never become part of the session's transcript. > Otherwise, it throws `SystemLanguageModel.Error.dynamicInstructionsUnavailable` > for a session with dynamic instructions. +### Foundation Models 27 request options + +`ContextOptions` mirrors Foundation Models 27's schema-in-prompt and reasoning-level +settings for compatibility-layer request construction. `GenerationOptions` also +provides the OS 27 `samplingMode` spelling and `ToolCallingMode` values: + +```swift +let context = ContextOptions( + includeSchemaInPrompt: true, + reasoningLevel: .moderate +) +let options = GenerationOptions( + samplingMode: .greedy, + toolCallingMode: .required +) +``` + +The older `sampling` property and initializer remain source-compatible. +Tool-calling mode is preserved in Codable transcripts and forwarded by the +Foundation Models bridge on OS 27. Other model providers can read the same +option from their request's `GenerationOptions`. + +### Dynamic Profiles + +`LanguageModelSession.DynamicProfile` mirrors the Foundation Models 27 profile +API on supported AnyLanguageModel deployment targets. A profile chooses one +active `Profile` for each actual model request and can change its model, +instructions, tools, generation options, context options, and transcript +projection without rebuilding the session: + +```swift +extension SessionPropertyValues { + @SessionPropertyEntry var isReviewing = false +} + +struct EditingProfile: LanguageModelSession.DynamicProfile { + let model: any LanguageModel + @SessionProperty(\.isReviewing) private var isReviewing + + var body: some LanguageModelSession.DynamicProfile { + if isReviewing { + Profile { + Instructions("Review the current draft.") + ReviewTool() + } + .model(model) + .temperature(0.1) + } else { + Profile { + Instructions("Help edit the current draft.") + EditTool() + } + .model(model) + .temperature(0.7) + } + } +} + +let session = LanguageModelSession( + profile: EditingProfile(model: model), + history: savedHistory +) +session.properties.isReviewing = true +``` + +Value modifiers use call-site, inner-profile, then outer-profile precedence. +Lifecycle callbacks accumulate. `historyTransform` changes only the transcript +projection for that request; `session.transcript` remains the canonical history. +`@SessionProperty` exposes isolated session state to profiles, dynamic +instructions, and tools. The built-in `history` property is writable from a +profile or lifecycle callback and read-only while dynamic instructions or a Tool +is active. + +External compatibility models that own Tool continuation use +`resolvedRequestContext(including:options:)` before every provider request. The +`including:` entries are the canonical Tool calls and outputs completed in the +current response but not yet committed to the session transcript. Calls execute +against the tools in the producing request context; the next resolved context is +used only for the continuation request. + ### Reasoning in the transcript Reasoning is transcript content, separate from the answer in `response.content`. @@ -654,6 +734,17 @@ say which API they follow. - `DynamicInstructions`, its builder, and `LanguageModelSession.init(model:dynamicInstructions:history:)`: [dynamic instructions](#dynamic-instructions), which follow the Foundation Models 27 API. +- `ContextOptions`, `GenerationOptions.samplingMode`, and + `GenerationOptions.ToolCallingMode`: + [request options](#foundation-models-27-request-options), which follow the + Foundation Models 27 API while remaining available on the package's existing + deployment targets. +- `LanguageModelSession.DynamicProfile`, profile modifiers, lifecycle callbacks, + session properties, and `LanguageModelSession.init(profile:history:)`: + [dynamic profiles](#dynamic-profiles), which mirror Foundation Models 27 on + earlier deployment targets. `resolvedRequestContext(including:options:)` and + the explicit compatibility lifecycle hooks are AnyLanguageModel extensions for + external model implementations until the package adopts the executor contract. - `Transcript.Entry.reasoning` and `Transcript.Reasoning`: [reasoning in the transcript](#reasoning-in-the-transcript), which follows the Foundation Models 27 API. diff --git a/Sources/AnyLanguageModel/ContextOptions.swift b/Sources/AnyLanguageModel/ContextOptions.swift new file mode 100644 index 00000000..f63020c3 --- /dev/null +++ b/Sources/AnyLanguageModel/ContextOptions.swift @@ -0,0 +1,27 @@ +/// Options that configure a model's request context. +/// +/// This mirrors the Foundation Models 27 request-context surface while keeping +/// the package available on its existing deployment targets. +public struct ContextOptions: Sendable, Equatable { + /// The amount of reasoning the model should perform for a request. + public enum ReasoningLevel: Sendable, Equatable { + case light + case moderate + case deep + case custom(String) + } + + /// Whether a structured-generation schema should also appear in the prompt. + public var includeSchemaInPrompt: Bool? + + /// The amount of reasoning the model should perform. + public var reasoningLevel: ReasoningLevel? + + public init( + includeSchemaInPrompt: Bool? = nil, + reasoningLevel: ReasoningLevel? = nil + ) { + self.includeSchemaInPrompt = includeSchemaInPrompt + self.reasoningLevel = reasoningLevel + } +} diff --git a/Sources/AnyLanguageModel/DynamicInstructions.swift b/Sources/AnyLanguageModel/DynamicInstructions.swift index 1f4fe4bf..e83fda4d 100644 --- a/Sources/AnyLanguageModel/DynamicInstructions.swift +++ b/Sources/AnyLanguageModel/DynamicInstructions.swift @@ -337,7 +337,7 @@ extension Instructions: PrimitiveDynamicInstructions { } } -private func resolveDynamicInstructions( +func resolveDynamicInstructions( _ dynamicInstructions: any DynamicInstructions ) -> ResolvedDynamicInstructions { func resolve(_ content: Content) -> ResolvedDynamicInstructions diff --git a/Sources/AnyLanguageModel/DynamicProfile.swift b/Sources/AnyLanguageModel/DynamicProfile.swift new file mode 100644 index 00000000..90e4f129 --- /dev/null +++ b/Sources/AnyLanguageModel/DynamicProfile.swift @@ -0,0 +1,639 @@ +extension LanguageModelSession { + /// A declarative, per-model-request session profile. + /// + /// - Note: This API is exclusive to AnyLanguageModel on OS 26 and mirrors + /// Foundation Models 27. + @_typeEraser(AnyDynamicProfile) + public protocol DynamicProfile { + associatedtype Body: DynamicProfile + + @DynamicProfileBuilder + var body: Body { get } + } + + /// A concrete profile containing dynamic instructions and Tools. + public struct Profile: DynamicProfile { + public typealias Body = Never + + fileprivate let dynamicInstructions: AnyDynamicInstructions + fileprivate let identity: String + + public init( + @DynamicInstructionsBuilder _ dynamicInstructions: () -> some DynamicInstructions + ) { + let value = dynamicInstructions() + self.dynamicInstructions = AnyDynamicInstructions(value) + self.identity = String(reflecting: type(of: value)) + } + + public var body: Never { + fatalError("LanguageModelSession.Profile has no body") + } + } + + /// A type-erased dynamic profile. + public struct AnyDynamicProfile: DynamicProfile { + public typealias Body = Never + + fileprivate let resolveValue: () -> ResolvedDynamicProfile + + public init(_ dynamicProfile: any DynamicProfile) { + resolveValue = { resolveDynamicProfile(dynamicProfile) } + } + + public init(erasing dynamicProfile: some DynamicProfile) { + self.init(dynamicProfile) + } + + public var body: Never { + fatalError("LanguageModelSession.AnyDynamicProfile has no body") + } + + func resolveForRequest() -> ResolvedDynamicProfile { + resolveValue() + } + } + + public struct ConditionalDynamicProfile: DynamicProfile + where TrueContent: DynamicProfile, FalseContent: DynamicProfile { + public enum Branch { + case trueContent(TrueContent) + case falseContent(FalseContent) + } + + public typealias Body = Never + fileprivate let branch: Branch + + public init(_ branch: Branch) { + self.branch = branch + } + + public var body: Never { + fatalError("ConditionalDynamicProfile has no body") + } + } + + @resultBuilder + public struct DynamicProfileBuilder { + public static func buildBlock(_ content: Content) -> Content + where Content: DynamicProfile { + content + } + + public static func buildEither( + first content: TrueContent + ) -> ConditionalDynamicProfile + where TrueContent: DynamicProfile, FalseContent: DynamicProfile { + ConditionalDynamicProfile(.trueContent(content)) + } + + public static func buildEither( + second content: FalseContent + ) -> ConditionalDynamicProfile + where TrueContent: DynamicProfile, FalseContent: DynamicProfile { + ConditionalDynamicProfile(.falseContent(content)) + } + + @available(*, unavailable, message: "The body of a 'DynamicProfile' must evaluate to a single active profile") + public static func buildBlock(_ contents: repeat each Content) -> Never + where repeat each Content: DynamicProfile { + fatalError("A DynamicProfile must have one active profile") + } + + public static func buildLimitedAvailability( + _ component: some DynamicProfile + ) -> AnyDynamicProfile { + AnyDynamicProfile(component) + } + } + + public protocol DynamicProfileModifier { + associatedtype Body: DynamicProfile + typealias Content = DynamicProfileModifierContent + + @DynamicProfileBuilder + func body(content: Content) -> Body + } + + public struct DynamicProfileModifierContent: DynamicProfile + where Modifier: DynamicProfileModifier { + public typealias Body = Never + fileprivate let content: AnyDynamicProfile + + fileprivate init(_ content: Content) where Content: DynamicProfile { + self.content = AnyDynamicProfile(content) + } + + public var body: Never { + fatalError("DynamicProfileModifierContent has no body") + } + } + + public struct ModifiedDynamicProfile: DynamicProfile + where Content: DynamicProfile, Modifier: DynamicProfileModifier { + public typealias Body = Never + fileprivate let content: Content + fileprivate let modifier: Modifier + + fileprivate init(content: Content, modifier: Modifier) { + self.content = content + self.modifier = modifier + } + + public var body: Never { + fatalError("ModifiedDynamicProfile has no body") + } + } +} + +extension LanguageModelSession.DynamicProfile { + public typealias Profile = LanguageModelSession.Profile + public typealias DynamicProfile = LanguageModelSession.DynamicProfile + public typealias SessionProperty = LanguageModelSession.SessionProperty + + public func modifier( + _ modifier: Modifier + ) -> some LanguageModelSession.DynamicProfile + where Modifier: LanguageModelSession.DynamicProfileModifier { + LanguageModelSession.ModifiedDynamicProfile(content: self, modifier: modifier) + } + + public func model( + _ model: any LanguageModel + ) -> some LanguageModelSession.DynamicProfile { + BuiltinModifiedDynamicProfile(content: self, modifier: .model(model)) + } + + public func model( + _ model: Model + ) -> some LanguageModelSession.DynamicProfile where Model: LanguageModel { + BuiltinModifiedDynamicProfile(content: self, modifier: .model(model)) + } + + public func temperature(_ temperature: Double?) -> some LanguageModelSession.DynamicProfile { + BuiltinModifiedDynamicProfile(content: self, modifier: .temperature(temperature)) + } + + public func samplingMode( + _ samplingMode: GenerationOptions.SamplingMode? + ) -> some LanguageModelSession.DynamicProfile { + BuiltinModifiedDynamicProfile(content: self, modifier: .samplingMode(samplingMode)) + } + + public func maximumResponseTokens( + _ maximumResponseTokens: Int? + ) -> some LanguageModelSession.DynamicProfile { + BuiltinModifiedDynamicProfile( + content: self, + modifier: .maximumResponseTokens(maximumResponseTokens) + ) + } + + public func reasoningLevel( + _ reasoningLevel: ContextOptions.ReasoningLevel? + ) -> some LanguageModelSession.DynamicProfile { + BuiltinModifiedDynamicProfile(content: self, modifier: .reasoningLevel(reasoningLevel)) + } + + public func includeSchemaInPrompt( + _ includeSchemaInPrompt: Bool? + ) -> some LanguageModelSession.DynamicProfile { + BuiltinModifiedDynamicProfile( + content: self, + modifier: .includeSchemaInPrompt(includeSchemaInPrompt) + ) + } + + public func toolCallingMode( + _ toolCallingMode: GenerationOptions.ToolCallingMode? + ) -> some LanguageModelSession.DynamicProfile { + BuiltinModifiedDynamicProfile(content: self, modifier: .toolCallingMode(toolCallingMode)) + } + + public func historyTransform( + _ transform: @Sendable @escaping ([Transcript.Entry]) -> [Transcript.Entry] + ) -> some LanguageModelSession.DynamicProfile { + BuiltinModifiedDynamicProfile(content: self, modifier: .historyTransform(transform)) + } + + public func transcriptErrorHandlingPolicy( + _ policy: TranscriptErrorHandlingPolicy? + ) -> some LanguageModelSession.DynamicProfile { + BuiltinModifiedDynamicProfile(content: self, modifier: .transcriptErrorHandlingPolicy(policy)) + } + + public func onPrompt( + @_inheritActorContext perform action: + nonisolated(nonsending) sending @escaping ( + Transcript.Prompt + ) async throws -> Void + ) -> some LanguageModelSession.DynamicProfile { + BuiltinModifiedDynamicProfile(content: self, modifier: .onPrompt(action)) + } + + public func onPrompt( + @_inheritActorContext perform action: nonisolated(nonsending) sending @escaping () async throws -> Void + ) -> some LanguageModelSession.DynamicProfile { + onPrompt { _ in try await action() } + } + + public func onResponse( + @_inheritActorContext perform action: + nonisolated(nonsending) sending @escaping ( + Transcript.Response + ) async throws -> Void + ) -> some LanguageModelSession.DynamicProfile { + BuiltinModifiedDynamicProfile(content: self, modifier: .onResponse(action)) + } + + public func onResponse( + @_inheritActorContext perform action: nonisolated(nonsending) sending @escaping () async throws -> Void + ) -> some LanguageModelSession.DynamicProfile { + onResponse { _ in try await action() } + } + + public func onReasoning( + @_inheritActorContext perform action: + nonisolated(nonsending) sending @escaping ( + Transcript.Reasoning + ) async throws -> Void + ) -> some LanguageModelSession.DynamicProfile { + BuiltinModifiedDynamicProfile(content: self, modifier: .onReasoning(action)) + } + + public func onReasoning( + @_inheritActorContext perform action: nonisolated(nonsending) sending @escaping () async throws -> Void + ) -> some LanguageModelSession.DynamicProfile { + onReasoning { _ in try await action() } + } + + public func onToolCall( + @_inheritActorContext perform action: + nonisolated(nonsending) sending @escaping ( + Transcript.ToolCall + ) async throws -> Void + ) -> some LanguageModelSession.DynamicProfile { + BuiltinModifiedDynamicProfile(content: self, modifier: .onToolCall(action)) + } + + public func onToolCall( + @_inheritActorContext perform action: nonisolated(nonsending) sending @escaping () async throws -> Void + ) -> some LanguageModelSession.DynamicProfile { + onToolCall { _ in try await action() } + } + + public func onToolOutput( + @_inheritActorContext perform action: + nonisolated(nonsending) sending @escaping ( + Transcript.ToolCall, + Transcript.ToolOutput + ) async throws -> Void + ) -> some LanguageModelSession.DynamicProfile { + BuiltinModifiedDynamicProfile(content: self, modifier: .onToolOutput(action)) + } + + public func onToolOutput( + @_inheritActorContext perform action: nonisolated(nonsending) sending @escaping () async throws -> Void + ) -> some LanguageModelSession.DynamicProfile { + onToolOutput { _, _ in try await action() } + } + + public func onActivate( + @_inheritActorContext perform action: sending @escaping @isolated(any) () async -> Void + ) -> some LanguageModelSession.DynamicProfile { + let wrapped: () async -> Void = { await action() } + return BuiltinModifiedDynamicProfile(content: self, modifier: .onActivate(wrapped)) + } + + public func onDeactivate( + @_inheritActorContext perform action: sending @escaping @isolated(any) () async -> Void + ) -> some LanguageModelSession.DynamicProfile { + let wrapped: () async -> Void = { await action() } + return BuiltinModifiedDynamicProfile(content: self, modifier: .onDeactivate(wrapped)) + } +} + +extension LanguageModelSession.DynamicProfileModifier { + public typealias DynamicProfile = LanguageModelSession.DynamicProfile + public typealias SessionProperty = LanguageModelSession.SessionProperty +} + +extension Never: LanguageModelSession.DynamicProfile {} + +private protocol PrimitiveDynamicProfile { + func resolve() -> ResolvedDynamicProfile +} + +struct ResolvedDynamicProfile: @unchecked Sendable { + var identity: String + var model: (any LanguageModel)? + var instructions: Instructions? + var tools: [any Tool] + var samplingMode: GenerationOptions.SamplingMode? + var hasSamplingMode = false + var temperature: Double? + var hasTemperature = false + var maximumResponseTokens: Int? + var hasMaximumResponseTokens = false + var reasoningLevel: ContextOptions.ReasoningLevel? + var hasReasoningLevel = false + var includeSchemaInPrompt: Bool? + var hasIncludeSchemaInPrompt = false + var toolCallingMode: GenerationOptions.ToolCallingMode? + var hasToolCallingMode = false + var transcriptErrorHandlingPolicy: TranscriptErrorHandlingPolicy? + var hasTranscriptErrorHandlingPolicy = false + var historyTransforms: [@Sendable ([Transcript.Entry]) -> [Transcript.Entry]] = [] + var onPrompt: [(Transcript.Prompt) async throws -> Void] = [] + var onResponse: [(Transcript.Response) async throws -> Void] = [] + var onReasoning: [(Transcript.Reasoning) async throws -> Void] = [] + var onToolCall: [(Transcript.ToolCall) async throws -> Void] = [] + var onToolOutput: [(Transcript.ToolCall, Transcript.ToolOutput) async throws -> Void] = [] + var onActivate: [() async -> Void] = [] + var onDeactivate: [() async -> Void] = [] +} + +extension LanguageModelSession.Profile: PrimitiveDynamicProfile { + fileprivate func resolve() -> ResolvedDynamicProfile { + let dynamic = dynamicInstructions.resolveForRequest() + return ResolvedDynamicProfile( + identity: identity, + model: nil, + instructions: dynamic.instructions, + tools: dynamic.tools + ) + } +} + +extension LanguageModelSession.AnyDynamicProfile: PrimitiveDynamicProfile { + fileprivate func resolve() -> ResolvedDynamicProfile { + resolveForRequest() + } +} + +extension LanguageModelSession.ConditionalDynamicProfile: PrimitiveDynamicProfile { + fileprivate func resolve() -> ResolvedDynamicProfile { + var resolved: ResolvedDynamicProfile + switch branch { + case .trueContent(let content): + resolved = resolveDynamicProfile(content) + resolved.identity = "true/" + resolved.identity + case .falseContent(let content): + resolved = resolveDynamicProfile(content) + resolved.identity = "false/" + resolved.identity + } + return resolved + } +} + +extension LanguageModelSession.DynamicProfileModifierContent: PrimitiveDynamicProfile { + fileprivate func resolve() -> ResolvedDynamicProfile { + content.resolveForRequest() + } +} + +extension LanguageModelSession.ModifiedDynamicProfile: PrimitiveDynamicProfile { + fileprivate func resolve() -> ResolvedDynamicProfile { + resolveDynamicProfile(modifier.body(content: .init(content))) + } +} + +private struct BuiltinModifiedDynamicProfile: LanguageModelSession.DynamicProfile, + PrimitiveDynamicProfile +where Content: LanguageModelSession.DynamicProfile { + typealias Body = Never + let content: Content + let modifier: BuiltinDynamicProfileModifier + + var body: Never { + fatalError("BuiltinModifiedDynamicProfile has no body") + } + + func resolve() -> ResolvedDynamicProfile { + var resolved = resolveDynamicProfile(content) + modifier.apply(to: &resolved) + return resolved + } +} + +private enum BuiltinDynamicProfileModifier: @unchecked Sendable { + case model(any LanguageModel) + case temperature(Double?) + case samplingMode(GenerationOptions.SamplingMode?) + case maximumResponseTokens(Int?) + case reasoningLevel(ContextOptions.ReasoningLevel?) + case includeSchemaInPrompt(Bool?) + case toolCallingMode(GenerationOptions.ToolCallingMode?) + case historyTransform(@Sendable ([Transcript.Entry]) -> [Transcript.Entry]) + case transcriptErrorHandlingPolicy(TranscriptErrorHandlingPolicy?) + case onPrompt((Transcript.Prompt) async throws -> Void) + case onResponse((Transcript.Response) async throws -> Void) + case onReasoning((Transcript.Reasoning) async throws -> Void) + case onToolCall((Transcript.ToolCall) async throws -> Void) + case onToolOutput((Transcript.ToolCall, Transcript.ToolOutput) async throws -> Void) + case onActivate(() async -> Void) + case onDeactivate(() async -> Void) + + func apply(to resolved: inout ResolvedDynamicProfile) { + switch self { + case .model(let model): + if resolved.model == nil { resolved.model = model } + case .temperature(let value): + if !resolved.hasTemperature { + resolved.temperature = value + resolved.hasTemperature = true + } + case .samplingMode(let value): + if !resolved.hasSamplingMode { + resolved.samplingMode = value + resolved.hasSamplingMode = true + } + case .maximumResponseTokens(let value): + if !resolved.hasMaximumResponseTokens { + resolved.maximumResponseTokens = value + resolved.hasMaximumResponseTokens = true + } + case .reasoningLevel(let value): + if !resolved.hasReasoningLevel { + resolved.reasoningLevel = value + resolved.hasReasoningLevel = true + } + case .includeSchemaInPrompt(let value): + if !resolved.hasIncludeSchemaInPrompt { + resolved.includeSchemaInPrompt = value + resolved.hasIncludeSchemaInPrompt = true + } + case .toolCallingMode(let value): + if !resolved.hasToolCallingMode { + resolved.toolCallingMode = value + resolved.hasToolCallingMode = true + } + case .transcriptErrorHandlingPolicy(let value): + if !resolved.hasTranscriptErrorHandlingPolicy { + resolved.transcriptErrorHandlingPolicy = value + resolved.hasTranscriptErrorHandlingPolicy = true + } + case .historyTransform(let transform): + resolved.historyTransforms.insert(transform, at: 0) + case .onPrompt(let action): + resolved.onPrompt.insert(action, at: 0) + case .onResponse(let action): + resolved.onResponse.insert(action, at: 0) + case .onReasoning(let action): + resolved.onReasoning.insert(action, at: 0) + case .onToolCall(let action): + resolved.onToolCall.insert(action, at: 0) + case .onToolOutput(let action): + resolved.onToolOutput.insert(action, at: 0) + case .onActivate(let action): + resolved.onActivate.insert(action, at: 0) + case .onDeactivate(let action): + resolved.onDeactivate.append(action) + } + } +} + +private func resolveDynamicProfile( + _ dynamicProfile: any LanguageModelSession.DynamicProfile +) -> ResolvedDynamicProfile { + func resolve(_ content: Content) -> ResolvedDynamicProfile + where Content: LanguageModelSession.DynamicProfile { + if let primitive = content as? any PrimitiveDynamicProfile { + return primitive.resolve() + } + return resolve(content.body) + } + + return resolve(dynamicProfile) +} + +actor ProfileLifecycle { + private var activeProfile: ResolvedDynamicProfile? + private var lastPromptID: String? + private var pendingPreparation: (id: Int, task: Task)? + private var nextPreparationID = 0 + + func prepare( + _ profile: ResolvedDynamicProfile, + properties: SessionPropertyValues, + history: [Transcript.Entry], + protectedEntryIDs: Set = [] + ) async throws { + let previous = pendingPreparation?.task + nextPreparationID += 1 + let id = nextPreparationID + let task = Task { + _ = try? await previous?.value + try await self.performPreparation( + profile, + properties: properties, + history: history, + protectedEntryIDs: protectedEntryIDs + ) + } + pendingPreparation = (id, task) + do { + try await task.value + if pendingPreparation?.id == id { pendingPreparation = nil } + } catch { + if pendingPreparation?.id == id { pendingPreparation = nil } + throw error + } + } + + private func performPreparation( + _ profile: ResolvedDynamicProfile, + properties: SessionPropertyValues, + history: [Transcript.Entry], + protectedEntryIDs: Set + ) async throws { + if activeProfile?.identity != profile.identity { + if let previous = activeProfile { + await withBindings( + properties: properties, + history: history, + protectedEntryIDs: protectedEntryIDs + ) { + for action in previous.onDeactivate { await action() } + } + } + await withBindings( + properties: properties, + history: history, + protectedEntryIDs: protectedEntryIDs + ) { + for action in profile.onActivate { await action() } + } + activeProfile = profile + } else { + activeProfile = profile + } + + guard + let prompt = history.reversed().compactMap({ entry -> Transcript.Prompt? in + if case .prompt(let prompt) = entry { return prompt } + return nil + }).first, + prompt.id != lastPromptID + else { return } + + try await withBindings( + properties: properties, + history: history, + protectedEntryIDs: protectedEntryIDs + ) { + for action in profile.onPrompt { try await action(prompt) } + } + lastPromptID = prompt.id + } + + private func withBindings( + properties: SessionPropertyValues, + history: [Transcript.Entry], + protectedEntryIDs: Set, + operation: @Sendable () async throws -> Result + ) async rethrows -> Result { + let history = properties.historyBinding( + history, + isWritable: true, + protecting: protectedEntryIDs + ) + return try await SessionPropertyBinding.$values.withValue(properties) { + try await SessionPropertyBinding.$history.withValue(history) { + try await operation() + } + } + } +} + +struct MissingDynamicProfileModel: LanguageModel { + typealias UnavailableReason = Never + + func respond( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) async throws -> LanguageModelSession.Response where Content: Generable { + throw MissingDynamicProfileModelError.missingModelModifier + } + + func streamResponse( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) -> sending LanguageModelSession.ResponseStream where Content: Generable { + let stream = AsyncThrowingStream.Snapshot, any Error> { + continuation in + continuation.finish(throwing: MissingDynamicProfileModelError.missingModelModifier) + } + return LanguageModelSession.ResponseStream(stream: stream) + } +} + +enum MissingDynamicProfileModelError: Error { + case missingModelModifier +} diff --git a/Sources/AnyLanguageModel/GenerationOptions.swift b/Sources/AnyLanguageModel/GenerationOptions.swift index cd2352a9..f02e7fb6 100644 --- a/Sources/AnyLanguageModel/GenerationOptions.swift +++ b/Sources/AnyLanguageModel/GenerationOptions.swift @@ -93,6 +93,15 @@ public struct GenerationOptions: Sendable, Equatable { /// a reasonable default on your behalf. public var sampling: SamplingMode? + /// The sampling strategy for generation. + /// + /// This is the Foundation Models 27 spelling. ``sampling`` remains for + /// source compatibility with earlier AnyLanguageModel releases. + public var samplingMode: SamplingMode? { + get { sampling } + set { sampling = newValue } + } + /// Temperature influences the confidence of the models response. /// /// The value of this property must be a number between `0` and `1` inclusive. @@ -121,6 +130,9 @@ public struct GenerationOptions: Sendable, Equatable { /// an error will be thrown. public var maximumResponseTokens: Int? + /// Whether tool calling is allowed, required, or disallowed. + public var toolCallingMode: ToolCallingMode? + /// Storage for model-specific custom options. private var customOptionsStorage: CustomOptionsStorage = .init() @@ -169,6 +181,43 @@ public struct GenerationOptions: Sendable, Equatable { self.sampling = sampling self.temperature = temperature self.maximumResponseTokens = maximumResponseTokens + self.toolCallingMode = nil + } + + /// Creates Foundation Models 27-style generation options. + public init( + samplingMode: SamplingMode? = nil, + temperature: Double? = nil, + maximumResponseTokens: Int? = nil, + toolCallingMode: ToolCallingMode? + ) { + self.init( + sampling: samplingMode, + temperature: temperature, + maximumResponseTokens: maximumResponseTokens + ) + self.toolCallingMode = toolCallingMode + } +} + +extension GenerationOptions { + /// A tool-calling policy for a generation request. + public struct ToolCallingMode: Sendable, Equatable { + public enum Kind: Sendable, Equatable, Hashable { + case allowed + case required + case disallowed + } + + public var kind: Kind + + private init(kind: Kind) { + self.kind = kind + } + + public static let allowed = Self(kind: .allowed) + public static let required = Self(kind: .required) + public static let disallowed = Self(kind: .disallowed) } } @@ -180,7 +229,8 @@ extension GenerationOptions { /// `GenerationOptions` isn't `Codable`, matching Foundation Models, /// but ``Transcript/Prompt`` is, so it codes its options through this type. /// It codes the sampling mode, temperature, and maximum response tokens - /// in the same format as earlier releases. + /// in the same format as earlier releases, and adds the optional OS 27 + /// tool-calling mode without changing older transcript payloads. /// Custom options aren't coded, /// because decoding them would need a registry of every model's option types. struct TranscriptCoding: Codable { @@ -188,21 +238,43 @@ extension GenerationOptions { var mode: SamplingMode.Mode } + enum ToolCallingModeCoding: String, Codable { + case allowed + case required + case disallowed + } + var sampling: Sampling? var temperature: Double? var maximumResponseTokens: Int? + var toolCallingMode: ToolCallingModeCoding? init(_ options: GenerationOptions) { self.sampling = options.sampling.map { Sampling(mode: $0.mode) } self.temperature = options.temperature self.maximumResponseTokens = options.maximumResponseTokens + self.toolCallingMode = + switch options.toolCallingMode?.kind { + case .allowed: .allowed + case .required: .required + case .disallowed: .disallowed + case nil: nil + } } var options: GenerationOptions { - GenerationOptions( - sampling: sampling.map { SamplingMode(mode: $0.mode) }, + let toolCallingMode: GenerationOptions.ToolCallingMode? = + switch toolCallingMode { + case .allowed: .allowed + case .required: .required + case .disallowed: .disallowed + case nil: nil + } + return GenerationOptions( + samplingMode: sampling.map { SamplingMode(mode: $0.mode) }, temperature: temperature, - maximumResponseTokens: maximumResponseTokens + maximumResponseTokens: maximumResponseTokens, + toolCallingMode: toolCallingMode ) } } diff --git a/Sources/AnyLanguageModel/LanguageModelSession.swift b/Sources/AnyLanguageModel/LanguageModelSession.swift index 61ae70c9..17167488 100644 --- a/Sources/AnyLanguageModel/LanguageModelSession.swift +++ b/Sources/AnyLanguageModel/LanguageModelSession.swift @@ -108,11 +108,17 @@ public final class LanguageModelSession: @unchecked Sendable { public let instructions: Instructions? private let dynamicInstructions: AnyDynamicInstructions? + private let dynamicProfile: AnyDynamicProfile? private let dynamicInstructionsLock = NSLock() + private let dynamicProfileLock = NSLock() + private let profileLifecycle = ProfileLifecycle() + + /// Session-scoped values available to profiles and dynamic instructions. + public let properties: SessionPropertyValues /// Whether the session resolves its instructions and tools before each request. nonisolated var usesDynamicInstructions: Bool { - dynamicInstructions != nil + dynamicInstructions != nil || dynamicProfile != nil } /// A delegate that observes and controls tool execution. @@ -148,10 +154,59 @@ public final class LanguageModelSession: @unchecked Sendable { /// The tools that the model can call in response to the request. public let tools: [any Tool] - fileprivate init(transcript: Transcript, instructions: Instructions?, tools: [any Tool]) { + /// The model selected for this request. + public let model: any LanguageModel + + /// Generation options after profile defaults are combined with call-site options. + public let options: GenerationOptions + + /// Context options selected by the active profile. + public let contextOptions: ContextOptions + + /// Transcript failure behavior captured for this request. + public let transcriptErrorHandlingPolicy: TranscriptErrorHandlingPolicy? + + fileprivate let profile: ResolvedDynamicProfile? + fileprivate let canonicalTranscript: [Transcript.Entry] + fileprivate let callbackState: ProfileCallbackState + + fileprivate init( + transcript: Transcript, + instructions: Instructions?, + tools: [any Tool], + model: any LanguageModel, + options: GenerationOptions = .init(), + contextOptions: ContextOptions = .init(), + transcriptErrorHandlingPolicy: TranscriptErrorHandlingPolicy? = nil, + profile: ResolvedDynamicProfile? = nil, + canonicalTranscript: [Transcript.Entry]? = nil, + callbackState: ProfileCallbackState = ProfileCallbackState() + ) { self.transcript = transcript self.instructions = instructions self.tools = tools + self.model = model + self.options = options + self.contextOptions = contextOptions + self.transcriptErrorHandlingPolicy = transcriptErrorHandlingPolicy + self.profile = profile + self.canonicalTranscript = canonicalTranscript ?? Array(transcript) + self.callbackState = callbackState + } + + fileprivate func sharingCallbackState(with context: RequestContext) -> RequestContext { + RequestContext( + transcript: transcript, + instructions: instructions, + tools: tools, + model: model, + options: options, + contextOptions: contextOptions, + transcriptErrorHandlingPolicy: transcriptErrorHandlingPolicy, + profile: profile, + canonicalTranscript: canonicalTranscript, + callbackState: context.callbackState + ) } } @@ -165,29 +220,339 @@ public final class LanguageModelSession: @unchecked Sendable { /// It's public so that language models outside this module /// can read the inputs for each request. nonisolated public func resolvedRequestContext() -> RequestContext { - guard let dynamicInstructions else { - return RequestContext(transcript: transcript, instructions: instructions, tools: tools) - } - - // Evaluate the body for this request. - // The resolved instructions go into the request's transcript, - // never into the session's. - let resolved = dynamicInstructionsLock.withLock { - dynamicInstructions.resolveForRequest() - } - var requestTranscript = transcript - if let instructions = resolved.instructions { - let instructionsEntry = Transcript.Entry.instructions( - Transcript.Instructions( - segments: [.text(Transcript.TextSegment(content: instructions.description))], - toolDefinitions: resolved.tools - .filter(\.includesSchemaInInstructions) - .map { Transcript.ToolDefinition(tool: $0) } + if let prepared = ProfileRequestBinding.prepared, + prepared.session === self + { + return prepared.context + } + return resolveRequestContext(additionalEntries: [], options: .init()) + } + + nonisolated func toolRoundLimit(for provider: String) -> ToolRoundLimit { + guard let prepared = ProfileRequestBinding.prepared, + prepared.session === self + else { + return ToolRoundLimit(provider: provider) + } + return prepared.toolRoundLimit(for: provider) + } + + /// Resolves the active profile for one actual model request. + /// + /// `additionalEntries` supplies canonical entries produced in the current Tool + /// round but not yet committed to the durable Session transcript. A profile's + /// history transform sees those entries before selecting the request projection. + /// + /// - Note: This overload is exclusive to AnyLanguageModel. Compatibility + /// executors use it to preserve Foundation Models 27 request timing while + /// AnyLanguageModel still exposes its legacy model interface. + nonisolated public func resolvedRequestContext( + including additionalEntries: [Transcript.Entry], + options: GenerationOptions = .init() + ) async throws -> RequestContext { + let prepared = ProfileRequestBinding.prepared + if additionalEntries.isEmpty, let prepared, prepared.session === self { + return prepared.context + } + let callSiteOptions = + if let prepared, prepared.session === self { + prepared.callSiteOptions + } else { + options + } + var context = resolveRequestContext( + additionalEntries: additionalEntries, + options: callSiteOptions + ) + if let prepared, prepared.session === self { + context = context.sharingCallbackState(with: prepared.context) + } + if let profile = context.profile { + if let prepared, prepared.session === self { + prepared.context = context + } + try await profileLifecycle.prepare( + profile, + properties: properties, + history: context.canonicalTranscript, + protectedEntryIDs: Set(additionalEntries.map(\.id)) + ) + } + if let prepared, prepared.session === self { + prepared.context = context + } + return context + } + + nonisolated private func resolveRequestContext( + additionalEntries: [Transcript.Entry], + options: GenerationOptions + ) -> RequestContext { + var canonical = Array(transcript) + additionalEntries + let sessionErrorPolicy = transcriptErrorHandlingPolicy + guard let dynamicProfile else { + guard let dynamicInstructions else { + return RequestContext( + transcript: Transcript(entries: canonical), + instructions: instructions, + tools: tools, + model: model, + options: options, + transcriptErrorHandlingPolicy: sessionErrorPolicy, + canonicalTranscript: canonical ) + } + + // Dynamic instructions are request-only. Their body can observe + // session properties and canonical history, but cannot rewrite history. + let history = properties.historyBinding(canonical, isWritable: false) + let resolved = dynamicInstructionsLock.withLock { + SessionPropertyBinding.$values.withValue(properties) { + SessionPropertyBinding.$history.withValue(history) { + dynamicInstructions.resolveForRequest() + } + } + } + var requestTranscript = Transcript(entries: canonical) + if let currentInstructions = resolved.instructions { + requestTranscript = Transcript( + entries: [instructionsEntry(currentInstructions, tools: resolved.tools)] + canonical + ) + } + return RequestContext( + transcript: requestTranscript, + instructions: resolved.instructions, + tools: resolved.tools, + model: model, + options: options, + transcriptErrorHandlingPolicy: sessionErrorPolicy, + canonicalTranscript: canonical ) - requestTranscript = Transcript(entries: [instructionsEntry] + requestTranscript) } - return RequestContext(transcript: requestTranscript, instructions: resolved.instructions, tools: resolved.tools) + + let protectedEntryIDs = Set(additionalEntries.map(\.id)) + let history = properties.historyBinding( + canonical, + isWritable: true, + protecting: protectedEntryIDs + ) + let resolved = dynamicProfileLock.withLock { + SessionPropertyBinding.$values.withValue(properties) { + SessionPropertyBinding.$history.withValue(history) { + dynamicProfile.resolveForRequest() + } + } + } + canonical = history.entries + var projected = canonical + for transform in resolved.historyTransforms { + let transformHistory = properties.historyBinding( + canonical, + isWritable: true, + protecting: protectedEntryIDs + ) + projected = SessionPropertyBinding.$values.withValue(properties) { + SessionPropertyBinding.$history.withValue(transformHistory) { + transform(projected) + } + } + canonical = transformHistory.entries + } + var requestTranscript = Transcript(entries: projected) + if let currentInstructions = resolved.instructions { + requestTranscript = Transcript( + entries: [instructionsEntry(currentInstructions, tools: resolved.tools)] + projected + ) + } + + var resolvedOptions = options + if resolvedOptions.sampling == nil, resolved.hasSamplingMode { + resolvedOptions.sampling = resolved.samplingMode + } + if resolvedOptions.temperature == nil, resolved.hasTemperature { + resolvedOptions.temperature = resolved.temperature + } + if resolvedOptions.maximumResponseTokens == nil, resolved.hasMaximumResponseTokens { + resolvedOptions.maximumResponseTokens = resolved.maximumResponseTokens + } + if resolvedOptions.toolCallingMode == nil, resolved.hasToolCallingMode { + resolvedOptions.toolCallingMode = resolved.toolCallingMode + } + return RequestContext( + transcript: requestTranscript, + instructions: resolved.instructions, + tools: resolved.tools, + model: resolved.model ?? model, + options: resolvedOptions, + contextOptions: .init( + includeSchemaInPrompt: resolved.includeSchemaInPrompt, + reasoningLevel: resolved.reasoningLevel + ), + transcriptErrorHandlingPolicy: resolved.hasTranscriptErrorHandlingPolicy + ? resolved.transcriptErrorHandlingPolicy + : sessionErrorPolicy, + profile: resolved, + canonicalTranscript: canonical + ) + } + + nonisolated private func instructionsEntry( + _ instructions: Instructions, + tools: [any Tool] + ) -> Transcript.Entry { + .instructions( + .init( + segments: [.text(.init(content: instructions.description))], + toolDefinitions: + tools + .filter(\.includesSchemaInInstructions) + .map { Transcript.ToolDefinition(tool: $0) } + ) + ) + } + + /// Runs the active request snapshot's Tool-call callbacks. + nonisolated public func profileWillExecuteToolCall( + _ call: Transcript.ToolCall, + requestContext: RequestContext, + currentRoundEntries: [Transcript.Entry] + ) async throws { + guard let profile = requestContext.profile else { return } + let history = profileHistory( + requestContext: requestContext, + currentRoundEntries: currentRoundEntries + ) + try await withSessionProperties( + history: history.entries, + isHistoryWritable: true, + protectedEntryIDs: history.protectedEntryIDs + ) { + for action in profile.onToolCall { try await action(call) } + } + } + + /// Runs the active request snapshot's Tool-output callbacks. + nonisolated public func profileDidProduceToolOutput( + for call: Transcript.ToolCall, + output: Transcript.ToolOutput, + requestContext: RequestContext, + currentRoundEntries: [Transcript.Entry] + ) async throws { + guard let profile = requestContext.profile else { return } + let history = profileHistory( + requestContext: requestContext, + currentRoundEntries: currentRoundEntries + ) + try await withSessionProperties( + history: history.entries, + isHistoryWritable: true, + protectedEntryIDs: history.protectedEntryIDs + ) { + for action in profile.onToolOutput { try await action(call, output) } + } + } + + /// Runs reasoning and response callbacks from the profile snapshot that + /// produced the entries. Each stable entry ID is dispatched at most once + /// for the active request, including when streaming snapshots repeat it. + /// + /// - Note: This API is exclusive to AnyLanguageModel's compatibility model + /// seam. Ordinary callers compose callbacks through ``DynamicProfile``. + nonisolated public func profileDidProduceEntries( + _ entries: [Transcript.Entry], + requestContext: RequestContext, + currentRoundEntries: [Transcript.Entry] + ) async throws { + guard let profile = requestContext.profile else { return } + let history = profileHistory( + requestContext: requestContext, + currentRoundEntries: currentRoundEntries + ) + for entry in entries { + switch entry { + case .reasoning(let reasoning): + guard requestContext.callbackState.reserve(entry.id) else { continue } + do { + try await withSessionProperties( + history: history.entries, + isHistoryWritable: true, + protectedEntryIDs: history.protectedEntryIDs + ) { + for action in profile.onReasoning { try await action(reasoning) } + } + } catch { + requestContext.callbackState.release(entry.id) + throw error + } + case .response(let response): + guard requestContext.callbackState.reserve(entry.id) else { continue } + do { + try await withSessionProperties( + history: history.entries, + isHistoryWritable: true, + protectedEntryIDs: history.protectedEntryIDs + ) { + for action in profile.onResponse { try await action(response) } + } + } catch { + requestContext.callbackState.release(entry.id) + throw error + } + default: + continue + } + } + } + + /// Executes compatibility-model Tool work with session properties bound to + /// the producing request. Custom properties remain mutable, while the + /// canonical history is read-only in a Tool, matching Foundation Models 27. + nonisolated public func withProfileToolExecution( + requestContext: RequestContext, + currentRoundEntries: [Transcript.Entry], + operation: @Sendable () async throws -> Result + ) async rethrows -> Result { + let history = profileHistory( + requestContext: requestContext, + currentRoundEntries: currentRoundEntries + ) + return try await withSessionProperties( + history: history.entries, + isHistoryWritable: false, + protectedEntryIDs: history.protectedEntryIDs + ) { + try await operation() + } + } + + nonisolated private func profileHistory( + requestContext: RequestContext, + currentRoundEntries: [Transcript.Entry] + ) -> (entries: [Transcript.Entry], protectedEntryIDs: Set) { + var entryIDs = Set(requestContext.canonicalTranscript.map(\.id)) + let additionalEntries = currentRoundEntries.filter { entryIDs.insert($0.id).inserted } + return ( + requestContext.canonicalTranscript + additionalEntries, + Set(additionalEntries.map(\.id)) + ) + } + + nonisolated private func withSessionProperties( + history: [Transcript.Entry], + isHistoryWritable: Bool, + protectedEntryIDs: Set = [], + operation: @Sendable () async throws -> Result + ) async rethrows -> Result { + let history = properties.historyBinding( + history, + isWritable: isHistoryWritable, + protecting: protectedEntryIDs + ) + return try await SessionPropertyBinding.$values.withValue(properties) { + try await SessionPropertyBinding.$history.withValue(history) { + try await operation() + } + } } /// Creates a session with a model, tools, @@ -282,7 +647,28 @@ public final class LanguageModelSession: @unchecked Sendable { tools: [], instructions: nil, transcript: Transcript(entries: history), - dynamicInstructions: AnyDynamicInstructions(dynamicInstructions) + dynamicInstructions: AnyDynamicInstructions(dynamicInstructions), + dynamicProfile: nil + ) + } + + /// Creates a session whose active model, instructions, Tools, and request + /// projection are resolved before every model request. + public convenience init( + profile: sending some DynamicProfile, + history: some Collection = [] + ) { + let history = history.filter { entry in + if case .instructions = entry { return false } + return true + } + self.init( + model: MissingDynamicProfileModel(), + tools: [], + instructions: nil, + transcript: Transcript(entries: history), + dynamicInstructions: nil, + dynamicProfile: AnyDynamicProfile(profile) ) } @@ -291,11 +677,13 @@ public final class LanguageModelSession: @unchecked Sendable { tools: [any Tool], instructions: Instructions?, transcript: Transcript, - dynamicInstructions: AnyDynamicInstructions? = nil + dynamicInstructions: AnyDynamicInstructions? = nil, + dynamicProfile: AnyDynamicProfile? = nil ) { self.model = model self.tools = tools self.dynamicInstructions = dynamicInstructions + self.dynamicProfile = dynamicProfile let resolvedInstructions = instructions ?? Self.instructions(from: transcript) self.instructions = resolvedInstructions @@ -322,7 +710,46 @@ public final class LanguageModelSession: @unchecked Sendable { } } - self.state = .init(.init(finalTranscript)) + let state = Locked(State(finalTranscript)) + self.state = state + let setHistory: @Sendable (Transcript.HistoryView) -> Void = { history in + state.withLock { locked in + let instructions = locked.transcript.prefix { + if case .instructions = $0 { return true } + return false + } + locked.transcript = Transcript(entries: Array(instructions) + history) + } + } + let observedHistorySetter = Locked<(@Sendable (Transcript.HistoryView) -> Void)?>(nil) + self.properties = SessionPropertyValues( + historyGetter: { + state.withLock { locked in + locked.transcript.filter { entry in + if case .instructions = entry { return false } + return true + } + } + }, + historySetter: { history in + if let setter = observedHistorySetter.withLock({ $0 }) { + setter(history) + } else { + setHistory(history) + } + } + ) + observedHistorySetter.withLock { setter in + setter = { [weak self] history in + guard let self else { + setHistory(history) + return + } + self.withMutation(keyPath: \.transcript) { + setHistory(history) + } + } + } } private static func instructions(from transcript: Transcript) -> Instructions? { @@ -358,17 +785,28 @@ public final class LanguageModelSession: @unchecked Sendable { } } + nonisolated private func withTranscriptMutation( + _ mutation: () throws -> Result + ) rethrows -> Result { + try properties._withSessionPropertyMutation(keyPath: \.history) { + try withMutation(keyPath: \.transcript, mutation) + } + } + // The prompt is the only entry committed before a request succeeds. Checkpoints and // the final response commit atomically at completion, so rollback owns only this ID. nonisolated private func removePrompt(id: String) { - withMutation(keyPath: \.transcript) { + withTranscriptMutation { state.withLock { state in state.transcript = Transcript(entries: state.transcript.filter { $0.id != id }) } } } - nonisolated private func wrapRespond(_ operation: (String) async throws -> T) async throws -> T { + nonisolated private func wrapRespond( + profileTracker: PreparedProfileRequestTracker? = nil, + _ operation: (String) async throws -> T + ) async throws -> T { let promptID = UUID().uuidString beginResponding() do { @@ -376,7 +814,13 @@ public final class LanguageModelSession: @unchecked Sendable { endResponding() return result } catch { - if transcriptErrorHandlingPolicy == .revertTranscript { + let policy: TranscriptErrorHandlingPolicy? + if let prepared = profileTracker?.prepared { + policy = prepared.context.transcriptErrorHandlingPolicy + } else { + policy = transcriptErrorHandlingPolicy + } + if policy == .revertTranscript { removePrompt(id: promptID) } endResponding() @@ -405,7 +849,8 @@ public final class LanguageModelSession: @unchecked Sendable { nonisolated private func wrapStream( _ upstream: sending ResponseStream, - promptEntry: Transcript.Entry + promptEntry: Transcript.Entry, + profileTracker: PreparedProfileRequestTracker? = nil ) -> ResponseStream where Content: Generable, Content.PartiallyGenerated: Sendable { let session = self let relay = AsyncThrowingStream.Snapshot, any Error> { continuation in @@ -419,6 +864,7 @@ public final class LanguageModelSession: @unchecked Sendable { session.beginResponding() var lastSnapshot: ResponseStream.Snapshot? var accountedUsage = Usage.zero + var committedEntryIDs: Set = [] do { for try await snapshot in stream { lastSnapshot = snapshot @@ -470,20 +916,43 @@ public final class LanguageModelSession: @unchecked Sendable { providerMetadata: lastSnapshot.providerMetadata ) ) - session.withMutation(keyPath: \.transcript) { + session.withTranscriptMutation { session.state.withLock { $0.transcript.append(contentsOf: lastSnapshot.transcriptEntries) $0.transcript.append(responseEntry) } } + committedEntryIDs = Set( + lastSnapshot.transcriptEntries.map(\.id) + [responseEntry.id] + ) + if let prepared = profileTracker?.prepared { + try await session.profileDidProduceEntries( + Array(lastSnapshot.transcriptEntries) + [responseEntry], + requestContext: prepared.context, + currentRoundEntries: [] + ) + } session.endResponding() continuation.finish() } catch { - session.withMutation(keyPath: \.transcript) { + session.withTranscriptMutation { session.state.withLock { state in - if state.transcriptErrorHandlingPolicy == .preserveTranscript { + if !committedEntryIDs.isEmpty { + state.transcript = Transcript( + entries: state.transcript.filter { + !committedEntryIDs.contains($0.id) + } + ) + } + let policy: TranscriptErrorHandlingPolicy? + if let prepared = profileTracker?.prepared { + policy = prepared.context.transcriptErrorHandlingPolicy + } else { + policy = state.transcriptErrorHandlingPolicy + } + if policy == .preserveTranscript { state.transcript.append(contentsOf: lastSnapshot?.transcriptEntries ?? []) - } else if state.transcriptErrorHandlingPolicy == .revertTranscript { + } else if policy == .revertTranscript { state.transcript = Transcript( entries: state.transcript.filter { $0.id != promptEntry.id } ) @@ -702,28 +1171,72 @@ public final class LanguageModelSession: @unchecked Sendable { includeSchemaInPrompt: Bool = true, options: GenerationOptions = GenerationOptions() ) async throws -> Response where Content: Generable { - try await respond( + let profileTracker = dynamicProfile == nil ? nil : PreparedProfileRequestTracker() + return try await respond( to: prompt, responseFormat: type == String.self ? nil : .init(type: type), - options: options + options: options, + profileTracker: profileTracker ) { - try await model.respond( - within: self, - to: prompt, - generating: type, - includeSchemaInPrompt: includeSchemaInPrompt, - options: options + try await profiledResponse(options: options, tracker: profileTracker) { + model, + resolvedOptions, + contextOptions in + try await model.respond( + within: self, + to: prompt, + generating: type, + includeSchemaInPrompt: contextOptions.includeSchemaInPrompt + ?? includeSchemaInPrompt, + options: resolvedOptions + ) + } + } + } + + nonisolated private func profiledResponse( + options: GenerationOptions, + tracker: PreparedProfileRequestTracker?, + generate: ( + any LanguageModel, + GenerationOptions, + ContextOptions + ) async throws -> Response + ) async throws -> ProfiledModelResponse where Content: Generable { + guard dynamicProfile != nil else { + return ProfiledModelResponse( + response: try await generate(model, options, .init()), + prepared: nil ) } + let context = resolveRequestContext(additionalEntries: [], options: options) + let prepared = PreparedProfileRequest( + session: self, + context: context, + callSiteOptions: options + ) + tracker?.prepared = prepared + let response = try await ProfileRequestBinding.$prepared.withValue(prepared) { + if let profile = context.profile { + try await profileLifecycle.prepare( + profile, + properties: properties, + history: context.canonicalTranscript + ) + } + return try await generate(context.model, context.options, context.contextOptions) + } + return ProfiledModelResponse(response: response, prepared: prepared) } nonisolated private func respond( to prompt: Prompt, responseFormat: Transcript.ResponseFormat?, options: GenerationOptions, - generate: () async throws -> Response + profileTracker: PreparedProfileRequestTracker?, + generate: () async throws -> ProfiledModelResponse ) async throws -> Response { - try await wrapRespond { promptID in + try await wrapRespond(profileTracker: profileTracker) { promptID in // Add prompt to transcript let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( @@ -733,11 +1246,15 @@ public final class LanguageModelSession: @unchecked Sendable { responseFormat: responseFormat ) ) - withMutation(keyPath: \.transcript) { + withTranscriptMutation { state.withLock { $0.transcript.append(promptEntry) } } - let response = Self.representingNilExplicitly(in: try await generate(), responseFormat: responseFormat) + let profiled = try await generate() + let response = Self.representingNilExplicitly( + in: profiled.response, + responseFormat: responseFormat + ) recordUsage(response.usage) @@ -757,15 +1274,45 @@ public final class LanguageModelSession: @unchecked Sendable { ) ) - // Add tool entries and response to transcript - withMutation(keyPath: \.transcript) { + try await commitResponse( + response, + entry: responseEntry, + prepared: profiled.prepared + ) + + return response + } + } + + nonisolated private func commitResponse( + _ response: Response, + entry responseEntry: Transcript.Entry, + prepared: PreparedProfileRequest? + ) async throws where Content: Generable { + withTranscriptMutation { + state.withLock { lockedState in + lockedState.transcript.append(contentsOf: response.transcriptEntries) + lockedState.transcript.append(responseEntry) + } + } + + guard let prepared else { return } + do { + try await profileDidProduceEntries( + Array(response.transcriptEntries) + [responseEntry], + requestContext: prepared.context, + currentRoundEntries: [] + ) + } catch { + let ids = Set(response.transcriptEntries.map(\.id) + [responseEntry.id]) + withTranscriptMutation { state.withLock { lockedState in - lockedState.transcript.append(contentsOf: response.transcriptEntries) - lockedState.transcript.append(responseEntry) + lockedState.transcript = Transcript( + entries: lockedState.transcript.filter { !ids.contains($0.id) } + ) } } - - return response + throw error } } @@ -791,21 +1338,110 @@ public final class LanguageModelSession: @unchecked Sendable { responseFormat: type == String.self ? nil : .init(type: type) ) ) - withMutation(keyPath: \.transcript) { + withTranscriptMutation { state.withLock { $0.transcript.append(promptEntry) } } - return wrapStream( - model.streamResponse( + let upstream: ResponseStream + let profileTracker: PreparedProfileRequestTracker? + if dynamicProfile != nil { + let tracker = PreparedProfileRequestTracker() + profileTracker = tracker + upstream = profileDispatchedStream( + prompt: prompt, + type: type, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options, + tracker: tracker + ) + } else { + profileTracker = nil + upstream = model.streamResponse( within: self, to: prompt, generating: type, includeSchemaInPrompt: includeSchemaInPrompt, options: options - ), - promptEntry: promptEntry + ) + } + return wrapStream( + upstream, + promptEntry: promptEntry, + profileTracker: profileTracker ) } + + nonisolated private func profileDispatchedStream( + prompt: Prompt, + type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions, + tracker: PreparedProfileRequestTracker + ) -> ResponseStream where Content: Generable { + profileDispatchedStream(options: options, tracker: tracker) { + model, + resolvedOptions, + contextOptions in + model.streamResponse( + within: self, + to: prompt, + generating: type, + includeSchemaInPrompt: contextOptions.includeSchemaInPrompt + ?? includeSchemaInPrompt, + options: resolvedOptions + ) + } + } + + nonisolated private func profileDispatchedStream( + options: GenerationOptions, + tracker: PreparedProfileRequestTracker, + generate: + @Sendable @escaping ( + any LanguageModel, + GenerationOptions, + ContextOptions + ) -> ResponseStream + ) -> ResponseStream where Content: Generable { + let stream = AsyncThrowingStream.Snapshot, any Error> { continuation in + let task = Task { + do { + let context = self.resolveRequestContext( + additionalEntries: [], + options: options + ) + let prepared = PreparedProfileRequest( + session: self, + context: context, + callSiteOptions: options + ) + tracker.prepared = prepared + try await ProfileRequestBinding.$prepared.withValue(prepared) { + if let profile = context.profile { + try await self.profileLifecycle.prepare( + profile, + properties: self.properties, + history: context.canonicalTranscript + ) + } + let upstream = generate( + context.model, + context.options, + context.contextOptions + ) + for try await snapshot in upstream { + continuation.yield(snapshot) + } + } + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { _ in task.cancel() } + } + return ResponseStream(stream: stream) + } } // MARK: - String Response Convenience Methods @@ -879,14 +1515,26 @@ extension LanguageModelSession { includeSchemaInPrompt: Bool = true, options: GenerationOptions = GenerationOptions() ) async throws -> Response { - try await respond(to: prompt, responseFormat: .init(schema: schema), options: options) { - try await model.respond( - within: self, - to: prompt, - schema: schema, - includeSchemaInPrompt: includeSchemaInPrompt, - options: options - ) + let profileTracker = dynamicProfile == nil ? nil : PreparedProfileRequestTracker() + return try await respond( + to: prompt, + responseFormat: .init(schema: schema), + options: options, + profileTracker: profileTracker + ) { + try await profiledResponse(options: options, tracker: profileTracker) { + model, + resolvedOptions, + contextOptions in + try await model.respond( + within: self, + to: prompt, + schema: schema, + includeSchemaInPrompt: contextOptions.includeSchemaInPrompt + ?? includeSchemaInPrompt, + options: resolvedOptions + ) + } } } @@ -947,19 +1595,42 @@ extension LanguageModelSession { responseFormat: .init(schema: schema) ) ) - withMutation(keyPath: \.transcript) { + withTranscriptMutation { state.withLock { $0.transcript.append(promptEntry) } } - return wrapStream( - model.streamResponse( + let upstream: ResponseStream + let profileTracker: PreparedProfileRequestTracker? + if dynamicProfile != nil { + let tracker = PreparedProfileRequestTracker() + profileTracker = tracker + upstream = profileDispatchedStream(options: options, tracker: tracker) { + model, + resolvedOptions, + contextOptions in + model.streamResponse( + within: self, + to: prompt, + schema: schema, + includeSchemaInPrompt: contextOptions.includeSchemaInPrompt + ?? includeSchemaInPrompt, + options: resolvedOptions + ) + } + } else { + profileTracker = nil + upstream = model.streamResponse( within: self, to: prompt, schema: schema, includeSchemaInPrompt: includeSchemaInPrompt, options: options - ), - promptEntry: promptEntry + ) + } + return wrapStream( + upstream, + promptEntry: promptEntry, + profileTracker: profileTracker ) } @@ -1129,7 +1800,8 @@ extension LanguageModelSession { includeSchemaInPrompt: Bool = true, options: GenerationOptions = GenerationOptions() ) async throws -> Response where Content: Generable { - try await wrapRespond { promptID in + let profileTracker = dynamicProfile == nil ? nil : PreparedProfileRequestTracker() + return try await wrapRespond(profileTracker: profileTracker) { promptID in // Build segments from text and images var segments: [Transcript.Segment] = [] if !prompt.isEmpty { @@ -1146,21 +1818,28 @@ extension LanguageModelSession { responseFormat: type == String.self ? nil : .init(type: type) ) ) - withMutation(keyPath: \.transcript) { + withTranscriptMutation { state.withLock { $0.transcript.append(promptEntry) } } // Extract text content for the Prompt parameter let textPrompt = Prompt(prompt) - let response = Self.representingNilExplicitly( - in: try await model.respond( + let profiled = try await profiledResponse(options: options, tracker: profileTracker) { + model, + resolvedOptions, + contextOptions in + try await model.respond( within: self, to: textPrompt, generating: type, - includeSchemaInPrompt: includeSchemaInPrompt, - options: options - ), + includeSchemaInPrompt: contextOptions.includeSchemaInPrompt + ?? includeSchemaInPrompt, + options: resolvedOptions + ) + } + let response = Self.representingNilExplicitly( + in: profiled.response, responseFormat: type == String.self ? nil : .init(type: type) ) @@ -1182,13 +1861,11 @@ extension LanguageModelSession { ) ) - // Add tool entries and response to transcript - withMutation(keyPath: \.transcript) { - state.withLock { lockedState in - lockedState.transcript.append(contentsOf: response.transcriptEntries) - lockedState.transcript.append(responseEntry) - } - } + try await commitResponse( + response, + entry: responseEntry, + prepared: profiled.prepared + ) return response } @@ -1278,22 +1955,45 @@ extension LanguageModelSession { responseFormat: type == String.self ? nil : .init(type: type) ) ) - withMutation(keyPath: \.transcript) { + withTranscriptMutation { state.withLock { $0.transcript.append(promptEntry) } } // Extract text content for the Prompt parameter let textPrompt = Prompt(prompt) - return wrapStream( - model.streamResponse( + let upstream: ResponseStream + let profileTracker: PreparedProfileRequestTracker? + if dynamicProfile != nil { + let tracker = PreparedProfileRequestTracker() + profileTracker = tracker + upstream = profileDispatchedStream(options: options, tracker: tracker) { + model, + resolvedOptions, + contextOptions in + model.streamResponse( + within: self, + to: textPrompt, + generating: type, + includeSchemaInPrompt: contextOptions.includeSchemaInPrompt + ?? includeSchemaInPrompt, + options: resolvedOptions + ) + } + } else { + profileTracker = nil + upstream = model.streamResponse( within: self, to: textPrompt, generating: type, includeSchemaInPrompt: includeSchemaInPrompt, options: options - ), - promptEntry: promptEntry + ) + } + return wrapStream( + upstream, + promptEntry: promptEntry, + profileTracker: profileTracker ) } } @@ -1676,6 +2376,71 @@ private enum ResponseStreamError: Error, LocalizedError { } } +private struct ProfiledModelResponse where Content: Generable { + let response: LanguageModelSession.Response + let prepared: PreparedProfileRequest? +} + +fileprivate final class ProfileCallbackState: @unchecked Sendable { + private let dispatchedEntryIDs = Locked>([]) + + func reserve(_ id: String) -> Bool { + dispatchedEntryIDs.withLock { $0.insert(id).inserted } + } + + func release(_ id: String) { + dispatchedEntryIDs.withLock { _ = $0.remove(id) } + } +} + +private final class PreparedProfileRequest: @unchecked Sendable { + let session: LanguageModelSession + let callSiteOptions: GenerationOptions + private let state: Locked + private let toolRoundLimitStorage = Locked(nil) + + init( + session: LanguageModelSession, + context: LanguageModelSession.RequestContext, + callSiteOptions: GenerationOptions + ) { + self.session = session + self.callSiteOptions = callSiteOptions + state = Locked(State(context: context)) + } + + var context: LanguageModelSession.RequestContext { + get { state.withLock { $0.context } } + set { state.withLock { $0.context = newValue } } + } + + func toolRoundLimit(for provider: String) -> ToolRoundLimit { + toolRoundLimitStorage.withLock { limit in + if let limit { return limit } + let created = ToolRoundLimit(provider: provider) + limit = created + return created + } + } + + private struct State: Sendable { + var context: LanguageModelSession.RequestContext + } +} + +private final class PreparedProfileRequestTracker: @unchecked Sendable { + private let storage = Locked(nil) + + var prepared: PreparedProfileRequest? { + get { storage.withLock { $0 } } + set { storage.withLock { $0 = newValue } } + } +} + +private enum ProfileRequestBinding { + @TaskLocal static var prepared: PreparedProfileRequest? +} + // MARK: - private struct State: Equatable, Sendable { diff --git a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift index 6a618a79..cf7a3c7e 100644 --- a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift @@ -27,7 +27,7 @@ import OrderedCollections /// betas: ["beta1", "beta2"] /// ) /// ``` -public struct AnthropicLanguageModel: LanguageModel { +public struct AnthropicLanguageModel: LanguageModel, ProfileRequestIdentifiedLanguageModel { /// Custom generation options specific to Anthropic's Claude API. /// /// Reached through `GenerationOptions[custom: AnthropicLanguageModel.self]`, @@ -370,6 +370,7 @@ public struct AnthropicLanguageModel: LanguageModel { public let model: String private let httpSession: HTTPSession + let profileRequestIdentity = UUID() /// Creates an Anthropic language model. /// @@ -445,99 +446,136 @@ public struct AnthropicLanguageModel: LanguageModel { ) async throws -> LanguageModelSession.Response where Content: Generable { let url = baseURL.appendingPathComponent("v1/messages") let headers = buildHeaders() - let requestContext = session.resolvedRequestContext() - - // Convert available tools to Anthropic format - let anthropicTools: [AnthropicTool] = try requestContext.tools.map { tool in - try convertToolToAnthropicFormat(tool) - } - let responseSchema = type == String.self ? nil : try convertSchemaToAnthropicFormat(schema) - let params = try createMessageParams( - model: model, - system: nil, - messages: try requestContext.transcript.toAnthropicMessages(), - tools: anthropicTools.isEmpty ? nil : anthropicTools, - responseSchema: responseSchema, - options: options - ) - - let body = try JSONEncoder().encode(params) - - let message: AnthropicMessageResponse = try await httpSession.fetch( - .post, - url: url, - headers: headers, - body: body - ) - - var entries: [Transcript.Entry] = message.content.compactMap { block in - switch block { - case .thinking(let thinking): return .reasoning(thinking.transcriptReasoning()) - case .redactedThinking(let redacted): return .reasoning(redacted.transcriptReasoning()) - default: return nil + var entries: [Transcript.Entry] = [] + var usage = LanguageModelSession.Usage.zero + var earlierText = "" + var inFlightMessages: [AnthropicMessage] = [] + let toolRounds = session.toolRoundLimit(for: "Anthropic") + + while true { + let requestContext = try await session.resolvedRequestContext( + including: entries, + options: options + ) + if let continuation = try await session.profileContinuationResponse( + ifSelectedModelDiffersFrom: profileRequestIdentity, + requestContext: requestContext, + prompt: prompt, + generating: type, + schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, + prefixEntries: entries, + prefixUsage: usage, + prefixText: earlierText + ) { + return continuation } - } - let usage = message.usage?.reportedUsage?.value ?? .zero - - // Handle tool calls, if present - let toolUses: [AnthropicToolUse] = message.content.compactMap { block in - if case .toolUse(let u) = block { return u } - return nil - } - - if !toolUses.isEmpty { - let resolution = try await resolveToolUses( - toolUses, - tools: requestContext.tools, - session: session + let anthropicTools: [AnthropicTool] = try requestContext.tools.map { + try convertToolToAnthropicFormat($0) + } + let params = try createMessageParams( + model: model, + system: nil, + messages: try anthropicMessages( + from: requestContext.transcript, + replacing: entries, + with: inFlightMessages + ), + tools: anthropicTools.isEmpty ? nil : anthropicTools, + responseSchema: responseSchema, + options: requestContext.options ) - switch resolution { - case .stop(let calls): - if !calls.isEmpty { - entries.append(.toolCalls(Transcript.ToolCalls(calls))) + let message: AnthropicMessageResponse = try await httpSession.fetch( + .post, + url: url, + headers: headers, + body: try JSONEncoder().encode(params) + ) + usage.add(message.usage?.reportedUsage?.value ?? .zero) + + let reasoningEntries: [Transcript.Entry] = message.content.compactMap { block in + switch block { + case .thinking(let thinking): return .reasoning(thinking.transcriptReasoning()) + case .redactedThinking(let redacted): return .reasoning(redacted.transcriptReasoning()) + default: return nil } - let empty = try emptyResponseContent(for: type) - return LanguageModelSession.Response( - content: empty.content, - rawContent: empty.rawContent, - transcriptEntries: ArraySlice(entries), - usage: usage + } + entries.append(contentsOf: reasoningEntries) + let toolUses: [AnthropicToolUse] = message.content.compactMap { block in + if case .toolUse(let use) = block { return use } + return nil + } + if !toolUses.isEmpty { + try toolRounds.record(toolUses.map(\.roundCall)) + try await session.profileDidProduceEntries( + reasoningEntries, + requestContext: requestContext, + currentRoundEntries: entries ) - case .invocations(let invocations): - if !invocations.isEmpty { - entries.append(.toolCalls(Transcript.ToolCalls(invocations.map(\.call)))) + switch try await resolveToolUses( + toolUses, + requestContext: requestContext, + currentRoundEntries: entries, + session: session + ) { + case .stop(let calls): + entries.append(.toolCalls(.init(calls))) + let empty = try emptyResponseContent(for: type) + return .init( + content: empty.content, + rawContent: empty.rawContent, + transcriptEntries: ArraySlice(entries), + usage: usage + ) + case .invocations(let invocations): + inFlightMessages.append(.init(role: .assistant, content: message.content)) + entries.append(.toolCalls(.init(invocations.map(\.call)))) + var results: [AnthropicContent] = [] for invocation in invocations { entries.append(.toolOutput(invocation.output)) + results.append( + .toolResult( + .init( + toolUseId: invocation.call.id, + content: convertSegmentsToAnthropicContent(invocation.output.segments) + ) + ) + ) } + inFlightMessages.append(.init(role: .user, content: results)) + if type == String.self { + earlierText += message.content.compactMap { block -> String? in + if case .text(let text) = block { return text.text } + return nil + }.joined() + } + continue } } - } - let text = message.content.compactMap { block -> String? in - switch block { - case .text(let t): return t.text - default: return nil + let text = + earlierText + + message.content.compactMap { block -> String? in + if case .text(let text) = block { return text.text } + return nil + }.joined() + if type == String.self { + return .init( + content: text as! Content, + rawContent: GeneratedContent(text), + transcriptEntries: ArraySlice(entries), + usage: usage + ) } - }.joined() - - if type == String.self { - return LanguageModelSession.Response( - content: text as! Content, - rawContent: GeneratedContent(text), + let rawContent = try GeneratedContent(json: text) + return .init( + content: try Content(rawContent), + rawContent: rawContent, transcriptEntries: ArraySlice(entries), usage: usage ) } - - let rawContent = try GeneratedContent(json: text) - let content = try Content(rawContent) - return LanguageModelSession.Response( - content: content, - rawContent: rawContent, - transcriptEntries: ArraySlice(entries), - usage: usage - ) } public func streamResponse( @@ -593,21 +631,43 @@ public struct AnthropicLanguageModel: LanguageModel { type == String.self ? nil : try convertSchemaToAnthropicFormat(schema) var inFlightMessages: [AnthropicMessage] = [] var state = StreamingResponseState() - var toolRounds = ToolRoundLimit(provider: "Anthropic") + let toolRounds = session.toolRoundLimit(for: "Anthropic") while true { try Task.checkCancellation() - let requestContext = session.resolvedRequestContext() + let requestContext = try await session.resolvedRequestContext( + including: state.entries, + options: options + ) + if let handoff = session.profileContinuationStream( + ifSelectedModelDiffersFrom: profileRequestIdentity, + requestContext: requestContext, + prompt: prompt, + generating: type, + schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, + prefixEntries: state.entries, + prefixUsage: state.totalUsage, + prefixText: state.responseText + ) { + for try await snapshot in handoff { continuation.yield(snapshot) } + continuation.finish() + return + } let anthropicTools: [AnthropicTool] = try requestContext.tools.map { try convertToolToAnthropicFormat($0) } - let messages = try requestContext.transcript.toAnthropicMessages() + inFlightMessages + let messages = try anthropicMessages( + from: requestContext.transcript, + replacing: state.entries, + with: inFlightMessages + ) var params = try createMessageParams( model: model, system: nil, messages: messages, tools: anthropicTools.isEmpty ? nil : anthropicTools, responseSchema: responseSchema, - options: options + options: requestContext.options ) params["stream"] = .bool(true) let body = try JSONEncoder().encode(params) @@ -696,9 +756,15 @@ public struct AnthropicLanguageModel: LanguageModel { guard !toolUses.isEmpty else { break } try Task.checkCancellation() try toolRounds.record(toolUses.map(\.roundCall)) + try await session.profileDidProduceEntries( + state.entries, + requestContext: requestContext, + currentRoundEntries: state.entries + ) switch try await resolveToolUses( toolUses, - tools: requestContext.tools, + requestContext: requestContext, + currentRoundEntries: state.entries, session: session ) { case .stop(let calls): @@ -856,6 +922,31 @@ private func createMessageParams( // MARK: - Tool Invocation Handling +private func anthropicMessages( + from transcript: Transcript, + replacing currentRoundEntries: [Transcript.Entry], + with continuation: [AnthropicMessage] +) throws -> [AnthropicMessage] { + guard !continuation.isEmpty, !currentRoundEntries.isEmpty else { + return try transcript.toAnthropicMessages() + } + let projected = Array(transcript) + let count = currentRoundEntries.count + let ranges = projected.indices.compactMap { start -> Range? in + let end = start + count + guard end <= projected.count, + Array(projected[start ..< end]) == currentRoundEntries + else { return nil } + return start ..< end + } + guard ranges.count == 1, let range = ranges.first else { + return try transcript.toAnthropicMessages() + } + return try Transcript(entries: projected[.. ToolResolutionOutcome { if toolUses.isEmpty { return .invocations([]) } - var toolsByName: [String: any Tool] = [:] - for tool in tools { - if toolsByName[tool.name] == nil { - toolsByName[tool.name] = tool - } - } - var transcriptCalls: [Transcript.ToolCall] = [] transcriptCalls.reserveCapacity(toolUses.count) for use in toolUses { @@ -927,82 +1012,18 @@ private func resolveToolUses( ) } - if let delegate = session.toolExecutionDelegate { - await delegate.didGenerateToolCalls(transcriptCalls, in: session) - } - guard !transcriptCalls.isEmpty else { return .invocations([]) } - - var decisions: [ToolExecutionDecision] = [] - decisions.reserveCapacity(transcriptCalls.count) - - if let delegate = session.toolExecutionDelegate { - for call in transcriptCalls { - let decision = await delegate.toolCallDecision(for: call, in: session) - if case .stop = decision { - return .stop(calls: transcriptCalls) - } - decisions.append(decision) - } - } else { - decisions = Array(repeating: .execute, count: transcriptCalls.count) - } - - var results: [ToolInvocationResult] = [] - results.reserveCapacity(transcriptCalls.count) - - for (index, call) in transcriptCalls.enumerated() { - switch decisions[index] { - case .stop: - // This branch should be unreachable because `.stop` returns during decision collection. - // Keep it as a defensive guard in case that logic changes. - return .stop(calls: transcriptCalls) - case .provideOutput(let segments): - let output = Transcript.ToolOutput( - id: call.id, - toolName: call.toolName, - segments: segments - ) - if let delegate = session.toolExecutionDelegate { - await delegate.didExecuteToolCall(call, output: output, in: session) - } - results.append(ToolInvocationResult(call: call, output: output)) - case .execute: - guard let tool = toolsByName[call.toolName] else { - let message = Transcript.Segment.text(.init(content: "Tool not found: \(call.toolName)")) - let output = Transcript.ToolOutput( - id: call.id, - toolName: call.toolName, - segments: [message] - ) - if let delegate = session.toolExecutionDelegate { - await delegate.didExecuteToolCall(call, output: output, in: session) - } - results.append(ToolInvocationResult(call: call, output: output)) - continue - } - - do { - let segments = try await tool.makeOutputSegments(from: call.arguments) - let output = Transcript.ToolOutput( - id: call.id, - toolName: tool.name, - segments: segments - ) - if let delegate = session.toolExecutionDelegate { - await delegate.didExecuteToolCall(call, output: output, in: session) - } - results.append(ToolInvocationResult(call: call, output: output)) - } catch { - if let delegate = session.toolExecutionDelegate { - await delegate.didFailToolCall(call, error: error, in: session) - } - throw LanguageModelSession.ToolCallError(tool: tool, underlyingError: error) - } - } + let entries = currentRoundEntries + [.toolCalls(.init(transcriptCalls))] + switch try await session.resolveProfileToolCalls( + transcriptCalls, + requestContext: requestContext, + currentRoundEntries: entries + ) { + case .stop(let calls): + return .stop(calls: calls) + case .invocations(let invocations): + return .invocations(invocations.map { .init(call: $0.call, output: $0.output) }) } - - return .invocations(results) } // Convert our GenerationSchema into Anthropic's expected JSON Schema payload diff --git a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift index 5be74601..5a3cceb3 100644 --- a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift @@ -7,7 +7,7 @@ import OrderedCollections import FoundationNetworking #endif -public struct GeminiLanguageModel: LanguageModel { +public struct GeminiLanguageModel: LanguageModel, ProfileRequestIdentifiedLanguageModel { public typealias UnavailableReason = Never public static let defaultBaseURL = URL(string: "https://generativelanguage.googleapis.com")! @@ -148,6 +148,7 @@ public struct GeminiLanguageModel: LanguageModel { public let model: String private let httpSession: HTTPSession + let profileRequestIdentity = UUID() /// Creates a new Gemini language model. /// @@ -218,47 +219,54 @@ public struct GeminiLanguageModel: LanguageModel { includeSchemaInPrompt: Bool, options: GenerationOptions ) async throws -> LanguageModelSession.Response where Content: Generable { - // Extract effective configuration from custom options or fall back to model defaults - let customOptions = options[custom: GeminiLanguageModel.self] - let effectiveThinking = customOptions?.thinking ?? .disabled - let effectiveServerTools = customOptions?.serverTools ?? [] - let effectiveJsonMode = customOptions?.jsonMode // Gemini's response schemas can be objects, primitives, or arrays, but not unions. let responseSchema = ProviderResponseSchema(schema, generating: type, wrapping: .unionRoots) - let url = baseURL .appendingPathComponent(apiVersion) .appendingPathComponent("models/\(model):generateContent") let headers = buildHeaders() - var inFlightEntries: [Transcript.Entry] = [] - - // The entries this call adds, which is what the response reports. `inFlightEntries` - // preserves tool rounds while each iteration rebuilds the request from a fresh context. + // The entries this call adds, which is what the response reports. var entries: [Transcript.Entry] = [] var usage = ReportedUsage() // The text of earlier tool rounds, which string responses include. var earlierText = "" - var toolRounds = ToolRoundLimit(provider: "Gemini") + let toolRounds = session.toolRoundLimit(for: "Gemini") // Multi-turn conversation loop for tool calling while true { - let requestContext = session.resolvedRequestContext() + let requestContext = try await session.resolvedRequestContext( + including: entries, + options: options + ) + if let continuation = try await session.profileContinuationResponse( + ifSelectedModelDiffersFrom: profileRequestIdentity, + requestContext: requestContext, + prompt: prompt, + generating: type, + schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, + prefixEntries: entries, + prefixUsage: usage.value, + prefixText: earlierText + ) { + return continuation + } + let customOptions = requestContext.options[custom: GeminiLanguageModel.self] + let effectiveThinking = customOptions?.thinking ?? .disabled + let effectiveServerTools = customOptions?.serverTools ?? [] + let effectiveJsonMode = customOptions?.jsonMode let geminiTools = try buildTools( from: requestContext.tools, serverTools: effectiveServerTools ) - var requestTranscript = requestContext.transcript - for entry in inFlightEntries { - requestTranscript.append(entry) - } let params = try createGenerateContentParams( - contents: requestTranscript.toGeminiContent(), + contents: requestContext.transcript.toGeminiContent(), tools: geminiTools, generating: type, schema: responseSchema.schema, - options: options, + options: requestContext.options, thinking: effectiveThinking, jsonMode: effectiveJsonMode ) @@ -293,7 +301,8 @@ public struct GeminiLanguageModel: LanguageModel { try toolRounds.record(functionCalls.map(\.roundCall)) let resolution = try await resolveFunctionCalls( functionCalls, - tools: requestContext.tools, + requestContext: requestContext, + currentRoundEntries: entries, session: session ) switch resolution { @@ -313,12 +322,10 @@ public struct GeminiLanguageModel: LanguageModel { let calls = Transcript.Entry.toolCalls( Transcript.ToolCalls(invocations.map(\.call), providerMetadata: providerMetadata) ) - inFlightEntries.append(calls) entries.append(calls) for invocation in invocations { let output = Transcript.Entry.toolOutput(invocation.output) - inFlightEntries.append(output) entries.append(output) } } @@ -397,14 +404,8 @@ public struct GeminiLanguageModel: LanguageModel { includeSchemaInPrompt: Bool, options: GenerationOptions ) -> sending LanguageModelSession.ResponseStream where Content: Generable { - // Extract effective configuration from custom options or fall back to model defaults - let customOptions = options[custom: GeminiLanguageModel.self] - let effectiveThinking = customOptions?.thinking ?? .disabled - let effectiveServerTools = customOptions?.serverTools ?? [] - let effectiveJsonMode = customOptions?.jsonMode // Gemini's response schemas can be objects, primitives, or arrays, but not unions. let responseSchema = ProviderResponseSchema(schema, generating: type, wrapping: .unionRoots) - var streamURL = baseURL .appendingPathComponent(apiVersion) @@ -417,26 +418,43 @@ public struct GeminiLanguageModel: LanguageModel { let task = Task { @Sendable in do { let headers = buildHeaders() - var inFlightEntries: [Transcript.Entry] = [] var state = StreamingResponseState(wrapperProperty: responseSchema.wrapperProperty) - var toolRounds = ToolRoundLimit(provider: "Gemini") + let toolRounds = session.toolRoundLimit(for: "Gemini") while true { try Task.checkCancellation() - let requestContext = session.resolvedRequestContext() + let requestContext = try await session.resolvedRequestContext( + including: state.entries, + options: options + ) + if let handoff = session.profileContinuationStream( + ifSelectedModelDiffersFrom: profileRequestIdentity, + requestContext: requestContext, + prompt: prompt, + generating: type, + schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, + prefixEntries: state.entries, + prefixUsage: state.totalUsage, + prefixText: state.responseText + ) { + for try await snapshot in handoff { continuation.yield(snapshot) } + continuation.finish() + return + } + let customOptions = requestContext.options[custom: GeminiLanguageModel.self] + let effectiveThinking = customOptions?.thinking ?? .disabled + let effectiveServerTools = customOptions?.serverTools ?? [] + let effectiveJsonMode = customOptions?.jsonMode let geminiTools = try buildTools( from: requestContext.tools, serverTools: effectiveServerTools ) - var requestTranscript = requestContext.transcript - for entry in inFlightEntries { - requestTranscript.append(entry) - } let params = try createGenerateContentParams( - contents: requestTranscript.toGeminiContent(), + contents: requestContext.transcript.toGeminiContent(), tools: geminiTools, generating: type, schema: responseSchema.schema, - options: options, + options: requestContext.options, thinking: effectiveThinking, jsonMode: effectiveJsonMode ) @@ -475,7 +493,8 @@ public struct GeminiLanguageModel: LanguageModel { try toolRounds.record(functionCalls.map(\.roundCall)) switch try await resolveFunctionCalls( functionCalls, - tools: requestContext.tools, + requestContext: requestContext, + currentRoundEntries: state.entries, session: session ) { case .stop(let calls): @@ -487,11 +506,9 @@ public struct GeminiLanguageModel: LanguageModel { let calls = Transcript.Entry.toolCalls( Transcript.ToolCalls(invocations.map(\.call), providerMetadata: metadata) ) - inFlightEntries.append(calls) state.entries.append(calls) for invocation in invocations { let output = Transcript.Entry.toolOutput(invocation.output) - inFlightEntries.append(output) state.entries.append(output) } } @@ -639,18 +656,12 @@ private enum ToolResolutionOutcome { private func resolveFunctionCalls( _ functionCalls: [GeminiFunctionCall], - tools: [any Tool], + requestContext: LanguageModelSession.RequestContext, + currentRoundEntries: [Transcript.Entry], session: LanguageModelSession ) async throws -> ToolResolutionOutcome { if functionCalls.isEmpty { return .invocations([]) } - var toolsByName: [String: any Tool] = [:] - for tool in tools { - if toolsByName[tool.name] == nil { - toolsByName[tool.name] = tool - } - } - var transcriptCalls: [Transcript.ToolCall] = [] transcriptCalls.reserveCapacity(functionCalls.count) for call in functionCalls { @@ -666,82 +677,18 @@ private func resolveFunctionCalls( ) } - if let delegate = session.toolExecutionDelegate { - await delegate.didGenerateToolCalls(transcriptCalls, in: session) - } - guard !transcriptCalls.isEmpty else { return .invocations([]) } - - var decisions: [ToolExecutionDecision] = [] - decisions.reserveCapacity(transcriptCalls.count) - - if let delegate = session.toolExecutionDelegate { - for call in transcriptCalls { - let decision = await delegate.toolCallDecision(for: call, in: session) - if case .stop = decision { - return .stop(calls: transcriptCalls) - } - decisions.append(decision) - } - } else { - decisions = Array(repeating: .execute, count: transcriptCalls.count) - } - - var results: [ToolInvocationResult] = [] - results.reserveCapacity(transcriptCalls.count) - - for (index, call) in transcriptCalls.enumerated() { - switch decisions[index] { - case .stop: - // This branch should be unreachable because `.stop` returns during decision collection. - // Keep it as a defensive guard in case that logic changes. - return .stop(calls: transcriptCalls) - case .provideOutput(let segments): - let output = Transcript.ToolOutput( - id: call.id, - toolName: call.toolName, - segments: segments - ) - if let delegate = session.toolExecutionDelegate { - await delegate.didExecuteToolCall(call, output: output, in: session) - } - results.append(ToolInvocationResult(call: call, output: output)) - case .execute: - guard let tool = toolsByName[call.toolName] else { - let message = Transcript.Segment.text(.init(content: "Tool not found: \(call.toolName)")) - let output = Transcript.ToolOutput( - id: call.id, - toolName: call.toolName, - segments: [message] - ) - if let delegate = session.toolExecutionDelegate { - await delegate.didExecuteToolCall(call, output: output, in: session) - } - results.append(ToolInvocationResult(call: call, output: output)) - continue - } - - do { - let segments = try await tool.makeOutputSegments(from: call.arguments) - let output = Transcript.ToolOutput( - id: call.id, - toolName: tool.name, - segments: segments - ) - if let delegate = session.toolExecutionDelegate { - await delegate.didExecuteToolCall(call, output: output, in: session) - } - results.append(ToolInvocationResult(call: call, output: output)) - } catch { - if let delegate = session.toolExecutionDelegate { - await delegate.didFailToolCall(call, error: error, in: session) - } - throw LanguageModelSession.ToolCallError(tool: tool, underlyingError: error) - } - } + let entries = currentRoundEntries + [.toolCalls(.init(transcriptCalls))] + switch try await session.resolveProfileToolCalls( + transcriptCalls, + requestContext: requestContext, + currentRoundEntries: entries + ) { + case .stop(let calls): + return .stop(calls: calls) + case .invocations(let invocations): + return .invocations(invocations.map { .init(call: $0.call, output: $0.output) }) } - - return .invocations(results) } /// Joins the text parts of a Gemini response. diff --git a/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift index ac1a37e4..5c53f4a6 100644 --- a/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift @@ -13,7 +13,7 @@ import OrderedCollections /// ```swift /// let model = OllamaLanguageModel(model: "qwen2.5") /// ``` -public struct OllamaLanguageModel: LanguageModel { +public struct OllamaLanguageModel: LanguageModel, ProfileRequestIdentifiedLanguageModel { /// The reason the model is unavailable. /// This model is always available. public typealias UnavailableReason = Never @@ -55,6 +55,7 @@ public struct OllamaLanguageModel: LanguageModel { public let model: String private let httpSession: HTTPSession + let profileRequestIdentity = UUID() /// Creates an Ollama language model. /// @@ -119,7 +120,6 @@ public struct OllamaLanguageModel: LanguageModel { includeSchemaInPrompt: Bool, options: GenerationOptions ) async throws -> LanguageModelSession.Response where Content: Generable { - let ollamaOptions = convertOptions(options) let ollamaFormat: JSONValue? if type == String.self { ollamaFormat = nil @@ -128,79 +128,116 @@ public struct OllamaLanguageModel: LanguageModel { ollamaFormat = try JSONValue(schema) } - let requestContext = session.resolvedRequestContext() - let ollamaTools = try requestContext.tools.map(convertToolToOllamaFormat) - var messages = try requestContext.transcript.toOllamaMessages() - if messages.isEmpty { - messages.append(.init(role: .user, content: prompt.description)) - } - let params = try createChatParams( - model: model, - messages: messages, - tools: ollamaTools.isEmpty ? nil : ollamaTools, - options: ollamaOptions, - stream: false, - format: ollamaFormat, - parameters: extractTopLevelChatParameters(options) - ) - let url = baseURL.appendingPathComponent("api/chat") - let body = try encodeChatParams(params) - let chatResponse: ChatResponse = try await httpSession.fetch( - .post, - url: url, - body: body, - dateDecodingStrategy: .iso8601WithFractionalSeconds - ) - var entries: [Transcript.Entry] = [] - let usage = chatResponse.reportedUsage?.value ?? .zero - - if let toolCalls = chatResponse.message.toolCalls, !toolCalls.isEmpty { - let resolution = try await resolveToolCalls( - toolCalls, - tools: requestContext.tools, - session: session + var usage = LanguageModelSession.Usage.zero + var earlierText = "" + var inFlightMessages: [OllamaMessage] = [] + let toolRounds = session.toolRoundLimit(for: "Ollama") + + while true { + let requestContext = try await session.resolvedRequestContext( + including: entries, + options: options ) - switch resolution { - case .stop(let calls): - if !calls.isEmpty { - entries.append(.toolCalls(Transcript.ToolCalls(calls))) - } - return LanguageModelSession.Response( - content: "" as! Content, - rawContent: GeneratedContent(""), - transcriptEntries: ArraySlice(entries), - usage: usage - ) - case .invocations(let invocations): - if !invocations.isEmpty { - entries.append(.toolCalls(Transcript.ToolCalls(invocations.map(\.call)))) + if let continuation = try await session.profileContinuationResponse( + ifSelectedModelDiffersFrom: profileRequestIdentity, + requestContext: requestContext, + prompt: prompt, + generating: type, + schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, + prefixEntries: entries, + prefixUsage: usage, + prefixText: earlierText + ) { + return continuation + } + let ollamaTools = try requestContext.tools.map(convertToolToOllamaFormat) + var messages = try ollamaMessages( + from: requestContext.transcript, + replacing: entries, + with: inFlightMessages + ) + if messages.isEmpty { + messages.append(.init(role: .user, content: prompt.description)) + } + let params = try createChatParams( + model: model, + messages: messages, + tools: ollamaTools.isEmpty ? nil : ollamaTools, + options: convertOptions(requestContext.options), + stream: false, + format: ollamaFormat, + parameters: extractTopLevelChatParameters(requestContext.options) + ) + let chatResponse: ChatResponse = try await httpSession.fetch( + .post, + url: url, + body: try encodeChatParams(params), + dateDecodingStrategy: .iso8601WithFractionalSeconds + ) + usage.add(chatResponse.reportedUsage?.value ?? .zero) + + if let toolCalls = chatResponse.message.toolCalls, !toolCalls.isEmpty { + try toolRounds.record(toolCalls.map(\.roundCall)) + switch try await resolveToolCalls( + toolCalls, + requestContext: requestContext, + currentRoundEntries: entries, + session: session + ) { + case .stop(let calls): + entries.append(.toolCalls(.init(calls))) + return .init( + content: "" as! Content, + rawContent: GeneratedContent(""), + transcriptEntries: ArraySlice(entries), + usage: usage + ) + case .invocations(let invocations): + inFlightMessages.append( + .init( + role: .assistant, + content: chatResponse.message.content ?? "", + toolCalls: try toolCalls.map { try JSONValue($0) } + ) + ) + entries.append(.toolCalls(.init(invocations.map(\.call)))) for invocation in invocations { entries.append(.toolOutput(invocation.output)) + let (text, images) = convertSegmentsToOllama(invocation.output.segments) + inFlightMessages.append( + .init( + role: .tool, + content: text, + images: images.isEmpty ? nil : images, + toolName: invocation.call.toolName + ) + ) } + if type == String.self { earlierText += chatResponse.message.content ?? "" } + continue } } - } - let text = chatResponse.message.content ?? "" - if type == String.self { - return LanguageModelSession.Response( - content: text as! Content, - rawContent: GeneratedContent(text), + let text = earlierText + (chatResponse.message.content ?? "") + if type == String.self { + return .init( + content: text as! Content, + rawContent: GeneratedContent(text), + transcriptEntries: ArraySlice(entries), + usage: usage + ) + } + let generatedContent = try GeneratedContent(json: text) + return .init( + content: try type.init(generatedContent), + rawContent: generatedContent, transcriptEntries: ArraySlice(entries), usage: usage ) } - - let generatedContent = try GeneratedContent(json: text) - let content = try type.init(generatedContent) - return LanguageModelSession.Response( - content: content, - rawContent: generatedContent, - transcriptEntries: ArraySlice(entries), - usage: usage - ) } public func streamResponse( @@ -253,24 +290,45 @@ public struct OllamaLanguageModel: LanguageModel { let format = type == String.self ? nil : try JSONValue(convertSchemaToOllamaFormat(schema)) var inFlightMessages: [OllamaMessage] = [] var state = StreamingResponseState() - var toolRounds = ToolRoundLimit(provider: "Ollama") + let toolRounds = session.toolRoundLimit(for: "Ollama") while true { try Task.checkCancellation() - let requestContext = session.resolvedRequestContext() + let requestContext = try await session.resolvedRequestContext( + including: state.entries, + options: options + ) + if let handoff = session.profileContinuationStream( + ifSelectedModelDiffersFrom: profileRequestIdentity, + requestContext: requestContext, + prompt: prompt, + generating: type, + schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, + prefixEntries: state.entries, + prefixUsage: state.totalUsage, + prefixText: state.responseText + ) { + for try await snapshot in handoff { continuation.yield(snapshot) } + continuation.finish() + return + } let tools = try requestContext.tools.map(convertToolToOllamaFormat) - var messages = try requestContext.transcript.toOllamaMessages() + var messages = try ollamaMessages( + from: requestContext.transcript, + replacing: state.entries, + with: inFlightMessages + ) if messages.isEmpty { messages.append(.init(role: .user, content: prompt.description)) } - messages.append(contentsOf: inFlightMessages) let params = try createChatParams( model: model, messages: messages, tools: tools.isEmpty ? nil : tools, - options: convertOptions(options), + options: convertOptions(requestContext.options), stream: true, format: format, - parameters: extractTopLevelChatParameters(options) + parameters: extractTopLevelChatParameters(requestContext.options) ) let body = try encodeChatParams(params) let chunks: AsyncThrowingStream = httpSession.fetchStream( @@ -294,7 +352,8 @@ public struct OllamaLanguageModel: LanguageModel { try toolRounds.record(toolCalls.map(\.roundCall)) switch try await resolveToolCalls( toolCalls, - tools: requestContext.tools, + requestContext: requestContext, + currentRoundEntries: state.entries, session: session ) { case .stop(let calls): @@ -340,6 +399,31 @@ public struct OllamaLanguageModel: LanguageModel { // MARK: - Tool Invocation Handling +private func ollamaMessages( + from transcript: Transcript, + replacing currentRoundEntries: [Transcript.Entry], + with continuation: [OllamaMessage] +) throws -> [OllamaMessage] { + guard !continuation.isEmpty, !currentRoundEntries.isEmpty else { + return try transcript.toOllamaMessages() + } + let projected = Array(transcript) + let count = currentRoundEntries.count + let ranges = projected.indices.compactMap { start -> Range? in + let end = start + count + guard end <= projected.count, + Array(projected[start ..< end]) == currentRoundEntries + else { return nil } + return start ..< end + } + guard ranges.count == 1, let range = ranges.first else { + return try transcript.toOllamaMessages() + } + return try Transcript(entries: projected[.. ToolResolutionOutcome { if toolCalls.isEmpty { return .invocations([]) } - var toolsByName: [String: any Tool] = [:] - for tool in tools { - if toolsByName[tool.name] == nil { - toolsByName[tool.name] = tool - } - } - var transcriptCalls: [Transcript.ToolCall] = [] transcriptCalls.reserveCapacity(toolCalls.count) for call in toolCalls { @@ -380,82 +458,18 @@ private func resolveToolCalls( ) } - if let delegate = session.toolExecutionDelegate { - await delegate.didGenerateToolCalls(transcriptCalls, in: session) - } - guard !transcriptCalls.isEmpty else { return .invocations([]) } - - var decisions: [ToolExecutionDecision] = [] - decisions.reserveCapacity(transcriptCalls.count) - - if let delegate = session.toolExecutionDelegate { - for call in transcriptCalls { - let decision = await delegate.toolCallDecision(for: call, in: session) - if case .stop = decision { - return .stop(calls: transcriptCalls) - } - decisions.append(decision) - } - } else { - decisions = Array(repeating: .execute, count: transcriptCalls.count) - } - - var results: [ToolInvocationResult] = [] - results.reserveCapacity(transcriptCalls.count) - - for (index, call) in transcriptCalls.enumerated() { - switch decisions[index] { - case .stop: - // This branch should be unreachable because `.stop` returns during decision collection. - // Keep it as a defensive guard in case that logic changes. - return .stop(calls: transcriptCalls) - case .provideOutput(let segments): - let output = Transcript.ToolOutput( - id: call.id, - toolName: call.toolName, - segments: segments - ) - if let delegate = session.toolExecutionDelegate { - await delegate.didExecuteToolCall(call, output: output, in: session) - } - results.append(ToolInvocationResult(call: call, output: output)) - case .execute: - guard let tool = toolsByName[call.toolName] else { - let message = Transcript.Segment.text(.init(content: "Tool not found: \(call.toolName)")) - let output = Transcript.ToolOutput( - id: call.id, - toolName: call.toolName, - segments: [message] - ) - if let delegate = session.toolExecutionDelegate { - await delegate.didExecuteToolCall(call, output: output, in: session) - } - results.append(ToolInvocationResult(call: call, output: output)) - continue - } - - do { - let segments = try await tool.makeOutputSegments(from: call.arguments) - let output = Transcript.ToolOutput( - id: call.id, - toolName: tool.name, - segments: segments - ) - if let delegate = session.toolExecutionDelegate { - await delegate.didExecuteToolCall(call, output: output, in: session) - } - results.append(ToolInvocationResult(call: call, output: output)) - } catch { - if let delegate = session.toolExecutionDelegate { - await delegate.didFailToolCall(call, error: error, in: session) - } - throw LanguageModelSession.ToolCallError(tool: tool, underlyingError: error) - } - } + let entries = currentRoundEntries + [.toolCalls(.init(transcriptCalls))] + switch try await session.resolveProfileToolCalls( + transcriptCalls, + requestContext: requestContext, + currentRoundEntries: entries + ) { + case .stop(let calls): + return .stop(calls: calls) + case .invocations(let invocations): + return .invocations(invocations.map { .init(call: $0.call, output: $0.output) }) } - - return .invocations(results) } // MARK: - Conversions diff --git a/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift b/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift index 6952ab62..005da488 100644 --- a/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift @@ -17,7 +17,7 @@ import JSONSchema /// model: "gpt-4" /// ) /// ``` -public struct OpenAILanguageModel: LanguageModel { +public struct OpenAILanguageModel: LanguageModel, ProfileRequestIdentifiedLanguageModel { /// The reason the model is unavailable. /// This model is always available. public typealias UnavailableReason = Never @@ -398,6 +398,7 @@ public struct OpenAILanguageModel: LanguageModel { public let apiVariant: APIVariant private let httpSession: HTTPSession + let profileRequestIdentity = UUID() /// Creates an OpenAI language model. /// @@ -471,15 +472,19 @@ public struct OpenAILanguageModel: LanguageModel { switch apiVariant { case .chatCompletions: return try await respondWithChatCompletions( + prompt: prompt, generating: type, schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, options: options, session: session ) case .responses: return try await respondWithResponses( + prompt: prompt, generating: type, schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, options: options, session: session ) @@ -487,8 +492,10 @@ public struct OpenAILanguageModel: LanguageModel { } private func respondWithChatCompletions( + prompt: Prompt, generating type: Content.Type, schema: GenerationSchema, + includeSchemaInPrompt: Bool, options: GenerationOptions, session: LanguageModelSession ) async throws -> LanguageModelSession.Response where Content: Generable { @@ -501,21 +508,41 @@ public struct OpenAILanguageModel: LanguageModel { var earlierText = "" var inFlightMessages: [OpenAIMessage] = [] - var toolRounds = ToolRoundLimit(provider: "OpenAI") + let toolRounds = session.toolRoundLimit(for: "OpenAI") // Loop until no more tool calls while true { - let requestContext = session.resolvedRequestContext() + let requestContext = try await session.resolvedRequestContext( + including: entries, + options: options + ) + if let continuation = try await session.profileContinuationResponse( + ifSelectedModelDiffersFrom: profileRequestIdentity, + requestContext: requestContext, + prompt: prompt, + generating: type, + schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, + prefixEntries: entries, + prefixUsage: usage.value, + prefixText: earlierText + ) { + return continuation + } let tools = requestContext.tools.isEmpty ? nil : requestContext.tools.map(convertToolToOpenAIFormat) - let messages = requestContext.transcript.toOpenAIMessages() + inFlightMessages + let messages = openAIMessages( + from: requestContext.transcript, + replacing: entries, + with: inFlightMessages + ) let params = try ChatCompletions.createRequestBody( model: model, messages: messages, tools: tools, generating: type, schema: responseSchema.schema, - options: options, + options: requestContext.options, stream: false ) @@ -556,7 +583,8 @@ public struct OpenAILanguageModel: LanguageModel { try toolRounds.record(toolCalls.map(\.roundCall)) let resolution = try await resolveToolCalls( toolCalls, - tools: requestContext.tools, + requestContext: requestContext, + currentRoundEntries: entries, session: session ) switch resolution { @@ -614,8 +642,10 @@ public struct OpenAILanguageModel: LanguageModel { } private func respondWithResponses( + prompt: Prompt, generating type: Content.Type, schema: GenerationSchema, + includeSchemaInPrompt: Bool, options: GenerationOptions, session: LanguageModelSession ) async throws -> LanguageModelSession.Response where Content: Generable { @@ -631,21 +661,41 @@ public struct OpenAILanguageModel: LanguageModel { let url = baseURL.appendingPathComponent("responses") - var toolRounds = ToolRoundLimit(provider: "OpenAI") + let toolRounds = session.toolRoundLimit(for: "OpenAI") // Loop until no more tool calls while true { - let requestContext = session.resolvedRequestContext() + let requestContext = try await session.resolvedRequestContext( + including: entries, + options: options + ) + if let continuation = try await session.profileContinuationResponse( + ifSelectedModelDiffersFrom: profileRequestIdentity, + requestContext: requestContext, + prompt: prompt, + generating: type, + schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, + prefixEntries: entries, + prefixUsage: usage.value, + prefixText: earlierText + ) { + return continuation + } let tools = requestContext.tools.isEmpty ? nil : requestContext.tools.map(convertToolToOpenAIFormat) - let messages = requestContext.transcript.toOpenAIMessages() + inFlightMessages + let messages = openAIMessages( + from: requestContext.transcript, + replacing: entries, + with: inFlightMessages + ) let params = try Responses.createRequestBody( model: model, messages: messages, tools: tools, generating: type, schema: responseSchema.schema, - options: options, + options: requestContext.options, stream: false ) @@ -673,7 +723,8 @@ public struct OpenAILanguageModel: LanguageModel { try toolRounds.record(toolCalls.map(\.roundCall)) let resolution = try await resolveToolCalls( toolCalls, - tools: requestContext.tools, + requestContext: requestContext, + currentRoundEntries: entries, session: session ) switch resolution { @@ -787,14 +838,36 @@ public struct OpenAILanguageModel: LanguageModel { do { var inFlightMessages: [OpenAIMessage] = [] var state = StreamingResponseState(wrapperProperty: responseSchema.wrapperProperty) - var toolRounds = ToolRoundLimit(provider: "OpenAI") + let toolRounds = session.toolRoundLimit(for: "OpenAI") while true { try Task.checkCancellation() - let requestContext = session.resolvedRequestContext() + let requestContext = try await session.resolvedRequestContext( + including: state.entries, + options: options + ) + if let handoff = session.profileContinuationStream( + ifSelectedModelDiffersFrom: profileRequestIdentity, + requestContext: requestContext, + prompt: prompt, + generating: type, + schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, + prefixEntries: state.entries, + prefixUsage: state.totalUsage, + prefixText: state.responseText + ) { + for try await snapshot in handoff { continuation.yield(snapshot) } + continuation.finish() + return + } let tools = requestContext.tools.isEmpty ? nil : requestContext.tools.map(convertToolToOpenAIFormat) - let messages = requestContext.transcript.toOpenAIMessages() + inFlightMessages + let messages = openAIMessages( + from: requestContext.transcript, + replacing: state.entries, + with: inFlightMessages + ) let params: JSONValue let path: String switch apiVariant { @@ -806,7 +879,7 @@ public struct OpenAILanguageModel: LanguageModel { tools: tools, generating: type, schema: responseSchema.schema, - options: options, + options: requestContext.options, stream: true ) case .chatCompletions: @@ -817,7 +890,7 @@ public struct OpenAILanguageModel: LanguageModel { tools: tools, generating: type, schema: responseSchema.schema, - options: options, + options: requestContext.options, stream: true, includeUsage: baseURL.host == Self.defaultBaseURL.host ) @@ -895,7 +968,8 @@ public struct OpenAILanguageModel: LanguageModel { try toolRounds.record(toolCalls.map(\.roundCall)) switch try await resolveToolCalls( toolCalls, - tools: requestContext.tools, + requestContext: requestContext, + currentRoundEntries: state.entries, session: session ) { case .stop(let calls): @@ -1741,18 +1815,12 @@ private enum OpenAIToolResolutionOutcome { private func resolveToolCalls( _ toolCalls: [OpenAIToolCall], - tools: [any Tool], + requestContext: LanguageModelSession.RequestContext, + currentRoundEntries: [Transcript.Entry], session: LanguageModelSession ) async throws -> OpenAIToolResolutionOutcome { if toolCalls.isEmpty { return .invocations([]) } - var toolsByName: [String: any Tool] = [:] - for tool in tools { - if toolsByName[tool.name] == nil { - toolsByName[tool.name] = tool - } - } - var transcriptCalls: [Transcript.ToolCall] = [] transcriptCalls.reserveCapacity(toolCalls.count) for call in toolCalls { @@ -1768,86 +1836,51 @@ private func resolveToolCalls( ) } - if let delegate = session.toolExecutionDelegate { - await delegate.didGenerateToolCalls(transcriptCalls, in: session) - } - guard !transcriptCalls.isEmpty else { return .invocations([]) } - - var decisions: [ToolExecutionDecision] = [] - decisions.reserveCapacity(transcriptCalls.count) - - if let delegate = session.toolExecutionDelegate { - for call in transcriptCalls { - let decision = await delegate.toolCallDecision(for: call, in: session) - if case .stop = decision { - return .stop(calls: transcriptCalls) - } - decisions.append(decision) - } - } else { - decisions = Array(repeating: .execute, count: transcriptCalls.count) + let entries = currentRoundEntries + [.toolCalls(.init(transcriptCalls))] + switch try await session.resolveProfileToolCalls( + transcriptCalls, + requestContext: requestContext, + currentRoundEntries: entries + ) { + case .stop(let calls): + return .stop(calls: calls) + case .invocations(let invocations): + return .invocations( + invocations.map { .init(call: $0.call, output: $0.output) } + ) } +} - var results: [OpenAIToolInvocationResult] = [] - results.reserveCapacity(transcriptCalls.count) - - for (index, call) in transcriptCalls.enumerated() { - switch decisions[index] { - case .stop: - // This branch should be unreachable because `.stop` returns during decision collection. - // Keep it as a defensive guard in case that logic changes. - return .stop(calls: transcriptCalls) - case .provideOutput(let segments): - let output = Transcript.ToolOutput( - id: call.id, - toolName: call.toolName, - segments: segments - ) - if let delegate = session.toolExecutionDelegate { - await delegate.didExecuteToolCall(call, output: output, in: session) - } - results.append(OpenAIToolInvocationResult(call: call, output: output)) - case .execute: - guard let tool = toolsByName[call.toolName] else { - let message = Transcript.Segment.text(.init(content: "Tool not found: \(call.toolName)")) - let output = Transcript.ToolOutput( - id: call.id, - toolName: call.toolName, - segments: [message] - ) - if let delegate = session.toolExecutionDelegate { - await delegate.didExecuteToolCall(call, output: output, in: session) - } - results.append(OpenAIToolInvocationResult(call: call, output: output)) - continue - } +// MARK: - Converters - do { - let segments = try await tool.makeOutputSegments(from: call.arguments) - let output = Transcript.ToolOutput( - id: call.id, - toolName: tool.name, - segments: segments - ) - if let delegate = session.toolExecutionDelegate { - await delegate.didExecuteToolCall(call, output: output, in: session) - } - results.append(OpenAIToolInvocationResult(call: call, output: output)) - } catch { - if let delegate = session.toolExecutionDelegate { - await delegate.didFailToolCall(call, error: error, in: session) - } - throw LanguageModelSession.ToolCallError(tool: tool, underlyingError: error) - } - } +private func openAIMessages( + from transcript: Transcript, + replacing currentRoundEntries: [Transcript.Entry], + with continuation: [OpenAIMessage] +) -> [OpenAIMessage] { + guard !continuation.isEmpty, !currentRoundEntries.isEmpty else { + return transcript.toOpenAIMessages() } - - return .invocations(results) + let projected = Array(transcript) + let count = currentRoundEntries.count + let ranges = projected.indices.compactMap { start -> Range? in + let end = start + count + guard end <= projected.count, + Array(projected[start ..< end]) == currentRoundEntries + else { return nil } + return start ..< end + } + guard ranges.count == 1, let range = ranges.first else { + // A legal request projection may remove or reorder the current Tool round. + // In that case its mapped transcript is authoritative and opaque replay is reset. + return transcript.toOpenAIMessages() + } + return Transcript(entries: projected[.. OpenAITool { // Prefer passing through a JSONSchema value built from GenerationSchema // where possible; fallback to minimal type/required map. diff --git a/Sources/AnyLanguageModel/Models/OpenResponsesLanguageModel.swift b/Sources/AnyLanguageModel/Models/OpenResponsesLanguageModel.swift index 395b40b8..8028379e 100644 --- a/Sources/AnyLanguageModel/Models/OpenResponsesLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/OpenResponsesLanguageModel.swift @@ -20,7 +20,7 @@ import JSONSchema /// model: "openai/gpt-4o-mini" /// ) /// ``` -public struct OpenResponsesLanguageModel: LanguageModel { +public struct OpenResponsesLanguageModel: LanguageModel, ProfileRequestIdentifiedLanguageModel { /// The reason the model is unavailable. /// This model is always available. public typealias UnavailableReason = Never @@ -369,6 +369,7 @@ public struct OpenResponsesLanguageModel: LanguageModel { public let model: String private let httpSession: HTTPSession + let profileRequestIdentity = UUID() /// Creates an Open Responses language model. /// @@ -436,8 +437,10 @@ public struct OpenResponsesLanguageModel: LanguageModel { options: GenerationOptions ) async throws -> LanguageModelSession.Response where Content: Generable { return try await respondWithOpenResponses( + prompt: prompt, generating: type, schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, options: options, session: session ) @@ -494,21 +497,43 @@ public struct OpenResponsesLanguageModel: LanguageModel { do { var inFlightMessages: [OpenResponsesMessage] = [] var state = StreamingResponseState(wrapperProperty: responseSchema.wrapperProperty) - var toolRounds = ToolRoundLimit(provider: "Open Responses") + let toolRounds = session.toolRoundLimit(for: "Open Responses") while true { try Task.checkCancellation() - let requestContext = session.resolvedRequestContext() + let requestContext = try await session.resolvedRequestContext( + including: state.entries, + options: options + ) + if let handoff = session.profileContinuationStream( + ifSelectedModelDiffersFrom: profileRequestIdentity, + requestContext: requestContext, + prompt: prompt, + generating: type, + schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, + prefixEntries: state.entries, + prefixUsage: state.totalUsage, + prefixText: state.responseText + ) { + for try await snapshot in handoff { continuation.yield(snapshot) } + continuation.finish() + return + } let tools: [OpenResponsesTool]? = requestContext.tools.isEmpty ? nil : requestContext.tools.map(convertToolToOpenResponsesFormat) - let messages = requestContext.transcript.toOpenResponsesMessages() + inFlightMessages + let messages = openResponsesMessages( + from: requestContext.transcript, + replacing: state.entries, + with: inFlightMessages + ) let params = try OpenResponsesAPI.createRequestBody( model: model, messages: messages, tools: tools, generating: type, schema: responseSchema.schema, - options: options, + options: requestContext.options, stream: true ) let body = try JSONEncoder().encode(params) @@ -553,7 +578,8 @@ public struct OpenResponsesLanguageModel: LanguageModel { try toolRounds.record(toolCalls.map(\.roundCall)) switch try await resolveToolCalls( toolCalls, - tools: requestContext.tools, + requestContext: requestContext, + currentRoundEntries: state.entries, session: session ) { case .stop(let calls): @@ -590,8 +616,10 @@ public struct OpenResponsesLanguageModel: LanguageModel { /// Sends a non-streaming request to the Open Responses API and returns the parsed response. private func respondWithOpenResponses( + prompt: Prompt, generating type: Content.Type, schema: GenerationSchema, + includeSchemaInPrompt: Bool, options: GenerationOptions, session: LanguageModelSession ) async throws -> LanguageModelSession.Response where Content: Generable { @@ -606,20 +634,40 @@ public struct OpenResponsesLanguageModel: LanguageModel { var inFlightMessages: [OpenResponsesMessage] = [] let url = baseURL.appendingPathComponent("responses") - var toolRounds = ToolRoundLimit(provider: "Open Responses") + let toolRounds = session.toolRoundLimit(for: "Open Responses") while true { - let requestContext = session.resolvedRequestContext() + let requestContext = try await session.resolvedRequestContext( + including: entries, + options: options + ) + if let continuation = try await session.profileContinuationResponse( + ifSelectedModelDiffersFrom: profileRequestIdentity, + requestContext: requestContext, + prompt: prompt, + generating: type, + schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, + prefixEntries: entries, + prefixUsage: usage.value, + prefixText: earlierText + ) { + return continuation + } let tools: [OpenResponsesTool]? = requestContext.tools.isEmpty ? nil : requestContext.tools.map(convertToolToOpenResponsesFormat) - let messages = requestContext.transcript.toOpenResponsesMessages() + inFlightMessages + let messages = openResponsesMessages( + from: requestContext.transcript, + replacing: entries, + with: inFlightMessages + ) let params = try OpenResponsesAPI.createRequestBody( model: model, messages: messages, tools: tools, generating: type, schema: responseSchema.schema, - options: options, + options: requestContext.options, stream: false ) let body = try JSONEncoder().encode(params) @@ -645,7 +693,8 @@ public struct OpenResponsesLanguageModel: LanguageModel { try toolRounds.record(toolCalls.map(\.roundCall)) let resolution = try await resolveToolCalls( toolCalls, - tools: requestContext.tools, + requestContext: requestContext, + currentRoundEntries: entries, session: session ) switch resolution { @@ -1171,6 +1220,31 @@ private struct OpenResponsesToolInvocationResult: Sendable { let output: Transcript.ToolOutput } +private func openResponsesMessages( + from transcript: Transcript, + replacing currentRoundEntries: [Transcript.Entry], + with continuation: [OpenResponsesMessage] +) -> [OpenResponsesMessage] { + guard !continuation.isEmpty, !currentRoundEntries.isEmpty else { + return transcript.toOpenResponsesMessages() + } + let projected = Array(transcript) + let count = currentRoundEntries.count + let ranges = projected.indices.compactMap { start -> Range? in + let end = start + count + guard end <= projected.count, + Array(projected[start ..< end]) == currentRoundEntries + else { return nil } + return start ..< end + } + guard ranges.count == 1, let range = ranges.first else { + return transcript.toOpenResponsesMessages() + } + return Transcript(entries: projected[.. OpenResponsesToolResolutionOutcome { if toolCalls.isEmpty { return .invocations([]) } - var byName: [String: any Tool] = [:] - for t in tools { if byName[t.name] == nil { byName[t.name] = t } } var transcriptCalls: [Transcript.ToolCall] = [] for c in toolCalls { let args = (c.arguments.flatMap { try? GeneratedContent(json: $0) } ?? GeneratedContent(properties: [:])) transcriptCalls.append(Transcript.ToolCall(id: c.id, toolName: c.name, arguments: args)) } - if let d = session.toolExecutionDelegate { - await d.didGenerateToolCalls(transcriptCalls, in: session) - } guard !transcriptCalls.isEmpty else { return .invocations([]) } - var decisions: [ToolExecutionDecision] = [] - if let d = session.toolExecutionDelegate { - for call in transcriptCalls { - let dec = await d.toolCallDecision(for: call, in: session) - if case .stop = dec { return .stop(calls: transcriptCalls) } - decisions.append(dec) - } - } else { - decisions = Array(repeating: .execute, count: transcriptCalls.count) - } - var results: [OpenResponsesToolInvocationResult] = [] - for (i, call) in transcriptCalls.enumerated() { - switch decisions[i] { - case .stop: - return .stop(calls: transcriptCalls) - case .provideOutput(let segs): - let out = Transcript.ToolOutput(id: call.id, toolName: call.toolName, segments: segs) - if let d = session.toolExecutionDelegate { await d.didExecuteToolCall(call, output: out, in: session) } - results.append(OpenResponsesToolInvocationResult(call: call, output: out)) - case .execute: - guard let tool = byName[call.toolName] else { - let out = Transcript.ToolOutput( - id: call.id, - toolName: call.toolName, - segments: [.text(.init(content: "Tool not found: \(call.toolName)"))] - ) - if let d = session.toolExecutionDelegate { await d.didExecuteToolCall(call, output: out, in: session) } - results.append(OpenResponsesToolInvocationResult(call: call, output: out)) - continue - } - do { - let segs = try await tool.makeOutputSegments(from: call.arguments) - let out = Transcript.ToolOutput(id: call.id, toolName: tool.name, segments: segs) - if let d = session.toolExecutionDelegate { await d.didExecuteToolCall(call, output: out, in: session) } - results.append(OpenResponsesToolInvocationResult(call: call, output: out)) - } catch { - if let d = session.toolExecutionDelegate { await d.didFailToolCall(call, error: error, in: session) } - throw LanguageModelSession.ToolCallError(tool: tool, underlyingError: error) - } - } + let entries = currentRoundEntries + [.toolCalls(.init(transcriptCalls))] + switch try await session.resolveProfileToolCalls( + transcriptCalls, + requestContext: requestContext, + currentRoundEntries: entries + ) { + case .stop(let calls): + return .stop(calls: calls) + case .invocations(let invocations): + return .invocations( + invocations.map { .init(call: $0.call, output: $0.output) } + ) } - return .invocations(results) } // MARK: - Streaming events diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index 3c9e5b6e..194ab949 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -498,6 +498,21 @@ } #if compiler(>=6.4) && !os(tvOS) + if #available(macOS 27, iOS 27, visionOS 27, watchOS 27, *) { + let toolCallingMode: FoundationModels.GenerationOptions.ToolCallingMode? = + switch self.toolCallingMode?.kind { + case .allowed: .allowed + case .required: .required + case .disallowed: .disallowed + case nil: nil + } + return FoundationModels.GenerationOptions( + samplingMode: sampling, + temperature: temperature, + maximumResponseTokens: maximumResponseTokens, + toolCallingMode: toolCallingMode + ) + } return FoundationModels.GenerationOptions( samplingMode: sampling, temperature: temperature, diff --git a/Sources/AnyLanguageModel/ProfileToolExecution.swift b/Sources/AnyLanguageModel/ProfileToolExecution.swift new file mode 100644 index 00000000..d87a1ada --- /dev/null +++ b/Sources/AnyLanguageModel/ProfileToolExecution.swift @@ -0,0 +1,248 @@ +import Foundation + +protocol ProfileRequestIdentifiedLanguageModel { + var profileRequestIdentity: UUID { get } +} + +struct ProfileToolInvocation: Sendable { + let call: Transcript.ToolCall + let output: Transcript.ToolOutput +} + +enum ProfileToolResolution: Sendable { + case stop(calls: [Transcript.ToolCall]) + case invocations([ProfileToolInvocation]) +} + +extension LanguageModelSession { + func profileContinuationResponse( + ifSelectedModelDiffersFrom currentModelIdentity: UUID, + requestContext: RequestContext, + prompt: Prompt, + generating type: Content.Type, + schema: GenerationSchema, + includeSchemaInPrompt: Bool, + prefixEntries: [Transcript.Entry], + prefixUsage: Usage, + prefixText: String + ) async throws -> Response? { + guard !prefixEntries.isEmpty else { return nil } + if let selected = requestContext.model as? any ProfileRequestIdentifiedLanguageModel, + selected.profileRequestIdentity == currentModelIdentity + { + return nil + } + + let response: Response + if type == GeneratedContent.self { + let generated = try await requestContext.model.respond( + within: self, + to: prompt, + schema: schema, + includeSchemaInPrompt: requestContext.contextOptions.includeSchemaInPrompt + ?? includeSchemaInPrompt, + options: requestContext.options + ) + guard let typed = generated as? Response else { + preconditionFailure("GeneratedContent response type mismatch") + } + response = typed + } else { + response = try await requestContext.model.respond( + within: self, + to: prompt, + generating: type, + includeSchemaInPrompt: requestContext.contextOptions.includeSchemaInPrompt + ?? includeSchemaInPrompt, + options: requestContext.options + ) + } + + var usage = prefixUsage + usage.add(response.usage) + var content = response.content + var rawContent = response.rawContent + if type == String.self, case .string(let suffix) = response.rawContent.kind { + let combined = prefixText + suffix + content = combined as! Content + rawContent = GeneratedContent(combined) + } + return Response( + content: content, + rawContent: rawContent, + transcriptEntries: ArraySlice(prefixEntries + response.transcriptEntries), + usage: usage, + providerMetadata: response.providerMetadata + ) + } + + func profileContinuationStream( + ifSelectedModelDiffersFrom currentModelIdentity: UUID, + requestContext: RequestContext, + prompt: Prompt, + generating type: Content.Type, + schema: GenerationSchema, + includeSchemaInPrompt: Bool, + prefixEntries: [Transcript.Entry], + prefixUsage: Usage, + prefixText: String + ) -> ResponseStream? { + guard !prefixEntries.isEmpty else { return nil } + if let selected = requestContext.model as? any ProfileRequestIdentifiedLanguageModel, + selected.profileRequestIdentity == currentModelIdentity + { + return nil + } + + let upstream: ResponseStream + if type == GeneratedContent.self { + let generated = requestContext.model.streamResponse( + within: self, + to: prompt, + schema: schema, + includeSchemaInPrompt: requestContext.contextOptions.includeSchemaInPrompt + ?? includeSchemaInPrompt, + options: requestContext.options + ) + guard let typed = generated as? ResponseStream else { + preconditionFailure("GeneratedContent response stream type mismatch") + } + upstream = typed + } else { + upstream = requestContext.model.streamResponse( + within: self, + to: prompt, + generating: type, + includeSchemaInPrompt: requestContext.contextOptions.includeSchemaInPrompt + ?? includeSchemaInPrompt, + options: requestContext.options + ) + } + + let stream = AsyncThrowingStream.Snapshot, any Error> { continuation in + let task = Task { + do { + for try await snapshot in upstream { + var usage = prefixUsage + usage.add(snapshot.usage) + var content = snapshot.content + var rawContent = snapshot.rawContent + if type == String.self, case .string(let suffix) = snapshot.rawContent.kind { + let combined = prefixText + suffix + content = combined as! Content.PartiallyGenerated + rawContent = GeneratedContent(combined) + } + continuation.yield( + .init( + content: content, + rawContent: rawContent, + transcriptEntries: ArraySlice(prefixEntries + snapshot.transcriptEntries), + usage: usage, + providerMetadata: snapshot.providerMetadata + ) + ) + } + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { _ in task.cancel() } + } + return ResponseStream(stream: stream) + } + + func resolveProfileToolCalls( + _ calls: [Transcript.ToolCall], + requestContext: RequestContext, + currentRoundEntries: [Transcript.Entry], + missingToolOutput: @Sendable (Transcript.ToolCall) -> [Transcript.Segment] = { call in + [.text(.init(content: "Tool not found: \(call.toolName)"))] + } + ) async throws -> ProfileToolResolution { + guard !calls.isEmpty else { return .invocations([]) } + + if let delegate = toolExecutionDelegate { + await delegate.didGenerateToolCalls(calls, in: self) + } + + var decisions: [ToolExecutionDecision] = [] + decisions.reserveCapacity(calls.count) + for call in calls { + try await profileWillExecuteToolCall( + call, + requestContext: requestContext, + currentRoundEntries: currentRoundEntries + ) + let decision = + await toolExecutionDelegate?.toolCallDecision(for: call, in: self) ?? .execute + if case .stop = decision { + return .stop(calls: calls) + } + decisions.append(decision) + } + + var toolsByName: [String: any Tool] = [:] + for tool in requestContext.tools where toolsByName[tool.name] == nil { + toolsByName[tool.name] = tool + } + + var invocations: [ProfileToolInvocation] = [] + invocations.reserveCapacity(calls.count) + var completedEntries: [Transcript.Entry] = [] + for (call, decision) in zip(calls, decisions) { + try Task.checkCancellation() + let output: Transcript.ToolOutput + switch decision { + case .stop: + return .stop(calls: calls) + case .provideOutput(let segments): + output = Transcript.ToolOutput( + id: call.id, + toolName: call.toolName, + segments: segments + ) + case .execute: + guard let tool = toolsByName[call.toolName] else { + output = Transcript.ToolOutput( + id: call.id, + toolName: call.toolName, + segments: missingToolOutput(call) + ) + break + } + do { + let segments = try await withProfileToolExecution( + requestContext: requestContext, + currentRoundEntries: currentRoundEntries + completedEntries + ) { + try await tool.makeOutputSegments(from: call.arguments) + } + output = Transcript.ToolOutput( + id: call.id, + toolName: tool.name, + segments: segments + ) + } catch { + if let delegate = toolExecutionDelegate { + await delegate.didFailToolCall(call, error: error, in: self) + } + throw ToolCallError(tool: tool, underlyingError: error) + } + } + + invocations.append(.init(call: call, output: output)) + completedEntries.append(.toolOutput(output)) + try await profileDidProduceToolOutput( + for: call, + output: output, + requestContext: requestContext, + currentRoundEntries: currentRoundEntries + completedEntries + ) + if let delegate = toolExecutionDelegate { + await delegate.didExecuteToolCall(call, output: output, in: self) + } + } + return .invocations(invocations) + } +} diff --git a/Sources/AnyLanguageModel/SessionProperties.swift b/Sources/AnyLanguageModel/SessionProperties.swift new file mode 100644 index 00000000..23bb8e43 --- /dev/null +++ b/Sources/AnyLanguageModel/SessionProperties.swift @@ -0,0 +1,208 @@ +import Foundation +import Observation + +extension Transcript { + /// The mutable, instruction-free history exposed through session properties. + /// + /// - Note: This API is exclusive to AnyLanguageModel on OS 26. It mirrors + /// Foundation Models 27's `Transcript.HistoryView` role with value semantics. + public typealias HistoryView = [Entry] +} + +/// A type-keyed session property. +/// +/// - Note: This API is exclusive to AnyLanguageModel on OS 26 and mirrors the +/// Foundation Models 27 session-property contract. +public protocol SessionPropertyKey: SendableMetatype { + associatedtype Value + static var defaultValue: Value { get } +} + +/// Declares a stored value on ``SessionPropertyValues``. +@attached(accessor) +@attached(peer, names: prefixed(__Key_)) +public macro SessionPropertyEntry() = + #externalMacro(module: "AnyLanguageModelMacros", type: "SessionPropertyEntryMacro") + +/// Session-scoped storage shared by profiles, dynamic instructions, and tools. +/// +/// - Note: This API is exclusive to AnyLanguageModel on OS 26 and mirrors +/// Foundation Models 27. +@Observable +public final class SessionPropertyValues: @unchecked Sendable { + @ObservationIgnored private let sessionPropertyRegistrar = ObservationRegistrar() + @ObservationIgnored private let customValues = Locked<[ObjectIdentifier: Any]>([:]) + @ObservationIgnored private let historyGetter: @Sendable () -> Transcript.HistoryView + @ObservationIgnored private let historySetter: @Sendable (Transcript.HistoryView) -> Void + + init( + historyGetter: @escaping @Sendable () -> Transcript.HistoryView, + historySetter: @escaping @Sendable (Transcript.HistoryView) -> Void + ) { + self.historyGetter = historyGetter + self.historySetter = historySetter + } + + /// The canonical session history after dynamic instructions. + public var history: Transcript.HistoryView { + get { + _accessSessionProperty(keyPath: \.history) + return SessionPropertyBinding.history?.entries ?? historyGetter() + } + set { + _withSessionPropertyMutation(keyPath: \.history) { + let newValue = newValue.filter { entry in + if case .instructions = entry { return false } + return true + } + if let history = SessionPropertyBinding.history { + precondition( + history.isWritable, + "Session history is read-only while dynamic instructions or a Tool is active" + ) + history.entries = newValue + history.persist(newValue) + } else { + historySetter(newValue) + } + } + } + } + + public subscript(key: Key.Type) -> Key.Value where Key: SessionPropertyKey { + get { + customValues.withLock { values in + if let value = values[ObjectIdentifier(key)] as? Key.Value { return value } + let value = Key.defaultValue + values[ObjectIdentifier(key)] = value + return value + } + } + set { + customValues.withLock { values in + values[ObjectIdentifier(key)] = newValue + } + } + } + + func historyBinding( + _ entries: Transcript.HistoryView, + isWritable: Bool, + protecting protectedEntryIDs: Set = [] + ) -> SessionHistoryBinding { + let persist: (@Sendable (Transcript.HistoryView) -> Void)? + if isWritable { + persist = { [historySetter] updated in + historySetter( + updated.filter { !protectedEntryIDs.contains($0.id) } + ) + } + } else { + persist = nil + } + return SessionHistoryBinding( + entries, + isWritable: isWritable, + protectedEntryIDs: protectedEntryIDs, + persist: persist + ) + } + + @_documentation(visibility: internal) + public func _accessSessionProperty( + keyPath: KeyPath + ) { + sessionPropertyRegistrar.access(self, keyPath: keyPath) + } + + @_documentation(visibility: internal) + public func _withSessionPropertyMutation( + keyPath: KeyPath, + _ mutation: () throws -> Result + ) rethrows -> Result { + try sessionPropertyRegistrar.withMutation( + of: self, + keyPath: keyPath, + mutation + ) + } +} + +enum SessionPropertyBinding { + @TaskLocal static var values: SessionPropertyValues? + @TaskLocal static var history: SessionHistoryBinding? +} + +final class SessionHistoryBinding: @unchecked Sendable { + private let storage: Locked + let isWritable: Bool + private let protectedEntryIDs: Set + private let persistValue: (@Sendable (Transcript.HistoryView) -> Void)? + + init( + _ entries: Transcript.HistoryView, + isWritable: Bool, + protectedEntryIDs: Set = [], + persist: (@Sendable (Transcript.HistoryView) -> Void)? = nil + ) { + storage = Locked(entries) + self.isWritable = isWritable + self.protectedEntryIDs = protectedEntryIDs + self.persistValue = persist + } + + var entries: Transcript.HistoryView { + get { storage.withLock { $0 } } + set { + storage.withLock { current in + let protected = current.filter { protectedEntryIDs.contains($0.id) } + current = newValue.filter { !protectedEntryIDs.contains($0.id) } + protected + } + } + } + + func persist(_ entries: Transcript.HistoryView) { + persistValue?(entries) + } +} + +extension LanguageModelSession { + /// Accesses a value in the currently evaluating session. + @propertyWrapper + public struct SessionProperty { + private let keyPath: ReferenceWritableKeyPath + + public init(_ keyPath: ReferenceWritableKeyPath) { + self.keyPath = keyPath + } + + public var wrappedValue: Value { + get { + guard let values = SessionPropertyBinding.values else { + preconditionFailure( + "SessionProperty can only be accessed while a session profile, dynamic instructions, or a Tool is active" + ) + } + return values[keyPath: keyPath] + } + nonmutating set { + guard let values = SessionPropertyBinding.values else { + preconditionFailure( + "SessionProperty can only be accessed while a session profile, dynamic instructions, or a Tool is active" + ) + } + values[keyPath: keyPath] = newValue + } + } + } +} + +extension LanguageModelSession.SessionProperty: @unchecked Sendable where Value: Sendable {} + +extension DynamicInstructions { + public typealias SessionProperty = LanguageModelSession.SessionProperty +} + +extension Tool { + public typealias SessionProperty = LanguageModelSession.SessionProperty +} diff --git a/Sources/AnyLanguageModel/Shared/ToolRoundLimit.swift b/Sources/AnyLanguageModel/Shared/ToolRoundLimit.swift index 4ce43717..b6a1299d 100644 --- a/Sources/AnyLanguageModel/Shared/ToolRoundLimit.swift +++ b/Sources/AnyLanguageModel/Shared/ToolRoundLimit.swift @@ -7,7 +7,7 @@ import Foundation /// Like the MLX and llama.cpp providers, /// a response allows at most ``maximumRounds`` rounds of tool calls /// and stops when a round repeats the previous round's calls exactly. -struct ToolRoundLimit { +final class ToolRoundLimit: @unchecked Sendable { /// A tool call, reduced to what identifies a repeated round. struct Call: Equatable { var name: String @@ -39,8 +39,7 @@ struct ToolRoundLimit { /// The provider name used in error messages. let provider: String - private var rounds = 0 - private var previousCalls: [Call]? + private let state = Locked(State()) init(provider: String) { self.provider = provider @@ -51,24 +50,31 @@ struct ToolRoundLimit { /// - Throws: A decoding failure /// when the round exceeds ``maximumRounds`` /// or repeats the previous round's calls. - mutating func record(_ calls: [Call]) throws { - rounds += 1 - if rounds > Self.maximumRounds { - throw LanguageModelSession.GenerationError.decodingFailure( - .init( - debugDescription: - "Exceeded maximum tool iterations (\(Self.maximumRounds)) while processing \(provider) tool calls." + func record(_ calls: [Call]) throws { + try state.withLock { state in + state.rounds += 1 + if state.rounds > Self.maximumRounds { + throw LanguageModelSession.GenerationError.decodingFailure( + .init( + debugDescription: + "Exceeded maximum tool iterations (\(Self.maximumRounds)) while processing \(provider) tool calls." + ) ) - ) - } - if calls == previousCalls { - throw LanguageModelSession.GenerationError.decodingFailure( - .init( - debugDescription: - "Detected repeated \(provider) tool-call signature and aborted to avoid an infinite tool loop." + } + if calls == state.previousCalls { + throw LanguageModelSession.GenerationError.decodingFailure( + .init( + debugDescription: + "Detected repeated \(provider) tool-call signature and aborted to avoid an infinite tool loop." + ) ) - ) + } + state.previousCalls = calls } - previousCalls = calls + } + + private struct State { + var rounds = 0 + var previousCalls: [Call]? } } diff --git a/Sources/AnyLanguageModelMacros/Plugin.swift b/Sources/AnyLanguageModelMacros/Plugin.swift index 917d2deb..dee43755 100644 --- a/Sources/AnyLanguageModelMacros/Plugin.swift +++ b/Sources/AnyLanguageModelMacros/Plugin.swift @@ -6,5 +6,6 @@ struct AnyLanguageModelMacrosPlugin: CompilerPlugin { let providingMacros: [any Macro.Type] = [ GenerableMacro.self, GuideMacro.self, + SessionPropertyEntryMacro.self, ] } diff --git a/Sources/AnyLanguageModelMacros/SessionPropertyEntryMacro.swift b/Sources/AnyLanguageModelMacros/SessionPropertyEntryMacro.swift new file mode 100644 index 00000000..e6625660 --- /dev/null +++ b/Sources/AnyLanguageModelMacros/SessionPropertyEntryMacro.swift @@ -0,0 +1,74 @@ +import SwiftSyntax +import SwiftSyntaxBuilder +import SwiftSyntaxMacros + +/// Synthesizes a `SessionPropertyKey` and forwards the declared property through +/// `SessionPropertyValues`' type-keyed storage. +public struct SessionPropertyEntryMacro: AccessorMacro, PeerMacro { + public static func expansion( + of node: AttributeSyntax, + providingAccessorsOf declaration: some DeclSyntaxProtocol, + in context: some MacroExpansionContext + ) throws -> [AccessorDeclSyntax] { + guard let name = propertyName(in: declaration) else { return [] } + return [ + """ + get { + self._accessSessionProperty(keyPath: \\.\(raw: name)) + return self[__Key_\(raw: name).self] + } + """, + """ + set { + self._withSessionPropertyMutation(keyPath: \\.\(raw: name)) { + self[__Key_\(raw: name).self] = newValue + } + } + """, + ] + } + + public static func expansion( + of node: AttributeSyntax, + providingPeersOf declaration: some DeclSyntaxProtocol, + in context: some MacroExpansionContext + ) throws -> [DeclSyntax] { + guard + let variable = declaration.as(VariableDeclSyntax.self), + variable.bindings.count == 1, + let binding = variable.bindings.first, + let name = binding.pattern.as(IdentifierPatternSyntax.self)?.identifier.text, + let initializer = binding.initializer?.value + else { return [] } + + let defaultValue: DeclSyntax + if let typeAnnotation = binding.typeAnnotation { + defaultValue = + """ + private struct __Key_\(raw: name): SessionPropertyKey { + static let defaultValue \(typeAnnotation) = \(initializer) + } + """ + } else { + defaultValue = + """ + private struct __Key_\(raw: name): SessionPropertyKey { + static let defaultValue = \(initializer) + } + """ + } + return [defaultValue] + } + + private static func propertyName(in declaration: some DeclSyntaxProtocol) -> String? { + guard + let variable = declaration.as(VariableDeclSyntax.self), + variable.bindings.count == 1, + let binding = variable.bindings.first, + binding.accessorBlock == nil, + let name = binding.pattern.as(IdentifierPatternSyntax.self)?.identifier.text, + binding.initializer != nil + else { return nil } + return name + } +} diff --git a/Tests/AnyLanguageModelTests/APICompatibilityAnyLanguageModelTests.swift b/Tests/AnyLanguageModelTests/APICompatibilityAnyLanguageModelTests.swift index 18bf1cba..c1e27220 100644 --- a/Tests/AnyLanguageModelTests/APICompatibilityAnyLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/APICompatibilityAnyLanguageModelTests.swift @@ -10,18 +10,44 @@ import Testing return false }() + @available(macOS 27.0, iOS 27.0, visionOS 27.0, *) + extension SessionPropertyValues { + @SessionPropertyEntry var compatibilityRevision = 0 + } + @available(macOS 27.0, iOS 27.0, visionOS 27.0, *) private struct CompatibilityDynamicInstructions: DynamicInstructions { let includeDetail: Bool + @SessionProperty(\.compatibilityRevision) private var revision var body: some DynamicInstructions { - Instructions("You are a helpful assistant.") + Instructions("You are a helpful assistant. Revision \(revision).") if includeDetail { Instructions("Include useful detail.") } } } + @available(macOS 27.0, iOS 27.0, visionOS 27.0, *) + private struct CompatibilityDynamicProfile: LanguageModelSession.DynamicProfile { + let model: any LanguageModel + @SessionProperty(\.compatibilityRevision) private var revision + + var body: some LanguageModelSession.DynamicProfile { + Profile { + Instructions("Revision \(revision)") + } + .model(model) + .samplingMode(.greedy) + .temperature(0.2) + .maximumResponseTokens(32) + .reasoningLevel(.light) + .toolCallingMode(.allowed) + .historyTransform { $0 } + .transcriptErrorHandlingPolicy(nil) + } + } + @available(macOS 26.0, iOS 26.0, tvOS 26.0, visionOS 26.0, *) @Test("AnyLanguageModel Drop-In Compatibility", .enabled(if: isSystemLanguageModelAvailable)) func anyLanguageModelCompatibility() async throws { @@ -32,11 +58,17 @@ import Testing ) if #available(macOS 27.0, iOS 27.0, visionOS 27.0, *) { - _ = LanguageModelSession( + let dynamicSession = LanguageModelSession( model: model, dynamicInstructions: CompatibilityDynamicInstructions(includeDetail: true), history: session.transcript ) + dynamicSession.properties.compatibilityRevision = 1 + _ = dynamicSession.properties.history + _ = LanguageModelSession( + profile: CompatibilityDynamicProfile(model: model), + history: session.transcript + ) } let options = GenerationOptions(temperature: 0.7) diff --git a/Tests/AnyLanguageModelTests/APICompatibilityFoundationModelsTests.swift b/Tests/AnyLanguageModelTests/APICompatibilityFoundationModelsTests.swift index 8142648e..532f90ec 100644 --- a/Tests/AnyLanguageModelTests/APICompatibilityFoundationModelsTests.swift +++ b/Tests/AnyLanguageModelTests/APICompatibilityFoundationModelsTests.swift @@ -12,17 +12,43 @@ import Testing #if compiler(>=6.4) #if os(macOS) || os(iOS) || os(visionOS) + @available(macOS 27.0, iOS 27.0, visionOS 27.0, *) + extension SessionPropertyValues { + @SessionPropertyEntry var compatibilityRevision = 0 + } + @available(macOS 27.0, iOS 27.0, visionOS 27.0, *) private struct CompatibilityDynamicInstructions: DynamicInstructions { let includeDetail: Bool + @SessionProperty(\.compatibilityRevision) private var revision var body: some DynamicInstructions { - Instructions("You are a helpful assistant.") + Instructions("You are a helpful assistant. Revision \(revision).") if includeDetail { Instructions("Include useful detail.") } } } + + @available(macOS 27.0, iOS 27.0, visionOS 27.0, *) + private struct CompatibilityDynamicProfile: LanguageModelSession.DynamicProfile { + let model: any LanguageModel + @SessionProperty(\.compatibilityRevision) private var revision + + var body: some LanguageModelSession.DynamicProfile { + Profile { + Instructions("Revision \(revision)") + } + .model(model) + .samplingMode(.greedy) + .temperature(0.2) + .maximumResponseTokens(32) + .reasoningLevel(.light) + .toolCallingMode(.allowed) + .historyTransform { $0 } + .transcriptErrorHandlingPolicy(nil) + } + } #endif #endif @@ -41,11 +67,17 @@ import Testing #if compiler(>=6.4) #if os(macOS) || os(iOS) || os(visionOS) if #available(macOS 27.0, iOS 27.0, visionOS 27.0, *) { - _ = LanguageModelSession( + let dynamicSession = LanguageModelSession( model: model, dynamicInstructions: CompatibilityDynamicInstructions(includeDetail: true), history: session.transcript ) + dynamicSession.properties.compatibilityRevision = 1 + _ = dynamicSession.properties.history + _ = LanguageModelSession( + profile: CompatibilityDynamicProfile(model: model), + history: session.transcript + ) } #endif #endif diff --git a/Tests/AnyLanguageModelTests/AnthropicReasoningTests.swift b/Tests/AnyLanguageModelTests/AnthropicReasoningTests.swift index 8506ba9e..f8f4f7d4 100644 --- a/Tests/AnyLanguageModelTests/AnthropicReasoningTests.swift +++ b/Tests/AnyLanguageModelTests/AnthropicReasoningTests.swift @@ -65,26 +65,16 @@ import Testing result = try await session.streamResponse(to: "Weather").collect() } else { ReasoningURLProtocol.enqueue(json: response(tool: true)) + ReasoningURLProtocol.enqueue(json: response()) result = try await session.respond(to: "Weather") } - #expect(result.content == (streaming ? "Answer" : "")) - #expect(result.transcriptEntries.count == (streaming ? 4 : 3)) + #expect(result.content == "Answer") + #expect(result.transcriptEntries.count == 4) #expect(Set(result.transcriptEntries.map(\.id)).count == result.transcriptEntries.count) - #expect(ReasoningURLProtocol.recordedBodies.count == (streaming ? 2 : 1)) - if streaming { - let body = String(decoding: ReasoningURLProtocol.recordedBodies[1], as: UTF8.self) - #expect(body.contains("opaque-signature")) - #expect(body.contains("tool_result")) - } else { - // Nonstreaming Anthropic keeps its existing one-request tool behavior. - let restored = try JSONDecoder().decode(Transcript.self, from: JSONEncoder().encode(session.transcript)) - ReasoningURLProtocol.enqueue(json: response()) - _ = try await LanguageModelSession(model: model(), tools: [WeatherTool()], transcript: restored) - .respond(to: "Continue") - let body = String(decoding: ReasoningURLProtocol.recordedBodies[1], as: UTF8.self) - #expect(body.contains("opaque-signature")) - #expect(body.contains("tool_result")) - } + #expect(ReasoningURLProtocol.recordedBodies.count == 2) + let body = String(decoding: ReasoningURLProtocol.recordedBodies[1], as: UTF8.self) + #expect(body.contains("opaque-signature")) + #expect(body.contains("tool_result")) } @Test(arguments: [false, true]) diff --git a/Tests/AnyLanguageModelTests/DynamicProfileTests.swift b/Tests/AnyLanguageModelTests/DynamicProfileTests.swift new file mode 100644 index 00000000..1178d592 --- /dev/null +++ b/Tests/AnyLanguageModelTests/DynamicProfileTests.swift @@ -0,0 +1,837 @@ +import Foundation +import Observation +import Testing + +@_spi(Compatibility) @testable import AnyLanguageModel + +extension SessionPropertyValues { + @SessionPropertyEntry var profileRevision = 0 +} + +@Suite("Dynamic profiles") +struct DynamicProfileTests { + @Test func sameSessionReevaluatesModelInstructionsOptionsAndProperties() async throws { + let state = ProfileFixtureState() + let first = ProfileFixtureModel(id: "model-a", state: state) + let second = ProfileFixtureModel(id: "model-b", state: state) + let session = LanguageModelSession( + profile: FixtureProfile(state: state, first: first, second: second) + ) + + _ = try await session.respond(to: "First") + state.selectSecond() + session.properties.profileRevision = 7 + _ = try await session.respond(to: "Second") + + let requests = state.requests.withLock { $0 } + #expect(requests.map(\.modelID) == ["model-a", "model-b"]) + #expect(requests.map(\.instructions) == ["Profile A revision 0", "Profile B revision 7"]) + #expect(requests.map(\.temperature) == [0.2, 0.8]) + #expect(requests.map(\.maximumResponseTokens) == [64, 128]) + #expect(requests.map(\.reasoningLevel) == [.light, .deep]) + #expect(requests.map(\.includeSchemaInPrompt) == [false, true]) + #expect(session.transcript.count == 4) + #expect(!session.transcript.contains { if case .instructions = $0 { true } else { false } }) + } + + @Test func callSiteOptionsOverrideProfileDefaults() async throws { + let state = ProfileFixtureState() + let model = ProfileFixtureModel(id: "model", state: state) + let session = LanguageModelSession( + profile: FixtureProfile(state: state, first: model, second: model) + ) + + _ = try await session.respond( + to: "Override", + options: .init( + temperature: 0.45, + maximumResponseTokens: 17, + toolCallingMode: .required + ) + ) + + let request = try #require(state.requests.withLock { $0.last }) + #expect(request.temperature == 0.45) + #expect(request.maximumResponseTokens == 17) + #expect(request.toolCallingMode == .required) + #expect(request.reasoningLevel == .light) + #expect(request.includeSchemaInPrompt == false) + } + + @Test func lifecycleCallbacksAccumulateInOrderForResponseAndStream() async throws { + let state = LifecycleFixtureState() + let model = LifecycleFixtureModel() + let session = LanguageModelSession(profile: LifecycleFixtureProfile(state: state, model: model)) + + _ = try await session.respond(to: "One") + _ = try await session.streamResponse(to: "Two").collect() + state.useSecond.withLock { $0 = true } + _ = try await session.respond(to: "Three") + + #expect( + state.events.withLock { $0 } == [ + "outer-activate:A", "inner-activate:A", "outer-prompt:A", "inner-prompt:A", + "outer-reasoning:A", "inner-reasoning:A", "outer-response:A", "inner-response:A", + "outer-prompt:A", "inner-prompt:A", "outer-reasoning:A", "inner-reasoning:A", + "outer-response:A", "inner-response:A", "inner-deactivate:A", "outer-deactivate:A", + "outer-activate:B", "inner-activate:B", "outer-prompt:B", "inner-prompt:B", + "outer-reasoning:B", "inner-reasoning:B", "outer-response:B", "inner-response:B", + ] + ) + } + + @MainActor + @Test func appleConcurrencyAndObservationSurfaceCompiles() { + var callbackCount = 0 + _ = LanguageModelSession.Profile { Instructions("Actor callbacks") } + .model(LifecycleFixtureModel()) + .onActivate { callbackCount += 1 } + .onDeactivate { callbackCount += 1 } + .onPrompt { (_: Transcript.Prompt) in callbackCount += 1 } + .onResponse { (_: Transcript.Response) in callbackCount += 1 } + .onReasoning { (_: Transcript.Reasoning) in callbackCount += 1 } + .onToolCall { (_: Transcript.ToolCall) in callbackCount += 1 } + .onToolOutput { (_: Transcript.ToolCall, _: Transcript.ToolOutput) in + callbackCount += 1 + } + + let session = LanguageModelSession(model: LifecycleFixtureModel()) + let _: any Observable = session.properties + } + + @Test func concurrentSnapshotsDispatchEachCallbackOnce() async throws { + let probe = ConcurrentCallbackProbe() + let model = LifecycleFixtureModel() + let session = LanguageModelSession( + profile: LanguageModelSession.Profile { Instructions("Concurrent callback") } + .model(model) + .onResponse { _ in await probe.recordInvocation() } + ) + let context = try await session.resolvedRequestContext(including: [], options: .init()) + let entry = Transcript.Entry.response( + .init(assetIDs: [], segments: [.text(.init(content: "response"))]) + ) + + let first = Task { + try await session.profileDidProduceEntries( + [entry], + requestContext: context, + currentRoundEntries: [entry] + ) + } + await probe.waitUntilFirstInvocation() + let second = Task { + try await session.profileDidProduceEntries( + [entry], + requestContext: context, + currentRoundEntries: [entry] + ) + } + + try await second.value + let invocationCount = await probe.invocationCount + await probe.releaseFirstInvocation() + try await first.value + + #expect(invocationCount == 1) + } + + @Test(arguments: [false, true]) + func callbackErrorsPropagateWithoutCommittingAResponse(streaming: Bool) async throws { + let model = LifecycleFixtureModel() + let session = LanguageModelSession(profile: ThrowingResponseProfile(model: model)) + + if streaming { + await #expect(throws: LifecycleFixtureError.self) { + _ = try await session.streamResponse(to: "Fail").collect() + } + } else { + await #expect(throws: LifecycleFixtureError.self) { + _ = try await session.respond(to: "Fail") + } + } + + #expect(session.transcript.count == 1) + #expect(session.transcript.allSatisfy { if case .prompt = $0 { true } else { false } }) + } + + @Test(arguments: [false, true]) + func promptFailureUsesTheProducingRequestPolicy(streaming: Bool) async throws { + let state = PromptFailureProfileState() + let session = LanguageModelSession( + profile: PromptFailureProfile(state: state, model: LifecycleFixtureModel()) + ) + + if streaming { + await #expect(throws: LifecycleFixtureError.self) { + _ = try await session.streamResponse(to: "Revert").collect() + } + } else { + await #expect(throws: LifecycleFixtureError.self) { + _ = try await session.respond(to: "Revert") + } + } + #expect(session.transcript.isEmpty) + #expect(session.transcriptErrorHandlingPolicy == nil) + + state.useRevert.withLock { $0 = false } + if streaming { + await #expect(throws: LifecycleFixtureError.self) { + _ = try await session.streamResponse(to: "Keep").collect() + } + } else { + await #expect(throws: LifecycleFixtureError.self) { + _ = try await session.respond(to: "Keep") + } + } + #expect(session.transcript.count == 1) + guard case .prompt = session.transcript[0] else { + Issue.record("Expected the second request prompt to be retained") + return + } + #expect(session.transcriptErrorHandlingPolicy == nil) + } + + @Test func dynamicInstructionsAndToolsShareSessionProperties() async throws { + let model = ProfileFixtureModel(id: "model", state: ProfileFixtureState()) + let session = LanguageModelSession( + model: model, + dynamicInstructions: PropertyInstructions() + ) + session.properties.profileRevision = 4 + + let context = session.resolvedRequestContext() + #expect(context.instructions?.description == "Revision 4") + + let tool = PropertyFixtureTool() + _ = try await session.withProfileToolExecution( + requestContext: context, + currentRoundEntries: [] + ) { + try await tool.makeOutputSegments(from: GeneratedContent(properties: [:])) + } + #expect(session.properties.profileRevision == 5) + } + + @Test func rehydrationKeepsCanonicalHistoryAndUsesTheCurrentProfile() async throws { + let oldState = ProfileFixtureState() + let model = ProfileFixtureModel(id: "model", state: oldState) + let oldSession = LanguageModelSession( + profile: FixtureProfile(state: oldState, first: model, second: model) + ) + _ = try await oldSession.respond(to: "Old") + let saved = oldSession.transcript + + let restoredState = ProfileFixtureState() + restoredState.selectSecond() + let restoredModel = ProfileFixtureModel(id: "restored", state: restoredState) + let restored = LanguageModelSession( + profile: FixtureProfile( + state: restoredState, + first: restoredModel, + second: restoredModel + ), + history: saved + ) + _ = try await restored.respond(to: "New") + + #expect(restoredState.requests.withLock { $0.last?.instructions } == "Profile B revision 0") + #expect(restored.transcript.count == saved.count + 2) + #expect(!restored.transcript.contains { if case .instructions = $0 { true } else { false } }) + } + + @Test func transformedCurrentRoundDoesNotReplaceCanonicalTranscript() async throws { + let state = ProfileFixtureState() + let model = ProfileFixtureModel(id: "model-a", state: state) + let session = LanguageModelSession( + profile: FixtureProfile(state: state, first: model, second: model) + ) + _ = try await session.respond(to: "First") + let durableBefore = session.transcript + let call = Transcript.ToolCall( + id: "call-1", + toolName: "fixture", + arguments: GeneratedContent(properties: [:]) + ) + let output = Transcript.ToolOutput( + id: call.id, + toolName: call.toolName, + segments: [.text(.init(content: "output"))] + ) + let inFlight: [Transcript.Entry] = [ + .toolCalls(.init([call])), + .toolOutput(output), + ] + + let context = try await session.resolvedRequestContext( + including: inFlight, + options: .init() + ) + + #expect(context.transcript.contains { $0.id == output.id }) + #expect(state.transformedHistory.withLock { $0.last?.contains(output.id) } == true) + #expect(session.transcript == durableBefore) + } + + @Test func toolCallbacksUseProducingSnapshotAndSessionPropertiesStayIsolated() async throws { + let state = ProfileFixtureState() + let model = ProfileFixtureModel(id: "model-a", state: state) + let first = LanguageModelSession( + profile: FixtureProfile(state: state, first: model, second: model) + ) + let second = LanguageModelSession( + profile: FixtureProfile(state: state, first: model, second: model) + ) + let call = Transcript.ToolCall( + id: "call", + toolName: "fixture", + arguments: GeneratedContent(properties: [:]) + ) + let output = Transcript.ToolOutput( + id: call.id, + toolName: call.toolName, + segments: [.text(.init(content: "done"))] + ) + let request = try await first.resolvedRequestContext(including: [], options: .init()) + let unrelated = Transcript.Entry.prompt( + .init(id: "unrelated", segments: [.text(.init(content: "Other request"))]) + ) + first.properties.history = [unrelated] + let callEntry = Transcript.Entry.toolCalls(.init([call])) + let outputEntry = Transcript.Entry.toolOutput(output) + + try await first.profileWillExecuteToolCall( + call, + requestContext: request, + currentRoundEntries: [callEntry] + ) + try await first.profileDidProduceToolOutput( + for: call, + output: output, + requestContext: request, + currentRoundEntries: [callEntry, outputEntry] + ) + let toolHistory = await first.withProfileToolExecution( + requestContext: request, + currentRoundEntries: [callEntry, outputEntry] + ) { + first.properties.history.map(\.id) + } + + #expect(first.properties.profileRevision == 1) + #expect(second.properties.profileRevision == 0) + #expect(state.callbackEvents.withLock { $0 } == ["call:call", "output:call"]) + #expect( + state.callbackHistories.withLock { $0 } + == [[callEntry.id], [callEntry.id, outputEntry.id]] + ) + #expect(toolHistory == [callEntry.id, outputEntry.id]) + } + + @Test func callbackHistoryDoesNotDuplicateCanonicalEntries() async throws { + let state = ProfileFixtureState() + let model = ProfileFixtureModel(id: "model-a", state: state) + let call = Transcript.ToolCall( + id: "call", + toolName: "fixture", + arguments: GeneratedContent(properties: [:]) + ) + let output = Transcript.ToolOutput( + id: call.id, + toolName: call.toolName, + segments: [.text(.init(content: "done"))] + ) + let callEntry = Transcript.Entry.toolCalls(.init([call])) + let outputEntry = Transcript.Entry.toolOutput(output) + let session = LanguageModelSession( + profile: FixtureProfile(state: state, first: model, second: model), + history: [callEntry] + ) + let request = try await session.resolvedRequestContext(including: [], options: .init()) + + try await session.profileDidProduceToolOutput( + for: call, + output: output, + requestContext: request, + currentRoundEntries: [callEntry, outputEntry] + ) + let toolHistory = await session.withProfileToolExecution( + requestContext: request, + currentRoundEntries: [callEntry, outputEntry] + ) { + session.properties.history.map(\.id) + } + + #expect(state.callbackHistories.withLock { $0 } == [[callEntry.id, outputEntry.id]]) + #expect(toolHistory == [callEntry.id, outputEntry.id]) + } + + @Test func continuationHistoryMutationDoesNotCommitPendingEntries() async throws { + let state = ProfileFixtureState() + let model = ProfileFixtureModel(id: "model", state: state) + let session = LanguageModelSession(profile: HistoryWritingProfile(model: model)) + _ = try await session.respond(to: "Durable") + let durableBefore = session.transcript + let call = Transcript.ToolCall( + id: "pending-call", + toolName: "fixture", + arguments: GeneratedContent(properties: [:]) + ) + let output = Transcript.ToolOutput( + id: call.id, + toolName: call.toolName, + segments: [.text(.init(content: "pending"))] + ) + let pending: [Transcript.Entry] = [ + .toolCalls(.init([call])), + .toolOutput(output), + ] + + let context = try await session.resolvedRequestContext( + including: pending, + options: .init() + ) + + #expect(context.transcript.contains { $0.id == output.id }) + #expect(session.transcript == durableBefore) + } + + @Test func modelHandoffIsSkippedForTheInitialRequest() async throws { + let session = LanguageModelSession( + profile: LifecycleFixtureProfile( + state: LifecycleFixtureState(), + model: LifecycleFixtureModel() + ) + ) + let context = try await session.resolvedRequestContext(including: [], options: .init()) + let response: LanguageModelSession.Response? = try await session.profileContinuationResponse( + ifSelectedModelDiffersFrom: UUID(), + requestContext: context, + prompt: Prompt("Initial"), + generating: String.self, + schema: String.generationSchema, + includeSchemaInPrompt: true, + prefixEntries: [], + prefixUsage: .zero, + prefixText: "" + ) + let stream: LanguageModelSession.ResponseStream? = session.profileContinuationStream( + ifSelectedModelDiffersFrom: UUID(), + requestContext: context, + prompt: Prompt("Initial"), + generating: String.self, + schema: String.generationSchema, + includeSchemaInPrompt: true, + prefixEntries: [], + prefixUsage: .zero, + prefixText: "" + ) + + #expect(response == nil) + #expect(stream == nil) + } + + @Test func profileErrorPolicyDoesNotLeakIntoTheNextProfile() async throws { + let state = LifecycleFixtureState() + let session = LanguageModelSession( + profile: ErrorPolicyProfile(state: state, model: FailingProfileModel()) + ) + session.transcriptErrorHandlingPolicy = .preserveTranscript + + await #expect(throws: LifecycleFixtureError.self) { + _ = try await session.respond(to: "Revert") + } + #expect(session.transcript.isEmpty) + + state.useSecond.withLock { $0 = true } + await #expect(throws: LifecycleFixtureError.self) { + _ = try await session.streamResponse(to: "Preserve").collect() + } + #expect(session.transcript.count == 1) + #expect(session.transcript.allSatisfy { if case .prompt = $0 { true } else { false } }) + #expect(session.transcriptErrorHandlingPolicy == .preserveTranscript) + } +} + +private struct FixtureProfile: LanguageModelSession.DynamicProfile, @unchecked Sendable { + let state: ProfileFixtureState + let first: ProfileFixtureModel + let second: ProfileFixtureModel + @SessionProperty(\.profileRevision) private var revision + @SessionProperty(\.history) private var history + + var body: some LanguageModelSession.DynamicProfile { + if state.usesSecond { + LanguageModelSession.Profile { + Instructions("Profile B revision \(revision)") + FixtureProfileTool() + } + .model(second) + .temperature(0.8) + .maximumResponseTokens(128) + .reasoningLevel(.deep) + .includeSchemaInPrompt(true) + .historyTransform(recordingTransform) + .onToolCall(perform: recordCall) + .onToolOutput(perform: recordOutput) + } else { + LanguageModelSession.Profile { + Instructions("Profile A revision \(revision)") + FixtureProfileTool() + } + .model(first) + .temperature(0.2) + .maximumResponseTokens(64) + .reasoningLevel(.light) + .includeSchemaInPrompt(false) + .historyTransform(recordingTransform) + .onToolCall(perform: recordCall) + .onToolOutput(perform: recordOutput) + } + } + + private func recordingTransform(_ entries: [Transcript.Entry]) -> [Transcript.Entry] { + state.transformedHistory.withLock { $0.append(entries.map(\.id)) } + return entries + } + + private func recordCall(_ call: Transcript.ToolCall) async throws { + state.callbackEvents.withLock { $0.append("call:\(call.id)") } + state.callbackHistories.withLock { $0.append(history.map(\.id)) } + } + + private func recordOutput( + _ call: Transcript.ToolCall, + _ output: Transcript.ToolOutput + ) async throws { + revision += 1 + state.callbackEvents.withLock { $0.append("output:\(output.id)") } + state.callbackHistories.withLock { $0.append(history.map(\.id)) } + } +} + +private struct FixtureProfileTool: Tool { + let name = "fixture" + let description = "Fixture" + + @Generable + struct Arguments {} + + func call(arguments: Arguments) async throws -> String { "done" } +} + +private struct ProfileFixtureModel: LanguageModel { + typealias UnavailableReason = Never + let id: String + let state: ProfileFixtureState + + func respond( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) async throws -> LanguageModelSession.Response where Content: Generable { + let context = try await session.resolvedRequestContext(including: [], options: options) + state.requests.withLock { + $0.append( + .init( + modelID: id, + instructions: context.instructions?.description, + temperature: context.options.temperature, + maximumResponseTokens: context.options.maximumResponseTokens, + toolCallingMode: context.options.toolCallingMode, + reasoningLevel: context.contextOptions.reasoningLevel, + includeSchemaInPrompt: includeSchemaInPrompt + ) + ) + } + let raw = GeneratedContent("response") + return .init(content: try Content(raw), rawContent: raw, transcriptEntries: []) + } + + func streamResponse( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) -> sending LanguageModelSession.ResponseStream where Content: Generable { + let stream = AsyncThrowingStream.Snapshot, any Error> { + continuation in + let task = Task { + do { + let response: LanguageModelSession.Response = try await respond( + within: session, + to: prompt, + generating: type, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ) + continuation.yield( + .init( + content: response.content.asPartiallyGenerated(), + rawContent: response.rawContent + ) + ) + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { _ in task.cancel() } + } + return .init(stream: stream) + } +} + +private final class ProfileFixtureState: @unchecked Sendable { + struct Request: Equatable { + let modelID: String + let instructions: String? + let temperature: Double? + let maximumResponseTokens: Int? + let toolCallingMode: GenerationOptions.ToolCallingMode? + let reasoningLevel: ContextOptions.ReasoningLevel? + let includeSchemaInPrompt: Bool + } + + private let selection = Locked(false) + let requests = Locked<[Request]>([]) + let transformedHistory = Locked<[[String]]>([]) + let callbackEvents = Locked<[String]>([]) + let callbackHistories = Locked<[[String]]>([]) + + var usesSecond: Bool { selection.withLock { $0 } } + func selectSecond() { selection.withLock { $0 = true } } +} + +private struct PropertyInstructions: DynamicInstructions, @unchecked Sendable { + @SessionProperty(\.profileRevision) private var revision + + var body: some DynamicInstructions { + Instructions("Revision \(revision)") + } +} + +private struct PropertyFixtureTool: Tool, @unchecked Sendable { + let name = "property" + let description = "Updates a session property" + @SessionProperty(\.profileRevision) private var revision + + @Generable struct Arguments {} + + func call(arguments: Arguments) async throws -> String { + revision += 1 + return "updated" + } +} + +private enum LifecycleFixtureError: Error { + case rejected +} + +private actor ConcurrentCallbackProbe { + private(set) var invocationCount = 0 + private var firstInvocationWaiter: CheckedContinuation? + private var firstInvocationRelease: CheckedContinuation? + + func recordInvocation() async { + invocationCount += 1 + guard invocationCount == 1 else { return } + firstInvocationWaiter?.resume() + firstInvocationWaiter = nil + await withCheckedContinuation { firstInvocationRelease = $0 } + } + + func waitUntilFirstInvocation() async { + guard invocationCount == 0 else { return } + await withCheckedContinuation { firstInvocationWaiter = $0 } + } + + func releaseFirstInvocation() { + firstInvocationRelease?.resume() + firstInvocationRelease = nil + } +} + +private final class LifecycleFixtureState: @unchecked Sendable { + let useSecond = Locked(false) + let events = Locked<[String]>([]) + + func record(_ value: String) { + events.withLock { $0.append(value) } + } +} + +private struct LifecycleFixtureProfile: LanguageModelSession.DynamicProfile, @unchecked Sendable { + let state: LifecycleFixtureState + let model: LifecycleFixtureModel + + var body: some LanguageModelSession.DynamicProfile { + if state.useSecond.withLock({ $0 }) { + profile(label: "B") + } else { + profile(label: "A") + } + } + + private func profile(label: String) -> some LanguageModelSession.DynamicProfile { + LanguageModelSession.Profile { Instructions("Lifecycle \(label)") } + .model(model) + .onActivate { state.record("inner-activate:\(label)") } + .onDeactivate { state.record("inner-deactivate:\(label)") } + .onPrompt { state.record("inner-prompt:\(label)") } + .onReasoning { state.record("inner-reasoning:\(label)") } + .onResponse { state.record("inner-response:\(label)") } + .onActivate { state.record("outer-activate:\(label)") } + .onDeactivate { state.record("outer-deactivate:\(label)") } + .onPrompt { state.record("outer-prompt:\(label)") } + .onReasoning { state.record("outer-reasoning:\(label)") } + .onResponse { state.record("outer-response:\(label)") } + } +} + +private struct ThrowingResponseProfile: LanguageModelSession.DynamicProfile { + let model: LifecycleFixtureModel + + var body: some LanguageModelSession.DynamicProfile { + LanguageModelSession.Profile { Instructions("Throw after response") } + .model(model) + .onResponse { throw LifecycleFixtureError.rejected } + } +} + +private struct HistoryWritingProfile: LanguageModelSession.DynamicProfile { + let model: ProfileFixtureModel + @SessionProperty(\.history) private var history + + var body: some LanguageModelSession.DynamicProfile { + history = history.filter { entry in + if case .toolCalls = entry { return false } + if case .toolOutput = entry { return false } + return true + } + return LanguageModelSession.Profile { Instructions("History") } + .model(model) + } +} + +private struct ErrorPolicyProfile: LanguageModelSession.DynamicProfile, @unchecked Sendable { + let state: LifecycleFixtureState + let model: FailingProfileModel + + var body: some LanguageModelSession.DynamicProfile { + if state.useSecond.withLock({ $0 }) { + LanguageModelSession.Profile { Instructions("Caller policy") } + .model(model) + } else { + LanguageModelSession.Profile { Instructions("Profile policy") } + .model(model) + .transcriptErrorHandlingPolicy(.revertTranscript) + } + } +} + +private final class PromptFailureProfileState: @unchecked Sendable { + let useRevert = Locked(true) +} + +private struct PromptFailureProfile: LanguageModelSession.DynamicProfile, @unchecked Sendable { + let state: PromptFailureProfileState + let model: LifecycleFixtureModel + + var body: some LanguageModelSession.DynamicProfile { + if state.useRevert.withLock({ $0 }) { + LanguageModelSession.Profile { Instructions("Revert prompt failures") } + .model(model) + .transcriptErrorHandlingPolicy(.revertTranscript) + .onPrompt { throw LifecycleFixtureError.rejected } + } else { + LanguageModelSession.Profile { Instructions("Keep prompt failures") } + .model(model) + .onPrompt { throw LifecycleFixtureError.rejected } + } + } +} + +private struct FailingProfileModel: LanguageModel { + typealias UnavailableReason = Never + + func respond( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) async throws -> LanguageModelSession.Response where Content: Generable { + throw LifecycleFixtureError.rejected + } + + func streamResponse( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) -> sending LanguageModelSession.ResponseStream where Content: Generable { + .init(stream: AsyncThrowingStream { $0.finish(throwing: LifecycleFixtureError.rejected) }) + } +} + +private struct LifecycleFixtureModel: LanguageModel { + typealias UnavailableReason = Never + + func respond( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) async throws -> LanguageModelSession.Response where Content: Generable { + let raw = GeneratedContent("response") + let reasoning = Transcript.Entry.reasoning( + .init(segments: [.text(.init(content: "reasoning"))]) + ) + return .init( + content: try Content(raw), + rawContent: raw, + transcriptEntries: [reasoning] + ) + } + + func streamResponse( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) -> sending LanguageModelSession.ResponseStream where Content: Generable { + let stream = AsyncThrowingStream.Snapshot, any Error> { + continuation in + let task = Task { + do { + let response: LanguageModelSession.Response = try await respond( + within: session, + to: prompt, + generating: type, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ) + continuation.yield( + .init( + content: response.content.asPartiallyGenerated(), + rawContent: response.rawContent, + transcriptEntries: response.transcriptEntries + ) + ) + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { _ in task.cancel() } + } + return .init(stream: stream) + } +} diff --git a/Tests/AnyLanguageModelTests/FoundationModelsGenerationOptionsTests.swift b/Tests/AnyLanguageModelTests/FoundationModelsGenerationOptionsTests.swift index 06c723fc..976903f5 100644 --- a/Tests/AnyLanguageModelTests/FoundationModelsGenerationOptionsTests.swift +++ b/Tests/AnyLanguageModelTests/FoundationModelsGenerationOptionsTests.swift @@ -67,6 +67,42 @@ import Testing #endif #expect(converted.temperature == nil) #expect(converted.maximumResponseTokens == nil) + #if compiler(>=6.4) && !os(tvOS) + if #available(macOS 27, iOS 27, visionOS 27, watchOS 27, *) { + #expect(converted.toolCallingMode == nil) + } + #endif } + + #if compiler(>=6.4) && !os(tvOS) + @available(macOS 27, iOS 27, visionOS 27, watchOS 27, *) + @Test( + "Forwards tool calling modes", + arguments: [ + ( + AnyLanguageModel.GenerationOptions.ToolCallingMode.allowed, + FoundationModels.GenerationOptions.ToolCallingMode.Kind.allowed + ), + ( + AnyLanguageModel.GenerationOptions.ToolCallingMode.required, + FoundationModels.GenerationOptions.ToolCallingMode.Kind.required + ), + ( + AnyLanguageModel.GenerationOptions.ToolCallingMode.disallowed, + FoundationModels.GenerationOptions.ToolCallingMode.Kind.disallowed + ), + ] + ) + func forwardsToolCallingMode( + mode: AnyLanguageModel.GenerationOptions.ToolCallingMode, + expected: FoundationModels.GenerationOptions.ToolCallingMode.Kind + ) { + let converted = AnyLanguageModel.GenerationOptions( + toolCallingMode: mode + ).toFoundationModels() + + #expect(converted.toolCallingMode?.kind == expected) + } + #endif } #endif diff --git a/Tests/AnyLanguageModelTests/ProviderUsageTests.swift b/Tests/AnyLanguageModelTests/ProviderUsageTests.swift index c4331da1..3b71d2d0 100644 --- a/Tests/AnyLanguageModelTests/ProviderUsageTests.swift +++ b/Tests/AnyLanguageModelTests/ProviderUsageTests.swift @@ -9,32 +9,34 @@ import Testing enum Provider: CaseIterable, Equatable, Sendable { case chat, responses, openResponses, anthropic, gemini, ollama - func makeSession(tools: [any Tool] = []) -> LanguageModelSession { + func makeModel() -> any LanguageModel { let http = UsageURLProtocol.makeSession() - let model: any LanguageModel switch self { case .chat, .responses: - model = OpenAILanguageModel( + return OpenAILanguageModel( apiKey: "test", model: "test", apiVariant: self == .chat ? .chatCompletions : .responses, session: http ) case .openResponses: - model = OpenResponsesLanguageModel( + return OpenResponsesLanguageModel( baseURL: URL(string: "https://example.com/v1")!, apiKey: "test", model: "test", session: http ) case .anthropic: - model = AnthropicLanguageModel(apiKey: "test", model: "test", session: http) + return AnthropicLanguageModel(apiKey: "test", model: "test", session: http) case .gemini: - model = GeminiLanguageModel(apiKey: "test", model: "test", session: http) + return GeminiLanguageModel(apiKey: "test", model: "test", session: http) case .ollama: - model = OllamaLanguageModel(model: "test", session: http) + return OllamaLanguageModel(model: "test", session: http) } - return LanguageModelSession(model: model, tools: tools) + } + + func makeSession(tools: [any Tool] = []) -> LanguageModelSession { + LanguageModelSession(model: makeModel(), tools: tools) } var counts: [String: Any] { @@ -460,6 +462,127 @@ import Testing #expect(String(decoding: followUp, as: UTF8.self).contains("Checking. ")) } + @Test(arguments: Provider.allCases, [false, true]) + func providerProfileReevaluatesBeforeEveryToolContinuation( + _ provider: Provider, + _ streaming: Bool + ) async throws { + UsageURLProtocol.reset() + if streaming { + UsageURLProtocol.enqueue(json: try provider.toolStream()) + UsageURLProtocol.enqueue(json: try provider.stream(text: "Profile done")) + } else { + UsageURLProtocol.enqueue(json: try Self.json(provider.response(text: "", tool: true))) + UsageURLProtocol.enqueue(json: try Self.json(provider.response(text: "Profile done"))) + } + let model = provider.makeModel() + let state = ProviderProfileState() + let session = LanguageModelSession( + profile: ProviderProfile(state: state, model: model) + ) + + let response = + try await streaming + ? session.streamResponse(to: "Weather?").collect() + : session.respond(to: "Weather?") + + #expect(response.content == "Profile done") + #expect(state.executions.withLock { $0 } == ["Paris"]) + #expect(state.transformSawOutput.withLock { $0 } == [false, true]) + #expect( + state.events.withLock { $0 } == [ + "call:A", "execute:A", "output:A", + ] + ) + #expect(UsageURLProtocol.recordedBodies.count == 2) + let first = String(decoding: UsageURLProtocol.recordedBodies[0], as: UTF8.self) + let second = String(decoding: UsageURLProtocol.recordedBodies[1], as: UTF8.self) + #expect(first.contains("Profile A")) + #expect(first.contains("getWeather")) + #expect(first.contains("0.2")) + #expect(second.contains("Profile B")) + #expect(second.contains("otherTool")) + #expect(second.contains("0.8")) + } + + @Test(arguments: Provider.allCases, [false, true]) + func providerProfileCanSwitchModelForToolContinuation( + _ provider: Provider, + _ streaming: Bool + ) async throws { + UsageURLProtocol.reset() + if streaming { + UsageURLProtocol.enqueue(json: try provider.toolStream()) + } else { + UsageURLProtocol.enqueue(json: try Self.json(provider.response(text: "", tool: true))) + } + let state = ProviderProfileState() + let session = LanguageModelSession( + profile: ProviderProfile( + state: state, + model: provider.makeModel(), + continuationModel: ProviderProfileTerminalModel(state: state) + ) + ) + + let response = + try await streaming + ? session.streamResponse(to: "Weather?").collect() + : session.respond(to: "Weather?") + + #expect(response.content == "Switched") + #expect(UsageURLProtocol.recordedBodies.count == 1) + #expect(response.transcriptEntries.contains { if case .toolCalls = $0 { true } else { false } }) + #expect(response.transcriptEntries.contains { if case .toolOutput = $0 { true } else { false } }) + let selected = try #require(state.selectedRequests.withLock { $0.last }) + #expect(selected.instructions == "Profile B") + #expect(selected.temperature == 0.8) + #expect(selected.tools == ["otherTool"]) + #expect(selected.sawToolOutput) + } + + @Test(arguments: [false, true]) + func providerModelHandoffsShareToolRoundLimit(_ streaming: Bool) async throws { + let provider = Provider.chat + UsageURLProtocol.reset() + let limit = ToolRoundLimit.maximumRounds + for round in 0 ... limit { + if streaming { + UsageURLProtocol.enqueue( + json: try provider.toolStream( + city: "City \(round)", + callID: "call_\(round)" + ) + ) + } else { + let response = try Self.json(provider.response(text: "", tool: true)) + .replacingOccurrences(of: "Paris", with: "City \(round)") + .replacingOccurrences(of: "call_1", with: "call_\(round)") + UsageURLProtocol.enqueue(json: response) + } + } + let state = AlternatingProviderProfileState() + let session = LanguageModelSession( + profile: AlternatingProviderProfile( + state: state, + first: provider.makeModel(), + second: provider.makeModel() + ) + ) + + let description = await Self.decodingFailureDescription { + if streaming { + _ = try await session.streamResponse(to: "Weather?").collect() + } else { + _ = try await session.respond(to: "Weather?") + } + } + + #expect(description?.contains("Exceeded maximum tool iterations (\(limit))") == true) + #expect(state.executions.withLock { $0.count } == limit) + #expect(UsageURLProtocol.recordedBodies.count == limit + 1) + } + @Test(arguments: Provider.allCases, [false, true]) func streamedToolsThenStructuredContent(_ provider: Provider, _ dynamicSchema: Bool) async throws { UsageURLProtocol.reset() @@ -905,6 +1028,186 @@ import Testing #expect(response.usage == provider.expected) #expect(UsageURLProtocol.recordedBodies.count == 1) } + + private final class ProviderProfileState: @unchecked Sendable { + struct SelectedRequest: Sendable { + let instructions: String? + let temperature: Double? + let tools: [String] + let sawToolOutput: Bool + } + + let useSecond = Locked(false) + let executions = Locked<[String]>([]) + let transformSawOutput = Locked<[Bool]>([]) + let events = Locked<[String]>([]) + let selectedRequests = Locked<[SelectedRequest]>([]) + } + + private struct ProviderProfile: LanguageModelSession.DynamicProfile, @unchecked Sendable { + let state: ProviderProfileState + let model: any LanguageModel + var continuationModel: (any LanguageModel)? = nil + + var body: some LanguageModelSession.DynamicProfile { + if state.useSecond.withLock({ $0 }) { + LanguageModelSession.Profile { + Instructions("Profile B") + ProviderProfileOtherTool() + } + .model(continuationModel ?? model) + .temperature(0.8) + .historyTransform(transform) + .onToolCall { _ in state.events.withLock { $0.append("call:B") } } + .onToolOutput { _, _ in state.events.withLock { $0.append("output:B") } } + } else { + LanguageModelSession.Profile { + Instructions("Profile A") + ProviderProfileWeatherTool(state: state) + } + .model(model) + .temperature(0.2) + .historyTransform(transform) + .onToolCall { _ in state.events.withLock { $0.append("call:A") } } + .onToolOutput { _, _ in state.events.withLock { $0.append("output:A") } } + } + } + + private func transform(_ entries: [Transcript.Entry]) -> [Transcript.Entry] { + state.transformSawOutput.withLock { values in + values.append(entries.contains { if case .toolOutput = $0 { true } else { false } }) + } + return entries + } + } + + private final class AlternatingProviderProfileState: @unchecked Sendable { + let useSecond = Locked(false) + let executions = Locked<[String]>([]) + + func toggleModel() { + useSecond.withLock { $0.toggle() } + } + } + + private struct AlternatingProviderProfile: LanguageModelSession.DynamicProfile, @unchecked Sendable { + let state: AlternatingProviderProfileState + let first: any LanguageModel + let second: any LanguageModel + + var body: some LanguageModelSession.DynamicProfile { + if state.useSecond.withLock({ $0 }) { + LanguageModelSession.Profile { + AlternatingProviderProfileTool(state: state) + } + .model(second) + } else { + LanguageModelSession.Profile { + AlternatingProviderProfileTool(state: state) + } + .model(first) + } + } + } + + private struct AlternatingProviderProfileTool: Tool { + let name = "getWeather" + let description = "Get weather and switch models" + let state: AlternatingProviderProfileState + + func call(arguments: WeatherTool.Arguments) async throws -> String { + state.executions.withLock { $0.append(arguments.city) } + state.toggleModel() + return "Sunny" + } + } + + private struct ProviderProfileWeatherTool: Tool { + let name = "getWeather" + let description = "Get weather" + let state: ProviderProfileState + + func call(arguments: WeatherTool.Arguments) async throws -> String { + state.executions.withLock { $0.append(arguments.city) } + state.events.withLock { $0.append("execute:A") } + state.useSecond.withLock { $0 = true } + return "Sunny" + } + } + + struct ProviderProfileOtherTool: Tool { + @Generable struct Arguments {} + let name = "otherTool" + let description = "Only available in profile B" + func call(arguments: Arguments) async throws -> String { "unused" } + } + + private struct ProviderProfileTerminalModel: LanguageModel { + typealias UnavailableReason = Never + let state: ProviderProfileState + + func respond( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) async throws -> LanguageModelSession.Response where Content: Generable { + let context = session.resolvedRequestContext() + state.selectedRequests.withLock { + $0.append( + .init( + instructions: context.instructions?.description, + temperature: context.options.temperature, + tools: context.tools.map(\.name), + sawToolOutput: context.transcript.contains { + if case .toolOutput = $0 { true } else { false } + } + ) + ) + } + let raw = GeneratedContent("Switched") + return .init( + content: try Content(raw), + rawContent: raw, + transcriptEntries: [] + ) + } + + func streamResponse( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) -> sending LanguageModelSession.ResponseStream where Content: Generable { + let stream = AsyncThrowingStream.Snapshot, any Error> { + continuation in + let task = Task { + do { + let response: LanguageModelSession.Response = try await respond( + within: session, + to: prompt, + generating: type, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ) + continuation.yield( + .init( + content: response.content.asPartiallyGenerated(), + rawContent: response.rawContent + ) + ) + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { _ in task.cancel() } + } + return .init(stream: stream) + } + } } /// A `URLProtocol` that answers requests from a queue of canned responses /// and records every request body it sees, diff --git a/Tests/AnyLanguageModelTests/RequestOptionsTests.swift b/Tests/AnyLanguageModelTests/RequestOptionsTests.swift new file mode 100644 index 00000000..7822a614 --- /dev/null +++ b/Tests/AnyLanguageModelTests/RequestOptionsTests.swift @@ -0,0 +1,66 @@ +import Foundation +import Testing + +@testable import AnyLanguageModel + +@Suite("Foundation Models 27 request options") +struct RequestOptionsTests { + @Test func contextOptionsRetainValues() { + #expect(ContextOptions() == ContextOptions()) + #expect( + ContextOptions(includeSchemaInPrompt: true, reasoningLevel: .deep) + == ContextOptions(includeSchemaInPrompt: true, reasoningLevel: .deep) + ) + #expect( + ContextOptions(reasoningLevel: .custom("provider-level")) + != ContextOptions(reasoningLevel: .moderate) + ) + } + + @Test( + arguments: [ + (GenerationOptions.ToolCallingMode.allowed, GenerationOptions.ToolCallingMode.Kind.allowed), + (.required, .required), + (.disallowed, .disallowed), + ] + ) + func toolCallingModesExposeTheirKind( + mode: GenerationOptions.ToolCallingMode, + expected: GenerationOptions.ToolCallingMode.Kind + ) { + #expect(mode.kind == expected) + } + + @Test func generationOptionsKeepOldAndNewSamplingSpellingsInSync() { + var options = GenerationOptions(samplingMode: .greedy, toolCallingMode: .required) + #expect(options.sampling == .greedy) + #expect(options.toolCallingMode == .required) + + options.sampling = .random(top: 4, seed: 7) + #expect(options.samplingMode == .random(top: 4, seed: 7)) + + options.samplingMode = .random(probabilityThreshold: 0.8, seed: 9) + #expect(options.sampling == .random(probabilityThreshold: 0.8, seed: 9)) + } + + @Test func transcriptRoundTripsToolCallingMode() throws { + let options = GenerationOptions( + samplingMode: .greedy, + temperature: 0.5, + maximumResponseTokens: 64, + toolCallingMode: .required + ) + let prompt = Transcript.Prompt( + id: "prompt", + segments: [.text(.init(id: "text", content: "Hello"))], + options: options + ) + + let encoder = JSONEncoder() + encoder.outputFormatting = .sortedKeys + let data = try encoder.encode(prompt) + #expect(String(decoding: data, as: UTF8.self).contains(#""toolCallingMode":"required""#)) + let decoded = try JSONDecoder().decode(Transcript.Prompt.self, from: data) + #expect(decoded.options == options) + } +} diff --git a/Tests/AnyLanguageModelTests/SessionPropertyTests.swift b/Tests/AnyLanguageModelTests/SessionPropertyTests.swift new file mode 100644 index 00000000..7cb7db2d --- /dev/null +++ b/Tests/AnyLanguageModelTests/SessionPropertyTests.swift @@ -0,0 +1,285 @@ +import Foundation +import Observation +import Testing + +@testable import AnyLanguageModel + +extension SessionPropertyValues { + @SessionPropertyEntry var revision = 0 + @SessionPropertyEntry var optionalSelection: String? = nil + @SessionPropertyEntry var items: [Int] = [] + @SessionPropertyEntry var mode: SessionPropertyMode = .first +} + +enum SessionPropertyMode { + case first + case second +} + +@Suite("Session properties") +struct SessionPropertyTests { + @Test func dynamicInstructionsReadAndUpdateSessionProperties() { + let session = LanguageModelSession( + model: SessionPropertyFixtureModel(), + dynamicInstructions: SessionPropertyFixtureInstructions() + ) + + #expect(session.properties.revision == 0) + let first = session.resolvedRequestContext() + let second = session.resolvedRequestContext() + + #expect(first.instructions?.description == "Revision 0") + #expect(second.instructions?.description == "Revision 1") + #expect(session.properties.revision == 2) + } + + @Test func historyIsInstructionFreeAndUsesTheCurrentRequestSnapshot() { + let prompt = Transcript.Entry.prompt( + Transcript.Prompt(segments: [.text(.init(content: "Earlier"))]) + ) + let session = LanguageModelSession( + model: SessionPropertyFixtureModel(), + dynamicInstructions: SessionPropertyHistoryInstructions(), + history: [prompt] + ) + + let context = session.resolvedRequestContext() + + #expect(context.instructions?.description == "History count 1") + #expect(context.transcript.count == 2) + #expect(session.properties.history == [prompt]) + } + + @Test func requestUsesOneHistorySnapshotDuringDynamicResolution() async { + let initial = Transcript.Entry.prompt( + Transcript.Prompt(segments: [.text(.init(content: "Initial"))]) + ) + let replacement = Transcript.Entry.prompt( + Transcript.Prompt(segments: [.text(.init(content: "Replacement"))]) + ) + let (started, startedContinuation) = AsyncStream.makeStream() + let resume = DispatchSemaphore(value: 0) + let session = LanguageModelSession( + model: SessionPropertyFixtureModel(), + dynamicInstructions: BlockingHistoryInstructions( + started: startedContinuation, + resume: resume + ), + history: [initial] + ) + + let request = Task.detached { session.resolvedRequestContext() } + for await _ in started.prefix(1) {} + session.properties.history = [replacement] + resume.signal() + let context = await request.value + + #expect(context.instructions?.description == "History count 1") + #expect( + context.transcript.filter { entry in + if case .instructions = entry { return false } + return true + } == [initial] + ) + #expect(session.properties.history == [replacement]) + } + + @Test func replacingHistoryNotifiesTranscriptObservers() { + let session = LanguageModelSession(model: SessionPropertyFixtureModel()) + let changed = Locked(false) + let prompt = Transcript.Entry.prompt( + Transcript.Prompt(segments: [.text(.init(content: "Replacement"))]) + ) + + withObservationTracking { + _ = session.transcript + } onChange: { + changed.withLock { $0 = true } + } + session.properties.history = [prompt] + + #expect(changed.withLock { $0 }) + } + + @Test func generatedPropertiesParticipateInObservation() { + let session = LanguageModelSession(model: SessionPropertyFixtureModel()) + let changed = Locked(false) + + withObservationTracking { + _ = session.properties.revision + } onChange: { + changed.withLock { $0 = true } + } + session.properties.revision = 1 + + #expect(changed.withLock { $0 }) + } + + @Test func generatedPropertiesPreserveExplicitTypeContext() { + let session = LanguageModelSession(model: SessionPropertyFixtureModel()) + + #expect(session.properties.optionalSelection == nil) + #expect(session.properties.items.isEmpty) + #expect(session.properties.mode == .first) + session.properties.optionalSelection = "selected" + session.properties.items = [1, 2] + session.properties.mode = .second + #expect(session.properties.optionalSelection == "selected") + #expect(session.properties.items == [1, 2]) + #expect(session.properties.mode == .second) + } + + @Test func historyObservationTracksSessionTranscriptChanges() async throws { + let session = LanguageModelSession(model: CompletingSessionPropertyModel()) + let changed = Locked(false) + + withObservationTracking { + _ = session.properties.history + } onChange: { + changed.withLock { $0 = true } + } + _ = try await session.respond(to: "Observe history") + + #expect(changed.withLock { $0 }) + #expect(session.properties.history.count == 2) + } + + @Test func propertiesAndHistoryAreIsolatedBetweenSessions() { + let first = LanguageModelSession(model: SessionPropertyFixtureModel()) + let second = LanguageModelSession(model: SessionPropertyFixtureModel()) + let prompt = Transcript.Entry.prompt( + Transcript.Prompt(segments: [.text(.init(content: "Replacement"))]) + ) + + first.properties.revision = 7 + first.properties.history = [prompt] + + #expect(first.properties.revision == 7) + #expect(first.transcript == Transcript(entries: [prompt])) + #expect(second.properties.revision == 0) + #expect(second.transcript.isEmpty) + let _: any Observable = first.properties + } + + @Test func replacingHistoryPreservesStaticInstructions() { + let session = LanguageModelSession( + model: SessionPropertyFixtureModel(), + instructions: "Keep these instructions" + ) + let prompt = Transcript.Entry.prompt( + Transcript.Prompt(segments: [.text(.init(content: "Replacement"))]) + ) + + session.properties.history = [prompt] + + #expect(session.properties.history == [prompt]) + #expect(session.transcript.count == 2) + guard case .instructions = session.transcript[0] else { + Issue.record("Expected the static instructions entry to remain first") + return + } + #expect(session.transcript[1] == prompt) + } + + @Test func replacingHistoryDropsInstructionEntries() { + let session = LanguageModelSession(model: SessionPropertyFixtureModel()) + let instructions = Transcript.Entry.instructions( + .init( + segments: [.text(.init(content: "Do not persist"))], + toolDefinitions: [] + ) + ) + let prompt = Transcript.Entry.prompt( + .init(segments: [.text(.init(content: "Keep"))]) + ) + + session.properties.history = [instructions, prompt] + + #expect(session.properties.history == [prompt]) + #expect(Array(session.transcript) == [prompt]) + } +} + +private struct SessionPropertyFixtureInstructions: DynamicInstructions { + @SessionProperty(\.revision) private var revision + + var body: some DynamicInstructions { + let currentRevision = revision + revision += 1 + return Instructions("Revision \(currentRevision)") + } +} + +private struct SessionPropertyHistoryInstructions: DynamicInstructions { + @SessionProperty(\.history) private var history + + var body: some DynamicInstructions { + Instructions("History count \(history.count)") + } +} + +private struct BlockingHistoryInstructions: DynamicInstructions { + @SessionProperty(\.history) private var history + let started: AsyncStream.Continuation + let resume: DispatchSemaphore + + var body: some DynamicInstructions { + let count = history.count + started.yield() + resume.wait() + return Instructions("History count \(count)") + } +} + +private struct SessionPropertyFixtureModel: LanguageModel { + typealias UnavailableReason = Never + + var availability: Availability { .available } + + func supports(locale: Locale) -> Bool { true } + + func respond( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) async throws -> LanguageModelSession.Response where Content: Generable { + fatalError("SessionPropertyFixtureModel does not generate responses") + } + + func streamResponse( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) -> sending LanguageModelSession.ResponseStream where Content: Generable { + fatalError("SessionPropertyFixtureModel does not generate responses") + } +} + +private struct CompletingSessionPropertyModel: LanguageModel { + typealias UnavailableReason = Never + + func respond( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) async throws -> LanguageModelSession.Response where Content: Generable { + let raw = GeneratedContent("Done") + return .init(content: try Content(raw), rawContent: raw, transcriptEntries: []) + } + + func streamResponse( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) -> sending LanguageModelSession.ResponseStream where Content: Generable { + fatalError("This fixture only supports non-streaming responses") + } +}