From d8ed385a2d75361f8f3a3ff1b0145288798abca9 Mon Sep 17 00:00:00 2001 From: Magdalena Majta Date: Tue, 17 Mar 2026 14:23:20 +0100 Subject: [PATCH 01/10] Add provider inference params and adapt services Introduce IProviderInferenceParams and provider-specific parameter types (LocalInferenceParams, AnthropicParams, OpenAiParams, GeminiParams, GroqCloudParams, DeepSeekParams, OllamaParams, XaiParams). Replace usages of the old InferenceParams with IProviderInferenceParams/LocalInferenceParams across the codebase: Chat now stores ProviderParams (with LocalParams and InferenceGrammar helpers), mappers updated to handle LocalInferenceParams, interfaces and AgentService adjusted to accept IProviderInferenceParams, and tests updated accordingly. LLM and OpenAI-compatible services now apply provider-specific settings (temperature, max_tokens, top_p, etc.) via ApplyProviderParams, and grammar handling was unified to use Chat.InferenceGrammar. Examples updated to pass LocalInferenceParams. These changes enable multi-backend provider support and provider-specific configuration handling. --- .../Examples/Chat/ChatCustomGrammarExample.cs | 2 +- .../Examples/Chat/ChatGrammarExampleGemini.cs | 2 +- src/MaIN.Core.UnitTests/AgentContextTests.cs | 4 +- src/MaIN.Core/Hub/Contexts/AgentContext.cs | 4 +- src/MaIN.Core/Hub/Contexts/ChatContext.cs | 4 +- .../IAgentConfigurationBuilder.cs | 4 +- .../ChatContext/IChatConfigurationBuilder.cs | 4 +- src/MaIN.Domain/Entities/Chat.cs | 35 ++++++++++- .../Entities/IProviderInferenceParams.cs | 8 +++ ...renceParams.cs => LocalInferenceParams.cs} | 11 ++-- .../ProviderParams/AnthropicParams.cs | 15 +++++ .../Entities/ProviderParams/DeepSeekParams.cs | 17 ++++++ .../Entities/ProviderParams/GeminiParams.cs | 16 +++++ .../ProviderParams/GroqCloudParams.cs | 16 +++++ .../Entities/ProviderParams/OllamaParams.cs | 17 ++++++ .../Entities/ProviderParams/OpenAiParams.cs | 17 ++++++ .../Entities/ProviderParams/XaiParams.cs | 16 +++++ src/MaIN.Services/Mappers/ChatMapper.cs | 10 ++-- .../Services/Abstract/IAgentService.cs | 2 +- src/MaIN.Services/Services/AgentService.cs | 4 +- .../Services/LLMService/AnthropicService.cs | 58 +++++++++++++------ .../Services/LLMService/DeepSeekService.cs | 12 ++++ .../Services/LLMService/GeminiService.cs | 11 ++++ .../Services/LLMService/GroqCloudService.cs | 11 ++++ .../Services/LLMService/LLMService.cs | 22 +++---- .../Services/LLMService/OllamaService.cs | 11 ++++ .../LLMService/OpenAiCompatibleService.cs | 12 +++- .../Services/LLMService/OpenAiService.cs | 13 +++++ .../Services/LLMService/Utils/ChatHelper.cs | 10 ++-- .../Services/LLMService/XaiService.cs | 11 ++++ .../Steps/Commands/AnswerCommandHandler.cs | 6 +- .../Services/Steps/FechDataStepHandler.cs | 2 +- 32 files changed, 322 insertions(+), 65 deletions(-) create mode 100644 src/MaIN.Domain/Entities/IProviderInferenceParams.cs rename src/MaIN.Domain/Entities/{InferenceParams.cs => LocalInferenceParams.cs} (82%) create mode 100644 src/MaIN.Domain/Entities/ProviderParams/AnthropicParams.cs create mode 100644 src/MaIN.Domain/Entities/ProviderParams/DeepSeekParams.cs create mode 100644 src/MaIN.Domain/Entities/ProviderParams/GeminiParams.cs create mode 100644 src/MaIN.Domain/Entities/ProviderParams/GroqCloudParams.cs create mode 100644 src/MaIN.Domain/Entities/ProviderParams/OllamaParams.cs create mode 100644 src/MaIN.Domain/Entities/ProviderParams/OpenAiParams.cs create mode 100644 src/MaIN.Domain/Entities/ProviderParams/XaiParams.cs diff --git a/Examples/Examples/Chat/ChatCustomGrammarExample.cs b/Examples/Examples/Chat/ChatCustomGrammarExample.cs index 7863fd7c..b16ff825 100644 --- a/Examples/Examples/Chat/ChatCustomGrammarExample.cs +++ b/Examples/Examples/Chat/ChatCustomGrammarExample.cs @@ -24,7 +24,7 @@ public async Task Start() await AIHub.Chat() .WithModel() .WithMessage("Generate random person") - .WithInferenceParams(new InferenceParams + .WithInferenceParams(new LocalInferenceParams { Grammar = personGrammar }) diff --git a/Examples/Examples/Chat/ChatGrammarExampleGemini.cs b/Examples/Examples/Chat/ChatGrammarExampleGemini.cs index eea49de8..0c3b152b 100644 --- a/Examples/Examples/Chat/ChatGrammarExampleGemini.cs +++ b/Examples/Examples/Chat/ChatGrammarExampleGemini.cs @@ -41,7 +41,7 @@ public async Task Start() await AIHub.Chat() .WithModel() .WithMessage("Generate random person") - .WithInferenceParams(new InferenceParams + .WithInferenceParams(new LocalInferenceParams { Grammar = new Grammar(grammarValue, GrammarFormat.JSONSchema) }) diff --git a/src/MaIN.Core.UnitTests/AgentContextTests.cs b/src/MaIN.Core.UnitTests/AgentContextTests.cs index a055ba46..e8d39ace 100644 --- a/src/MaIN.Core.UnitTests/AgentContextTests.cs +++ b/src/MaIN.Core.UnitTests/AgentContextTests.cs @@ -137,7 +137,7 @@ public async Task CreateAsync_ShouldCallAgentServiceCreateAgent() It.IsAny(), It.IsAny(), It.IsAny(), - It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(agent); @@ -151,7 +151,7 @@ public async Task CreateAsync_ShouldCallAgentServiceCreateAgent() It.IsAny(), It.Is(f => f == true), It.Is(r => r == false), - It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); diff --git a/src/MaIN.Core/Hub/Contexts/AgentContext.cs b/src/MaIN.Core/Hub/Contexts/AgentContext.cs index a5dec747..3b5bb3f0 100644 --- a/src/MaIN.Core/Hub/Contexts/AgentContext.cs +++ b/src/MaIN.Core/Hub/Contexts/AgentContext.cs @@ -18,7 +18,7 @@ namespace MaIN.Core.Hub.Contexts; public sealed class AgentContext : IAgentBuilderEntryPoint, IAgentConfigurationBuilder, IAgentContextExecutor { private readonly IAgentService _agentService; - private InferenceParams? _inferenceParams; + private IProviderInferenceParams? _inferenceParams; private MemoryParams? _memoryParams; private bool _disableCache; private bool _ensureModelDownloaded; @@ -152,7 +152,7 @@ public IAgentConfigurationBuilder WithMcpConfig(Mcp mcpConfig) return this; } - public IAgentConfigurationBuilder WithInferenceParams(InferenceParams inferenceParams) + public IAgentConfigurationBuilder WithInferenceParams(IProviderInferenceParams inferenceParams) { _inferenceParams = inferenceParams; return this; diff --git a/src/MaIN.Core/Hub/Contexts/ChatContext.cs b/src/MaIN.Core/Hub/Contexts/ChatContext.cs index 3cb0073b..db32bf4c 100644 --- a/src/MaIN.Core/Hub/Contexts/ChatContext.cs +++ b/src/MaIN.Core/Hub/Contexts/ChatContext.cs @@ -92,9 +92,9 @@ public IChatMessageBuilder EnsureModelDownloaded() return this; } - public IChatConfigurationBuilder WithInferenceParams(InferenceParams inferenceParams) + public IChatConfigurationBuilder WithInferenceParams(IProviderInferenceParams inferenceParams) { - _chat.InterferenceParams = inferenceParams; + _chat.ProviderParams = inferenceParams; return this; } diff --git a/src/MaIN.Core/Hub/Contexts/Interfaces/AgentContext/IAgentConfigurationBuilder.cs b/src/MaIN.Core/Hub/Contexts/Interfaces/AgentContext/IAgentConfigurationBuilder.cs index d5fda2d1..2edbed35 100644 --- a/src/MaIN.Core/Hub/Contexts/Interfaces/AgentContext/IAgentConfigurationBuilder.cs +++ b/src/MaIN.Core/Hub/Contexts/Interfaces/AgentContext/IAgentConfigurationBuilder.cs @@ -79,10 +79,10 @@ public interface IAgentConfigurationBuilder : IAgentActions /// based on specific parameters. Inference parameters can influence various aspects of the chat, such as response length, /// temperature, and other model-specific settings. /// - /// An object that holds the parameters for inference, such as Temperature, MaxTokens, + /// An object that holds the parameters for inference, such as Temperature, MaxTokens, /// TopP, etc. These parameters control the generation behavior of the agent. /// The context instance implementing for method chaining. - IAgentConfigurationBuilder WithInferenceParams(InferenceParams inferenceParams); + IAgentConfigurationBuilder WithInferenceParams(IProviderInferenceParams inferenceParams); /// /// Sets the memory parameters for the chat session, allowing you to customize how the AI accesses and uses its memory diff --git a/src/MaIN.Core/Hub/Contexts/Interfaces/ChatContext/IChatConfigurationBuilder.cs b/src/MaIN.Core/Hub/Contexts/Interfaces/ChatContext/IChatConfigurationBuilder.cs index 5c3c1788..3c9e663a 100644 --- a/src/MaIN.Core/Hub/Contexts/Interfaces/ChatContext/IChatConfigurationBuilder.cs +++ b/src/MaIN.Core/Hub/Contexts/Interfaces/ChatContext/IChatConfigurationBuilder.cs @@ -14,10 +14,10 @@ public interface IChatConfigurationBuilder : IChatActions /// responses based on specific parameters. Inference parameters can influence various aspects of the chat, such as response length, /// temperature, and other model-specific settings. /// - /// An object that holds the parameters for inference, such as Temperature, + /// An object that holds the parameters for inference, such as Temperature, /// MaxTokens, TopP, etc. These parameters control the generation behavior of the chat. /// The context instance implementing for method chaining. - IChatConfigurationBuilder WithInferenceParams(InferenceParams inferenceParams); + IChatConfigurationBuilder WithInferenceParams(IProviderInferenceParams inferenceParams); /// /// Attaches external tools/functions that the model can invoke during the conversation. diff --git a/src/MaIN.Domain/Entities/Chat.cs b/src/MaIN.Domain/Entities/Chat.cs index 2f905775..2d81bf65 100644 --- a/src/MaIN.Domain/Entities/Chat.cs +++ b/src/MaIN.Domain/Entities/Chat.cs @@ -1,7 +1,9 @@ using LLama.Batched; using MaIN.Domain.Configuration; +using MaIN.Domain.Entities.ProviderParams; using MaIN.Domain.Entities.Tools; using MaIN.Domain.Models.Abstract; +using Grammar = MaIN.Domain.Models.Grammar; namespace MaIN.Domain.Entities; @@ -38,7 +40,38 @@ public AIModel? ModelInstance public List Messages { get; set; } = []; public ChatType Type { get; set; } = ChatType.Conversation; public bool ImageGen { get; set; } - public InferenceParams InterferenceParams { get; set; } = new(); + public IProviderInferenceParams ProviderParams { get; set; } = new LocalInferenceParams(); + public LocalInferenceParams? LocalParams => ProviderParams as LocalInferenceParams; + + public Grammar? InferenceGrammar + { + get => ProviderParams switch + { + LocalInferenceParams p => p.Grammar, + OpenAiParams p => p.Grammar, + DeepSeekParams p => p.Grammar, + GroqCloudParams p => p.Grammar, + XaiParams p => p.Grammar, + GeminiParams p => p.Grammar, + AnthropicParams p => p.Grammar, + OllamaParams p => p.Grammar, + _ => null + }; + set + { + switch (ProviderParams) + { + case LocalInferenceParams p: p.Grammar = value; break; + case OpenAiParams p: p.Grammar = value; break; + case DeepSeekParams p: p.Grammar = value; break; + case GroqCloudParams p: p.Grammar = value; break; + case XaiParams p: p.Grammar = value; break; + case GeminiParams p: p.Grammar = value; break; + case AnthropicParams p: p.Grammar = value; break; + case OllamaParams p: p.Grammar = value; break; + } + } + } public MemoryParams MemoryParams { get; set; } = new(); public ToolsConfiguration? ToolsConfiguration { get; set; } public TextToSpeechParams? TextToSpeechParams { get; set; } diff --git a/src/MaIN.Domain/Entities/IProviderInferenceParams.cs b/src/MaIN.Domain/Entities/IProviderInferenceParams.cs new file mode 100644 index 00000000..de970e2d --- /dev/null +++ b/src/MaIN.Domain/Entities/IProviderInferenceParams.cs @@ -0,0 +1,8 @@ +using MaIN.Domain.Configuration; + +namespace MaIN.Domain.Entities; + +public interface IProviderInferenceParams +{ + BackendType Backend { get; } +} diff --git a/src/MaIN.Domain/Entities/InferenceParams.cs b/src/MaIN.Domain/Entities/LocalInferenceParams.cs similarity index 82% rename from src/MaIN.Domain/Entities/InferenceParams.cs rename to src/MaIN.Domain/Entities/LocalInferenceParams.cs index 13b591f0..8f6ea3f4 100644 --- a/src/MaIN.Domain/Entities/InferenceParams.cs +++ b/src/MaIN.Domain/Entities/LocalInferenceParams.cs @@ -1,9 +1,12 @@ +using MaIN.Domain.Configuration; using Grammar = MaIN.Domain.Models.Grammar; namespace MaIN.Domain.Entities; -public class InferenceParams +public class LocalInferenceParams : IProviderInferenceParams { + public BackendType Backend => BackendType.Self; + public float Temperature { get; init; } = 0.8f; public int ContextSize { get; init; } = 1024; public int GpuLayerCount { get; init; } = 30; @@ -13,11 +16,11 @@ public class InferenceParams public bool Embeddings { get; init; } = false; public int TypeK { get; init; } = 0; public int TypeV { get; init; } = 0; - + public int TokensKeep { get; set; } public int MaxTokens { get; set; } = -1; - + public int TopK { get; init; } = 40; public float TopP { get; init; } = 0.9f; public Grammar? Grammar { get; set; } -} \ No newline at end of file +} diff --git a/src/MaIN.Domain/Entities/ProviderParams/AnthropicParams.cs b/src/MaIN.Domain/Entities/ProviderParams/AnthropicParams.cs new file mode 100644 index 00000000..1fda444f --- /dev/null +++ b/src/MaIN.Domain/Entities/ProviderParams/AnthropicParams.cs @@ -0,0 +1,15 @@ +using MaIN.Domain.Configuration; +using Grammar = MaIN.Domain.Models.Grammar; + +namespace MaIN.Domain.Entities.ProviderParams; + +public class AnthropicParams : IProviderInferenceParams +{ + public BackendType Backend => BackendType.Anthropic; + + public float Temperature { get; init; } = 1.0f; + public int MaxTokens { get; init; } = 4096; + public int TopK { get; init; } = -1; + public float TopP { get; init; } = 1.0f; + public Grammar? Grammar { get; set; } +} diff --git a/src/MaIN.Domain/Entities/ProviderParams/DeepSeekParams.cs b/src/MaIN.Domain/Entities/ProviderParams/DeepSeekParams.cs new file mode 100644 index 00000000..1b62807d --- /dev/null +++ b/src/MaIN.Domain/Entities/ProviderParams/DeepSeekParams.cs @@ -0,0 +1,17 @@ +using MaIN.Domain.Configuration; +using Grammar = MaIN.Domain.Models.Grammar; + +namespace MaIN.Domain.Entities.ProviderParams; + +public class DeepSeekParams : IProviderInferenceParams +{ + public BackendType Backend => BackendType.DeepSeek; + + public float Temperature { get; init; } = 0.7f; + public int MaxTokens { get; init; } = 4096; + public float TopP { get; init; } = 1.0f; + public float FrequencyPenalty { get; init; } + public float PresencePenalty { get; init; } + public string? ResponseFormat { get; init; } + public Grammar? Grammar { get; set; } +} diff --git a/src/MaIN.Domain/Entities/ProviderParams/GeminiParams.cs b/src/MaIN.Domain/Entities/ProviderParams/GeminiParams.cs new file mode 100644 index 00000000..981c1bc9 --- /dev/null +++ b/src/MaIN.Domain/Entities/ProviderParams/GeminiParams.cs @@ -0,0 +1,16 @@ +using MaIN.Domain.Configuration; +using Grammar = MaIN.Domain.Models.Grammar; + +namespace MaIN.Domain.Entities.ProviderParams; + +public class GeminiParams : IProviderInferenceParams +{ + public BackendType Backend => BackendType.Gemini; + + public float Temperature { get; init; } = 0.7f; + public int MaxTokens { get; init; } = 4096; + public int TopK { get; init; } = 40; + public float TopP { get; init; } = 0.95f; + public string[]? StopSequences { get; init; } + public Grammar? Grammar { get; set; } +} diff --git a/src/MaIN.Domain/Entities/ProviderParams/GroqCloudParams.cs b/src/MaIN.Domain/Entities/ProviderParams/GroqCloudParams.cs new file mode 100644 index 00000000..60940730 --- /dev/null +++ b/src/MaIN.Domain/Entities/ProviderParams/GroqCloudParams.cs @@ -0,0 +1,16 @@ +using MaIN.Domain.Configuration; +using Grammar = MaIN.Domain.Models.Grammar; + +namespace MaIN.Domain.Entities.ProviderParams; + +public class GroqCloudParams : IProviderInferenceParams +{ + public BackendType Backend => BackendType.GroqCloud; + + public float Temperature { get; init; } = 0.7f; + public int MaxTokens { get; init; } = 4096; + public float TopP { get; init; } = 1.0f; + public float FrequencyPenalty { get; init; } + public string? ResponseFormat { get; init; } + public Grammar? Grammar { get; set; } +} diff --git a/src/MaIN.Domain/Entities/ProviderParams/OllamaParams.cs b/src/MaIN.Domain/Entities/ProviderParams/OllamaParams.cs new file mode 100644 index 00000000..7cbee160 --- /dev/null +++ b/src/MaIN.Domain/Entities/ProviderParams/OllamaParams.cs @@ -0,0 +1,17 @@ +using MaIN.Domain.Configuration; +using Grammar = MaIN.Domain.Models.Grammar; + +namespace MaIN.Domain.Entities.ProviderParams; + +public class OllamaParams : IProviderInferenceParams +{ + public BackendType Backend => BackendType.Ollama; + + public float Temperature { get; init; } = 0.8f; + public int MaxTokens { get; init; } = 4096; + public int TopK { get; init; } = 40; + public float TopP { get; init; } = 0.9f; + public int NumCtx { get; init; } = 2048; + public int NumGpu { get; init; } = 30; + public Grammar? Grammar { get; set; } +} diff --git a/src/MaIN.Domain/Entities/ProviderParams/OpenAiParams.cs b/src/MaIN.Domain/Entities/ProviderParams/OpenAiParams.cs new file mode 100644 index 00000000..78548da9 --- /dev/null +++ b/src/MaIN.Domain/Entities/ProviderParams/OpenAiParams.cs @@ -0,0 +1,17 @@ +using MaIN.Domain.Configuration; +using Grammar = MaIN.Domain.Models.Grammar; + +namespace MaIN.Domain.Entities.ProviderParams; + +public class OpenAiParams : IProviderInferenceParams +{ + public BackendType Backend => BackendType.OpenAi; + + public float Temperature { get; init; } = 0.7f; + public int MaxTokens { get; init; } = 4096; + public float TopP { get; init; } = 1.0f; + public float FrequencyPenalty { get; init; } + public float PresencePenalty { get; init; } + public string? ResponseFormat { get; init; } + public Grammar? Grammar { get; set; } +} diff --git a/src/MaIN.Domain/Entities/ProviderParams/XaiParams.cs b/src/MaIN.Domain/Entities/ProviderParams/XaiParams.cs new file mode 100644 index 00000000..1bdc6814 --- /dev/null +++ b/src/MaIN.Domain/Entities/ProviderParams/XaiParams.cs @@ -0,0 +1,16 @@ +using MaIN.Domain.Configuration; +using Grammar = MaIN.Domain.Models.Grammar; + +namespace MaIN.Domain.Entities.ProviderParams; + +public class XaiParams : IProviderInferenceParams +{ + public BackendType Backend => BackendType.Xai; + + public float Temperature { get; init; } = 0.7f; + public int MaxTokens { get; init; } = 4096; + public float TopP { get; init; } = 1.0f; + public float FrequencyPenalty { get; init; } + public float PresencePenalty { get; init; } + public Grammar? Grammar { get; set; } +} diff --git a/src/MaIN.Services/Mappers/ChatMapper.cs b/src/MaIN.Services/Mappers/ChatMapper.cs index 1bdf0398..0d0743fd 100644 --- a/src/MaIN.Services/Mappers/ChatMapper.cs +++ b/src/MaIN.Services/Mappers/ChatMapper.cs @@ -93,7 +93,7 @@ public static ChatDocument ToDocument(this Chat chat) Backend = chat.Backend, ToolsConfiguration = chat.ToolsConfiguration, MemoryParams = chat.MemoryParams.ToDocument(), - InferenceParams = chat.InterferenceParams.ToDocument(), + InferenceParams = (chat.ProviderParams as LocalInferenceParams)?.ToDocument(), ConvState = chat.ConversationState, Properties = chat.Properties, Interactive = chat.Interactive, @@ -114,7 +114,7 @@ public static Chat ToDomain(this ChatDocument chat) ToolsConfiguration = chat.ToolsConfiguration, ConversationState = chat.ConvState as Conversation.State, MemoryParams = chat.MemoryParams!.ToDomain(), - InterferenceParams = chat.InferenceParams!.ToDomain(), + ProviderParams = chat.InferenceParams?.ToDomain() ?? new LocalInferenceParams(), Interactive = chat.Interactive, Translate = chat.Translate, Type = Enum.Parse(chat.Type.ToString()) @@ -147,8 +147,8 @@ private static LLMTokenValue ToDomain(this LLMTokenValueDocument llmTokenValue) Type = llmTokenValue.Type }; - private static InferenceParams ToDomain(this InferenceParamsDocument inferenceParams) - => new InferenceParams + private static LocalInferenceParams ToDomain(this InferenceParamsDocument inferenceParams) + => new LocalInferenceParams { Temperature = inferenceParams.Temperature, ContextSize = inferenceParams.ContextSize, @@ -179,7 +179,7 @@ private static MemoryParams ToDomain(this MemoryParamsDocument memoryParams) Grammar = memoryParams.Grammar }; - private static InferenceParamsDocument ToDocument(this InferenceParams inferenceParams) + private static InferenceParamsDocument ToDocument(this LocalInferenceParams inferenceParams) => new InferenceParamsDocument { Temperature = inferenceParams.Temperature, diff --git a/src/MaIN.Services/Services/Abstract/IAgentService.cs b/src/MaIN.Services/Services/Abstract/IAgentService.cs index 25c0449e..785920ca 100644 --- a/src/MaIN.Services/Services/Abstract/IAgentService.cs +++ b/src/MaIN.Services/Services/Abstract/IAgentService.cs @@ -11,7 +11,7 @@ public interface IAgentService Task Process(Chat chat, string agentId, Knowledge? knowledge, bool translatePrompt = false, Func? callbackToken = null, Func? callbackTool = null); Task CreateAgent(Agent agent, bool flow = false, bool interactiveResponse = false, - InferenceParams? inferenceParams = null, MemoryParams? memoryParams = null, bool disableCache = false); + IProviderInferenceParams? inferenceParams = null, MemoryParams? memoryParams = null, bool disableCache = false); Task GetChatByAgent(string agentId); Task Restart(string agentId); Task> GetAgents(); diff --git a/src/MaIN.Services/Services/AgentService.cs b/src/MaIN.Services/Services/AgentService.cs index fc06c719..bc80bc6b 100644 --- a/src/MaIN.Services/Services/AgentService.cs +++ b/src/MaIN.Services/Services/AgentService.cs @@ -94,7 +94,7 @@ await notificationService.DispatchNotification( } public async Task CreateAgent(Agent agent, bool flow = false, bool interactiveResponse = false, - InferenceParams? inferenceParams = null, MemoryParams? memoryParams = null, bool disableCache = false) + IProviderInferenceParams? inferenceParams = null, MemoryParams? memoryParams = null, bool disableCache = false) { var chat = new Chat { @@ -103,7 +103,7 @@ public async Task CreateAgent(Agent agent, bool flow = false, bool intera Name = agent.Name, ImageGen = agent.Model == ImageGenService.LocalImageModels.FLUX, ToolsConfiguration = agent.ToolsConfiguration, - InterferenceParams = inferenceParams ?? new InferenceParams(), + ProviderParams = inferenceParams ?? new LocalInferenceParams(), MemoryParams = memoryParams ?? new MemoryParams(), Messages = new List(), Interactive = interactiveResponse, diff --git a/src/MaIN.Services/Services/LLMService/AnthropicService.cs b/src/MaIN.Services/Services/LLMService/AnthropicService.cs index bbf1d879..942bfb80 100644 --- a/src/MaIN.Services/Services/LLMService/AnthropicService.cs +++ b/src/MaIN.Services/Services/LLMService/AnthropicService.cs @@ -1,4 +1,5 @@ using MaIN.Domain.Entities; +using MaIN.Domain.Entities.ProviderParams; using MaIN.Domain.Models; using MaIN.Services.Constants; using MaIN.Services.Services.Abstract; @@ -506,14 +507,23 @@ private async Task HandleApiError(HttpResponseMessage response, CancellationToke private object BuildAnthropicRequestBody(Chat chat, List conversation, bool stream) { + var anthParams = chat.ProviderParams as AnthropicParams; + var requestBody = new Dictionary { ["model"] = chat.ModelId, - ["max_tokens"] = chat.InterferenceParams.MaxTokens < 0 ? 4096 : chat.InterferenceParams.MaxTokens, + ["max_tokens"] = anthParams?.MaxTokens ?? 4096, ["stream"] = stream, ["messages"] = BuildAnthropicMessages(conversation) }; + if (anthParams != null) + { + requestBody["temperature"] = anthParams.Temperature; + if (anthParams.TopP < 1.0f) requestBody["top_p"] = anthParams.TopP; + if (anthParams.TopK > 0) requestBody["top_k"] = anthParams.TopK; + } + var systemMessage = conversation.FirstOrDefault(m => m.Role.Equals("system", StringComparison.OrdinalIgnoreCase)); @@ -522,10 +532,10 @@ private object BuildAnthropicRequestBody(Chat chat, List conversati requestBody["system"] = systemContent; } - if (chat.InterferenceParams.Grammar is not null) + if (chat.InferenceGrammar is not null) { requestBody["system"] = - $"Respond only using the following grammar format: \n{chat.InterferenceParams.Grammar.Value}\n. Do not add explanations, code tags, or any extra content."; + $"Respond only using the following grammar format: \n{chat.InferenceGrammar.Value}\n. Do not add explanations, code tags, or any extra content."; } if (chat.ToolsConfiguration?.Tools != null && chat.ToolsConfiguration.Tools.Any()) @@ -683,16 +693,23 @@ private async Task ProcessStreamingChatAsync( { var httpClient = CreateAnthropicHttpClient(); - var requestBody = new + var anthParams2 = chat.ProviderParams as AnthropicParams; + var requestBody = new Dictionary { - model = chat.ModelId, - max_tokens = chat.InterferenceParams.MaxTokens < 0 ? 4096 : chat.InterferenceParams.MaxTokens, - stream = true, - system = chat.InterferenceParams.Grammar is not null - ? $"Respond only using the following grammar format: \n{chat.InterferenceParams.Grammar.Value}\n. Do not add explanations, code tags, or any extra content." + ["model"] = chat.ModelId, + ["max_tokens"] = anthParams2?.MaxTokens ?? 4096, + ["stream"] = true, + ["system"] = chat.InferenceGrammar is not null + ? $"Respond only using the following grammar format: \n{chat.InferenceGrammar.Value}\n. Do not add explanations, code tags, or any extra content." : "", - messages = await OpenAiCompatibleService.BuildMessagesArray(conversation, chat, ImageType.AsBase64) + ["messages"] = await OpenAiCompatibleService.BuildMessagesArray(conversation, chat, ImageType.AsBase64) }; + if (anthParams2 != null) + { + requestBody["temperature"] = anthParams2.Temperature; + if (anthParams2.TopP < 1.0f) requestBody["top_p"] = anthParams2.TopP; + if (anthParams2.TopK > 0) requestBody["top_k"] = anthParams2.TopK; + } var requestJson = JsonSerializer.Serialize(requestBody); var content = new StringContent(requestJson, Encoding.UTF8, "application/json"); @@ -771,16 +788,23 @@ private async Task ProcessNonStreamingChatAsync( { var httpClient = CreateAnthropicHttpClient(); - var requestBody = new + var anthParams3 = chat.ProviderParams as AnthropicParams; + var requestBody = new Dictionary { - model = chat.ModelId, - max_tokens = chat.InterferenceParams.MaxTokens < 0 ? 4096 : chat.InterferenceParams.MaxTokens, - stream = false, - system = chat.InterferenceParams.Grammar is not null - ? $"Respond only using the following grammar format: \n{chat.InterferenceParams.Grammar.Value}\n. Do not add explanations, code tags, or any extra content." + ["model"] = chat.ModelId, + ["max_tokens"] = anthParams3?.MaxTokens ?? 4096, + ["stream"] = false, + ["system"] = chat.InferenceGrammar is not null + ? $"Respond only using the following grammar format: \n{chat.InferenceGrammar.Value}\n. Do not add explanations, code tags, or any extra content." : "", - messages = await OpenAiCompatibleService.BuildMessagesArray(conversation, chat, ImageType.AsBase64) + ["messages"] = await OpenAiCompatibleService.BuildMessagesArray(conversation, chat, ImageType.AsBase64) }; + if (anthParams3 != null) + { + requestBody["temperature"] = anthParams3.Temperature; + if (anthParams3.TopP < 1.0f) requestBody["top_p"] = anthParams3.TopP; + if (anthParams3.TopK > 0) requestBody["top_k"] = anthParams3.TopK; + } var requestJson = JsonSerializer.Serialize(requestBody); var content = new StringContent(requestJson, Encoding.UTF8, "application/json"); diff --git a/src/MaIN.Services/Services/LLMService/DeepSeekService.cs b/src/MaIN.Services/Services/LLMService/DeepSeekService.cs index 4c5f303a..86f12ed2 100644 --- a/src/MaIN.Services/Services/LLMService/DeepSeekService.cs +++ b/src/MaIN.Services/Services/LLMService/DeepSeekService.cs @@ -1,6 +1,7 @@ using System.Text; using MaIN.Domain.Configuration; using MaIN.Domain.Entities; +using MaIN.Domain.Entities.ProviderParams; using MaIN.Services.Services.Abstract; using Microsoft.Extensions.Logging; using MaIN.Services.Services.LLMService.Memory; @@ -49,6 +50,17 @@ protected override void ValidateApiKey() } } + protected override void ApplyProviderParams(Dictionary requestBody, Chat chat) + { + if (chat.ProviderParams is not DeepSeekParams p) return; + requestBody["temperature"] = p.Temperature; + requestBody["max_tokens"] = p.MaxTokens; + requestBody["top_p"] = p.TopP; + if (p.FrequencyPenalty != 0) requestBody["frequency_penalty"] = p.FrequencyPenalty; + if (p.PresencePenalty != 0) requestBody["presence_penalty"] = p.PresencePenalty; + if (p.ResponseFormat != null) requestBody["response_format"] = new { type = p.ResponseFormat }; + } + public override async Task AskMemory( Chat chat, ChatMemoryOptions memoryOptions, diff --git a/src/MaIN.Services/Services/LLMService/GeminiService.cs b/src/MaIN.Services/Services/LLMService/GeminiService.cs index 630d76ed..96c23a6a 100644 --- a/src/MaIN.Services/Services/LLMService/GeminiService.cs +++ b/src/MaIN.Services/Services/LLMService/GeminiService.cs @@ -9,6 +9,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using MaIN.Domain.Entities; +using MaIN.Domain.Entities.ProviderParams; using MaIN.Domain.Exceptions; using MaIN.Domain.Models; using MaIN.Domain.Models.Concrete; @@ -75,6 +76,16 @@ protected override void ValidateApiKey() } } + protected override void ApplyProviderParams(Dictionary requestBody, Chat chat) + { + if (chat.ProviderParams is not GeminiParams p) return; + requestBody["temperature"] = p.Temperature; + requestBody["max_tokens"] = p.MaxTokens; + requestBody["top_p"] = p.TopP; + if (p.TopK > 0) requestBody["top_k"] = p.TopK; + if (p.StopSequences is { Length: > 0 }) requestBody["stop"] = p.StopSequences; + } + public override async Task AskMemory( Chat chat, ChatMemoryOptions memoryOptions, diff --git a/src/MaIN.Services/Services/LLMService/GroqCloudService.cs b/src/MaIN.Services/Services/LLMService/GroqCloudService.cs index bab7221e..e1072d9e 100644 --- a/src/MaIN.Services/Services/LLMService/GroqCloudService.cs +++ b/src/MaIN.Services/Services/LLMService/GroqCloudService.cs @@ -1,6 +1,7 @@ using System.Text; using MaIN.Domain.Configuration; using MaIN.Domain.Entities; +using MaIN.Domain.Entities.ProviderParams; using MaIN.Domain.Exceptions; using MaIN.Domain.Models.Concrete; using MaIN.Services.Services.Abstract; @@ -43,6 +44,16 @@ protected override void ValidateApiKey() } } + protected override void ApplyProviderParams(Dictionary requestBody, Chat chat) + { + if (chat.ProviderParams is not GroqCloudParams p) return; + requestBody["temperature"] = p.Temperature; + requestBody["max_tokens"] = p.MaxTokens; + requestBody["top_p"] = p.TopP; + if (p.FrequencyPenalty != 0) requestBody["frequency_penalty"] = p.FrequencyPenalty; + if (p.ResponseFormat != null) requestBody["response_format"] = new { type = p.ResponseFormat }; + } + public override async Task AskMemory( Chat chat, ChatMemoryOptions memoryOptions, diff --git a/src/MaIN.Services/Services/LLMService/LLMService.cs b/src/MaIN.Services/Services/LLMService/LLMService.cs index fb8664f5..3ed431ff 100644 --- a/src/MaIN.Services/Services/LLMService/LLMService.cs +++ b/src/MaIN.Services/Services/LLMService/LLMService.cs @@ -20,7 +20,7 @@ using MaIN.Services.Utils; using Microsoft.KernelMemory; using Grammar = LLama.Sampling.Grammar; -using InferenceParams = MaIN.Domain.Entities.InferenceParams; +using LocalInferenceParams = MaIN.Domain.Entities.LocalInferenceParams; #pragma warning disable KMEXP00 namespace MaIN.Services.Services.LLMService; @@ -319,14 +319,14 @@ private ModelParams CreateModelParameters(Chat chat, string modelKey, string? cu { return new ModelParams(ResolvePath(customPath, modelKey)) { - ContextSize = (uint?)chat.InterferenceParams.ContextSize, - GpuLayerCount = chat.InterferenceParams.GpuLayerCount, - SeqMax = chat.InterferenceParams.SeqMax, - BatchSize = chat.InterferenceParams.BatchSize, - UBatchSize = chat.InterferenceParams.UBatchSize, - Embeddings = chat.InterferenceParams.Embeddings, - TypeK = (GGMLType)chat.InterferenceParams.TypeK, - TypeV = (GGMLType)chat.InterferenceParams.TypeV, + ContextSize = (uint?)chat.LocalParams!.ContextSize, + GpuLayerCount = chat.LocalParams!.GpuLayerCount, + SeqMax = chat.LocalParams!.SeqMax, + BatchSize = chat.LocalParams!.BatchSize, + UBatchSize = chat.LocalParams!.UBatchSize, + Embeddings = chat.LocalParams!.Embeddings, + TypeK = (GGMLType)chat.LocalParams!.TypeK, + TypeV = (GGMLType)chat.LocalParams!.TypeV, }; } @@ -461,7 +461,7 @@ private static string FormatToolsForPrompt(ToolsConfiguration toolsConfig) var isComplete = false; var hasFailed = false; - using var sampler = LLMService.CreateSampler(chat.InterferenceParams); + using var sampler = LLMService.CreateSampler(chat.LocalParams!); var decoder = new StreamingTokenDecoder(executor.Context); var inferenceParams = ChatHelper.CreateInferenceParams(chat, llmModel); @@ -527,7 +527,7 @@ private static string FormatToolsForPrompt(ToolsConfiguration toolsConfig) return (tokens, isComplete, hasFailed); } - private static BaseSamplingPipeline CreateSampler(InferenceParams interferenceParams) + private static BaseSamplingPipeline CreateSampler(LocalInferenceParams interferenceParams) { if (interferenceParams.Temperature == 0) { diff --git a/src/MaIN.Services/Services/LLMService/OllamaService.cs b/src/MaIN.Services/Services/LLMService/OllamaService.cs index cd59661a..bd511efb 100644 --- a/src/MaIN.Services/Services/LLMService/OllamaService.cs +++ b/src/MaIN.Services/Services/LLMService/OllamaService.cs @@ -1,6 +1,7 @@ using System.Text; using MaIN.Domain.Configuration; using MaIN.Domain.Entities; +using MaIN.Domain.Entities.ProviderParams; using MaIN.Domain.Models.Concrete; using MaIN.Services.Constants; using MaIN.Services.Services.Abstract; @@ -40,6 +41,16 @@ protected override void ValidateApiKey() // Cloud Ollama will fail at runtime if the key is missing } + protected override void ApplyProviderParams(Dictionary requestBody, Chat chat) + { + if (chat.ProviderParams is not OllamaParams p) return; + requestBody["temperature"] = p.Temperature; + requestBody["max_tokens"] = p.MaxTokens; + requestBody["top_p"] = p.TopP; + if (p.TopK > 0) requestBody["top_k"] = p.TopK; + if (p.NumCtx > 0) requestBody["options"] = new { num_ctx = p.NumCtx, num_gpu = p.NumGpu }; + } + public override async Task AskMemory( Chat chat, ChatMemoryOptions memoryOptions, diff --git a/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs b/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs index 355e8391..fff4c415 100644 --- a/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs +++ b/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs @@ -856,6 +856,8 @@ private object BuildRequestBody(Chat chat, List conversation, bool ["stream"] = stream }; + ApplyProviderParams(requestBody, chat); + if (chat.ToolsConfiguration?.Tools != null && chat.ToolsConfiguration.Tools.Any()) { requestBody["tools"] = chat.ToolsConfiguration.Tools.Select(t => new @@ -868,7 +870,7 @@ private object BuildRequestBody(Chat chat, List conversation, bool parameters = t.Function.Parameters } : null }).ToList(); - + if (!string.IsNullOrEmpty(chat.ToolsConfiguration.ToolChoice)) { requestBody["tool_choice"] = chat.ToolsConfiguration.ToolChoice; @@ -878,6 +880,10 @@ private object BuildRequestBody(Chat chat, List conversation, bool return requestBody; } + protected virtual void ApplyProviderParams(Dictionary requestBody, Chat chat) + { + } + internal static void MergeMessages(List conversation, List messages) { var existing = new HashSet<(string, object)>(conversation.Select(m => (m.Role, m.Content))); @@ -950,10 +956,10 @@ internal static async Task BuildMessagesArray(List conver foreach (var msg in conversation) { var content = msg.OriginalMessage != null ? BuildMessageContent(msg.OriginalMessage, imageType) : msg.Content; - if (chat.InterferenceParams.Grammar != null && msg.Role == "user") + if (chat.InferenceGrammar != null && msg.Role == "user") { var jsonGrammarConverter = new GrammarToJsonConverter(); - string jsonGrammar = jsonGrammarConverter.ConvertToJson(chat.InterferenceParams.Grammar); + string jsonGrammar = jsonGrammarConverter.ConvertToJson(chat.InferenceGrammar); var grammarInstruction = $" | Respond only using the following JSON format: \n{jsonGrammar}\n. Do not add explanations, code tags, or any extra content."; diff --git a/src/MaIN.Services/Services/LLMService/OpenAiService.cs b/src/MaIN.Services/Services/LLMService/OpenAiService.cs index 203d52b1..14af65e8 100644 --- a/src/MaIN.Services/Services/LLMService/OpenAiService.cs +++ b/src/MaIN.Services/Services/LLMService/OpenAiService.cs @@ -1,4 +1,6 @@ using MaIN.Domain.Configuration; +using MaIN.Domain.Entities; +using MaIN.Domain.Entities.ProviderParams; using MaIN.Domain.Exceptions; using MaIN.Domain.Models.Concrete; using MaIN.Services.Services.Abstract; @@ -34,6 +36,17 @@ protected override void ValidateApiKey() } } + protected override void ApplyProviderParams(Dictionary requestBody, Chat chat) + { + if (chat.ProviderParams is not OpenAiParams p) return; + requestBody["temperature"] = p.Temperature; + requestBody["max_tokens"] = p.MaxTokens; + requestBody["top_p"] = p.TopP; + if (p.FrequencyPenalty != 0) requestBody["frequency_penalty"] = p.FrequencyPenalty; + if (p.PresencePenalty != 0) requestBody["presence_penalty"] = p.PresencePenalty; + if (p.ResponseFormat != null) requestBody["response_format"] = new { type = p.ResponseFormat }; + } + public override async Task GetCurrentModels() { var allModels = await base.GetCurrentModels(); diff --git a/src/MaIN.Services/Services/LLMService/Utils/ChatHelper.cs b/src/MaIN.Services/Services/LLMService/Utils/ChatHelper.cs index 7c33858c..36f7efef 100644 --- a/src/MaIN.Services/Services/LLMService/Utils/ChatHelper.cs +++ b/src/MaIN.Services/Services/LLMService/Utils/ChatHelper.cs @@ -79,13 +79,13 @@ public static InferenceParams CreateInferenceParams(Chat chat, LLamaWeights mode { SamplingPipeline = new DefaultSamplingPipeline { - Temperature = chat.InterferenceParams.Temperature, - TopK = chat.InterferenceParams.TopK, - TopP = chat.InterferenceParams.TopP + Temperature = chat.LocalParams!.Temperature, + TopK = chat.LocalParams!.TopK, + TopP = chat.LocalParams!.TopP }, AntiPrompts = [model.Vocab.EOT?.ToString() ?? "User:"], - TokensKeep = chat.InterferenceParams.TokensKeep, - MaxTokens = chat.InterferenceParams.MaxTokens + TokensKeep = chat.LocalParams!.TokensKeep, + MaxTokens = chat.LocalParams!.MaxTokens }; } diff --git a/src/MaIN.Services/Services/LLMService/XaiService.cs b/src/MaIN.Services/Services/LLMService/XaiService.cs index 632b3e55..267a7bc7 100644 --- a/src/MaIN.Services/Services/LLMService/XaiService.cs +++ b/src/MaIN.Services/Services/LLMService/XaiService.cs @@ -2,6 +2,7 @@ using MaIN.Services.Constants; using MaIN.Services.Services.Models; using MaIN.Domain.Entities; +using MaIN.Domain.Entities.ProviderParams; using MaIN.Services.Services.Abstract; using MaIN.Services.Services.LLMService.Memory; using Microsoft.Extensions.Logging; @@ -42,6 +43,16 @@ protected override void ValidateApiKey() } } + protected override void ApplyProviderParams(Dictionary requestBody, Chat chat) + { + if (chat.ProviderParams is not XaiParams p) return; + requestBody["temperature"] = p.Temperature; + requestBody["max_tokens"] = p.MaxTokens; + requestBody["top_p"] = p.TopP; + if (p.FrequencyPenalty != 0) requestBody["frequency_penalty"] = p.FrequencyPenalty; + if (p.PresencePenalty != 0) requestBody["presence_penalty"] = p.PresencePenalty; + } + public override async Task AskMemory( Chat chat, ChatMemoryOptions memoryOptions, diff --git a/src/MaIN.Services/Services/Steps/Commands/AnswerCommandHandler.cs b/src/MaIN.Services/Services/Steps/Commands/AnswerCommandHandler.cs index 4178f6d4..3a2ad2c3 100644 --- a/src/MaIN.Services/Services/Steps/Commands/AnswerCommandHandler.cs +++ b/src/MaIN.Services/Services/Steps/Commands/AnswerCommandHandler.cs @@ -66,7 +66,7 @@ private async Task ShouldUseKnowledge(Knowledge? knowledge, Chat chat) var indexAsKnowledge = knowledge?.Index.Items.ToDictionary(x => x.Name, x => x.Tags); var index = JsonSerializer.Serialize(indexAsKnowledge, JsonOptions); - chat.InterferenceParams.Grammar = new Grammar(ServiceConstants.Grammars.DecisionGrammar, GrammarFormat.GBNF); + chat.InferenceGrammar = new Grammar(ServiceConstants.Grammars.DecisionGrammar, GrammarFormat.GBNF); chat.Messages.Last().Content = $""" KNOWLEDGE: @@ -86,7 +86,7 @@ private async Task ShouldUseKnowledge(Knowledge? knowledge, Chat chat) }); var decision = JsonSerializer.Deserialize(result!.Message.Content, JsonOptions); var decisionValue = decision.GetProperty("decision").GetRawText(); - chat.InterferenceParams.Grammar = null; + chat.InferenceGrammar = null; var shouldUseKnowledge = bool.Parse(decisionValue.Trim('"')); chat.Messages.Last().Content = originalContent; return shouldUseKnowledge!; @@ -98,7 +98,7 @@ private async Task ShouldUseKnowledge(Knowledge? knowledge, Chat chat) var indexAsKnowledge = knowledge?.Index.Items.ToDictionary(x => x.Name, x => x.Tags); var index = JsonSerializer.Serialize(indexAsKnowledge, JsonOptions); - chat.InterferenceParams.Grammar = new Grammar(ServiceConstants.Grammars.KnowledgeGrammar, GrammarFormat.GBNF); + chat.InferenceGrammar = new Grammar(ServiceConstants.Grammars.KnowledgeGrammar, GrammarFormat.GBNF); chat.Messages.Last().Content = $""" KNOWLEDGE: diff --git a/src/MaIN.Services/Services/Steps/FechDataStepHandler.cs b/src/MaIN.Services/Services/Steps/FechDataStepHandler.cs index 28af3856..a1f13c3e 100644 --- a/src/MaIN.Services/Services/Steps/FechDataStepHandler.cs +++ b/src/MaIN.Services/Services/Steps/FechDataStepHandler.cs @@ -68,7 +68,7 @@ private static Chat CreateMemoryChat(StepContext context, string? filterVal) ModelId = context.Chat.ModelId, Properties = context.Chat.Properties, MemoryParams = context.Chat.MemoryParams, - InterferenceParams = context.Chat.InterferenceParams, + ProviderParams = context.Chat.ProviderParams, Backend = context.Chat.Backend, Name = "Memory Chat", Id = Guid.NewGuid().ToString() From 85793d632f0b4af1b29e79c336dfaab96bda2e3e Mon Sep 17 00:00:00 2001 From: Magdalena Majta Date: Tue, 17 Mar 2026 22:32:52 +0100 Subject: [PATCH 02/10] Validate provider params and add exception Introduce InvalidProviderParamsException for clearer bad-request errors when provider params types mismatch. Add an ExpectedParamsType contract to OpenAiCompatibleService and implement it in several providers (Gemini, GroqCloud, Ollama, OpenAi, Xai, DeepSeek). Perform runtime type checks in OpenAiCompatibleService, LLMService (local inference), and AnthropicService to throw the new exception when chat.ProviderParams is the wrong type. Misc: remove TopK from GeminiParams and stop emitting top_k in Gemini requests; preserve Chat.Backend if already set; minor cleanup (pragmas/using/whitespace). --- .../Entities/ProviderParams/GeminiParams.cs | 1 - .../Exceptions/InvalidProviderParamsException.cs | 10 ++++++++++ src/MaIN.Services/Services/ChatService.cs | 2 +- .../Services/LLMService/AnthropicService.cs | 5 +++++ .../Services/LLMService/DeepSeekService.cs | 1 + src/MaIN.Services/Services/LLMService/GeminiService.cs | 4 +--- .../Services/LLMService/GroqCloudService.cs | 1 + src/MaIN.Services/Services/LLMService/LLMService.cs | 6 ++++++ src/MaIN.Services/Services/LLMService/OllamaService.cs | 1 + .../Services/LLMService/OpenAiCompatibleService.cs | 7 ++++++- src/MaIN.Services/Services/LLMService/OpenAiService.cs | 4 +++- src/MaIN.Services/Services/LLMService/XaiService.cs | 1 + 12 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 src/MaIN.Domain/Exceptions/InvalidProviderParamsException.cs diff --git a/src/MaIN.Domain/Entities/ProviderParams/GeminiParams.cs b/src/MaIN.Domain/Entities/ProviderParams/GeminiParams.cs index 981c1bc9..b0a2f503 100644 --- a/src/MaIN.Domain/Entities/ProviderParams/GeminiParams.cs +++ b/src/MaIN.Domain/Entities/ProviderParams/GeminiParams.cs @@ -9,7 +9,6 @@ public class GeminiParams : IProviderInferenceParams public float Temperature { get; init; } = 0.7f; public int MaxTokens { get; init; } = 4096; - public int TopK { get; init; } = 40; public float TopP { get; init; } = 0.95f; public string[]? StopSequences { get; init; } public Grammar? Grammar { get; set; } diff --git a/src/MaIN.Domain/Exceptions/InvalidProviderParamsException.cs b/src/MaIN.Domain/Exceptions/InvalidProviderParamsException.cs new file mode 100644 index 00000000..4368c9a1 --- /dev/null +++ b/src/MaIN.Domain/Exceptions/InvalidProviderParamsException.cs @@ -0,0 +1,10 @@ +using System.Net; + +namespace MaIN.Domain.Exceptions; + +public class InvalidProviderParamsException(string serviceName, string expectedType, string receivedType) + : MaINCustomException($"{serviceName} service requires {expectedType}, but received {receivedType}.") +{ + public override string PublicErrorMessage => Message; + public override HttpStatusCode HttpStatusCode => HttpStatusCode.BadRequest; +} diff --git a/src/MaIN.Services/Services/ChatService.cs b/src/MaIN.Services/Services/ChatService.cs index d805f4a0..6ffed5c9 100644 --- a/src/MaIN.Services/Services/ChatService.cs +++ b/src/MaIN.Services/Services/ChatService.cs @@ -37,7 +37,7 @@ public async Task Completions( { chat.ImageGen = true; } - chat.Backend = settings.BackendType; + chat.Backend = chat.Backend ?? settings.BackendType; chat.Messages.Where(x => x.Type == MessageType.NotSet).ToList() .ForEach(x => x.Type = chat.Backend != BackendType.Self ? MessageType.CloudLLM : MessageType.LocalLLM); diff --git a/src/MaIN.Services/Services/LLMService/AnthropicService.cs b/src/MaIN.Services/Services/LLMService/AnthropicService.cs index 942bfb80..be11de87 100644 --- a/src/MaIN.Services/Services/LLMService/AnthropicService.cs +++ b/src/MaIN.Services/Services/LLMService/AnthropicService.cs @@ -60,6 +60,11 @@ private void ValidateApiKey() public async Task Send(Chat chat, ChatRequestOptions options, CancellationToken cancellationToken = default) { + if (chat.ProviderParams is not AnthropicParams) + { + throw new InvalidProviderParamsException(LLMApiRegistry.Anthropic.ApiName, nameof(AnthropicParams), chat.ProviderParams.GetType().Name); + } + ValidateApiKey(); if (!chat.Messages.Any()) diff --git a/src/MaIN.Services/Services/LLMService/DeepSeekService.cs b/src/MaIN.Services/Services/LLMService/DeepSeekService.cs index 86f12ed2..456ec164 100644 --- a/src/MaIN.Services/Services/LLMService/DeepSeekService.cs +++ b/src/MaIN.Services/Services/LLMService/DeepSeekService.cs @@ -32,6 +32,7 @@ public sealed class DeepSeekService( protected override string HttpClientName => ServiceConstants.HttpClients.DeepSeekClient; protected override string ChatCompletionsUrl => ServiceConstants.ApiUrls.DeepSeekOpenAiChatCompletions; protected override string ModelsUrl => ServiceConstants.ApiUrls.DeepSeekModels; + protected override Type ExpectedParamsType => typeof(DeepSeekParams); protected override string GetApiKey() { diff --git a/src/MaIN.Services/Services/LLMService/GeminiService.cs b/src/MaIN.Services/Services/LLMService/GeminiService.cs index 96c23a6a..dcd01c53 100644 --- a/src/MaIN.Services/Services/LLMService/GeminiService.cs +++ b/src/MaIN.Services/Services/LLMService/GeminiService.cs @@ -24,9 +24,7 @@ public sealed class GeminiService( IMemoryFactory memoryFactory, IMemoryService memoryService, ILogger? logger = null) -#pragma warning disable CS9107 // Parameter is captured into the state of the enclosing type and its value is also passed to the base constructor. The value might be captured by the base class as well. : OpenAiCompatibleService(notificationService, httpClientFactory, memoryFactory, memoryService, logger) -#pragma warning restore CS9107 // Parameter is captured into the state of the enclosing type and its value is also passed to the base constructor. The value might be captured by the base class as well. { private readonly MaINSettings _settings = settings ?? throw new ArgumentNullException(nameof(settings)); @@ -38,6 +36,7 @@ public sealed class GeminiService( protected override string HttpClientName => ServiceConstants.HttpClients.GeminiClient; protected override string ChatCompletionsUrl => ServiceConstants.ApiUrls.GeminiOpenAiChatCompletions; + protected override Type ExpectedParamsType => typeof(GeminiParams); public override async Task GetCurrentModels() { @@ -82,7 +81,6 @@ protected override void ApplyProviderParams(Dictionary requestBo requestBody["temperature"] = p.Temperature; requestBody["max_tokens"] = p.MaxTokens; requestBody["top_p"] = p.TopP; - if (p.TopK > 0) requestBody["top_k"] = p.TopK; if (p.StopSequences is { Length: > 0 }) requestBody["stop"] = p.StopSequences; } diff --git a/src/MaIN.Services/Services/LLMService/GroqCloudService.cs b/src/MaIN.Services/Services/LLMService/GroqCloudService.cs index e1072d9e..f0abda19 100644 --- a/src/MaIN.Services/Services/LLMService/GroqCloudService.cs +++ b/src/MaIN.Services/Services/LLMService/GroqCloudService.cs @@ -26,6 +26,7 @@ public sealed class GroqCloudService( protected override string HttpClientName => ServiceConstants.HttpClients.GroqCloudClient; protected override string ChatCompletionsUrl => ServiceConstants.ApiUrls.GroqCloudOpenAiChatCompletions; protected override string ModelsUrl => ServiceConstants.ApiUrls.GroqCloudModels; + protected override Type ExpectedParamsType => typeof(GroqCloudParams); protected override string GetApiKey() { diff --git a/src/MaIN.Services/Services/LLMService/LLMService.cs b/src/MaIN.Services/Services/LLMService/LLMService.cs index 3ed431ff..fcfad169 100644 --- a/src/MaIN.Services/Services/LLMService/LLMService.cs +++ b/src/MaIN.Services/Services/LLMService/LLMService.cs @@ -8,6 +8,7 @@ using LLama.Sampling; using MaIN.Domain.Configuration; using MaIN.Domain.Entities; +using MaIN.Domain.Exceptions; using MaIN.Domain.Exceptions.Models; using MaIN.Domain.Entities.Tools; using MaIN.Domain.Models; @@ -55,6 +56,11 @@ public LLMService( ChatRequestOptions requestOptions, CancellationToken cancellationToken = default) { + if (chat.ProviderParams is not LocalInferenceParams) + { + throw new InvalidProviderParamsException("Local LLM", nameof(LocalInferenceParams), chat.ProviderParams.GetType().Name); + } + if (chat.Messages.Count == 0) { return null; diff --git a/src/MaIN.Services/Services/LLMService/OllamaService.cs b/src/MaIN.Services/Services/LLMService/OllamaService.cs index bd511efb..09c98e63 100644 --- a/src/MaIN.Services/Services/LLMService/OllamaService.cs +++ b/src/MaIN.Services/Services/LLMService/OllamaService.cs @@ -27,6 +27,7 @@ public sealed class OllamaService( protected override string HttpClientName => HasApiKey ? ServiceConstants.HttpClients.OllamaClient : ServiceConstants.HttpClients.OllamaLocalClient; protected override string ChatCompletionsUrl => HasApiKey ? ServiceConstants.ApiUrls.OllamaOpenAiChatCompletions : ServiceConstants.ApiUrls.OllamaLocalOpenAiChatCompletions; protected override string ModelsUrl => HasApiKey ? ServiceConstants.ApiUrls.OllamaModels : ServiceConstants.ApiUrls.OllamaLocalModels; + protected override Type ExpectedParamsType => typeof(OllamaParams); protected override string GetApiKey() { diff --git a/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs b/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs index fff4c415..e15272eb 100644 --- a/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs +++ b/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs @@ -43,6 +43,7 @@ public abstract class OpenAiCompatibleService( protected abstract string GetApiKey(); protected abstract string GetApiName(); protected abstract void ValidateApiKey(); + protected abstract Type ExpectedParamsType { get; } protected virtual string HttpClientName => ServiceConstants.HttpClients.OpenAiClient; protected virtual string ChatCompletionsUrl => ServiceConstants.ApiUrls.OpenAiChatCompletions; protected virtual string ModelsUrl => ServiceConstants.ApiUrls.OpenAiModels; @@ -53,6 +54,11 @@ public abstract class OpenAiCompatibleService( ChatRequestOptions options, CancellationToken cancellationToken = default) { + if (chat.ProviderParams.GetType() != ExpectedParamsType) + { + throw new InvalidProviderParamsException(GetApiName(), ExpectedParamsType.Name, chat.ProviderParams.GetType().Name); + } + ValidateApiKey(); if (!chat.Messages.Any()) return null; @@ -682,7 +688,6 @@ private static void SetAuthorizationIfNeeded(HttpClient client, string apiKey) client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey); } } - private async Task ProcessStreamingChatAsync( Chat chat, List conversation, diff --git a/src/MaIN.Services/Services/LLMService/OpenAiService.cs b/src/MaIN.Services/Services/LLMService/OpenAiService.cs index 14af65e8..b7418dbd 100644 --- a/src/MaIN.Services/Services/LLMService/OpenAiService.cs +++ b/src/MaIN.Services/Services/LLMService/OpenAiService.cs @@ -19,7 +19,9 @@ public sealed class OpenAiService( : OpenAiCompatibleService(notificationService, httpClientFactory, memoryFactory, memoryService, logger) { private readonly MaINSettings _settings = settings ?? throw new ArgumentNullException(nameof(settings)); - + + protected override Type ExpectedParamsType => typeof(OpenAiParams); + protected override string GetApiKey() { return _settings.OpenAiKey ?? Environment.GetEnvironmentVariable(LLMApiRegistry.OpenAi.ApiKeyEnvName) ?? diff --git a/src/MaIN.Services/Services/LLMService/XaiService.cs b/src/MaIN.Services/Services/LLMService/XaiService.cs index 267a7bc7..564851c2 100644 --- a/src/MaIN.Services/Services/LLMService/XaiService.cs +++ b/src/MaIN.Services/Services/LLMService/XaiService.cs @@ -26,6 +26,7 @@ public sealed class XaiService( protected override string HttpClientName => ServiceConstants.HttpClients.XaiClient; protected override string ChatCompletionsUrl => ServiceConstants.ApiUrls.XaiOpenAiChatCompletions; protected override string ModelsUrl => ServiceConstants.ApiUrls.XaiModels; + protected override Type ExpectedParamsType => typeof(XaiParams); protected override string GetApiKey() { From 9ec7d2ff001294630666b85c78de095f4ef54e4b Mon Sep 17 00:00:00 2001 From: Magdalena Majta Date: Tue, 17 Mar 2026 23:31:57 +0100 Subject: [PATCH 03/10] Add provider params integration tests Add ProviderParamsTests integration test suite that verifies provider-specific inference parameters for OpenAI, Anthropic, Gemini, DeepSeek, GroqCloud, Xai, local (Gemma2), and Ollama models. Each passing test checks the model returns the expected answer when custom params are supplied; additional negative tests assert InvalidProviderParamsException is thrown when wrong provider params are used. Add Xunit.SkippableFact package reference to enable conditional skipping based on environment (API keys, local model file, or Ollama availability). --- .../MaIN.Core.IntegrationTests.csproj | 1 + .../ProviderParamsTests.cs | 319 ++++++++++++++++++ 2 files changed, 320 insertions(+) create mode 100644 MaIN.Core.IntegrationTests/ProviderParamsTests.cs diff --git a/MaIN.Core.IntegrationTests/MaIN.Core.IntegrationTests.csproj b/MaIN.Core.IntegrationTests/MaIN.Core.IntegrationTests.csproj index 94edd801..460c35cf 100644 --- a/MaIN.Core.IntegrationTests/MaIN.Core.IntegrationTests.csproj +++ b/MaIN.Core.IntegrationTests/MaIN.Core.IntegrationTests.csproj @@ -15,6 +15,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/MaIN.Core.IntegrationTests/ProviderParamsTests.cs b/MaIN.Core.IntegrationTests/ProviderParamsTests.cs new file mode 100644 index 00000000..383320d5 --- /dev/null +++ b/MaIN.Core.IntegrationTests/ProviderParamsTests.cs @@ -0,0 +1,319 @@ +using MaIN.Core.Hub; +using MaIN.Domain.Configuration; +using MaIN.Domain.Entities; +using MaIN.Domain.Entities.ProviderParams; +using MaIN.Domain.Exceptions; +using MaIN.Domain.Models.Concrete; + +namespace MaIN.Core.IntegrationTests; + +public class ProviderParamsTests : IntegrationTestBase +{ + private const string TestQuestion = "What is 2+2? Answer with just the number."; + + [SkippableFact] + public async Task OpenAi_Should_RespondWithParams() + { + SkipIfMissingKey(LLMApiRegistry.GetEntry(BackendType.OpenAi)?.ApiKeyEnvName!); + + var result = await AIHub.Chat() + .WithModel() + .WithMessage(TestQuestion) + .WithInferenceParams(new OpenAiParams + { + Temperature = 0.3f, + MaxTokens = 100, + TopP = 0.9f + }) + .CompleteAsync(); + + Assert.True(result.Done); + Assert.NotNull(result.Message); + Assert.NotEmpty(result.Message.Content); + Assert.Contains("4", result.Message.Content); + } + + [SkippableFact] + public async Task Anthropic_Should_RespondWithParams() + { + SkipIfMissingKey(LLMApiRegistry.GetEntry(BackendType.Anthropic)?.ApiKeyEnvName!); + + var result = await AIHub.Chat() + .WithModel() + .WithMessage(TestQuestion) + .WithInferenceParams(new AnthropicParams + { + Temperature = 0.3f, + MaxTokens = 100, + TopP = 0.9f + }) + .CompleteAsync(); + + Assert.True(result.Done); + Assert.NotNull(result.Message); + Assert.NotEmpty(result.Message.Content); + Assert.Contains("4", result.Message.Content); + } + + [SkippableFact] + public async Task Gemini_Should_RespondWithParams() + { + SkipIfMissingKey(LLMApiRegistry.GetEntry(BackendType.Gemini)?.ApiKeyEnvName!); + + var result = await AIHub.Chat() + .WithModel() + .WithMessage(TestQuestion) + .WithInferenceParams(new GeminiParams + { + Temperature = 0.3f, + MaxTokens = 100, + TopP = 0.9f + }) + .CompleteAsync(); + + Assert.True(result.Done); + Assert.NotNull(result.Message); + Assert.NotEmpty(result.Message.Content); + Assert.Contains("4", result.Message.Content); + } + + [SkippableFact] + public async Task DeepSeek_Should_RespondWithParams() + { + SkipIfMissingKey(LLMApiRegistry.GetEntry(BackendType.DeepSeek)?.ApiKeyEnvName!); + + var result = await AIHub.Chat() + .WithModel() + .WithMessage(TestQuestion) + .WithInferenceParams(new DeepSeekParams + { + Temperature = 0.3f, + MaxTokens = 100, + TopP = 0.9f + }) + .CompleteAsync(); + + Assert.True(result.Done); + Assert.NotNull(result.Message); + Assert.NotEmpty(result.Message.Content); + Assert.Contains("4", result.Message.Content); + } + + [SkippableFact] + public async Task GroqCloud_Should_RespondWithParams() + { + SkipIfMissingKey(LLMApiRegistry.GetEntry(BackendType.GroqCloud)?.ApiKeyEnvName!); + + var result = await AIHub.Chat() + .WithModel() + .WithMessage(TestQuestion) + .WithInferenceParams(new GroqCloudParams + { + Temperature = 0.3f, + MaxTokens = 100, + TopP = 0.9f + }) + .CompleteAsync(); + + Assert.True(result.Done); + Assert.NotNull(result.Message); + Assert.NotEmpty(result.Message.Content); + Assert.Contains("4", result.Message.Content); + } + + [SkippableFact] + public async Task Xai_Should_RespondWithParams() + { + SkipIfMissingKey(LLMApiRegistry.GetEntry(BackendType.Xai)?.ApiKeyEnvName!); + + var result = await AIHub.Chat() + .WithModel() + .WithMessage(TestQuestion) + .WithInferenceParams(new XaiParams + { + Temperature = 0.3f, + MaxTokens = 100, + TopP = 0.9f + }) + .CompleteAsync(); + + Assert.True(result.Done); + Assert.NotNull(result.Message); + Assert.NotEmpty(result.Message.Content); + Assert.Contains("4", result.Message.Content); + } + + [SkippableFact] + public async Task Self_Should_RespondWithParams() + { + Skip.If(!File.Exists("C:/Models/gemma2-2b.gguf"), "Local model not found at C:/Models/gemma2-2b.gguf"); + + var result = await AIHub.Chat() + .WithModel() + .WithMessage(TestQuestion) + .WithInferenceParams(new LocalInferenceParams + { + Temperature = 0.3f, + ContextSize = 8192, + MaxTokens = 100, + TopK = 40, + TopP = 0.9f + }) + .CompleteAsync(); + + Assert.True(result.Done); + Assert.NotNull(result.Message); + Assert.NotEmpty(result.Message.Content); + Assert.Contains("4", result.Message.Content); + } + + [SkippableFact] + public async Task LocalOllama_Should_RespondWithParams() + { + SkipIfOllamaNotRunning(); + + var result = await AIHub.Chat() + .WithModel() + .WithMessage(TestQuestion) + .WithInferenceParams(new OllamaParams + { + Temperature = 0.3f, + MaxTokens = 100, + TopK = 40, + TopP = 0.9f, + NumCtx = 2048 + }) + .CompleteAsync(); + + Assert.True(result.Done); + Assert.NotNull(result.Message); + Assert.NotEmpty(result.Message.Content); + Assert.Contains("4", result.Message.Content); + } + + [SkippableFact] + public async Task ClaudOllama_Should_RespondWithParams() + { + SkipIfMissingKey(LLMApiRegistry.GetEntry(BackendType.Ollama)?.ApiKeyEnvName!); + + var result = await AIHub.Chat() + .WithModel() + .WithMessage(TestQuestion) + .WithInferenceParams(new OllamaParams + { + Temperature = 0.3f, + MaxTokens = 100, + TopK = 40, + TopP = 0.9f, + NumCtx = 2048 + }) + .CompleteAsync(); + + Assert.True(result.Done); + Assert.NotNull(result.Message); + Assert.NotEmpty(result.Message.Content); + Assert.Contains("4", result.Message.Content); + } + + // --- Params mismatch validation (no API key required) --- + + [Fact] + public async Task Self_Should_ThrowWhenGivenWrongParams() + { + await Assert.ThrowsAsync(() => + AIHub.Chat() + .WithModel() + .WithMessage(TestQuestion) + .WithInferenceParams(new OpenAiParams()) + .CompleteAsync()); + } + + [Fact] + public async Task OpenAi_Should_ThrowWhenGivenWrongParams() + { + await Assert.ThrowsAsync(() => + AIHub.Chat() + .WithModel() + .WithMessage(TestQuestion) + .WithInferenceParams(new DeepSeekParams()) + .CompleteAsync()); + } + + [Fact] + public async Task Anthropic_Should_ThrowWhenGivenWrongParams() + { + await Assert.ThrowsAsync(() => + AIHub.Chat() + .WithModel() + .WithMessage(TestQuestion) + .WithInferenceParams(new OpenAiParams()) + .CompleteAsync()); + } + + [Fact] + public async Task Gemini_Should_ThrowWhenGivenWrongParams() + { + await Assert.ThrowsAsync(() => + AIHub.Chat() + .WithModel() + .WithMessage(TestQuestion) + .WithInferenceParams(new AnthropicParams()) + .CompleteAsync()); + } + + [Fact] + public async Task DeepSeek_Should_ThrowWhenGivenWrongParams() + { + await Assert.ThrowsAsync(() => + AIHub.Chat() + .WithModel() + .WithMessage(TestQuestion) + .WithInferenceParams(new GeminiParams()) + .CompleteAsync()); + } + + [Fact] + public async Task GroqCloud_Should_ThrowWhenGivenWrongParams() + { + await Assert.ThrowsAsync(() => + AIHub.Chat() + .WithModel() + .WithMessage(TestQuestion) + .WithInferenceParams(new OpenAiParams()) + .CompleteAsync()); + } + + [Fact] + public async Task Xai_Should_ThrowWhenGivenWrongParams() + { + await Assert.ThrowsAsync(() => + AIHub.Chat() + .WithModel() + .WithMessage(TestQuestion) + .WithInferenceParams(new AnthropicParams()) + .CompleteAsync()); + } + + [Fact] + public async Task Ollama_Should_ThrowWhenGivenWrongParams() + { + await Assert.ThrowsAsync(() => + AIHub.Chat() + .WithModel() + .WithMessage(TestQuestion) + .WithInferenceParams(new DeepSeekParams()) + .CompleteAsync()); + } + + private static void SkipIfMissingKey(string envName) + { + Skip.If(string.IsNullOrEmpty(Environment.GetEnvironmentVariable(envName)), + $"{envName} environment variable not set"); + } + + private static void SkipIfOllamaNotRunning() + { + Skip.If(!Helpers.NetworkHelper.PingHost("127.0.0.1", 11434, 3), + "Ollama is not running on localhost:11434"); + } +} From 5b3c3646dc479a59c51db706dcc42041d6d030ad Mon Sep 17 00:00:00 2001 From: Magdalena Majta Date: Wed, 18 Mar 2026 00:06:35 +0100 Subject: [PATCH 04/10] Add ProviderParamsFactory and wire inference params Introduce ProviderParamsFactory to create IProviderInferenceParams based on BackendType. Update Home.razor to set the backend and inference params on the new chat context (casting to IChatConfigurationBuilder) before preserving message history, ensuring the correct provider settings are applied when switching models. Also simplify ChatService by using the null-coalescing assignment (chat.Backend ??= settings.BackendType) to default the backend. --- .../ProviderParams/ProviderParamsFactory.cs | 19 +++++++++++++++++++ .../Components/Pages/Home.razor | 8 +++++++- src/MaIN.Services/Services/ChatService.cs | 2 +- 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 src/MaIN.Domain/Entities/ProviderParams/ProviderParamsFactory.cs diff --git a/src/MaIN.Domain/Entities/ProviderParams/ProviderParamsFactory.cs b/src/MaIN.Domain/Entities/ProviderParams/ProviderParamsFactory.cs new file mode 100644 index 00000000..457cb6be --- /dev/null +++ b/src/MaIN.Domain/Entities/ProviderParams/ProviderParamsFactory.cs @@ -0,0 +1,19 @@ +using MaIN.Domain.Configuration; + +namespace MaIN.Domain.Entities.ProviderParams; + +public static class ProviderParamsFactory +{ + public static IProviderInferenceParams Create(BackendType backend) => backend switch + { + BackendType.Self => new LocalInferenceParams(), + BackendType.OpenAi => new OpenAiParams(), + BackendType.DeepSeek => new DeepSeekParams(), + BackendType.GroqCloud => new GroqCloudParams(), + BackendType.Xai => new XaiParams(), + BackendType.Gemini => new GeminiParams(), + BackendType.Anthropic => new AnthropicParams(), + BackendType.Ollama => new OllamaParams(), + _ => new LocalInferenceParams() + }; +} diff --git a/src/MaIN.InferPage/Components/Pages/Home.razor b/src/MaIN.InferPage/Components/Pages/Home.razor index 9f9a7fe9..adccbf67 100644 --- a/src/MaIN.InferPage/Components/Pages/Home.razor +++ b/src/MaIN.InferPage/Components/Pages/Home.razor @@ -9,6 +9,7 @@ @using MaIN.Core.Hub.Contexts.Interfaces.ChatContext @using MaIN.Domain.Configuration @using MaIN.Domain.Entities +@using MaIN.Domain.Entities.ProviderParams @using MaIN.Domain.Exceptions @using MaIN.Domain.Models @using MaIN.Domain.Models.Abstract @@ -309,7 +310,12 @@ if (model != null) { var newCtx = AIHub.Chat().WithModel(model, imageGen: Utils.ImageGen); - // Preserve history on model switch; cast is safe — ChatContext implements both interfaces. + // Set backend and provider params before adding messages. + // Cast is safe — ChatContext implements both IChatMessageBuilder and IChatConfigurationBuilder. + ((IChatConfigurationBuilder)newCtx) + .WithBackend(Utils.BackendType) + .WithInferenceParams(ProviderParamsFactory.Create(Utils.BackendType)); + // Preserve history on model switch. ctx = Chat.Messages.Count > 0 ? (IChatMessageBuilder)newCtx.WithMessages(Chat.Messages) : newCtx; diff --git a/src/MaIN.Services/Services/ChatService.cs b/src/MaIN.Services/Services/ChatService.cs index 6ffed5c9..ef24d866 100644 --- a/src/MaIN.Services/Services/ChatService.cs +++ b/src/MaIN.Services/Services/ChatService.cs @@ -37,7 +37,7 @@ public async Task Completions( { chat.ImageGen = true; } - chat.Backend = chat.Backend ?? settings.BackendType; + chat.Backend ??= settings.BackendType; chat.Messages.Where(x => x.Type == MessageType.NotSet).ToList() .ForEach(x => x.Type = chat.Backend != BackendType.Self ? MessageType.CloudLLM : MessageType.LocalLLM); From fd9448e0d32a663c8e477e873216337e968591db Mon Sep 17 00:00:00 2001 From: Magdalena Majta Date: Wed, 18 Mar 2026 10:43:10 +0100 Subject: [PATCH 05/10] Rename provider to backend --- .../ProviderParamsTests.cs | 52 +++++++++---------- src/MaIN.Core.UnitTests/AgentContextTests.cs | 4 +- src/MaIN.Core/Hub/Contexts/AgentContext.cs | 4 +- src/MaIN.Core/Hub/Contexts/ChatContext.cs | 4 +- .../IAgentConfigurationBuilder.cs | 4 +- .../ChatContext/IChatConfigurationBuilder.cs | 4 +- .../AnthropicInferenceParams.cs} | 5 +- .../BackendParamsFactory.cs | 21 ++++++++ .../DeepSeekInferenceParams.cs} | 5 +- .../GeminiInferenceParams.cs} | 5 +- .../GroqCloudInferenceParams.cs} | 5 +- .../OllamaInferenceParams.cs} | 5 +- .../OpenAiInferenceParams.cs} | 5 +- .../XaiInferenceParams.cs} | 5 +- src/MaIN.Domain/Entities/Chat.cs | 38 +++++++------- ...ceParams.cs => IBackendInferenceParams.cs} | 2 +- .../Entities/LocalInferenceParams.cs | 2 +- .../ProviderParams/ProviderParamsFactory.cs | 19 ------- ...on.cs => InvalidBackendParamsException.cs} | 2 +- .../Components/Pages/Home.razor | 4 +- src/MaIN.Services/Mappers/ChatMapper.cs | 4 +- .../Services/Abstract/IAgentService.cs | 2 +- src/MaIN.Services/Services/AgentService.cs | 4 +- .../Services/LLMService/AnthropicService.cs | 12 ++--- .../Services/LLMService/DeepSeekService.cs | 8 +-- .../Services/LLMService/GeminiService.cs | 8 +-- .../Services/LLMService/GroqCloudService.cs | 8 +-- .../Services/LLMService/LLMService.cs | 4 +- .../Services/LLMService/OllamaService.cs | 8 +-- .../LLMService/OpenAiCompatibleService.cs | 8 +-- .../Services/LLMService/OpenAiService.cs | 8 +-- .../Services/LLMService/XaiService.cs | 8 +-- .../Services/Steps/FechDataStepHandler.cs | 2 +- 33 files changed, 144 insertions(+), 135 deletions(-) rename src/MaIN.Domain/{Entities/ProviderParams/AnthropicParams.cs => Configuration/BackendInferenceParams/AnthropicInferenceParams.cs} (70%) create mode 100644 src/MaIN.Domain/Configuration/BackendInferenceParams/BackendParamsFactory.cs rename src/MaIN.Domain/{Entities/ProviderParams/DeepSeekParams.cs => Configuration/BackendInferenceParams/DeepSeekInferenceParams.cs} (75%) rename src/MaIN.Domain/{Entities/ProviderParams/GeminiParams.cs => Configuration/BackendInferenceParams/GeminiInferenceParams.cs} (71%) rename src/MaIN.Domain/{Entities/ProviderParams/GroqCloudParams.cs => Configuration/BackendInferenceParams/GroqCloudInferenceParams.cs} (73%) rename src/MaIN.Domain/{Entities/ProviderParams/OllamaParams.cs => Configuration/BackendInferenceParams/OllamaInferenceParams.cs} (75%) rename src/MaIN.Domain/{Entities/ProviderParams/OpenAiParams.cs => Configuration/BackendInferenceParams/OpenAiInferenceParams.cs} (75%) rename src/MaIN.Domain/{Entities/ProviderParams/XaiParams.cs => Configuration/BackendInferenceParams/XaiInferenceParams.cs} (74%) rename src/MaIN.Domain/Entities/{IProviderInferenceParams.cs => IBackendInferenceParams.cs} (71%) delete mode 100644 src/MaIN.Domain/Entities/ProviderParams/ProviderParamsFactory.cs rename src/MaIN.Domain/Exceptions/{InvalidProviderParamsException.cs => InvalidBackendParamsException.cs} (74%) diff --git a/MaIN.Core.IntegrationTests/ProviderParamsTests.cs b/MaIN.Core.IntegrationTests/ProviderParamsTests.cs index 383320d5..666f66a0 100644 --- a/MaIN.Core.IntegrationTests/ProviderParamsTests.cs +++ b/MaIN.Core.IntegrationTests/ProviderParamsTests.cs @@ -1,13 +1,13 @@ using MaIN.Core.Hub; using MaIN.Domain.Configuration; using MaIN.Domain.Entities; -using MaIN.Domain.Entities.ProviderParams; +using MaIN.Domain.Configuration.BackendInferenceParams; using MaIN.Domain.Exceptions; using MaIN.Domain.Models.Concrete; namespace MaIN.Core.IntegrationTests; -public class ProviderParamsTests : IntegrationTestBase +public class BackendParamsTests : IntegrationTestBase { private const string TestQuestion = "What is 2+2? Answer with just the number."; @@ -19,7 +19,7 @@ public async Task OpenAi_Should_RespondWithParams() var result = await AIHub.Chat() .WithModel() .WithMessage(TestQuestion) - .WithInferenceParams(new OpenAiParams + .WithInferenceParams(new OpenAiInferenceParams { Temperature = 0.3f, MaxTokens = 100, @@ -41,7 +41,7 @@ public async Task Anthropic_Should_RespondWithParams() var result = await AIHub.Chat() .WithModel() .WithMessage(TestQuestion) - .WithInferenceParams(new AnthropicParams + .WithInferenceParams(new AnthropicInferenceParams { Temperature = 0.3f, MaxTokens = 100, @@ -63,7 +63,7 @@ public async Task Gemini_Should_RespondWithParams() var result = await AIHub.Chat() .WithModel() .WithMessage(TestQuestion) - .WithInferenceParams(new GeminiParams + .WithInferenceParams(new GeminiInferenceParams { Temperature = 0.3f, MaxTokens = 100, @@ -85,7 +85,7 @@ public async Task DeepSeek_Should_RespondWithParams() var result = await AIHub.Chat() .WithModel() .WithMessage(TestQuestion) - .WithInferenceParams(new DeepSeekParams + .WithInferenceParams(new DeepSeekInferenceParams { Temperature = 0.3f, MaxTokens = 100, @@ -107,7 +107,7 @@ public async Task GroqCloud_Should_RespondWithParams() var result = await AIHub.Chat() .WithModel() .WithMessage(TestQuestion) - .WithInferenceParams(new GroqCloudParams + .WithInferenceParams(new GroqCloudInferenceParams { Temperature = 0.3f, MaxTokens = 100, @@ -129,7 +129,7 @@ public async Task Xai_Should_RespondWithParams() var result = await AIHub.Chat() .WithModel() .WithMessage(TestQuestion) - .WithInferenceParams(new XaiParams + .WithInferenceParams(new XaiInferenceParams { Temperature = 0.3f, MaxTokens = 100, @@ -175,7 +175,7 @@ public async Task LocalOllama_Should_RespondWithParams() var result = await AIHub.Chat() .WithModel() .WithMessage(TestQuestion) - .WithInferenceParams(new OllamaParams + .WithInferenceParams(new OllamaInferenceParams { Temperature = 0.3f, MaxTokens = 100, @@ -199,7 +199,7 @@ public async Task ClaudOllama_Should_RespondWithParams() var result = await AIHub.Chat() .WithModel() .WithMessage(TestQuestion) - .WithInferenceParams(new OllamaParams + .WithInferenceParams(new OllamaInferenceParams { Temperature = 0.3f, MaxTokens = 100, @@ -220,88 +220,88 @@ public async Task ClaudOllama_Should_RespondWithParams() [Fact] public async Task Self_Should_ThrowWhenGivenWrongParams() { - await Assert.ThrowsAsync(() => + await Assert.ThrowsAsync(() => AIHub.Chat() .WithModel() .WithMessage(TestQuestion) - .WithInferenceParams(new OpenAiParams()) + .WithInferenceParams(new OpenAiInferenceParams()) .CompleteAsync()); } [Fact] public async Task OpenAi_Should_ThrowWhenGivenWrongParams() { - await Assert.ThrowsAsync(() => + await Assert.ThrowsAsync(() => AIHub.Chat() .WithModel() .WithMessage(TestQuestion) - .WithInferenceParams(new DeepSeekParams()) + .WithInferenceParams(new DeepSeekInferenceParams()) .CompleteAsync()); } [Fact] public async Task Anthropic_Should_ThrowWhenGivenWrongParams() { - await Assert.ThrowsAsync(() => + await Assert.ThrowsAsync(() => AIHub.Chat() .WithModel() .WithMessage(TestQuestion) - .WithInferenceParams(new OpenAiParams()) + .WithInferenceParams(new OpenAiInferenceParams()) .CompleteAsync()); } [Fact] public async Task Gemini_Should_ThrowWhenGivenWrongParams() { - await Assert.ThrowsAsync(() => + await Assert.ThrowsAsync(() => AIHub.Chat() .WithModel() .WithMessage(TestQuestion) - .WithInferenceParams(new AnthropicParams()) + .WithInferenceParams(new AnthropicInferenceParams()) .CompleteAsync()); } [Fact] public async Task DeepSeek_Should_ThrowWhenGivenWrongParams() { - await Assert.ThrowsAsync(() => + await Assert.ThrowsAsync(() => AIHub.Chat() .WithModel() .WithMessage(TestQuestion) - .WithInferenceParams(new GeminiParams()) + .WithInferenceParams(new GeminiInferenceParams()) .CompleteAsync()); } [Fact] public async Task GroqCloud_Should_ThrowWhenGivenWrongParams() { - await Assert.ThrowsAsync(() => + await Assert.ThrowsAsync(() => AIHub.Chat() .WithModel() .WithMessage(TestQuestion) - .WithInferenceParams(new OpenAiParams()) + .WithInferenceParams(new OpenAiInferenceParams()) .CompleteAsync()); } [Fact] public async Task Xai_Should_ThrowWhenGivenWrongParams() { - await Assert.ThrowsAsync(() => + await Assert.ThrowsAsync(() => AIHub.Chat() .WithModel() .WithMessage(TestQuestion) - .WithInferenceParams(new AnthropicParams()) + .WithInferenceParams(new AnthropicInferenceParams()) .CompleteAsync()); } [Fact] public async Task Ollama_Should_ThrowWhenGivenWrongParams() { - await Assert.ThrowsAsync(() => + await Assert.ThrowsAsync(() => AIHub.Chat() .WithModel() .WithMessage(TestQuestion) - .WithInferenceParams(new DeepSeekParams()) + .WithInferenceParams(new DeepSeekInferenceParams()) .CompleteAsync()); } diff --git a/src/MaIN.Core.UnitTests/AgentContextTests.cs b/src/MaIN.Core.UnitTests/AgentContextTests.cs index e8d39ace..eb17a6ea 100644 --- a/src/MaIN.Core.UnitTests/AgentContextTests.cs +++ b/src/MaIN.Core.UnitTests/AgentContextTests.cs @@ -137,7 +137,7 @@ public async Task CreateAsync_ShouldCallAgentServiceCreateAgent() It.IsAny(), It.IsAny(), It.IsAny(), - It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(agent); @@ -151,7 +151,7 @@ public async Task CreateAsync_ShouldCallAgentServiceCreateAgent() It.IsAny(), It.Is(f => f == true), It.Is(r => r == false), - It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); diff --git a/src/MaIN.Core/Hub/Contexts/AgentContext.cs b/src/MaIN.Core/Hub/Contexts/AgentContext.cs index 3b5bb3f0..80edc350 100644 --- a/src/MaIN.Core/Hub/Contexts/AgentContext.cs +++ b/src/MaIN.Core/Hub/Contexts/AgentContext.cs @@ -18,7 +18,7 @@ namespace MaIN.Core.Hub.Contexts; public sealed class AgentContext : IAgentBuilderEntryPoint, IAgentConfigurationBuilder, IAgentContextExecutor { private readonly IAgentService _agentService; - private IProviderInferenceParams? _inferenceParams; + private IBackendInferenceParams? _inferenceParams; private MemoryParams? _memoryParams; private bool _disableCache; private bool _ensureModelDownloaded; @@ -152,7 +152,7 @@ public IAgentConfigurationBuilder WithMcpConfig(Mcp mcpConfig) return this; } - public IAgentConfigurationBuilder WithInferenceParams(IProviderInferenceParams inferenceParams) + public IAgentConfigurationBuilder WithInferenceParams(IBackendInferenceParams inferenceParams) { _inferenceParams = inferenceParams; return this; diff --git a/src/MaIN.Core/Hub/Contexts/ChatContext.cs b/src/MaIN.Core/Hub/Contexts/ChatContext.cs index db32bf4c..18317c9e 100644 --- a/src/MaIN.Core/Hub/Contexts/ChatContext.cs +++ b/src/MaIN.Core/Hub/Contexts/ChatContext.cs @@ -92,9 +92,9 @@ public IChatMessageBuilder EnsureModelDownloaded() return this; } - public IChatConfigurationBuilder WithInferenceParams(IProviderInferenceParams inferenceParams) + public IChatConfigurationBuilder WithInferenceParams(IBackendInferenceParams inferenceParams) { - _chat.ProviderParams = inferenceParams; + _chat.BackendParams = inferenceParams; return this; } diff --git a/src/MaIN.Core/Hub/Contexts/Interfaces/AgentContext/IAgentConfigurationBuilder.cs b/src/MaIN.Core/Hub/Contexts/Interfaces/AgentContext/IAgentConfigurationBuilder.cs index 2edbed35..28ab7b2c 100644 --- a/src/MaIN.Core/Hub/Contexts/Interfaces/AgentContext/IAgentConfigurationBuilder.cs +++ b/src/MaIN.Core/Hub/Contexts/Interfaces/AgentContext/IAgentConfigurationBuilder.cs @@ -79,10 +79,10 @@ public interface IAgentConfigurationBuilder : IAgentActions /// based on specific parameters. Inference parameters can influence various aspects of the chat, such as response length, /// temperature, and other model-specific settings. /// - /// An object that holds the parameters for inference, such as Temperature, MaxTokens, + /// An object that holds the parameters for inference, such as Temperature, MaxTokens, /// TopP, etc. These parameters control the generation behavior of the agent. /// The context instance implementing for method chaining. - IAgentConfigurationBuilder WithInferenceParams(IProviderInferenceParams inferenceParams); + IAgentConfigurationBuilder WithInferenceParams(IBackendInferenceParams inferenceParams); /// /// Sets the memory parameters for the chat session, allowing you to customize how the AI accesses and uses its memory diff --git a/src/MaIN.Core/Hub/Contexts/Interfaces/ChatContext/IChatConfigurationBuilder.cs b/src/MaIN.Core/Hub/Contexts/Interfaces/ChatContext/IChatConfigurationBuilder.cs index 3c9e663a..5a31b843 100644 --- a/src/MaIN.Core/Hub/Contexts/Interfaces/ChatContext/IChatConfigurationBuilder.cs +++ b/src/MaIN.Core/Hub/Contexts/Interfaces/ChatContext/IChatConfigurationBuilder.cs @@ -14,10 +14,10 @@ public interface IChatConfigurationBuilder : IChatActions /// responses based on specific parameters. Inference parameters can influence various aspects of the chat, such as response length, /// temperature, and other model-specific settings. /// - /// An object that holds the parameters for inference, such as Temperature, + /// An object that holds the parameters for inference, such as Temperature, /// MaxTokens, TopP, etc. These parameters control the generation behavior of the chat. /// The context instance implementing for method chaining. - IChatConfigurationBuilder WithInferenceParams(IProviderInferenceParams inferenceParams); + IChatConfigurationBuilder WithInferenceParams(IBackendInferenceParams inferenceParams); /// /// Attaches external tools/functions that the model can invoke during the conversation. diff --git a/src/MaIN.Domain/Entities/ProviderParams/AnthropicParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/AnthropicInferenceParams.cs similarity index 70% rename from src/MaIN.Domain/Entities/ProviderParams/AnthropicParams.cs rename to src/MaIN.Domain/Configuration/BackendInferenceParams/AnthropicInferenceParams.cs index 1fda444f..f9394e0f 100644 --- a/src/MaIN.Domain/Entities/ProviderParams/AnthropicParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/AnthropicInferenceParams.cs @@ -1,9 +1,10 @@ using MaIN.Domain.Configuration; +using MaIN.Domain.Entities; using Grammar = MaIN.Domain.Models.Grammar; -namespace MaIN.Domain.Entities.ProviderParams; +namespace MaIN.Domain.Configuration.BackendInferenceParams; -public class AnthropicParams : IProviderInferenceParams +public class AnthropicInferenceParams : IBackendInferenceParams { public BackendType Backend => BackendType.Anthropic; diff --git a/src/MaIN.Domain/Configuration/BackendInferenceParams/BackendParamsFactory.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/BackendParamsFactory.cs new file mode 100644 index 00000000..63b77f84 --- /dev/null +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/BackendParamsFactory.cs @@ -0,0 +1,21 @@ +using MaIN.Domain.Configuration; +using MaIN.Domain.Entities; +using MaIN.Domain.Configuration.BackendInferenceParams; + +namespace MaIN.Domain.Configuration.BackendInferenceParams; + +public static class BackendParamsFactory +{ + public static IBackendInferenceParams Create(BackendType backend) => backend switch + { + BackendType.Self => new LocalInferenceParams(), + BackendType.OpenAi => new OpenAiInferenceParams(), + BackendType.DeepSeek => new DeepSeekInferenceParams(), + BackendType.GroqCloud => new GroqCloudInferenceParams(), + BackendType.Xai => new XaiInferenceParams(), + BackendType.Gemini => new GeminiInferenceParams(), + BackendType.Anthropic => new AnthropicInferenceParams(), + BackendType.Ollama => new OllamaInferenceParams(), + _ => new LocalInferenceParams() + }; +} diff --git a/src/MaIN.Domain/Entities/ProviderParams/DeepSeekParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/DeepSeekInferenceParams.cs similarity index 75% rename from src/MaIN.Domain/Entities/ProviderParams/DeepSeekParams.cs rename to src/MaIN.Domain/Configuration/BackendInferenceParams/DeepSeekInferenceParams.cs index 1b62807d..ad473884 100644 --- a/src/MaIN.Domain/Entities/ProviderParams/DeepSeekParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/DeepSeekInferenceParams.cs @@ -1,9 +1,10 @@ using MaIN.Domain.Configuration; +using MaIN.Domain.Entities; using Grammar = MaIN.Domain.Models.Grammar; -namespace MaIN.Domain.Entities.ProviderParams; +namespace MaIN.Domain.Configuration.BackendInferenceParams; -public class DeepSeekParams : IProviderInferenceParams +public class DeepSeekInferenceParams : IBackendInferenceParams { public BackendType Backend => BackendType.DeepSeek; diff --git a/src/MaIN.Domain/Entities/ProviderParams/GeminiParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/GeminiInferenceParams.cs similarity index 71% rename from src/MaIN.Domain/Entities/ProviderParams/GeminiParams.cs rename to src/MaIN.Domain/Configuration/BackendInferenceParams/GeminiInferenceParams.cs index b0a2f503..1586a3d7 100644 --- a/src/MaIN.Domain/Entities/ProviderParams/GeminiParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/GeminiInferenceParams.cs @@ -1,9 +1,10 @@ using MaIN.Domain.Configuration; +using MaIN.Domain.Entities; using Grammar = MaIN.Domain.Models.Grammar; -namespace MaIN.Domain.Entities.ProviderParams; +namespace MaIN.Domain.Configuration.BackendInferenceParams; -public class GeminiParams : IProviderInferenceParams +public class GeminiInferenceParams : IBackendInferenceParams { public BackendType Backend => BackendType.Gemini; diff --git a/src/MaIN.Domain/Entities/ProviderParams/GroqCloudParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/GroqCloudInferenceParams.cs similarity index 73% rename from src/MaIN.Domain/Entities/ProviderParams/GroqCloudParams.cs rename to src/MaIN.Domain/Configuration/BackendInferenceParams/GroqCloudInferenceParams.cs index 60940730..9549e93f 100644 --- a/src/MaIN.Domain/Entities/ProviderParams/GroqCloudParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/GroqCloudInferenceParams.cs @@ -1,9 +1,10 @@ using MaIN.Domain.Configuration; +using MaIN.Domain.Entities; using Grammar = MaIN.Domain.Models.Grammar; -namespace MaIN.Domain.Entities.ProviderParams; +namespace MaIN.Domain.Configuration.BackendInferenceParams; -public class GroqCloudParams : IProviderInferenceParams +public class GroqCloudInferenceParams : IBackendInferenceParams { public BackendType Backend => BackendType.GroqCloud; diff --git a/src/MaIN.Domain/Entities/ProviderParams/OllamaParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/OllamaInferenceParams.cs similarity index 75% rename from src/MaIN.Domain/Entities/ProviderParams/OllamaParams.cs rename to src/MaIN.Domain/Configuration/BackendInferenceParams/OllamaInferenceParams.cs index 7cbee160..403078b5 100644 --- a/src/MaIN.Domain/Entities/ProviderParams/OllamaParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/OllamaInferenceParams.cs @@ -1,9 +1,10 @@ using MaIN.Domain.Configuration; +using MaIN.Domain.Entities; using Grammar = MaIN.Domain.Models.Grammar; -namespace MaIN.Domain.Entities.ProviderParams; +namespace MaIN.Domain.Configuration.BackendInferenceParams; -public class OllamaParams : IProviderInferenceParams +public class OllamaInferenceParams : IBackendInferenceParams { public BackendType Backend => BackendType.Ollama; diff --git a/src/MaIN.Domain/Entities/ProviderParams/OpenAiParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/OpenAiInferenceParams.cs similarity index 75% rename from src/MaIN.Domain/Entities/ProviderParams/OpenAiParams.cs rename to src/MaIN.Domain/Configuration/BackendInferenceParams/OpenAiInferenceParams.cs index 78548da9..fbe43c1b 100644 --- a/src/MaIN.Domain/Entities/ProviderParams/OpenAiParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/OpenAiInferenceParams.cs @@ -1,9 +1,10 @@ using MaIN.Domain.Configuration; +using MaIN.Domain.Entities; using Grammar = MaIN.Domain.Models.Grammar; -namespace MaIN.Domain.Entities.ProviderParams; +namespace MaIN.Domain.Configuration.BackendInferenceParams; -public class OpenAiParams : IProviderInferenceParams +public class OpenAiInferenceParams : IBackendInferenceParams { public BackendType Backend => BackendType.OpenAi; diff --git a/src/MaIN.Domain/Entities/ProviderParams/XaiParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/XaiInferenceParams.cs similarity index 74% rename from src/MaIN.Domain/Entities/ProviderParams/XaiParams.cs rename to src/MaIN.Domain/Configuration/BackendInferenceParams/XaiInferenceParams.cs index 1bdc6814..b04edcf3 100644 --- a/src/MaIN.Domain/Entities/ProviderParams/XaiParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/XaiInferenceParams.cs @@ -1,9 +1,10 @@ using MaIN.Domain.Configuration; +using MaIN.Domain.Entities; using Grammar = MaIN.Domain.Models.Grammar; -namespace MaIN.Domain.Entities.ProviderParams; +namespace MaIN.Domain.Configuration.BackendInferenceParams; -public class XaiParams : IProviderInferenceParams +public class XaiInferenceParams : IBackendInferenceParams { public BackendType Backend => BackendType.Xai; diff --git a/src/MaIN.Domain/Entities/Chat.cs b/src/MaIN.Domain/Entities/Chat.cs index 2d81bf65..38d5594f 100644 --- a/src/MaIN.Domain/Entities/Chat.cs +++ b/src/MaIN.Domain/Entities/Chat.cs @@ -1,6 +1,6 @@ using LLama.Batched; using MaIN.Domain.Configuration; -using MaIN.Domain.Entities.ProviderParams; +using MaIN.Domain.Configuration.BackendInferenceParams; using MaIN.Domain.Entities.Tools; using MaIN.Domain.Models.Abstract; using Grammar = MaIN.Domain.Models.Grammar; @@ -40,35 +40,35 @@ public AIModel? ModelInstance public List Messages { get; set; } = []; public ChatType Type { get; set; } = ChatType.Conversation; public bool ImageGen { get; set; } - public IProviderInferenceParams ProviderParams { get; set; } = new LocalInferenceParams(); - public LocalInferenceParams? LocalParams => ProviderParams as LocalInferenceParams; + public IBackendInferenceParams BackendParams { get; set; } = new LocalInferenceParams(); + public LocalInferenceParams? LocalParams => BackendParams as LocalInferenceParams; public Grammar? InferenceGrammar { - get => ProviderParams switch + get => BackendParams switch { LocalInferenceParams p => p.Grammar, - OpenAiParams p => p.Grammar, - DeepSeekParams p => p.Grammar, - GroqCloudParams p => p.Grammar, - XaiParams p => p.Grammar, - GeminiParams p => p.Grammar, - AnthropicParams p => p.Grammar, - OllamaParams p => p.Grammar, + OpenAiInferenceParams p => p.Grammar, + DeepSeekInferenceParams p => p.Grammar, + GroqCloudInferenceParams p => p.Grammar, + XaiInferenceParams p => p.Grammar, + GeminiInferenceParams p => p.Grammar, + AnthropicInferenceParams p => p.Grammar, + OllamaInferenceParams p => p.Grammar, _ => null }; set { - switch (ProviderParams) + switch (BackendParams) { case LocalInferenceParams p: p.Grammar = value; break; - case OpenAiParams p: p.Grammar = value; break; - case DeepSeekParams p: p.Grammar = value; break; - case GroqCloudParams p: p.Grammar = value; break; - case XaiParams p: p.Grammar = value; break; - case GeminiParams p: p.Grammar = value; break; - case AnthropicParams p: p.Grammar = value; break; - case OllamaParams p: p.Grammar = value; break; + case OpenAiInferenceParams p: p.Grammar = value; break; + case DeepSeekInferenceParams p: p.Grammar = value; break; + case GroqCloudInferenceParams p: p.Grammar = value; break; + case XaiInferenceParams p: p.Grammar = value; break; + case GeminiInferenceParams p: p.Grammar = value; break; + case AnthropicInferenceParams p: p.Grammar = value; break; + case OllamaInferenceParams p: p.Grammar = value; break; } } } diff --git a/src/MaIN.Domain/Entities/IProviderInferenceParams.cs b/src/MaIN.Domain/Entities/IBackendInferenceParams.cs similarity index 71% rename from src/MaIN.Domain/Entities/IProviderInferenceParams.cs rename to src/MaIN.Domain/Entities/IBackendInferenceParams.cs index de970e2d..80db88d5 100644 --- a/src/MaIN.Domain/Entities/IProviderInferenceParams.cs +++ b/src/MaIN.Domain/Entities/IBackendInferenceParams.cs @@ -2,7 +2,7 @@ namespace MaIN.Domain.Entities; -public interface IProviderInferenceParams +public interface IBackendInferenceParams { BackendType Backend { get; } } diff --git a/src/MaIN.Domain/Entities/LocalInferenceParams.cs b/src/MaIN.Domain/Entities/LocalInferenceParams.cs index 8f6ea3f4..91716185 100644 --- a/src/MaIN.Domain/Entities/LocalInferenceParams.cs +++ b/src/MaIN.Domain/Entities/LocalInferenceParams.cs @@ -3,7 +3,7 @@ namespace MaIN.Domain.Entities; -public class LocalInferenceParams : IProviderInferenceParams +public class LocalInferenceParams : IBackendInferenceParams { public BackendType Backend => BackendType.Self; diff --git a/src/MaIN.Domain/Entities/ProviderParams/ProviderParamsFactory.cs b/src/MaIN.Domain/Entities/ProviderParams/ProviderParamsFactory.cs deleted file mode 100644 index 457cb6be..00000000 --- a/src/MaIN.Domain/Entities/ProviderParams/ProviderParamsFactory.cs +++ /dev/null @@ -1,19 +0,0 @@ -using MaIN.Domain.Configuration; - -namespace MaIN.Domain.Entities.ProviderParams; - -public static class ProviderParamsFactory -{ - public static IProviderInferenceParams Create(BackendType backend) => backend switch - { - BackendType.Self => new LocalInferenceParams(), - BackendType.OpenAi => new OpenAiParams(), - BackendType.DeepSeek => new DeepSeekParams(), - BackendType.GroqCloud => new GroqCloudParams(), - BackendType.Xai => new XaiParams(), - BackendType.Gemini => new GeminiParams(), - BackendType.Anthropic => new AnthropicParams(), - BackendType.Ollama => new OllamaParams(), - _ => new LocalInferenceParams() - }; -} diff --git a/src/MaIN.Domain/Exceptions/InvalidProviderParamsException.cs b/src/MaIN.Domain/Exceptions/InvalidBackendParamsException.cs similarity index 74% rename from src/MaIN.Domain/Exceptions/InvalidProviderParamsException.cs rename to src/MaIN.Domain/Exceptions/InvalidBackendParamsException.cs index 4368c9a1..e4bae40a 100644 --- a/src/MaIN.Domain/Exceptions/InvalidProviderParamsException.cs +++ b/src/MaIN.Domain/Exceptions/InvalidBackendParamsException.cs @@ -2,7 +2,7 @@ namespace MaIN.Domain.Exceptions; -public class InvalidProviderParamsException(string serviceName, string expectedType, string receivedType) +public class InvalidBackendParamsException(string serviceName, string expectedType, string receivedType) : MaINCustomException($"{serviceName} service requires {expectedType}, but received {receivedType}.") { public override string PublicErrorMessage => Message; diff --git a/src/MaIN.InferPage/Components/Pages/Home.razor b/src/MaIN.InferPage/Components/Pages/Home.razor index adccbf67..3847f833 100644 --- a/src/MaIN.InferPage/Components/Pages/Home.razor +++ b/src/MaIN.InferPage/Components/Pages/Home.razor @@ -9,7 +9,7 @@ @using MaIN.Core.Hub.Contexts.Interfaces.ChatContext @using MaIN.Domain.Configuration @using MaIN.Domain.Entities -@using MaIN.Domain.Entities.ProviderParams +@using MaIN.Domain.Configuration.BackendInferenceParams @using MaIN.Domain.Exceptions @using MaIN.Domain.Models @using MaIN.Domain.Models.Abstract @@ -314,7 +314,7 @@ // Cast is safe — ChatContext implements both IChatMessageBuilder and IChatConfigurationBuilder. ((IChatConfigurationBuilder)newCtx) .WithBackend(Utils.BackendType) - .WithInferenceParams(ProviderParamsFactory.Create(Utils.BackendType)); + .WithInferenceParams(BackendParamsFactory.Create(Utils.BackendType)); // Preserve history on model switch. ctx = Chat.Messages.Count > 0 ? (IChatMessageBuilder)newCtx.WithMessages(Chat.Messages) diff --git a/src/MaIN.Services/Mappers/ChatMapper.cs b/src/MaIN.Services/Mappers/ChatMapper.cs index 0d0743fd..ab7e14a0 100644 --- a/src/MaIN.Services/Mappers/ChatMapper.cs +++ b/src/MaIN.Services/Mappers/ChatMapper.cs @@ -93,7 +93,7 @@ public static ChatDocument ToDocument(this Chat chat) Backend = chat.Backend, ToolsConfiguration = chat.ToolsConfiguration, MemoryParams = chat.MemoryParams.ToDocument(), - InferenceParams = (chat.ProviderParams as LocalInferenceParams)?.ToDocument(), + InferenceParams = (chat.BackendParams as LocalInferenceParams)?.ToDocument(), ConvState = chat.ConversationState, Properties = chat.Properties, Interactive = chat.Interactive, @@ -114,7 +114,7 @@ public static Chat ToDomain(this ChatDocument chat) ToolsConfiguration = chat.ToolsConfiguration, ConversationState = chat.ConvState as Conversation.State, MemoryParams = chat.MemoryParams!.ToDomain(), - ProviderParams = chat.InferenceParams?.ToDomain() ?? new LocalInferenceParams(), + BackendParams = chat.InferenceParams?.ToDomain() ?? new LocalInferenceParams(), Interactive = chat.Interactive, Translate = chat.Translate, Type = Enum.Parse(chat.Type.ToString()) diff --git a/src/MaIN.Services/Services/Abstract/IAgentService.cs b/src/MaIN.Services/Services/Abstract/IAgentService.cs index 785920ca..16cb248a 100644 --- a/src/MaIN.Services/Services/Abstract/IAgentService.cs +++ b/src/MaIN.Services/Services/Abstract/IAgentService.cs @@ -11,7 +11,7 @@ public interface IAgentService Task Process(Chat chat, string agentId, Knowledge? knowledge, bool translatePrompt = false, Func? callbackToken = null, Func? callbackTool = null); Task CreateAgent(Agent agent, bool flow = false, bool interactiveResponse = false, - IProviderInferenceParams? inferenceParams = null, MemoryParams? memoryParams = null, bool disableCache = false); + IBackendInferenceParams? inferenceParams = null, MemoryParams? memoryParams = null, bool disableCache = false); Task GetChatByAgent(string agentId); Task Restart(string agentId); Task> GetAgents(); diff --git a/src/MaIN.Services/Services/AgentService.cs b/src/MaIN.Services/Services/AgentService.cs index bc80bc6b..41acba21 100644 --- a/src/MaIN.Services/Services/AgentService.cs +++ b/src/MaIN.Services/Services/AgentService.cs @@ -94,7 +94,7 @@ await notificationService.DispatchNotification( } public async Task CreateAgent(Agent agent, bool flow = false, bool interactiveResponse = false, - IProviderInferenceParams? inferenceParams = null, MemoryParams? memoryParams = null, bool disableCache = false) + IBackendInferenceParams? inferenceParams = null, MemoryParams? memoryParams = null, bool disableCache = false) { var chat = new Chat { @@ -103,7 +103,7 @@ public async Task CreateAgent(Agent agent, bool flow = false, bool intera Name = agent.Name, ImageGen = agent.Model == ImageGenService.LocalImageModels.FLUX, ToolsConfiguration = agent.ToolsConfiguration, - ProviderParams = inferenceParams ?? new LocalInferenceParams(), + BackendParams = inferenceParams ?? new LocalInferenceParams(), MemoryParams = memoryParams ?? new MemoryParams(), Messages = new List(), Interactive = interactiveResponse, diff --git a/src/MaIN.Services/Services/LLMService/AnthropicService.cs b/src/MaIN.Services/Services/LLMService/AnthropicService.cs index be11de87..3790ac48 100644 --- a/src/MaIN.Services/Services/LLMService/AnthropicService.cs +++ b/src/MaIN.Services/Services/LLMService/AnthropicService.cs @@ -1,5 +1,4 @@ using MaIN.Domain.Entities; -using MaIN.Domain.Entities.ProviderParams; using MaIN.Domain.Models; using MaIN.Services.Constants; using MaIN.Services.Services.Abstract; @@ -15,6 +14,7 @@ using MaIN.Domain.Exceptions; using MaIN.Domain.Models.Concrete; using MaIN.Services.Services.LLMService.Utils; +using MaIN.Domain.Configuration.BackendInferenceParams; namespace MaIN.Services.Services.LLMService; @@ -60,9 +60,9 @@ private void ValidateApiKey() public async Task Send(Chat chat, ChatRequestOptions options, CancellationToken cancellationToken = default) { - if (chat.ProviderParams is not AnthropicParams) + if (chat.BackendParams is not AnthropicInferenceParams) { - throw new InvalidProviderParamsException(LLMApiRegistry.Anthropic.ApiName, nameof(AnthropicParams), chat.ProviderParams.GetType().Name); + throw new InvalidBackendParamsException(LLMApiRegistry.Anthropic.ApiName, nameof(AnthropicInferenceParams), chat.BackendParams.GetType().Name); } ValidateApiKey(); @@ -512,7 +512,7 @@ private async Task HandleApiError(HttpResponseMessage response, CancellationToke private object BuildAnthropicRequestBody(Chat chat, List conversation, bool stream) { - var anthParams = chat.ProviderParams as AnthropicParams; + var anthParams = chat.BackendParams as AnthropicInferenceParams; var requestBody = new Dictionary { @@ -698,7 +698,7 @@ private async Task ProcessStreamingChatAsync( { var httpClient = CreateAnthropicHttpClient(); - var anthParams2 = chat.ProviderParams as AnthropicParams; + var anthParams2 = chat.BackendParams as AnthropicInferenceParams; var requestBody = new Dictionary { ["model"] = chat.ModelId, @@ -793,7 +793,7 @@ private async Task ProcessNonStreamingChatAsync( { var httpClient = CreateAnthropicHttpClient(); - var anthParams3 = chat.ProviderParams as AnthropicParams; + var anthParams3 = chat.BackendParams as AnthropicInferenceParams; var requestBody = new Dictionary { ["model"] = chat.ModelId, diff --git a/src/MaIN.Services/Services/LLMService/DeepSeekService.cs b/src/MaIN.Services/Services/LLMService/DeepSeekService.cs index 456ec164..23330760 100644 --- a/src/MaIN.Services/Services/LLMService/DeepSeekService.cs +++ b/src/MaIN.Services/Services/LLMService/DeepSeekService.cs @@ -1,7 +1,6 @@ using System.Text; using MaIN.Domain.Configuration; using MaIN.Domain.Entities; -using MaIN.Domain.Entities.ProviderParams; using MaIN.Services.Services.Abstract; using Microsoft.Extensions.Logging; using MaIN.Services.Services.LLMService.Memory; @@ -12,6 +11,7 @@ using System.Text.Json.Serialization; using MaIN.Domain.Exceptions; using MaIN.Domain.Models.Concrete; +using MaIN.Domain.Configuration.BackendInferenceParams; namespace MaIN.Services.Services.LLMService; @@ -32,7 +32,7 @@ public sealed class DeepSeekService( protected override string HttpClientName => ServiceConstants.HttpClients.DeepSeekClient; protected override string ChatCompletionsUrl => ServiceConstants.ApiUrls.DeepSeekOpenAiChatCompletions; protected override string ModelsUrl => ServiceConstants.ApiUrls.DeepSeekModels; - protected override Type ExpectedParamsType => typeof(DeepSeekParams); + protected override Type ExpectedParamsType => typeof(DeepSeekInferenceParams); protected override string GetApiKey() { @@ -51,9 +51,9 @@ protected override void ValidateApiKey() } } - protected override void ApplyProviderParams(Dictionary requestBody, Chat chat) + protected override void ApplyBackendParams(Dictionary requestBody, Chat chat) { - if (chat.ProviderParams is not DeepSeekParams p) return; + if (chat.BackendParams is not DeepSeekInferenceParams p) return; requestBody["temperature"] = p.Temperature; requestBody["max_tokens"] = p.MaxTokens; requestBody["top_p"] = p.TopP; diff --git a/src/MaIN.Services/Services/LLMService/GeminiService.cs b/src/MaIN.Services/Services/LLMService/GeminiService.cs index dcd01c53..2468bb4a 100644 --- a/src/MaIN.Services/Services/LLMService/GeminiService.cs +++ b/src/MaIN.Services/Services/LLMService/GeminiService.cs @@ -9,11 +9,11 @@ using System.Text.Json; using System.Text.Json.Serialization; using MaIN.Domain.Entities; -using MaIN.Domain.Entities.ProviderParams; using MaIN.Domain.Exceptions; using MaIN.Domain.Models; using MaIN.Domain.Models.Concrete; using MaIN.Services.Utils; +using MaIN.Domain.Configuration.BackendInferenceParams; namespace MaIN.Services.Services.LLMService; @@ -36,7 +36,7 @@ public sealed class GeminiService( protected override string HttpClientName => ServiceConstants.HttpClients.GeminiClient; protected override string ChatCompletionsUrl => ServiceConstants.ApiUrls.GeminiOpenAiChatCompletions; - protected override Type ExpectedParamsType => typeof(GeminiParams); + protected override Type ExpectedParamsType => typeof(GeminiInferenceParams); public override async Task GetCurrentModels() { @@ -75,9 +75,9 @@ protected override void ValidateApiKey() } } - protected override void ApplyProviderParams(Dictionary requestBody, Chat chat) + protected override void ApplyBackendParams(Dictionary requestBody, Chat chat) { - if (chat.ProviderParams is not GeminiParams p) return; + if (chat.BackendParams is not GeminiInferenceParams p) return; requestBody["temperature"] = p.Temperature; requestBody["max_tokens"] = p.MaxTokens; requestBody["top_p"] = p.TopP; diff --git a/src/MaIN.Services/Services/LLMService/GroqCloudService.cs b/src/MaIN.Services/Services/LLMService/GroqCloudService.cs index f0abda19..0fbb9e43 100644 --- a/src/MaIN.Services/Services/LLMService/GroqCloudService.cs +++ b/src/MaIN.Services/Services/LLMService/GroqCloudService.cs @@ -1,7 +1,6 @@ using System.Text; using MaIN.Domain.Configuration; using MaIN.Domain.Entities; -using MaIN.Domain.Entities.ProviderParams; using MaIN.Domain.Exceptions; using MaIN.Domain.Models.Concrete; using MaIN.Services.Services.Abstract; @@ -9,6 +8,7 @@ using MaIN.Services.Services.LLMService.Memory; using MaIN.Services.Constants; using MaIN.Services.Services.Models; +using MaIN.Domain.Configuration.BackendInferenceParams; namespace MaIN.Services.Services.LLMService; @@ -26,7 +26,7 @@ public sealed class GroqCloudService( protected override string HttpClientName => ServiceConstants.HttpClients.GroqCloudClient; protected override string ChatCompletionsUrl => ServiceConstants.ApiUrls.GroqCloudOpenAiChatCompletions; protected override string ModelsUrl => ServiceConstants.ApiUrls.GroqCloudModels; - protected override Type ExpectedParamsType => typeof(GroqCloudParams); + protected override Type ExpectedParamsType => typeof(GroqCloudInferenceParams); protected override string GetApiKey() { @@ -45,9 +45,9 @@ protected override void ValidateApiKey() } } - protected override void ApplyProviderParams(Dictionary requestBody, Chat chat) + protected override void ApplyBackendParams(Dictionary requestBody, Chat chat) { - if (chat.ProviderParams is not GroqCloudParams p) return; + if (chat.BackendParams is not GroqCloudInferenceParams p) return; requestBody["temperature"] = p.Temperature; requestBody["max_tokens"] = p.MaxTokens; requestBody["top_p"] = p.TopP; diff --git a/src/MaIN.Services/Services/LLMService/LLMService.cs b/src/MaIN.Services/Services/LLMService/LLMService.cs index fcfad169..dad7cc39 100644 --- a/src/MaIN.Services/Services/LLMService/LLMService.cs +++ b/src/MaIN.Services/Services/LLMService/LLMService.cs @@ -56,9 +56,9 @@ public LLMService( ChatRequestOptions requestOptions, CancellationToken cancellationToken = default) { - if (chat.ProviderParams is not LocalInferenceParams) + if (chat.BackendParams is not LocalInferenceParams) { - throw new InvalidProviderParamsException("Local LLM", nameof(LocalInferenceParams), chat.ProviderParams.GetType().Name); + throw new InvalidBackendParamsException("Local LLM", nameof(LocalInferenceParams), chat.BackendParams.GetType().Name); } if (chat.Messages.Count == 0) diff --git a/src/MaIN.Services/Services/LLMService/OllamaService.cs b/src/MaIN.Services/Services/LLMService/OllamaService.cs index 09c98e63..c2715458 100644 --- a/src/MaIN.Services/Services/LLMService/OllamaService.cs +++ b/src/MaIN.Services/Services/LLMService/OllamaService.cs @@ -1,7 +1,7 @@ using System.Text; using MaIN.Domain.Configuration; +using MaIN.Domain.Configuration.BackendInferenceParams; using MaIN.Domain.Entities; -using MaIN.Domain.Entities.ProviderParams; using MaIN.Domain.Models.Concrete; using MaIN.Services.Constants; using MaIN.Services.Services.Abstract; @@ -27,7 +27,7 @@ public sealed class OllamaService( protected override string HttpClientName => HasApiKey ? ServiceConstants.HttpClients.OllamaClient : ServiceConstants.HttpClients.OllamaLocalClient; protected override string ChatCompletionsUrl => HasApiKey ? ServiceConstants.ApiUrls.OllamaOpenAiChatCompletions : ServiceConstants.ApiUrls.OllamaLocalOpenAiChatCompletions; protected override string ModelsUrl => HasApiKey ? ServiceConstants.ApiUrls.OllamaModels : ServiceConstants.ApiUrls.OllamaLocalModels; - protected override Type ExpectedParamsType => typeof(OllamaParams); + protected override Type ExpectedParamsType => typeof(OllamaInferenceParams); protected override string GetApiKey() { @@ -42,9 +42,9 @@ protected override void ValidateApiKey() // Cloud Ollama will fail at runtime if the key is missing } - protected override void ApplyProviderParams(Dictionary requestBody, Chat chat) + protected override void ApplyBackendParams(Dictionary requestBody, Chat chat) { - if (chat.ProviderParams is not OllamaParams p) return; + if (chat.BackendParams is not OllamaInferenceParams p) return; requestBody["temperature"] = p.Temperature; requestBody["max_tokens"] = p.MaxTokens; requestBody["top_p"] = p.TopP; diff --git a/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs b/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs index e15272eb..257c1658 100644 --- a/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs +++ b/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs @@ -54,9 +54,9 @@ public abstract class OpenAiCompatibleService( ChatRequestOptions options, CancellationToken cancellationToken = default) { - if (chat.ProviderParams.GetType() != ExpectedParamsType) + if (chat.BackendParams.GetType() != ExpectedParamsType) { - throw new InvalidProviderParamsException(GetApiName(), ExpectedParamsType.Name, chat.ProviderParams.GetType().Name); + throw new InvalidBackendParamsException(GetApiName(), ExpectedParamsType.Name, chat.BackendParams.GetType().Name); } ValidateApiKey(); @@ -861,7 +861,7 @@ private object BuildRequestBody(Chat chat, List conversation, bool ["stream"] = stream }; - ApplyProviderParams(requestBody, chat); + ApplyBackendParams(requestBody, chat); if (chat.ToolsConfiguration?.Tools != null && chat.ToolsConfiguration.Tools.Any()) { @@ -885,7 +885,7 @@ private object BuildRequestBody(Chat chat, List conversation, bool return requestBody; } - protected virtual void ApplyProviderParams(Dictionary requestBody, Chat chat) + protected virtual void ApplyBackendParams(Dictionary requestBody, Chat chat) { } diff --git a/src/MaIN.Services/Services/LLMService/OpenAiService.cs b/src/MaIN.Services/Services/LLMService/OpenAiService.cs index b7418dbd..0f981de1 100644 --- a/src/MaIN.Services/Services/LLMService/OpenAiService.cs +++ b/src/MaIN.Services/Services/LLMService/OpenAiService.cs @@ -1,11 +1,11 @@ using MaIN.Domain.Configuration; using MaIN.Domain.Entities; -using MaIN.Domain.Entities.ProviderParams; using MaIN.Domain.Exceptions; using MaIN.Domain.Models.Concrete; using MaIN.Services.Services.Abstract; using Microsoft.Extensions.Logging; using MaIN.Services.Services.LLMService.Memory; +using MaIN.Domain.Configuration.BackendInferenceParams; namespace MaIN.Services.Services.LLMService; @@ -20,7 +20,7 @@ public sealed class OpenAiService( { private readonly MaINSettings _settings = settings ?? throw new ArgumentNullException(nameof(settings)); - protected override Type ExpectedParamsType => typeof(OpenAiParams); + protected override Type ExpectedParamsType => typeof(OpenAiInferenceParams); protected override string GetApiKey() { @@ -38,9 +38,9 @@ protected override void ValidateApiKey() } } - protected override void ApplyProviderParams(Dictionary requestBody, Chat chat) + protected override void ApplyBackendParams(Dictionary requestBody, Chat chat) { - if (chat.ProviderParams is not OpenAiParams p) return; + if (chat.BackendParams is not OpenAiInferenceParams p) return; requestBody["temperature"] = p.Temperature; requestBody["max_tokens"] = p.MaxTokens; requestBody["top_p"] = p.TopP; diff --git a/src/MaIN.Services/Services/LLMService/XaiService.cs b/src/MaIN.Services/Services/LLMService/XaiService.cs index 564851c2..14b656eb 100644 --- a/src/MaIN.Services/Services/LLMService/XaiService.cs +++ b/src/MaIN.Services/Services/LLMService/XaiService.cs @@ -2,13 +2,13 @@ using MaIN.Services.Constants; using MaIN.Services.Services.Models; using MaIN.Domain.Entities; -using MaIN.Domain.Entities.ProviderParams; using MaIN.Services.Services.Abstract; using MaIN.Services.Services.LLMService.Memory; using Microsoft.Extensions.Logging; using System.Text; using MaIN.Domain.Exceptions; using MaIN.Domain.Models.Concrete; +using MaIN.Domain.Configuration.BackendInferenceParams; namespace MaIN.Services.Services.LLMService; @@ -26,7 +26,7 @@ public sealed class XaiService( protected override string HttpClientName => ServiceConstants.HttpClients.XaiClient; protected override string ChatCompletionsUrl => ServiceConstants.ApiUrls.XaiOpenAiChatCompletions; protected override string ModelsUrl => ServiceConstants.ApiUrls.XaiModels; - protected override Type ExpectedParamsType => typeof(XaiParams); + protected override Type ExpectedParamsType => typeof(XaiInferenceParams); protected override string GetApiKey() { @@ -44,9 +44,9 @@ protected override void ValidateApiKey() } } - protected override void ApplyProviderParams(Dictionary requestBody, Chat chat) + protected override void ApplyBackendParams(Dictionary requestBody, Chat chat) { - if (chat.ProviderParams is not XaiParams p) return; + if (chat.BackendParams is not XaiInferenceParams p) return; requestBody["temperature"] = p.Temperature; requestBody["max_tokens"] = p.MaxTokens; requestBody["top_p"] = p.TopP; diff --git a/src/MaIN.Services/Services/Steps/FechDataStepHandler.cs b/src/MaIN.Services/Services/Steps/FechDataStepHandler.cs index a1f13c3e..82498a2d 100644 --- a/src/MaIN.Services/Services/Steps/FechDataStepHandler.cs +++ b/src/MaIN.Services/Services/Steps/FechDataStepHandler.cs @@ -68,7 +68,7 @@ private static Chat CreateMemoryChat(StepContext context, string? filterVal) ModelId = context.Chat.ModelId, Properties = context.Chat.Properties, MemoryParams = context.Chat.MemoryParams, - ProviderParams = context.Chat.ProviderParams, + BackendParams = context.Chat.BackendParams, Backend = context.Chat.Backend, Name = "Memory Chat", Id = Guid.NewGuid().ToString() From ac54712e061158f24be1026b238799285f52f52c Mon Sep 17 00:00:00 2001 From: Magdalena Majta Date: Wed, 18 Mar 2026 11:24:05 +0100 Subject: [PATCH 06/10] fix: don't send default parameters; Params may vary by model --- ...derParamsTests.cs => BackendParamsTests.cs} | 0 .../AnthropicInferenceParams.cs | 8 ++++---- .../DeepSeekInferenceParams.cs | 10 +++++----- .../GeminiInferenceParams.cs | 6 +++--- .../GroqCloudInferenceParams.cs | 8 ++++---- .../OllamaInferenceParams.cs | 12 ++++++------ .../OpenAiInferenceParams.cs | 10 +++++----- .../XaiInferenceParams.cs | 10 +++++----- src/MaIN.Domain/Entities/Chat.cs | 2 +- src/MaIN.Services/Services/ChatService.cs | 2 ++ .../Services/LLMService/AnthropicService.cs | 18 +++++++++--------- .../Services/LLMService/DeepSeekService.cs | 10 +++++----- .../Services/LLMService/GeminiService.cs | 6 +++--- .../Services/LLMService/GroqCloudService.cs | 8 ++++---- .../Services/LLMService/OllamaService.cs | 16 +++++++++++----- .../Services/LLMService/OpenAiService.cs | 10 +++++----- .../Services/LLMService/XaiService.cs | 10 +++++----- 17 files changed, 77 insertions(+), 69 deletions(-) rename MaIN.Core.IntegrationTests/{ProviderParamsTests.cs => BackendParamsTests.cs} (100%) diff --git a/MaIN.Core.IntegrationTests/ProviderParamsTests.cs b/MaIN.Core.IntegrationTests/BackendParamsTests.cs similarity index 100% rename from MaIN.Core.IntegrationTests/ProviderParamsTests.cs rename to MaIN.Core.IntegrationTests/BackendParamsTests.cs diff --git a/src/MaIN.Domain/Configuration/BackendInferenceParams/AnthropicInferenceParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/AnthropicInferenceParams.cs index f9394e0f..991e4b4f 100644 --- a/src/MaIN.Domain/Configuration/BackendInferenceParams/AnthropicInferenceParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/AnthropicInferenceParams.cs @@ -8,9 +8,9 @@ public class AnthropicInferenceParams : IBackendInferenceParams { public BackendType Backend => BackendType.Anthropic; - public float Temperature { get; init; } = 1.0f; - public int MaxTokens { get; init; } = 4096; - public int TopK { get; init; } = -1; - public float TopP { get; init; } = 1.0f; + public float? Temperature { get; init; } + public int? MaxTokens { get; init; } + public int? TopK { get; init; } + public float? TopP { get; init; } public Grammar? Grammar { get; set; } } diff --git a/src/MaIN.Domain/Configuration/BackendInferenceParams/DeepSeekInferenceParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/DeepSeekInferenceParams.cs index ad473884..3332680e 100644 --- a/src/MaIN.Domain/Configuration/BackendInferenceParams/DeepSeekInferenceParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/DeepSeekInferenceParams.cs @@ -8,11 +8,11 @@ public class DeepSeekInferenceParams : IBackendInferenceParams { public BackendType Backend => BackendType.DeepSeek; - public float Temperature { get; init; } = 0.7f; - public int MaxTokens { get; init; } = 4096; - public float TopP { get; init; } = 1.0f; - public float FrequencyPenalty { get; init; } - public float PresencePenalty { get; init; } + public float? Temperature { get; init; } + public int? MaxTokens { get; init; } + public float? TopP { get; init; } + public float? FrequencyPenalty { get; init; } + public float? PresencePenalty { get; init; } public string? ResponseFormat { get; init; } public Grammar? Grammar { get; set; } } diff --git a/src/MaIN.Domain/Configuration/BackendInferenceParams/GeminiInferenceParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/GeminiInferenceParams.cs index 1586a3d7..e9824a8a 100644 --- a/src/MaIN.Domain/Configuration/BackendInferenceParams/GeminiInferenceParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/GeminiInferenceParams.cs @@ -8,9 +8,9 @@ public class GeminiInferenceParams : IBackendInferenceParams { public BackendType Backend => BackendType.Gemini; - public float Temperature { get; init; } = 0.7f; - public int MaxTokens { get; init; } = 4096; - public float TopP { get; init; } = 0.95f; + public float? Temperature { get; init; } + public int? MaxTokens { get; init; } + public float? TopP { get; init; } public string[]? StopSequences { get; init; } public Grammar? Grammar { get; set; } } diff --git a/src/MaIN.Domain/Configuration/BackendInferenceParams/GroqCloudInferenceParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/GroqCloudInferenceParams.cs index 9549e93f..bd4bb581 100644 --- a/src/MaIN.Domain/Configuration/BackendInferenceParams/GroqCloudInferenceParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/GroqCloudInferenceParams.cs @@ -8,10 +8,10 @@ public class GroqCloudInferenceParams : IBackendInferenceParams { public BackendType Backend => BackendType.GroqCloud; - public float Temperature { get; init; } = 0.7f; - public int MaxTokens { get; init; } = 4096; - public float TopP { get; init; } = 1.0f; - public float FrequencyPenalty { get; init; } + public float? Temperature { get; init; } + public int? MaxTokens { get; init; } + public float? TopP { get; init; } + public float? FrequencyPenalty { get; init; } public string? ResponseFormat { get; init; } public Grammar? Grammar { get; set; } } diff --git a/src/MaIN.Domain/Configuration/BackendInferenceParams/OllamaInferenceParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/OllamaInferenceParams.cs index 403078b5..f0853ab7 100644 --- a/src/MaIN.Domain/Configuration/BackendInferenceParams/OllamaInferenceParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/OllamaInferenceParams.cs @@ -8,11 +8,11 @@ public class OllamaInferenceParams : IBackendInferenceParams { public BackendType Backend => BackendType.Ollama; - public float Temperature { get; init; } = 0.8f; - public int MaxTokens { get; init; } = 4096; - public int TopK { get; init; } = 40; - public float TopP { get; init; } = 0.9f; - public int NumCtx { get; init; } = 2048; - public int NumGpu { get; init; } = 30; + public float? Temperature { get; init; } + public int? MaxTokens { get; init; } + public int? TopK { get; init; } + public float? TopP { get; init; } + public int? NumCtx { get; init; } + public int? NumGpu { get; init; } public Grammar? Grammar { get; set; } } diff --git a/src/MaIN.Domain/Configuration/BackendInferenceParams/OpenAiInferenceParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/OpenAiInferenceParams.cs index fbe43c1b..4d802d9b 100644 --- a/src/MaIN.Domain/Configuration/BackendInferenceParams/OpenAiInferenceParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/OpenAiInferenceParams.cs @@ -8,11 +8,11 @@ public class OpenAiInferenceParams : IBackendInferenceParams { public BackendType Backend => BackendType.OpenAi; - public float Temperature { get; init; } = 0.7f; - public int MaxTokens { get; init; } = 4096; - public float TopP { get; init; } = 1.0f; - public float FrequencyPenalty { get; init; } - public float PresencePenalty { get; init; } + public float? Temperature { get; init; } + public int? MaxTokens { get; init; } + public float? TopP { get; init; } + public float? FrequencyPenalty { get; init; } + public float? PresencePenalty { get; init; } public string? ResponseFormat { get; init; } public Grammar? Grammar { get; set; } } diff --git a/src/MaIN.Domain/Configuration/BackendInferenceParams/XaiInferenceParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/XaiInferenceParams.cs index b04edcf3..5afc6316 100644 --- a/src/MaIN.Domain/Configuration/BackendInferenceParams/XaiInferenceParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/XaiInferenceParams.cs @@ -8,10 +8,10 @@ public class XaiInferenceParams : IBackendInferenceParams { public BackendType Backend => BackendType.Xai; - public float Temperature { get; init; } = 0.7f; - public int MaxTokens { get; init; } = 4096; - public float TopP { get; init; } = 1.0f; - public float FrequencyPenalty { get; init; } - public float PresencePenalty { get; init; } + public float? Temperature { get; init; } + public int? MaxTokens { get; init; } + public float? TopP { get; init; } + public float? FrequencyPenalty { get; init; } + public float? PresencePenalty { get; init; } public Grammar? Grammar { get; set; } } diff --git a/src/MaIN.Domain/Entities/Chat.cs b/src/MaIN.Domain/Entities/Chat.cs index 38d5594f..6a85c374 100644 --- a/src/MaIN.Domain/Entities/Chat.cs +++ b/src/MaIN.Domain/Entities/Chat.cs @@ -40,7 +40,7 @@ public AIModel? ModelInstance public List Messages { get; set; } = []; public ChatType Type { get; set; } = ChatType.Conversation; public bool ImageGen { get; set; } - public IBackendInferenceParams BackendParams { get; set; } = new LocalInferenceParams(); + public IBackendInferenceParams? BackendParams { get; set; } public LocalInferenceParams? LocalParams => BackendParams as LocalInferenceParams; public Grammar? InferenceGrammar diff --git a/src/MaIN.Services/Services/ChatService.cs b/src/MaIN.Services/Services/ChatService.cs index ef24d866..0c357270 100644 --- a/src/MaIN.Services/Services/ChatService.cs +++ b/src/MaIN.Services/Services/ChatService.cs @@ -1,4 +1,5 @@ using MaIN.Domain.Configuration; +using MaIN.Domain.Configuration.BackendInferenceParams; using MaIN.Domain.Entities; using MaIN.Domain.Exceptions.Chats; using MaIN.Domain.Models; @@ -38,6 +39,7 @@ public async Task Completions( chat.ImageGen = true; } chat.Backend ??= settings.BackendType; + chat.BackendParams ??= BackendParamsFactory.Create(chat.Backend.Value); chat.Messages.Where(x => x.Type == MessageType.NotSet).ToList() .ForEach(x => x.Type = chat.Backend != BackendType.Self ? MessageType.CloudLLM : MessageType.LocalLLM); diff --git a/src/MaIN.Services/Services/LLMService/AnthropicService.cs b/src/MaIN.Services/Services/LLMService/AnthropicService.cs index 3790ac48..d8038d16 100644 --- a/src/MaIN.Services/Services/LLMService/AnthropicService.cs +++ b/src/MaIN.Services/Services/LLMService/AnthropicService.cs @@ -524,9 +524,9 @@ private object BuildAnthropicRequestBody(Chat chat, List conversati if (anthParams != null) { - requestBody["temperature"] = anthParams.Temperature; - if (anthParams.TopP < 1.0f) requestBody["top_p"] = anthParams.TopP; - if (anthParams.TopK > 0) requestBody["top_k"] = anthParams.TopK; + if (anthParams.Temperature.HasValue) requestBody["temperature"] = anthParams.Temperature.Value; + if (anthParams.TopP.HasValue) requestBody["top_p"] = anthParams.TopP.Value; + if (anthParams.TopK.HasValue) requestBody["top_k"] = anthParams.TopK.Value; } var systemMessage = conversation.FirstOrDefault(m => @@ -711,9 +711,9 @@ private async Task ProcessStreamingChatAsync( }; if (anthParams2 != null) { - requestBody["temperature"] = anthParams2.Temperature; - if (anthParams2.TopP < 1.0f) requestBody["top_p"] = anthParams2.TopP; - if (anthParams2.TopK > 0) requestBody["top_k"] = anthParams2.TopK; + if (anthParams2.Temperature.HasValue) requestBody["temperature"] = anthParams2.Temperature.Value; + if (anthParams2.TopP.HasValue) requestBody["top_p"] = anthParams2.TopP.Value; + if (anthParams2.TopK.HasValue) requestBody["top_k"] = anthParams2.TopK.Value; } var requestJson = JsonSerializer.Serialize(requestBody); @@ -806,9 +806,9 @@ private async Task ProcessNonStreamingChatAsync( }; if (anthParams3 != null) { - requestBody["temperature"] = anthParams3.Temperature; - if (anthParams3.TopP < 1.0f) requestBody["top_p"] = anthParams3.TopP; - if (anthParams3.TopK > 0) requestBody["top_k"] = anthParams3.TopK; + if (anthParams3.Temperature.HasValue) requestBody["temperature"] = anthParams3.Temperature.Value; + if (anthParams3.TopP.HasValue) requestBody["top_p"] = anthParams3.TopP.Value; + if (anthParams3.TopK.HasValue) requestBody["top_k"] = anthParams3.TopK.Value; } var requestJson = JsonSerializer.Serialize(requestBody); diff --git a/src/MaIN.Services/Services/LLMService/DeepSeekService.cs b/src/MaIN.Services/Services/LLMService/DeepSeekService.cs index 23330760..2c70d92d 100644 --- a/src/MaIN.Services/Services/LLMService/DeepSeekService.cs +++ b/src/MaIN.Services/Services/LLMService/DeepSeekService.cs @@ -54,11 +54,11 @@ protected override void ValidateApiKey() protected override void ApplyBackendParams(Dictionary requestBody, Chat chat) { if (chat.BackendParams is not DeepSeekInferenceParams p) return; - requestBody["temperature"] = p.Temperature; - requestBody["max_tokens"] = p.MaxTokens; - requestBody["top_p"] = p.TopP; - if (p.FrequencyPenalty != 0) requestBody["frequency_penalty"] = p.FrequencyPenalty; - if (p.PresencePenalty != 0) requestBody["presence_penalty"] = p.PresencePenalty; + if (p.Temperature.HasValue) requestBody["temperature"] = p.Temperature.Value; + if (p.MaxTokens.HasValue) requestBody["max_tokens"] = p.MaxTokens.Value; + if (p.TopP.HasValue) requestBody["top_p"] = p.TopP.Value; + if (p.FrequencyPenalty.HasValue) requestBody["frequency_penalty"] = p.FrequencyPenalty.Value; + if (p.PresencePenalty.HasValue) requestBody["presence_penalty"] = p.PresencePenalty.Value; if (p.ResponseFormat != null) requestBody["response_format"] = new { type = p.ResponseFormat }; } diff --git a/src/MaIN.Services/Services/LLMService/GeminiService.cs b/src/MaIN.Services/Services/LLMService/GeminiService.cs index 2468bb4a..b8cb6ce5 100644 --- a/src/MaIN.Services/Services/LLMService/GeminiService.cs +++ b/src/MaIN.Services/Services/LLMService/GeminiService.cs @@ -78,9 +78,9 @@ protected override void ValidateApiKey() protected override void ApplyBackendParams(Dictionary requestBody, Chat chat) { if (chat.BackendParams is not GeminiInferenceParams p) return; - requestBody["temperature"] = p.Temperature; - requestBody["max_tokens"] = p.MaxTokens; - requestBody["top_p"] = p.TopP; + if (p.Temperature.HasValue) requestBody["temperature"] = p.Temperature.Value; + if (p.MaxTokens.HasValue) requestBody["max_tokens"] = p.MaxTokens.Value; + if (p.TopP.HasValue) requestBody["top_p"] = p.TopP.Value; if (p.StopSequences is { Length: > 0 }) requestBody["stop"] = p.StopSequences; } diff --git a/src/MaIN.Services/Services/LLMService/GroqCloudService.cs b/src/MaIN.Services/Services/LLMService/GroqCloudService.cs index 0fbb9e43..a19b8116 100644 --- a/src/MaIN.Services/Services/LLMService/GroqCloudService.cs +++ b/src/MaIN.Services/Services/LLMService/GroqCloudService.cs @@ -48,10 +48,10 @@ protected override void ValidateApiKey() protected override void ApplyBackendParams(Dictionary requestBody, Chat chat) { if (chat.BackendParams is not GroqCloudInferenceParams p) return; - requestBody["temperature"] = p.Temperature; - requestBody["max_tokens"] = p.MaxTokens; - requestBody["top_p"] = p.TopP; - if (p.FrequencyPenalty != 0) requestBody["frequency_penalty"] = p.FrequencyPenalty; + if (p.Temperature.HasValue) requestBody["temperature"] = p.Temperature.Value; + if (p.MaxTokens.HasValue) requestBody["max_tokens"] = p.MaxTokens.Value; + if (p.TopP.HasValue) requestBody["top_p"] = p.TopP.Value; + if (p.FrequencyPenalty.HasValue) requestBody["frequency_penalty"] = p.FrequencyPenalty.Value; if (p.ResponseFormat != null) requestBody["response_format"] = new { type = p.ResponseFormat }; } diff --git a/src/MaIN.Services/Services/LLMService/OllamaService.cs b/src/MaIN.Services/Services/LLMService/OllamaService.cs index c2715458..978ff6c8 100644 --- a/src/MaIN.Services/Services/LLMService/OllamaService.cs +++ b/src/MaIN.Services/Services/LLMService/OllamaService.cs @@ -45,11 +45,17 @@ protected override void ValidateApiKey() protected override void ApplyBackendParams(Dictionary requestBody, Chat chat) { if (chat.BackendParams is not OllamaInferenceParams p) return; - requestBody["temperature"] = p.Temperature; - requestBody["max_tokens"] = p.MaxTokens; - requestBody["top_p"] = p.TopP; - if (p.TopK > 0) requestBody["top_k"] = p.TopK; - if (p.NumCtx > 0) requestBody["options"] = new { num_ctx = p.NumCtx, num_gpu = p.NumGpu }; + if (p.Temperature.HasValue) requestBody["temperature"] = p.Temperature.Value; + if (p.MaxTokens.HasValue) requestBody["max_tokens"] = p.MaxTokens.Value; + if (p.TopP.HasValue) requestBody["top_p"] = p.TopP.Value; + if (p.TopK.HasValue) requestBody["top_k"] = p.TopK.Value; + if (p.NumCtx.HasValue || p.NumGpu.HasValue) + { + var options = new Dictionary(); + if (p.NumCtx.HasValue) options["num_ctx"] = p.NumCtx.Value; + if (p.NumGpu.HasValue) options["num_gpu"] = p.NumGpu.Value; + requestBody["options"] = options; + } } public override async Task AskMemory( diff --git a/src/MaIN.Services/Services/LLMService/OpenAiService.cs b/src/MaIN.Services/Services/LLMService/OpenAiService.cs index 0f981de1..e07d1fa8 100644 --- a/src/MaIN.Services/Services/LLMService/OpenAiService.cs +++ b/src/MaIN.Services/Services/LLMService/OpenAiService.cs @@ -41,11 +41,11 @@ protected override void ValidateApiKey() protected override void ApplyBackendParams(Dictionary requestBody, Chat chat) { if (chat.BackendParams is not OpenAiInferenceParams p) return; - requestBody["temperature"] = p.Temperature; - requestBody["max_tokens"] = p.MaxTokens; - requestBody["top_p"] = p.TopP; - if (p.FrequencyPenalty != 0) requestBody["frequency_penalty"] = p.FrequencyPenalty; - if (p.PresencePenalty != 0) requestBody["presence_penalty"] = p.PresencePenalty; + if (p.MaxTokens.HasValue) requestBody["max_completion_tokens"] = p.MaxTokens.Value; + if (p.Temperature.HasValue) requestBody["temperature"] = p.Temperature.Value; + if (p.TopP.HasValue) requestBody["top_p"] = p.TopP.Value; + if (p.FrequencyPenalty.HasValue) requestBody["frequency_penalty"] = p.FrequencyPenalty.Value; + if (p.PresencePenalty.HasValue) requestBody["presence_penalty"] = p.PresencePenalty.Value; if (p.ResponseFormat != null) requestBody["response_format"] = new { type = p.ResponseFormat }; } diff --git a/src/MaIN.Services/Services/LLMService/XaiService.cs b/src/MaIN.Services/Services/LLMService/XaiService.cs index 14b656eb..b1231005 100644 --- a/src/MaIN.Services/Services/LLMService/XaiService.cs +++ b/src/MaIN.Services/Services/LLMService/XaiService.cs @@ -47,11 +47,11 @@ protected override void ValidateApiKey() protected override void ApplyBackendParams(Dictionary requestBody, Chat chat) { if (chat.BackendParams is not XaiInferenceParams p) return; - requestBody["temperature"] = p.Temperature; - requestBody["max_tokens"] = p.MaxTokens; - requestBody["top_p"] = p.TopP; - if (p.FrequencyPenalty != 0) requestBody["frequency_penalty"] = p.FrequencyPenalty; - if (p.PresencePenalty != 0) requestBody["presence_penalty"] = p.PresencePenalty; + if (p.Temperature.HasValue) requestBody["temperature"] = p.Temperature.Value; + if (p.MaxTokens.HasValue) requestBody["max_tokens"] = p.MaxTokens.Value; + if (p.TopP.HasValue) requestBody["top_p"] = p.TopP.Value; + if (p.FrequencyPenalty.HasValue) requestBody["frequency_penalty"] = p.FrequencyPenalty.Value; + if (p.PresencePenalty.HasValue) requestBody["presence_penalty"] = p.PresencePenalty.Value; } public override async Task AskMemory( From 276b21a2bab324d3c3c37fe78234b96dd0587eb9 Mon Sep 17 00:00:00 2001 From: Magdalena Ruman <67785133+Madzionator@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:53:47 +0100 Subject: [PATCH 07/10] Adding additional parameters --- Examples/Examples/Chat/ChatExampleOpenAi.cs | 9 +++++++++ .../AnthropicInferenceParams.cs | 2 +- .../BackendParamsFactory.cs | 2 -- .../DeepSeekInferenceParams.cs | 2 +- .../GeminiInferenceParams.cs | 2 +- .../GroqCloudInferenceParams.cs | 2 +- .../OllamaInferenceParams.cs | 2 +- .../OpenAiInferenceParams.cs | 2 +- .../BackendInferenceParams/XaiInferenceParams.cs | 2 +- .../Entities/IBackendInferenceParams.cs | 1 + src/MaIN.Domain/Entities/LocalInferenceParams.cs | 1 + .../Services/LLMService/AnthropicService.cs | 16 ++++++++++++++++ .../LLMService/OpenAiCompatibleService.cs | 10 ++++++++++ .../Services/LLMService/OpenAiService.cs | 2 +- 14 files changed, 45 insertions(+), 10 deletions(-) diff --git a/Examples/Examples/Chat/ChatExampleOpenAi.cs b/Examples/Examples/Chat/ChatExampleOpenAi.cs index da03637a..12542c40 100644 --- a/Examples/Examples/Chat/ChatExampleOpenAi.cs +++ b/Examples/Examples/Chat/ChatExampleOpenAi.cs @@ -1,5 +1,6 @@ using Examples.Utils; using MaIN.Core.Hub; +using MaIN.Domain.Configuration.BackendInferenceParams; using MaIN.Domain.Models.Concrete; namespace Examples.Chat; @@ -15,6 +16,14 @@ public async Task Start() await AIHub.Chat() .WithModel() .WithMessage("What do you consider to be the greatest invention in history?") + .WithInferenceParams(new OpenAiInferenceParams // We could override some inference params + { + ResponseFormat = "text", + AdditionalParams = new Dictionary + { + ["max_completion_tokens"] = 2137 + } + }) .CompleteAsync(interactive: true); } } \ No newline at end of file diff --git a/src/MaIN.Domain/Configuration/BackendInferenceParams/AnthropicInferenceParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/AnthropicInferenceParams.cs index 991e4b4f..d649a233 100644 --- a/src/MaIN.Domain/Configuration/BackendInferenceParams/AnthropicInferenceParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/AnthropicInferenceParams.cs @@ -1,4 +1,3 @@ -using MaIN.Domain.Configuration; using MaIN.Domain.Entities; using Grammar = MaIN.Domain.Models.Grammar; @@ -13,4 +12,5 @@ public class AnthropicInferenceParams : IBackendInferenceParams public int? TopK { get; init; } public float? TopP { get; init; } public Grammar? Grammar { get; set; } + public Dictionary? AdditionalParams { get; init; } } diff --git a/src/MaIN.Domain/Configuration/BackendInferenceParams/BackendParamsFactory.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/BackendParamsFactory.cs index 63b77f84..fa8d729b 100644 --- a/src/MaIN.Domain/Configuration/BackendInferenceParams/BackendParamsFactory.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/BackendParamsFactory.cs @@ -1,6 +1,4 @@ -using MaIN.Domain.Configuration; using MaIN.Domain.Entities; -using MaIN.Domain.Configuration.BackendInferenceParams; namespace MaIN.Domain.Configuration.BackendInferenceParams; diff --git a/src/MaIN.Domain/Configuration/BackendInferenceParams/DeepSeekInferenceParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/DeepSeekInferenceParams.cs index 3332680e..00661c59 100644 --- a/src/MaIN.Domain/Configuration/BackendInferenceParams/DeepSeekInferenceParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/DeepSeekInferenceParams.cs @@ -1,4 +1,3 @@ -using MaIN.Domain.Configuration; using MaIN.Domain.Entities; using Grammar = MaIN.Domain.Models.Grammar; @@ -15,4 +14,5 @@ public class DeepSeekInferenceParams : IBackendInferenceParams public float? PresencePenalty { get; init; } public string? ResponseFormat { get; init; } public Grammar? Grammar { get; set; } + public Dictionary? AdditionalParams { get; init; } } diff --git a/src/MaIN.Domain/Configuration/BackendInferenceParams/GeminiInferenceParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/GeminiInferenceParams.cs index e9824a8a..9147b017 100644 --- a/src/MaIN.Domain/Configuration/BackendInferenceParams/GeminiInferenceParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/GeminiInferenceParams.cs @@ -1,4 +1,3 @@ -using MaIN.Domain.Configuration; using MaIN.Domain.Entities; using Grammar = MaIN.Domain.Models.Grammar; @@ -13,4 +12,5 @@ public class GeminiInferenceParams : IBackendInferenceParams public float? TopP { get; init; } public string[]? StopSequences { get; init; } public Grammar? Grammar { get; set; } + public Dictionary? AdditionalParams { get; init; } } diff --git a/src/MaIN.Domain/Configuration/BackendInferenceParams/GroqCloudInferenceParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/GroqCloudInferenceParams.cs index bd4bb581..b7c91c88 100644 --- a/src/MaIN.Domain/Configuration/BackendInferenceParams/GroqCloudInferenceParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/GroqCloudInferenceParams.cs @@ -1,4 +1,3 @@ -using MaIN.Domain.Configuration; using MaIN.Domain.Entities; using Grammar = MaIN.Domain.Models.Grammar; @@ -14,4 +13,5 @@ public class GroqCloudInferenceParams : IBackendInferenceParams public float? FrequencyPenalty { get; init; } public string? ResponseFormat { get; init; } public Grammar? Grammar { get; set; } + public Dictionary? AdditionalParams { get; init; } } diff --git a/src/MaIN.Domain/Configuration/BackendInferenceParams/OllamaInferenceParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/OllamaInferenceParams.cs index f0853ab7..92138e6b 100644 --- a/src/MaIN.Domain/Configuration/BackendInferenceParams/OllamaInferenceParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/OllamaInferenceParams.cs @@ -1,4 +1,3 @@ -using MaIN.Domain.Configuration; using MaIN.Domain.Entities; using Grammar = MaIN.Domain.Models.Grammar; @@ -15,4 +14,5 @@ public class OllamaInferenceParams : IBackendInferenceParams public int? NumCtx { get; init; } public int? NumGpu { get; init; } public Grammar? Grammar { get; set; } + public Dictionary? AdditionalParams { get; init; } } diff --git a/src/MaIN.Domain/Configuration/BackendInferenceParams/OpenAiInferenceParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/OpenAiInferenceParams.cs index 4d802d9b..b69d2640 100644 --- a/src/MaIN.Domain/Configuration/BackendInferenceParams/OpenAiInferenceParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/OpenAiInferenceParams.cs @@ -1,4 +1,3 @@ -using MaIN.Domain.Configuration; using MaIN.Domain.Entities; using Grammar = MaIN.Domain.Models.Grammar; @@ -15,4 +14,5 @@ public class OpenAiInferenceParams : IBackendInferenceParams public float? PresencePenalty { get; init; } public string? ResponseFormat { get; init; } public Grammar? Grammar { get; set; } + public Dictionary? AdditionalParams { get; init; } } diff --git a/src/MaIN.Domain/Configuration/BackendInferenceParams/XaiInferenceParams.cs b/src/MaIN.Domain/Configuration/BackendInferenceParams/XaiInferenceParams.cs index 5afc6316..b8a7c196 100644 --- a/src/MaIN.Domain/Configuration/BackendInferenceParams/XaiInferenceParams.cs +++ b/src/MaIN.Domain/Configuration/BackendInferenceParams/XaiInferenceParams.cs @@ -1,4 +1,3 @@ -using MaIN.Domain.Configuration; using MaIN.Domain.Entities; using Grammar = MaIN.Domain.Models.Grammar; @@ -14,4 +13,5 @@ public class XaiInferenceParams : IBackendInferenceParams public float? FrequencyPenalty { get; init; } public float? PresencePenalty { get; init; } public Grammar? Grammar { get; set; } + public Dictionary? AdditionalParams { get; init; } } diff --git a/src/MaIN.Domain/Entities/IBackendInferenceParams.cs b/src/MaIN.Domain/Entities/IBackendInferenceParams.cs index 80db88d5..8fe00b17 100644 --- a/src/MaIN.Domain/Entities/IBackendInferenceParams.cs +++ b/src/MaIN.Domain/Entities/IBackendInferenceParams.cs @@ -5,4 +5,5 @@ namespace MaIN.Domain.Entities; public interface IBackendInferenceParams { BackendType Backend { get; } + Dictionary? AdditionalParams { get; } } diff --git a/src/MaIN.Domain/Entities/LocalInferenceParams.cs b/src/MaIN.Domain/Entities/LocalInferenceParams.cs index 91716185..89f52621 100644 --- a/src/MaIN.Domain/Entities/LocalInferenceParams.cs +++ b/src/MaIN.Domain/Entities/LocalInferenceParams.cs @@ -23,4 +23,5 @@ public class LocalInferenceParams : IBackendInferenceParams public int TopK { get; init; } = 40; public float TopP { get; init; } = 0.9f; public Grammar? Grammar { get; set; } + public Dictionary? AdditionalParams { get; init; } } diff --git a/src/MaIN.Services/Services/LLMService/AnthropicService.cs b/src/MaIN.Services/Services/LLMService/AnthropicService.cs index d8038d16..51714c50 100644 --- a/src/MaIN.Services/Services/LLMService/AnthropicService.cs +++ b/src/MaIN.Services/Services/LLMService/AnthropicService.cs @@ -529,6 +529,12 @@ private object BuildAnthropicRequestBody(Chat chat, List conversati if (anthParams.TopK.HasValue) requestBody["top_k"] = anthParams.TopK.Value; } + if (chat.BackendParams?.AdditionalParams != null) + { + foreach (var (key, value) in chat.BackendParams.AdditionalParams) + requestBody[key] = value; + } + var systemMessage = conversation.FirstOrDefault(m => m.Role.Equals("system", StringComparison.OrdinalIgnoreCase)); @@ -715,6 +721,11 @@ private async Task ProcessStreamingChatAsync( if (anthParams2.TopP.HasValue) requestBody["top_p"] = anthParams2.TopP.Value; if (anthParams2.TopK.HasValue) requestBody["top_k"] = anthParams2.TopK.Value; } + if (chat.BackendParams?.AdditionalParams != null) + { + foreach (var (key, value) in chat.BackendParams.AdditionalParams) + requestBody[key] = value; + } var requestJson = JsonSerializer.Serialize(requestBody); var content = new StringContent(requestJson, Encoding.UTF8, "application/json"); @@ -810,6 +821,11 @@ private async Task ProcessNonStreamingChatAsync( if (anthParams3.TopP.HasValue) requestBody["top_p"] = anthParams3.TopP.Value; if (anthParams3.TopK.HasValue) requestBody["top_k"] = anthParams3.TopK.Value; } + if (chat.BackendParams?.AdditionalParams != null) + { + foreach (var (key, value) in chat.BackendParams.AdditionalParams) + requestBody[key] = value; + } var requestJson = JsonSerializer.Serialize(requestBody); var content = new StringContent(requestJson, Encoding.UTF8, "application/json"); diff --git a/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs b/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs index 257c1658..ea73a022 100644 --- a/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs +++ b/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs @@ -862,6 +862,7 @@ private object BuildRequestBody(Chat chat, List conversation, bool }; ApplyBackendParams(requestBody, chat); + ApplyAdditionalParams(requestBody, chat); if (chat.ToolsConfiguration?.Tools != null && chat.ToolsConfiguration.Tools.Any()) { @@ -889,6 +890,15 @@ protected virtual void ApplyBackendParams(Dictionary requestBody { } + private static void ApplyAdditionalParams(Dictionary requestBody, Chat chat) + { + if (chat.BackendParams?.AdditionalParams == null) return; + foreach (var (key, value) in chat.BackendParams.AdditionalParams) + { + requestBody[key] = value; + } + } + internal static void MergeMessages(List conversation, List messages) { var existing = new HashSet<(string, object)>(conversation.Select(m => (m.Role, m.Content))); diff --git a/src/MaIN.Services/Services/LLMService/OpenAiService.cs b/src/MaIN.Services/Services/LLMService/OpenAiService.cs index e07d1fa8..1848d58c 100644 --- a/src/MaIN.Services/Services/LLMService/OpenAiService.cs +++ b/src/MaIN.Services/Services/LLMService/OpenAiService.cs @@ -41,7 +41,7 @@ protected override void ValidateApiKey() protected override void ApplyBackendParams(Dictionary requestBody, Chat chat) { if (chat.BackendParams is not OpenAiInferenceParams p) return; - if (p.MaxTokens.HasValue) requestBody["max_completion_tokens"] = p.MaxTokens.Value; + if (p.MaxTokens.HasValue) requestBody["max_tokens"] = p.MaxTokens.Value; if (p.Temperature.HasValue) requestBody["temperature"] = p.Temperature.Value; if (p.TopP.HasValue) requestBody["top_p"] = p.TopP.Value; if (p.FrequencyPenalty.HasValue) requestBody["frequency_penalty"] = p.FrequencyPenalty.Value; From 12df45393e580d2c382369b748d9c3080e9046e0 Mon Sep 17 00:00:00 2001 From: Magdalena Ruman <67785133+Madzionator@users.noreply.github.com> Date: Wed, 18 Mar 2026 16:22:51 +0100 Subject: [PATCH 08/10] Refactor chat helper and AnhropicService Centralize message, image and tool handling into ChatHelper and update services to use it. Added ServiceConstants properties for ToolCalls/ToolCallId/ToolName and replaced hardcoded property keys in LLMService/OpenAiCompatibleService with ServiceConstants.Properties. Moved image extraction, message merging and message-array building logic out of Anthropic/OpenAi services into ChatHelper, made BuildAnthropicRequestBody asynchronous and now use ChatHelper.BuildMessagesArray. Removed duplicated helper methods and consolidated image MIME detection and message content construction. --- .../Constants/ServiceConstants.cs | 3 + .../Services/LLMService/AnthropicService.cs | 152 +---------- .../Services/LLMService/LLMService.cs | 37 ++- .../LLMService/OpenAiCompatibleService.cs | 215 +--------------- .../Services/LLMService/Utils/ChatHelper.cs | 238 +++++++++++++++--- 5 files changed, 247 insertions(+), 398 deletions(-) diff --git a/src/MaIN.Services/Constants/ServiceConstants.cs b/src/MaIN.Services/Constants/ServiceConstants.cs index a38ee412..ead2d5ba 100644 --- a/src/MaIN.Services/Constants/ServiceConstants.cs +++ b/src/MaIN.Services/Constants/ServiceConstants.cs @@ -63,6 +63,9 @@ public static class Properties public const string DisableCacheProperty = "DisableCache"; public const string AgentIdProperty = "AgentId"; public const string MmProjNameProperty = "MmProjName"; + public const string ToolCallsProperty = "ToolCalls"; + public const string ToolCallIdProperty = "ToolCallId"; + public const string ToolNameProperty = "ToolName"; } public static class Defaults diff --git a/src/MaIN.Services/Services/LLMService/AnthropicService.cs b/src/MaIN.Services/Services/LLMService/AnthropicService.cs index 51714c50..f2181c67 100644 --- a/src/MaIN.Services/Services/LLMService/AnthropicService.cs +++ b/src/MaIN.Services/Services/LLMService/AnthropicService.cs @@ -27,7 +27,6 @@ public sealed class AnthropicService( { private readonly MaINSettings _settings = settings ?? throw new ArgumentNullException(nameof(settings)); - private static readonly HashSet AnthropicImageExtensions = [".jpg", ".jpeg", ".png", ".gif", ".webp"]; private static readonly ConcurrentDictionary> SessionCache = new(); private const string CompletionsUrl = ServiceConstants.ApiUrls.AnthropicChatMessages; @@ -70,16 +69,14 @@ private void ValidateApiKey() if (!chat.Messages.Any()) return null; - var apiKey = GetApiKey(); - var lastMessage = chat.Messages.Last(); - await ExtractImageFromFiles(lastMessage); + await ChatHelper.ExtractImageFromFiles(lastMessage); var conversation = GetOrCreateConversation(chat, options.CreateSession); var resultBuilder = new StringBuilder(); var tokens = new List(); - if (HasFiles(lastMessage)) + if (ChatHelper.HasFiles(lastMessage)) { var result = ChatHelper.ExtractMemoryOptions(lastMessage); var memoryResult = await AskMemory(chat, result, options, cancellationToken); @@ -103,7 +100,6 @@ await options.TokenCallback(new LLMTokenValue() return await ProcessWithToolsAsync( chat, conversation, - apiKey, tokens, options, cancellationToken); @@ -114,7 +110,6 @@ await options.TokenCallback(new LLMTokenValue() await ProcessStreamingChatAsync( chat, conversation, - apiKey, tokens, resultBuilder, options.TokenCallback, @@ -126,7 +121,6 @@ await ProcessStreamingChatAsync( await ProcessNonStreamingChatAsync( chat, conversation, - apiKey, resultBuilder, cancellationToken); } @@ -150,7 +144,6 @@ await notificationService.DispatchNotification( private async Task ProcessWithToolsAsync( Chat chat, List conversation, - string apiKey, List tokens, ChatRequestOptions options, CancellationToken cancellationToken) @@ -183,7 +176,6 @@ await notificationService.DispatchNotification( currentToolUses = await ProcessStreamingChatWithToolsAsync( chat, conversation, - apiKey, tokens, resultBuilder, options, @@ -194,7 +186,6 @@ await notificationService.DispatchNotification( currentToolUses = await ProcessNonStreamingChatWithToolsAsync( chat, conversation, - apiKey, resultBuilder, cancellationToken); } @@ -319,7 +310,6 @@ await notificationService.DispatchNotification( private async Task?> ProcessStreamingChatWithToolsAsync( Chat chat, List conversation, - string apiKey, List tokens, StringBuilder resultBuilder, ChatRequestOptions options, @@ -327,7 +317,7 @@ await notificationService.DispatchNotification( { var httpClient = CreateAnthropicHttpClient(); - var requestBody = BuildAnthropicRequestBody(chat, conversation, true); + var requestBody = await BuildAnthropicRequestBody(chat, conversation, true); var requestJson = JsonSerializer.Serialize(requestBody); var content = new StringContent(requestJson, Encoding.UTF8, "application/json"); @@ -464,18 +454,17 @@ private async Task HandleApiError(HttpResponseMessage response, CancellationToke private async Task?> ProcessNonStreamingChatWithToolsAsync( Chat chat, List conversation, - string apiKey, StringBuilder resultBuilder, CancellationToken cancellationToken) { var httpClient = CreateAnthropicHttpClient(); - var requestBody = BuildAnthropicRequestBody(chat, conversation, false); + var requestBody = await BuildAnthropicRequestBody(chat, conversation, false); var requestJson = JsonSerializer.Serialize(requestBody); var content = new StringContent(requestJson, Encoding.UTF8, "application/json"); using var response = await httpClient.PostAsync(CompletionsUrl, content, cancellationToken); - + if (!response.IsSuccessStatusCode) { await HandleApiError(response, cancellationToken); @@ -510,7 +499,7 @@ private async Task HandleApiError(HttpResponseMessage response, CancellationToke return toolUses.Any() ? toolUses : null; } - private object BuildAnthropicRequestBody(Chat chat, List conversation, bool stream) + private async Task> BuildAnthropicRequestBody(Chat chat, List conversation, bool stream) { var anthParams = chat.BackendParams as AnthropicInferenceParams; @@ -519,7 +508,7 @@ private object BuildAnthropicRequestBody(Chat chat, List conversati ["model"] = chat.ModelId, ["max_tokens"] = anthParams?.MaxTokens ?? 4096, ["stream"] = stream, - ["messages"] = BuildAnthropicMessages(conversation) + ["messages"] = await ChatHelper.BuildMessagesArray(conversation, chat, ImageType.AsBase64) }; if (anthParams != null) @@ -562,40 +551,6 @@ private object BuildAnthropicRequestBody(Chat chat, List conversati return requestBody; } - private List BuildAnthropicMessages(List conversation) - { - var messages = new List(); - - foreach (var msg in conversation) - { - if (msg.Role.Equals("system", StringComparison.OrdinalIgnoreCase)) - continue; - - object content; - - if (msg.Content is string textContent) - { - content = textContent; - } - else if (msg.Content is List contentBlocks) - { - content = contentBlocks; - } - else - { - content = msg.Content; - } - - messages.Add(new - { - role = msg.Role, - content = content - }); - } - - return messages; - } - public async Task AskMemory(Chat chat, ChatMemoryOptions memoryOptions, ChatRequestOptions requestOptions, CancellationToken cancellationToken = default) { @@ -639,7 +594,7 @@ private List GetOrCreateConversation(Chat chat, bool createSession) conversation = new List(); } - OpenAiCompatibleService.MergeMessages(conversation, chat.Messages); + ChatHelper.MergeMessages(conversation, chat.Messages); return conversation; } @@ -651,51 +606,9 @@ private void UpdateSessionCache(string chatId, string assistantResponse, bool cr } } - private static bool HasFiles(Message message) - { - return message.Files != null && message.Files.Count > 0; - } - - private static async Task ExtractImageFromFiles(Message message) - { - if (message.Files == null || message.Files.Count == 0) - return; - - var imageFiles = message.Files - .Where(f => AnthropicImageExtensions.Contains(f.Extension.ToLowerInvariant())) - .ToList(); - - if (imageFiles.Count == 0) - return; - - var imageBytesList = new List(); - foreach (var imageFile in imageFiles) - { - if (imageFile.StreamContent != null) - { - using var ms = new MemoryStream(); - imageFile.StreamContent.Position = 0; - await imageFile.StreamContent.CopyToAsync(ms); - imageBytesList.Add(ms.ToArray()); - } - else if (imageFile.Path != null) - { - imageBytesList.Add(await File.ReadAllBytesAsync(imageFile.Path)); - } - - message.Files.Remove(imageFile); - } - - message.Images = imageBytesList; - - if (message.Files.Count == 0) - message.Files = null; - } - private async Task ProcessStreamingChatAsync( Chat chat, List conversation, - string apiKey, List tokens, StringBuilder resultBuilder, Func? tokenCallback, @@ -704,29 +617,7 @@ private async Task ProcessStreamingChatAsync( { var httpClient = CreateAnthropicHttpClient(); - var anthParams2 = chat.BackendParams as AnthropicInferenceParams; - var requestBody = new Dictionary - { - ["model"] = chat.ModelId, - ["max_tokens"] = anthParams2?.MaxTokens ?? 4096, - ["stream"] = true, - ["system"] = chat.InferenceGrammar is not null - ? $"Respond only using the following grammar format: \n{chat.InferenceGrammar.Value}\n. Do not add explanations, code tags, or any extra content." - : "", - ["messages"] = await OpenAiCompatibleService.BuildMessagesArray(conversation, chat, ImageType.AsBase64) - }; - if (anthParams2 != null) - { - if (anthParams2.Temperature.HasValue) requestBody["temperature"] = anthParams2.Temperature.Value; - if (anthParams2.TopP.HasValue) requestBody["top_p"] = anthParams2.TopP.Value; - if (anthParams2.TopK.HasValue) requestBody["top_k"] = anthParams2.TopK.Value; - } - if (chat.BackendParams?.AdditionalParams != null) - { - foreach (var (key, value) in chat.BackendParams.AdditionalParams) - requestBody[key] = value; - } - + var requestBody = await BuildAnthropicRequestBody(chat, conversation, true); var requestJson = JsonSerializer.Serialize(requestBody); var content = new StringContent(requestJson, Encoding.UTF8, "application/json"); @@ -798,35 +689,12 @@ await notificationService.DispatchNotification( private async Task ProcessNonStreamingChatAsync( Chat chat, List conversation, - string apiKey, StringBuilder resultBuilder, CancellationToken cancellationToken) { var httpClient = CreateAnthropicHttpClient(); - var anthParams3 = chat.BackendParams as AnthropicInferenceParams; - var requestBody = new Dictionary - { - ["model"] = chat.ModelId, - ["max_tokens"] = anthParams3?.MaxTokens ?? 4096, - ["stream"] = false, - ["system"] = chat.InferenceGrammar is not null - ? $"Respond only using the following grammar format: \n{chat.InferenceGrammar.Value}\n. Do not add explanations, code tags, or any extra content." - : "", - ["messages"] = await OpenAiCompatibleService.BuildMessagesArray(conversation, chat, ImageType.AsBase64) - }; - if (anthParams3 != null) - { - if (anthParams3.Temperature.HasValue) requestBody["temperature"] = anthParams3.Temperature.Value; - if (anthParams3.TopP.HasValue) requestBody["top_p"] = anthParams3.TopP.Value; - if (anthParams3.TopK.HasValue) requestBody["top_k"] = anthParams3.TopK.Value; - } - if (chat.BackendParams?.AdditionalParams != null) - { - foreach (var (key, value) in chat.BackendParams.AdditionalParams) - requestBody[key] = value; - } - + var requestBody = await BuildAnthropicRequestBody(chat, conversation, false); var requestJson = JsonSerializer.Serialize(requestBody); var content = new StringContent(requestJson, Encoding.UTF8, "application/json"); diff --git a/src/MaIN.Services/Services/LLMService/LLMService.cs b/src/MaIN.Services/Services/LLMService/LLMService.cs index dad7cc39..d121070b 100644 --- a/src/MaIN.Services/Services/LLMService/LLMService.cs +++ b/src/MaIN.Services/Services/LLMService/LLMService.cs @@ -391,7 +391,7 @@ private static void ProcessTextMessage(Conversation conversation, bool isNewConversation) { var template = new LLamaTemplate(llmModel); - var finalPrompt = ChatHelper.GetFinalPrompt(lastMsg, model, isNewConversation); + var finalPrompt = GetFinalPrompt(lastMsg, model, isNewConversation); var hasTools = chat.ToolsConfiguration?.Tools != null && chat.ToolsConfiguration.Tools.Count != 0; @@ -470,7 +470,19 @@ private static string FormatToolsForPrompt(ToolsConfiguration toolsConfig) using var sampler = LLMService.CreateSampler(chat.LocalParams!); var decoder = new StreamingTokenDecoder(executor.Context); - var inferenceParams = ChatHelper.CreateInferenceParams(chat, llmModel); + var inferenceParams = new InferenceParams + { + SamplingPipeline = new DefaultSamplingPipeline + { + Temperature = chat.LocalParams!.Temperature, + TopK = chat.LocalParams!.TopK, + TopP = chat.LocalParams!.TopP + }, + AntiPrompts = [llmModel.Vocab.EOT?.ToString() ?? "User:"], + TokensKeep = chat.LocalParams!.TokensKeep, + MaxTokens = chat.LocalParams!.MaxTokens + }; + var maxTokens = inferenceParams.MaxTokens == -1 ? int.MaxValue : inferenceParams.MaxTokens; var reasoningModel = model as IReasoningModel; @@ -704,7 +716,7 @@ private async Task ProcessWithToolsAsync( } var toolCalls = parseResult.ToolCalls!; - responseMessage.Properties[ToolCallsProperty] = JsonSerializer.Serialize(toolCalls); + responseMessage.Properties[ServiceConstants.Properties.ToolCallsProperty] = JsonSerializer.Serialize(toolCalls); foreach (var toolCall in toolCalls) { @@ -758,8 +770,8 @@ await requestOptions.ToolCallback.Invoke(new ToolInvocation Type = MessageType.LocalLLM, Tool = true }; - toolMessage.Properties[ToolCallIdProperty] = toolCall.Id; - toolMessage.Properties[ToolNameProperty] = toolCall.Function.Name; + toolMessage.Properties[ServiceConstants.Properties.ToolCallIdProperty] = toolCall.Id; + toolMessage.Properties[ServiceConstants.Properties.ToolNameProperty] = toolCall.Function.Name; chat.Messages.Add(toolMessage.MarkProcessed()); } catch (Exception ex) @@ -772,8 +784,8 @@ await requestOptions.ToolCallback.Invoke(new ToolInvocation Type = MessageType.LocalLLM, Tool = true }; - toolMessage.Properties[ToolCallIdProperty] = toolCall.Id; - toolMessage.Properties[ToolNameProperty] = toolCall.Function.Name; + toolMessage.Properties[ServiceConstants.Properties.ToolCallIdProperty] = toolCall.Id; + toolMessage.Properties[ServiceConstants.Properties.ToolNameProperty] = toolCall.Function.Name; chat.Messages.Add(toolMessage.MarkProcessed()); } } @@ -808,7 +820,12 @@ await requestOptions.ToolCallback.Invoke(new ToolInvocation }; } - private const string ToolCallsProperty = "ToolCalls"; - private const string ToolCallIdProperty = "ToolCallId"; - private const string ToolNameProperty = "ToolName"; + private static string GetFinalPrompt(Message message, AIModel model, bool startSession) + { + var additionalPrompt = (model as IReasoningModel)?.AdditionalPrompt; + return startSession && additionalPrompt != null + ? $"{message.Content}{additionalPrompt}" + : message.Content; + } + } diff --git a/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs b/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs index ea73a022..ff4b641f 100644 --- a/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs +++ b/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs @@ -36,9 +36,6 @@ public abstract class OpenAiCompatibleService( private static readonly JsonSerializerOptions DefaultJsonSerializerOptions = new() { PropertyNameCaseInsensitive = true }; - private const string ToolCallsProperty = "ToolCalls"; - private const string ToolCallIdProperty = "ToolCallId"; - private const string ToolNameProperty = "ToolName"; protected abstract string GetApiKey(); protected abstract string GetApiName(); @@ -463,7 +460,7 @@ await _notificationService.DispatchNotification( } // If there are images, use SearchAsync + regular chat with images - if (HasImages(lastMessage)) + if (ChatHelper.HasImages(lastMessage)) { var searchResult = await kernel.SearchAsync(userQuery, cancellationToken: cancellationToken); await kernel.DeleteIndexAsync(cancellationToken: cancellationToken); @@ -628,7 +625,7 @@ private List GetOrCreateConversation(Chat chat, bool createSession) conversation = new List(); } - MergeMessages(conversation, chat.Messages); + ChatHelper.MergeMessages(conversation, chat.Messages); return conversation; } @@ -857,7 +854,7 @@ private object BuildRequestBody(Chat chat, List conversation, bool var requestBody = new Dictionary { ["model"] = chat.ModelId, - ["messages"] = BuildMessagesArray(conversation, chat, ImageType.AsUrl).Result, + ["messages"] = ChatHelper.BuildMessagesArray(conversation, chat, ImageType.AsUrl).Result, ["stream"] = stream }; @@ -899,53 +896,6 @@ private static void ApplyAdditionalParams(Dictionary requestBody } } - internal static void MergeMessages(List conversation, List messages) - { - var existing = new HashSet<(string, object)>(conversation.Select(m => (m.Role, m.Content))); - foreach (var msg in messages) - { - var role = msg.Role.ToLowerInvariant(); - - if (HasImages(msg)) - { - var simplifiedContent = $"{msg.Content} [Contains image]"; - if (!existing.Contains((role, simplifiedContent))) - { - var chatMessage = new ChatMessage(role, msg.Content); - chatMessage.OriginalMessage = msg; - conversation.Add(chatMessage); - existing.Add((role, simplifiedContent)); - } - } - else - { - if (!existing.Contains((role, msg.Content))) - { - var chatMessage = new ChatMessage(role, msg.Content); - - // Extract tool-related data from Properties - if (msg.Tool && msg.Properties.ContainsKey(ToolCallsProperty)) - { - var toolCallsJson = msg.Properties[ToolCallsProperty]; - chatMessage.ToolCalls = JsonSerializer.Deserialize>(toolCallsJson); - } - - if (msg.Properties.ContainsKey(ToolCallIdProperty)) - { - chatMessage.ToolCallId = msg.Properties[ToolCallIdProperty]; - } - - if (msg.Properties.ContainsKey(ToolNameProperty)) - { - chatMessage.Name = msg.Properties[ToolNameProperty]; - } - - conversation.Add(chatMessage); - existing.Add((role, msg.Content)); - } - } - } - } protected static ChatResult CreateChatResult(Chat chat, string content, List tokens) { @@ -964,59 +914,6 @@ protected static ChatResult CreateChatResult(Chat chat, string content, List BuildMessagesArray(List conversation, Chat chat, ImageType imageType) - { - var messages = new List(); - - foreach (var msg in conversation) - { - var content = msg.OriginalMessage != null ? BuildMessageContent(msg.OriginalMessage, imageType) : msg.Content; - if (chat.InferenceGrammar != null && msg.Role == "user") - { - var jsonGrammarConverter = new GrammarToJsonConverter(); - string jsonGrammar = jsonGrammarConverter.ConvertToJson(chat.InferenceGrammar); - - var grammarInstruction = $" | Respond only using the following JSON format: \n{jsonGrammar}\n. Do not add explanations, code tags, or any extra content."; - - if (content is string textContent) - { - content = textContent + grammarInstruction; - } - else if (content is List contentParts) - { - var modifiedParts = contentParts.ToList(); - modifiedParts.Add(new { type = "text", text = grammarInstruction }); - content = modifiedParts; - } - } - - var messageObj = new Dictionary - { - ["role"] = msg.Role, - ["content"] = content ?? string.Empty - }; - - if (msg.ToolCalls != null && msg.ToolCalls.Any()) - { - messageObj["tool_calls"] = msg.ToolCalls; - } - - if (!string.IsNullOrEmpty(msg.ToolCallId)) - { - messageObj["tool_call_id"] = msg.ToolCallId; - - if (!string.IsNullOrEmpty(msg.Name)) - { - messageObj["name"] = msg.Name; - } - } - - messages.Add(messageObj); - } - - return messages.ToArray(); - } - private static async Task InvokeTokenCallbackAsync(Func? callback, LLMTokenValue token) { if (callback != null) @@ -1024,112 +921,6 @@ private static async Task InvokeTokenCallbackAsync(Func? ca await callback.Invoke(token); } } - - private static bool HasImages(Message message) - { - return message.Images?.Count > 0; - } - - private static object BuildMessageContent(Message message, ImageType imageType) - { - if (!HasImages(message)) - { - return message.Content; - } - - var contentParts = new List(); - - if (!string.IsNullOrEmpty(message.Content)) - { - contentParts.Add(new - { - type = "text", - text = message.Content - }); - } - - foreach (var imageBytes in message.Images!) - { - var base64Data = Convert.ToBase64String(imageBytes); - var mimeType = DetectImageMimeType(imageBytes); - - switch (imageType) - { - case ImageType.AsUrl: - contentParts.Add(new - { - type = "image_url", - image_url = new - { - url = $"data:{mimeType};base64,{base64Data}", - detail = "auto" - } - }); - break; - case ImageType.AsBase64: - contentParts.Add(new - { - type = "image", - source = new - { - data = base64Data, - media_type = mimeType, - type = "base64" - } - }); - break; - } - } - - return contentParts; - } - - private static string DetectImageMimeType(byte[] imageBytes) - { - if (imageBytes.Length < 4) - return "image/jpeg"; - - if (imageBytes[0] == 0xFF && imageBytes[1] == 0xD8) - return "image/jpeg"; - - if (imageBytes.Length >= 8 && - imageBytes[0] == 0x89 && imageBytes[1] == 0x50 && - imageBytes[2] == 0x4E && imageBytes[3] == 0x47) - return "image/png"; - - if (imageBytes.Length >= 6 && - imageBytes[0] == 0x47 && imageBytes[1] == 0x49 && - imageBytes[2] == 0x46 && imageBytes[3] == 0x38) - return "image/gif"; - - if (imageBytes.Length >= 12 && - imageBytes[0] == 0x52 && imageBytes[1] == 0x49 && - imageBytes[2] == 0x46 && imageBytes[3] == 0x46 && - imageBytes[8] == 0x57 && imageBytes[9] == 0x45 && - imageBytes[10] == 0x42 && imageBytes[11] == 0x50) - return "image/webp"; - - // HEIC/HEIF format (iPhone photos) - if (imageBytes.Length >= 12 && - imageBytes[4] == 0x66 && imageBytes[5] == 0x74 && - imageBytes[6] == 0x79 && imageBytes[7] == 0x70) - { - // Check for heic/heif brands - if ((imageBytes[8] == 0x68 && imageBytes[9] == 0x65 && imageBytes[10] == 0x69 && imageBytes[11] == 0x63) || - (imageBytes[8] == 0x68 && imageBytes[9] == 0x65 && imageBytes[10] == 0x69 && imageBytes[11] == 0x66)) - return "image/heic"; - } - - // AVIF format - if (imageBytes.Length >= 12 && - imageBytes[4] == 0x66 && imageBytes[5] == 0x74 && - imageBytes[6] == 0x79 && imageBytes[7] == 0x70 && - imageBytes[8] == 0x61 && imageBytes[9] == 0x76 && - imageBytes[10] == 0x69 && imageBytes[11] == 0x66) - return "image/avif"; - - return "image/jpeg"; - } } internal class ChatMessage diff --git a/src/MaIN.Services/Services/LLMService/Utils/ChatHelper.cs b/src/MaIN.Services/Services/LLMService/Utils/ChatHelper.cs index 36f7efef..d9c08bd9 100644 --- a/src/MaIN.Services/Services/LLMService/Utils/ChatHelper.cs +++ b/src/MaIN.Services/Services/LLMService/Utils/ChatHelper.cs @@ -1,9 +1,8 @@ -using LLama; -using LLama.Sampling; +using System.Text.Json; using MaIN.Domain.Entities; -using MaIN.Domain.Models.Abstract; +using MaIN.Domain.Entities.Tools; using MaIN.Services.Constants; -using InferenceParams = LLama.Common.InferenceParams; +using MaIN.Services.Utils; namespace MaIN.Services.Services.LLMService.Utils; @@ -59,36 +58,6 @@ public static async Task ExtractImageFromFiles(Message message) } - /// - /// Generates final prompt including additional prompt if needed - /// - public static string GetFinalPrompt(Message message, AIModel model, bool startSession) - { - var additionalPrompt = (model as IReasoningModel)?.AdditionalPrompt; - return startSession && additionalPrompt != null - ? $"{message.Content}{additionalPrompt}" - : message.Content; - } - - /// - /// Creates inference parameters for a chat - /// - public static InferenceParams CreateInferenceParams(Chat chat, LLamaWeights model) - { - return new InferenceParams - { - SamplingPipeline = new DefaultSamplingPipeline - { - Temperature = chat.LocalParams!.Temperature, - TopK = chat.LocalParams!.TopK, - TopP = chat.LocalParams!.TopP - }, - AntiPrompts = [model.Vocab.EOT?.ToString() ?? "User:"], - TokensKeep = chat.LocalParams!.TokensKeep, - MaxTokens = chat.LocalParams!.MaxTokens - }; - } - /// /// Checks if a message contains files /// @@ -97,6 +66,11 @@ public static bool HasFiles(Message message) return message.Files?.Any() ?? false; } + public static bool HasImages(Message message) + { + return message.Images?.Count > 0; + } + /// /// Extracts memory options from a message with files /// @@ -127,4 +101,200 @@ public static ChatMemoryOptions ExtractMemoryOptions(Message message) PreProcess = preProcess }; } + + /// + /// Builds an array of message objects for API requests, handling images and grammar injection. + /// + internal static void MergeMessages(List conversation, List messages) + { + var existing = new HashSet<(string, object)>(conversation.Select(m => (m.Role, m.Content))); + foreach (var msg in messages) + { + var role = msg.Role.ToLowerInvariant(); + + if (HasImages(msg)) + { + var simplifiedContent = $"{msg.Content} [Contains image]"; + if (!existing.Contains((role, simplifiedContent))) + { + var chatMessage = new ChatMessage(role, msg.Content) { OriginalMessage = msg }; + conversation.Add(chatMessage); + existing.Add((role, simplifiedContent)); + } + } + else + { + if (!existing.Contains((role, msg.Content))) + { + var chatMessage = new ChatMessage(role, msg.Content); + + if (msg.Tool && msg.Properties.ContainsKey(ServiceConstants.Properties.ToolCallsProperty)) + { + var toolCallsJson = msg.Properties[ServiceConstants.Properties.ToolCallsProperty]; + chatMessage.ToolCalls = JsonSerializer.Deserialize>(toolCallsJson); + } + + if (msg.Properties.ContainsKey(ServiceConstants.Properties.ToolCallIdProperty)) + chatMessage.ToolCallId = msg.Properties[ServiceConstants.Properties.ToolCallIdProperty]; + + if (msg.Properties.ContainsKey(ServiceConstants.Properties.ToolNameProperty)) + chatMessage.Name = msg.Properties[ServiceConstants.Properties.ToolNameProperty]; + + conversation.Add(chatMessage); + existing.Add((role, msg.Content)); + } + } + } + } + + internal static async Task BuildMessagesArray(List conversation, Chat chat, ImageType imageType) + { + var messages = new List(); + + foreach (var msg in conversation) + { + var content = msg.OriginalMessage != null ? BuildMessageContent(msg.OriginalMessage, imageType) : msg.Content; + if (chat.InferenceGrammar != null && msg.Role == "user") + { + var jsonGrammarConverter = new GrammarToJsonConverter(); + string jsonGrammar = jsonGrammarConverter.ConvertToJson(chat.InferenceGrammar); + + var grammarInstruction = $" | Respond only using the following JSON format: \n{jsonGrammar}\n. Do not add explanations, code tags, or any extra content."; + + if (content is string textContent) + { + content = textContent + grammarInstruction; + } + else if (content is List contentParts) + { + var modifiedParts = contentParts.ToList(); + modifiedParts.Add(new { type = "text", text = grammarInstruction }); + content = modifiedParts; + } + } + + var messageObj = new Dictionary + { + ["role"] = msg.Role, + ["content"] = content ?? string.Empty + }; + + if (msg.ToolCalls != null && msg.ToolCalls.Any()) + { + messageObj["tool_calls"] = msg.ToolCalls; + } + + if (!string.IsNullOrEmpty(msg.ToolCallId)) + { + messageObj["tool_call_id"] = msg.ToolCallId; + + if (!string.IsNullOrEmpty(msg.Name)) + { + messageObj["name"] = msg.Name; + } + } + + messages.Add(messageObj); + } + + return messages.ToArray(); + } + + private static object BuildMessageContent(Message message, ImageType imageType) + { + if (message.Images == null || message.Images.Count == 0) + { + return message.Content; + } + + var contentParts = new List(); + + if (!string.IsNullOrEmpty(message.Content)) + { + contentParts.Add(new + { + type = "text", + text = message.Content + }); + } + + foreach (var imageBytes in message.Images) + { + var base64Data = Convert.ToBase64String(imageBytes); + var mimeType = DetectImageMimeType(imageBytes); + + switch (imageType) + { + case ImageType.AsUrl: + contentParts.Add(new + { + type = "image_url", + image_url = new + { + url = $"data:{mimeType};base64,{base64Data}", + detail = "auto" + } + }); + break; + case ImageType.AsBase64: + contentParts.Add(new + { + type = "image", + source = new + { + data = base64Data, + media_type = mimeType, + type = "base64" + } + }); + break; + } + } + + return contentParts; + } + + private static string DetectImageMimeType(byte[] imageBytes) + { + if (imageBytes.Length < 4) + return "image/jpeg"; + + if (imageBytes[0] == 0xFF && imageBytes[1] == 0xD8) + return "image/jpeg"; + + if (imageBytes.Length >= 8 && + imageBytes[0] == 0x89 && imageBytes[1] == 0x50 && + imageBytes[2] == 0x4E && imageBytes[3] == 0x47) + return "image/png"; + + if (imageBytes.Length >= 6 && + imageBytes[0] == 0x47 && imageBytes[1] == 0x49 && + imageBytes[2] == 0x46 && imageBytes[3] == 0x38) + return "image/gif"; + + if (imageBytes.Length >= 12 && + imageBytes[0] == 0x52 && imageBytes[1] == 0x49 && + imageBytes[2] == 0x46 && imageBytes[3] == 0x46 && + imageBytes[8] == 0x57 && imageBytes[9] == 0x45 && + imageBytes[10] == 0x42 && imageBytes[11] == 0x50) + return "image/webp"; + + if (imageBytes.Length >= 12 && + imageBytes[4] == 0x66 && imageBytes[5] == 0x74 && + imageBytes[6] == 0x79 && imageBytes[7] == 0x70) + { + if ((imageBytes[8] == 0x68 && imageBytes[9] == 0x65 && imageBytes[10] == 0x69 && imageBytes[11] == 0x63) || + (imageBytes[8] == 0x68 && imageBytes[9] == 0x65 && imageBytes[10] == 0x69 && imageBytes[11] == 0x66)) + return "image/heic"; + } + + if (imageBytes.Length >= 12 && + imageBytes[4] == 0x66 && imageBytes[5] == 0x74 && + imageBytes[6] == 0x79 && imageBytes[7] == 0x70 && + imageBytes[8] == 0x61 && imageBytes[9] == 0x76 && + imageBytes[10] == 0x69 && imageBytes[11] == 0x66) + return "image/avif"; + + return "image/jpeg"; + } } \ No newline at end of file From 7fbf1d5e7578db80a39c71056cc5adbcef5c4530 Mon Sep 17 00:00:00 2001 From: Magdalena Ruman <67785133+Madzionator@users.noreply.github.com> Date: Thu, 19 Mar 2026 11:04:43 +0100 Subject: [PATCH 09/10] post merge fixes --- .../BackendParamsTests.cs | 35 ++++++++++--------- .../Mappers/ChatDocumentMapper.cs | 8 ++--- .../Services/LLMService/LLMService.cs | 18 +++++----- 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/MaIN.Core.IntegrationTests/BackendParamsTests.cs b/MaIN.Core.IntegrationTests/BackendParamsTests.cs index 666f66a0..cf240ff6 100644 --- a/MaIN.Core.IntegrationTests/BackendParamsTests.cs +++ b/MaIN.Core.IntegrationTests/BackendParamsTests.cs @@ -3,6 +3,7 @@ using MaIN.Domain.Entities; using MaIN.Domain.Configuration.BackendInferenceParams; using MaIN.Domain.Exceptions; +using MaIN.Domain.Models; using MaIN.Domain.Models.Concrete; namespace MaIN.Core.IntegrationTests; @@ -17,7 +18,7 @@ public async Task OpenAi_Should_RespondWithParams() SkipIfMissingKey(LLMApiRegistry.GetEntry(BackendType.OpenAi)?.ApiKeyEnvName!); var result = await AIHub.Chat() - .WithModel() + .WithModel(Models.OpenAi.Gpt4oMini) .WithMessage(TestQuestion) .WithInferenceParams(new OpenAiInferenceParams { @@ -39,7 +40,7 @@ public async Task Anthropic_Should_RespondWithParams() SkipIfMissingKey(LLMApiRegistry.GetEntry(BackendType.Anthropic)?.ApiKeyEnvName!); var result = await AIHub.Chat() - .WithModel() + .WithModel(Models.Anthropic.ClaudeSonnet4) .WithMessage(TestQuestion) .WithInferenceParams(new AnthropicInferenceParams { @@ -61,7 +62,7 @@ public async Task Gemini_Should_RespondWithParams() SkipIfMissingKey(LLMApiRegistry.GetEntry(BackendType.Gemini)?.ApiKeyEnvName!); var result = await AIHub.Chat() - .WithModel() + .WithModel(Models.Gemini.Gemini2_0Flash) .WithMessage(TestQuestion) .WithInferenceParams(new GeminiInferenceParams { @@ -83,7 +84,7 @@ public async Task DeepSeek_Should_RespondWithParams() SkipIfMissingKey(LLMApiRegistry.GetEntry(BackendType.DeepSeek)?.ApiKeyEnvName!); var result = await AIHub.Chat() - .WithModel() + .WithModel(Models.DeepSeek.Reasoner) .WithMessage(TestQuestion) .WithInferenceParams(new DeepSeekInferenceParams { @@ -105,7 +106,7 @@ public async Task GroqCloud_Should_RespondWithParams() SkipIfMissingKey(LLMApiRegistry.GetEntry(BackendType.GroqCloud)?.ApiKeyEnvName!); var result = await AIHub.Chat() - .WithModel() + .WithModel(Models.Groq.Llama3_1_8bInstant) .WithMessage(TestQuestion) .WithInferenceParams(new GroqCloudInferenceParams { @@ -127,7 +128,7 @@ public async Task Xai_Should_RespondWithParams() SkipIfMissingKey(LLMApiRegistry.GetEntry(BackendType.Xai)?.ApiKeyEnvName!); var result = await AIHub.Chat() - .WithModel() + .WithModel(Models.Xai.Grok3Beta) .WithMessage(TestQuestion) .WithInferenceParams(new XaiInferenceParams { @@ -149,7 +150,7 @@ public async Task Self_Should_RespondWithParams() Skip.If(!File.Exists("C:/Models/gemma2-2b.gguf"), "Local model not found at C:/Models/gemma2-2b.gguf"); var result = await AIHub.Chat() - .WithModel() + .WithModel(Models.Local.Gemma2_2b) .WithMessage(TestQuestion) .WithInferenceParams(new LocalInferenceParams { @@ -173,7 +174,7 @@ public async Task LocalOllama_Should_RespondWithParams() SkipIfOllamaNotRunning(); var result = await AIHub.Chat() - .WithModel() + .WithModel(Models.Ollama.Gemma3_4b) .WithMessage(TestQuestion) .WithInferenceParams(new OllamaInferenceParams { @@ -197,7 +198,7 @@ public async Task ClaudOllama_Should_RespondWithParams() SkipIfMissingKey(LLMApiRegistry.GetEntry(BackendType.Ollama)?.ApiKeyEnvName!); var result = await AIHub.Chat() - .WithModel() + .WithModel(Models.Ollama.Gemma3_4b) .WithMessage(TestQuestion) .WithInferenceParams(new OllamaInferenceParams { @@ -222,7 +223,7 @@ public async Task Self_Should_ThrowWhenGivenWrongParams() { await Assert.ThrowsAsync(() => AIHub.Chat() - .WithModel() + .WithModel(Models.Local.Gemma2_2b) .WithMessage(TestQuestion) .WithInferenceParams(new OpenAiInferenceParams()) .CompleteAsync()); @@ -233,7 +234,7 @@ public async Task OpenAi_Should_ThrowWhenGivenWrongParams() { await Assert.ThrowsAsync(() => AIHub.Chat() - .WithModel() + .WithModel(Models.OpenAi.Gpt4oMini) .WithMessage(TestQuestion) .WithInferenceParams(new DeepSeekInferenceParams()) .CompleteAsync()); @@ -244,7 +245,7 @@ public async Task Anthropic_Should_ThrowWhenGivenWrongParams() { await Assert.ThrowsAsync(() => AIHub.Chat() - .WithModel() + .WithModel(Models.Anthropic.ClaudeSonnet4) .WithMessage(TestQuestion) .WithInferenceParams(new OpenAiInferenceParams()) .CompleteAsync()); @@ -255,7 +256,7 @@ public async Task Gemini_Should_ThrowWhenGivenWrongParams() { await Assert.ThrowsAsync(() => AIHub.Chat() - .WithModel() + .WithModel(Models.Gemini.Gemini2_0Flash) .WithMessage(TestQuestion) .WithInferenceParams(new AnthropicInferenceParams()) .CompleteAsync()); @@ -266,7 +267,7 @@ public async Task DeepSeek_Should_ThrowWhenGivenWrongParams() { await Assert.ThrowsAsync(() => AIHub.Chat() - .WithModel() + .WithModel(Models.DeepSeek.Reasoner) .WithMessage(TestQuestion) .WithInferenceParams(new GeminiInferenceParams()) .CompleteAsync()); @@ -277,7 +278,7 @@ public async Task GroqCloud_Should_ThrowWhenGivenWrongParams() { await Assert.ThrowsAsync(() => AIHub.Chat() - .WithModel() + .WithModel(Models.Groq.Llama3_1_8bInstant) .WithMessage(TestQuestion) .WithInferenceParams(new OpenAiInferenceParams()) .CompleteAsync()); @@ -288,7 +289,7 @@ public async Task Xai_Should_ThrowWhenGivenWrongParams() { await Assert.ThrowsAsync(() => AIHub.Chat() - .WithModel() + .WithModel(Models.Xai.Grok3Beta) .WithMessage(TestQuestion) .WithInferenceParams(new AnthropicInferenceParams()) .CompleteAsync()); @@ -299,7 +300,7 @@ public async Task Ollama_Should_ThrowWhenGivenWrongParams() { await Assert.ThrowsAsync(() => AIHub.Chat() - .WithModel() + .WithModel(Models.Ollama.Gemma3_4b) .WithMessage(TestQuestion) .WithInferenceParams(new DeepSeekInferenceParams()) .CompleteAsync()); diff --git a/src/MaIN.Infrastructure/Mappers/ChatDocumentMapper.cs b/src/MaIN.Infrastructure/Mappers/ChatDocumentMapper.cs index 2d5f70db..3f7c6d87 100644 --- a/src/MaIN.Infrastructure/Mappers/ChatDocumentMapper.cs +++ b/src/MaIN.Infrastructure/Mappers/ChatDocumentMapper.cs @@ -16,7 +16,7 @@ internal static class ChatDocumentMapper ImageGen = chat.ImageGen, ToolsConfiguration = chat.ToolsConfiguration, MemoryParams = chat.MemoryParams.ToDocument(), - InferenceParams = chat.InterferenceParams.ToDocument(), + InferenceParams = (chat.BackendParams as LocalInferenceParams)?.ToDocument(), ConvState = chat.ConversationState, Properties = chat.Properties, Interactive = chat.Interactive, @@ -35,7 +35,7 @@ internal static class ChatDocumentMapper ToolsConfiguration = chat.ToolsConfiguration, ConversationState = chat.ConvState as Conversation.State, MemoryParams = chat.MemoryParams!.ToDomain(), - InterferenceParams = chat.InferenceParams!.ToDomain(), + BackendParams = chat.InferenceParams?.ToDomain() ?? new LocalInferenceParams(), Interactive = chat.Interactive, Translate = chat.Translate, Type = Enum.Parse(chat.Type.ToString()) @@ -78,7 +78,7 @@ internal static class ChatDocumentMapper Type = llmTokenValue.Type }; - private static InferenceParamsDocument ToDocument(this InferenceParams inferenceParams) => new() + private static InferenceParamsDocument ToDocument(this LocalInferenceParams inferenceParams) => new() { Temperature = inferenceParams.Temperature, ContextSize = inferenceParams.ContextSize, @@ -96,7 +96,7 @@ internal static class ChatDocumentMapper Grammar = inferenceParams.Grammar }; - private static InferenceParams ToDomain(this InferenceParamsDocument inferenceParams) => new() + private static LocalInferenceParams ToDomain(this InferenceParamsDocument inferenceParams) => new() { Temperature = inferenceParams.Temperature, ContextSize = inferenceParams.ContextSize, diff --git a/src/MaIN.Services/Services/LLMService/LLMService.cs b/src/MaIN.Services/Services/LLMService/LLMService.cs index 7659d8c9..791df6b2 100644 --- a/src/MaIN.Services/Services/LLMService/LLMService.cs +++ b/src/MaIN.Services/Services/LLMService/LLMService.cs @@ -546,22 +546,22 @@ private static string FormatToolsForPrompt(ToolsConfiguration toolsConfig) return (tokens, isComplete, hasFailed); } - private static BaseSamplingPipeline CreateSampler(LocalInferenceParams interferenceParams) + private static BaseSamplingPipeline CreateSampler(LocalInferenceParams inferenceParams) { - return interferenceParams.Temperature == 0 + return inferenceParams.Temperature == 0 ? new GreedySamplingPipeline() { - Grammar = interferenceParams.Grammar is not null - ? new Grammar(interferenceParams.Grammar.Value, "root") + Grammar = inferenceParams.Grammar is not null + ? new Grammar(inferenceParams.Grammar.Value, "root") : null } : new DefaultSamplingPipeline() { - Temperature = interferenceParams.Temperature, - TopP = interferenceParams.TopP, - TopK = interferenceParams.TopK, - Grammar = interferenceParams.Grammar is not null - ? new Grammar(interferenceParams.Grammar.Value, "root") + Temperature = inferenceParams.Temperature, + TopP = inferenceParams.TopP, + TopK = inferenceParams.TopK, + Grammar = inferenceParams.Grammar is not null + ? new Grammar(inferenceParams.Grammar.Value, "root") : null }; } From 89cbc2f905c925245097cb3704b223ebca95cda8 Mon Sep 17 00:00:00 2001 From: Magdalena Ruman <67785133+Madzionator@users.noreply.github.com> Date: Thu, 19 Mar 2026 11:15:31 +0100 Subject: [PATCH 10/10] versioning --- Releases/0.10.2.md | 3 +++ src/MaIN.Core/.nuspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Releases/0.10.2.md diff --git a/Releases/0.10.2.md b/Releases/0.10.2.md new file mode 100644 index 00000000..dd048c8a --- /dev/null +++ b/Releases/0.10.2.md @@ -0,0 +1,3 @@ +# 0.10.2 release + +Inference parameters are now backend-specific — each AI provider has its own typed params class where only explicitly set values are sent to the API, with an AdditionalParams dictionary for custom fields. \ No newline at end of file diff --git a/src/MaIN.Core/.nuspec b/src/MaIN.Core/.nuspec index 91376edf..30970341 100644 --- a/src/MaIN.Core/.nuspec +++ b/src/MaIN.Core/.nuspec @@ -2,7 +2,7 @@ MaIN.NET - 0.10.1 + 0.10.2 Wisedev Wisedev favicon.png