Background service for chunking and embedding data

app/MindWork AI Studio/Agents/AgentDataSourceSelection.cs

@@ -140,13 +140,21 @@ public async Task<List<SelectedDataSource>> PerformSelectionAsync(IProvider prov
         //
 
         // We start with the provider currently selected by the user:
+        var requiredDataSecurity = dataSources.AllowedDataSources.GetRequiredSecurityPolicy();
+        var requiredComplianceLevel = dataSources.AllowedDataSources.GetRequiredComplianceLevel();
         var agentProvider = this.SettingsManager.GetPreselectedProvider(Tools.Components.AGENT_DATA_SOURCE_SELECTION, provider.Id, true);
         if (agentProvider == Settings.Provider.NONE)
         {
             logger.LogWarning("No provider is selected for the agent. The agent cannot select data sources.");
             return [];
         }
 
+        if (!agentProvider.AllowsDataSourceAccess(this.SettingsManager, requiredDataSecurity, requiredComplianceLevel))
+        {
+            logger.LogWarning($"The agent for data source selection uses provider '{agentProvider.InstanceName}' with confidence '{agentProvider.GetConfidenceLevel(this.SettingsManager).GetName()}', but the available data sources require data security '{requiredDataSecurity}' and provider confidence '{requiredComplianceLevel.GetName()}'. The agent cannot select data sources.");
+            return [];
+        }
+
         // Assign the provider settings to the agent:
         logger.LogInformation($"The agent for the data source selection uses the provider '{agentProvider.InstanceName}' ({agentProvider.UsedLLMProvider.ToName()}, confidence={agentProvider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level.GetName()}).");
         this.ProviderSettings = agentProvider;

app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs

@@ -3,6 +3,7 @@
 using AIStudio.Chat;
 using AIStudio.Provider;
 using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
 using AIStudio.Tools.RAG;
 using AIStudio.Tools.Services;
 
@@ -129,19 +130,30 @@ public override async Task<ContentBlock> ProcessInput(ContentBlock input, IDicti
     /// you can set the provider once and then call the validation method in parallel.
     /// </remarks>
     /// <param name="provider">The current LLM provider. When the user doesn't preselect an agent provider, the agent uses this provider.</param>
-    public void SetLLMProvider(IProvider provider)
+    /// <param name="requiredDataSecurity">The data security required by the retrieved data.</param>
+    /// <param name="requiredComplianceLevel">The minimum provider confidence required by the retrieved data.</param>
+    public bool SetLLMProvider(IProvider provider, DataSourceSecurity requiredDataSecurity = DataSourceSecurity.NOT_SPECIFIED, ConfidenceLevel requiredComplianceLevel = ConfidenceLevel.NONE)
     {
         // We start with the provider currently selected by the user:
         var agentProvider = this.SettingsManager.GetPreselectedProvider(Tools.Components.AGENT_RETRIEVAL_CONTEXT_VALIDATION, provider.Id, true);
         if (agentProvider == Settings.Provider.NONE)
         {
             logger.LogWarning("No provider is selected for the agent.");
-            return;
+            this.ProviderSettings = Settings.Provider.NONE;
+            return false;
+        }
+
+        if (!agentProvider.AllowsDataSourceAccess(this.SettingsManager, requiredDataSecurity, requiredComplianceLevel))
+        {
+            logger.LogWarning($"The agent for retrieval context validation uses provider '{agentProvider.InstanceName}' with confidence '{agentProvider.GetConfidenceLevel(this.SettingsManager).GetName()}', but the retrieved data requires data security '{requiredDataSecurity}' and provider confidence '{requiredComplianceLevel.GetName()}'. The agent cannot validate retrieval contexts.");
+            this.ProviderSettings = Settings.Provider.NONE;
+            return false;
         }
 
         // Assign the provider settings to the agent:
         logger.LogInformation($"The agent for the retrieval context validation uses the provider '{agentProvider.InstanceName}' ({agentProvider.UsedLLMProvider.ToName()}, confidence={agentProvider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level.GetName()}).");
         this.ProviderSettings = agentProvider;
+        return true;
     }
 
     /// <summary>

app/MindWork AI Studio/Chat/ChatThread.cs

@@ -1,6 +1,7 @@
 using System.Globalization;
 
 using AIStudio.Components;
+using AIStudio.Provider;
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
 using AIStudio.Tools.ERIClient.DataModel;
@@ -76,6 +77,11 @@ public sealed record ChatThread
     /// </summary>
     public DataSourceSecurity DataSecurity { get; set; } = DataSourceSecurity.NOT_SPECIFIED;
 
+    /// <summary>
+    /// The minimum provider confidence required by data sources used so far.
+    /// </summary>
+    public ConfidenceLevel DataComplianceLevel { get; set; } = ConfidenceLevel.NONE;
+
     /// <summary>
     /// The name of the chat thread. Usually generated by an AI model or manually edited by the user.
     /// </summary>

app/MindWork AI Studio/Chat/ChatThreadExtensions.cs

@@ -11,12 +11,12 @@ public static class ChatThreadExtensions
     /// </summary>
     /// <remarks>
     /// We don't check if the provider is allowed to use the data sources of the chat thread.
-    /// That kind of check is done in the RAG process itself.<br/><br/>
+    /// That kind of check is done when the available data sources are resolved.<br/><br/>
     /// 
     /// One thing which is not so obvious: after RAG was used on this thread, the entire chat
     /// thread is kind of a data source by itself. Why? Because the augmentation data collected
     /// from the data sources is stored in the chat thread. This means we must check if the
-    /// selected provider is allowed to use this thread's data.
+    /// selected provider is allowed to use this thread's data security and compliance level.
     /// </remarks>
     /// <param name="chatThread">The chat thread to check.</param>
     /// <param name="provider">The provider to check.</param>
@@ -26,7 +26,19 @@ public static bool IsLLMProviderAllowed<T>(this ChatThread? chatThread, T provid
         // No chat thread available means we have a new chat. That's fine:
         if (chatThread is null)
             return true;
-
+
+        var settingsManager = Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>();
+        var providerConfidenceLevel = provider switch
+        {
+            IProvider p => p.GetConfidenceLevel(settingsManager),
+            AIStudio.Settings.Provider p => p.GetConfidenceLevel(settingsManager),
+
+            _ => ConfidenceLevel.NONE,
+        };
+
+        if (!providerConfidenceLevel.AllowsDataSourceComplianceLevel(chatThread.DataComplianceLevel))
+            return false;
+
         // The chat thread is available, but the data security is not specified.
         // Means, we never used RAG or RAG was enabled, but no data sources were selected.
         // That's fine as well:
@@ -36,7 +48,6 @@ public static bool IsLLMProviderAllowed<T>(this ChatThread? chatThread, T provid
         //
         // Is the provider trusted for data-source security checks?
         //
-        var settingsManager = Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>();
         var isTrustedProvider = provider switch
         {
             IProvider p => p.IsTrustedForDataSourceSecurityChecks(settingsManager),

app/MindWork AI Studio/Chat/ContentText.cs

@@ -57,7 +57,7 @@ public async Task<ChatThread> CreateFromProviderAsync(IProvider provider, Model
 
         if(!chatThread.IsLLMProviderAllowed(provider))
         {
-            LOGGER.LogError("The provider is not allowed for this chat thread due to data security reasons. Skipping the AI process.");
+            LOGGER.LogError("The provider is not allowed for this chat thread due to data security or compliance reasons. Skipping the AI process.");
             await this.CompleteWithoutStreaming();
             return chatThread;
         }

app/MindWork AI Studio/Components/ChatComponent.razor

@@ -35,7 +35,7 @@
     <FooterContent>
         <MediaTranscriptionStatus Owner="@this.CurrentMediaImportOwner"/>
         <MudElement Style="flex: 0 0 auto;">
-            <MudTextField
+            <UserPromptComponent
                 T="string"
                 @ref="@this.inputField"
                 @bind-Text="@this.UserInput"
@@ -51,8 +51,11 @@
                 Disabled="@this.IsInputForbidden()"
                 Immediate="@true"
                 OnKeyUp="@this.InputKeyEvent"
+                WhenTextChangedAsync="@(_ =>this.CalculateTokenCount())"
                 UserAttributes="@USER_INPUT_ATTRIBUTES"
                 Class="@this.UserInputClass"
+                DebounceTime="TimeSpan.FromSeconds(1)" 
+                HelperText="@this.TokenCountMessage"
                 Style="@this.UserInputStyle"/>
         </MudElement>
         <MudToolBar WrapContent="true" Gutters="@false" Class="border border-solid rounded" Style="border-color: lightgrey; gap: 2px;">
@@ -144,7 +147,7 @@
 
             @if (!this.ChatThread.IsLLMProviderAllowed(this.Provider))
             {
-                <MudTooltip Text="@T("The selected provider is not allowed in this chat due to data security reasons.")" Placement="@TOOLBAR_TOOLTIP_PLACEMENT">
+                <MudTooltip Text="@T("The selected provider is not allowed in this chat due to data security or compliance reasons.")" Placement="@TOOLBAR_TOOLTIP_PLACEMENT">
                     <MudIconButton Icon="@Icons.Material.Filled.Error" Color="Color.Error"/>
                 </MudTooltip>
             }

app/MindWork AI Studio/Components/ChatComponent.razor.cs

@@ -51,6 +51,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
     [Inject]
     private IDialogService DialogService { get; init; } = null!;
 
+    [Inject] 
+    private RustService RustService { get; init; } = null!;
     [Inject]
     private IJSRuntime JsRuntime { get; init; } = null!;
 
@@ -86,12 +88,18 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
     private Guid loadedParameterWorkspaceId = Guid.Empty;
     private Guid foregroundChatId = Guid.Empty;
     private int workspaceHeaderSyncVersion;
+    private HashSet<FileAttachment> chatDocumentPaths = [];
+    private string tokenCount = "0";
+    private bool HasCustomTokenizer => !string.IsNullOrWhiteSpace(this.Provider.TokenizerPath);
+    private string TokenCountMessage => this.HasCustomTokenizer
+        ? $"{this.T("Estimated amount of tokens:")} {this.tokenCount}"
+        : string.Empty;
 
     private MediaImportOwner CurrentMediaImportOwner => MediaImportOwner.ForChat(this.ChatThread?.ChatId ?? this.draftMediaOwnerId);
 
     // Unfortunately, we need the input field reference to blur the focus away. Without
     // this, we cannot clear the input field.
-    private MudTextField<string> inputField = null!;
+    private UserPromptComponent<string> inputField = null!;
 
     /// <summary>
     /// Represents the user's input in the chat interface.
@@ -356,15 +364,22 @@ protected override async Task OnAfterRenderAsync(bool firstRender)
     protected override async Task OnParametersSetAsync()
     {
         var incomingChatId = this.ChatThread?.ChatId ?? Guid.Empty;
+        var providerChanged = this.Provider != this.lastSeenProvider;
         if (incomingChatId != this.lastSeenChatId || this.Provider != this.lastSeenProvider)
         {
             this.lastSeenChatId = incomingChatId;
             this.lastSeenProvider = this.Provider;
+            if (providerChanged)
+                this.tokenCount = "0";
+
             this.previousInputForbidden = true;
         }
 
         await this.ApplyLoadedChatParameterAsync();
         await this.SyncForegroundChatAsync();
+        if (providerChanged && this.HasCustomTokenizer && this.inputField is not null)
+            await this.CalculateTokenCount();
+
         await this.ConsumeMediaOutcomeAsync();
         await base.OnParametersSetAsync();
     }
@@ -697,6 +712,9 @@ private async Task InputKeyEvent(KeyboardEventArgs keyEvent)
         // Was a modifier key pressed as well?
         var isModifier = keyEvent.AltKey || keyEvent.CtrlKey || keyEvent.MetaKey || keyEvent.ShiftKey;
 
+        if (isEnter)
+            await this.CalculateTokenCount();
+
         // Depending on the user's settings, might react to shortcuts:
         switch (this.SettingsManager.ConfigurationData.Chat.ShortcutSendBehavior)
         {
@@ -879,6 +897,7 @@ private async Task SendMessage(bool reuseLastUserPrompt = false)
         this.ComposerState.Clear();
 
         await this.inputField.BlurAsync();
+        this.tokenCount = "0";
 
         // Enable the stream state for the chat component:
         this.hasUnsavedChanges = true;
@@ -1221,6 +1240,37 @@ private void RestoreComposerFromTextBlock(ContentText textBlock)
         this.ComposerState.RestoreFromTextBlock(textBlock);
     }
 
+    private async Task CalculateTokenCount()
+    {
+        if (!this.HasCustomTokenizer)
+        {
+            if (this.tokenCount != "0")
+            {
+                this.tokenCount = "0";
+                this.StateHasChanged();
+            }
+
+            return;
+        }
+
+        if (this.inputField.Value is null)
+        {
+            this.tokenCount = "0";
+            return;
+        }
+
+        var response = await this.RustService.GetTokenCount(this.Provider, this.inputField.Value);
+        if (response is null)
+            return;
+        if (!response.Value.Success)
+        {
+            this.Logger.LogWarning("Failed to calculate token count: reason='{Reason}'", response.Value.Message);
+            return;
+        }
+        this.tokenCount = response.Value.TokenCount.ToString();
+        this.StateHasChanged();
+    }
+
     #region Overrides of MSGComponentBase
 
     protected override async Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default
@@ -1304,4 +1354,4 @@ public async ValueTask DisposeAsync()
     }
 
     #endregion
-}
+}

app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor

@@ -0,0 +1,9 @@
+@inherits MSGComponentBase
+
+<MudAlert Severity="Severity.Warning" Variant="Variant.Outlined" Dense="@true" Class="mb-3">
+    <MudText Typo="Typo.body2">
+        @this.WarningText
+    </MudText>
+</MudAlert>
+
+<MudTextSwitch Value="@this.UserAcknowledged" ValueChanged="@this.UserAcknowledgedChanged" Label="@T("I confirm that I have read and understood the above")" LabelOn="@T("Yes, please send my data to the external embedding provider")" LabelOff="@T("No, I will choose another embedding")" Validation="@this.Validation"/>

app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor.cs

@@ -0,0 +1,52 @@
+using AIStudio.Settings.DataModel;
+
+using Microsoft.AspNetCore.Components;
+
+namespace AIStudio.Components;
+
+public partial class DataSourceCloudEmbeddingWarning : MSGComponentBase
+{
+    [Parameter]
+    public DataSourceType DataSourceType { get; set; }
+
+    [Parameter]
+    public string SourcePath { get; set; } = string.Empty;
+
+    [Parameter]
+    public bool UserAcknowledged { get; set; }
+
+    [Parameter]
+    public EventCallback<bool> UserAcknowledgedChanged { get; set; }
+
+    [Parameter]
+    public Func<bool, string?> Validation { get; set; } = _ => null;
+
+    private string WarningText
+    {
+        get
+        {
+            var subject = this.GetSubjectText();
+            return string.Format(
+                T("Warning: The selected embedding provider is not self-hosted. Creating embeddings can cost money and may need to run multiple times, for example after errors or file changes. {0} will be sent to an external third party. MindWork AI Studio has no control over what that third party does with the data after it is sent."),
+                subject);
+        }
+    }
+
+    private string GetSubjectText()
+    {
+        if (string.IsNullOrWhiteSpace(this.SourcePath))
+            return this.DataSourceType switch
+            {
+                DataSourceType.LOCAL_DIRECTORY => T("All files in this folder and its subfolders"),
+                DataSourceType.LOCAL_FILE => T("The selected file"),
+                _ => T("The selected data")
+            };
+
+        return this.DataSourceType switch
+        {
+            DataSourceType.LOCAL_DIRECTORY => string.Format(T("All files in the folder '{0}' and its subfolders"), this.SourcePath),
+            DataSourceType.LOCAL_FILE => string.Format(T("The file '{0}'"), this.SourcePath),
+            _ => string.Format(T("The data source '{0}'"), this.SourcePath)
+        };
+    }
+}

app/MindWork AI Studio/Components/DataSourceSelection.razor

@@ -70,7 +70,7 @@
                             {
                                 case true when this.availableDataSources.Count == 0:
                                     <MudText Typo="Typo.body1" Class="mb-3">
-                                        @T("Your data sources cannot be used with the LLM provider you selected due to data privacy, or they are currently unavailable.")
+                                        @T("Your data sources cannot be used with the LLM provider you selected due to data privacy or compliance requirements, or they are currently unavailable.")
                                     </MudText>
                                     break; 
 
@@ -82,7 +82,7 @@
 
                                 case false when this.availableDataSources.Count == 0:
                                     <MudText Typo="Typo.body1" Class="mb-3">
-                                        @T("Your data sources cannot be used with the LLM provider you selected due to data privacy, or they are currently unavailable.")
+                                        @T("Your data sources cannot be used with the LLM provider you selected due to data privacy or compliance requirements, or they are currently unavailable.")
                                     </MudText>
                                     break;
 
@@ -177,4 +177,4 @@ else if (this.SelectionMode is DataSourceSelectionMode.CONFIGURATION_MODE)
             </MudField>
         }
     </MudPaper>
-}
+}