From 630d155bdbf2d3cef166b36f81c1ce432587a05f Mon Sep 17 00:00:00 2001 From: adamsitnik <6011991+adamsitnik@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:34:31 +0000 Subject: [PATCH] .Net: Remove migrated .NET Milvus connector Remove the legacy Milvus memory store and tests now that the VectorData provider is available in CommunityToolkit/AI. Point users to the published package and remove unused dependencies and the empty MEVD solution. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- dotnet/Directory.Packages.props | 2 - dotnet/MEVD.slnx | 29 - dotnet/docs/EXPERIMENTS.md | 1 - .../Process.IntegrationTestRunner.Dapr.csproj | 2 - .../IntegrationTests/IntegrationTests.csproj | 1 - dotnet/src/VectorData/Milvus/AssemblyInfo.cs | 6 - dotnet/src/VectorData/Milvus/Milvus.csproj | 34 - .../VectorData/Milvus/MilvusMemoryStore.cs | 615 ------------------ dotnet/src/VectorData/Milvus/README.md | 36 +- .../Milvus.UnitTests/Milvus.UnitTests.csproj | 24 - .../Milvus.UnitTests/MilvusFilterTests.cs | 84 --- .../Milvus.UnitTests/MilvusFixture.cs | 76 --- .../MilvusLegacyServerTests.cs | 50 -- .../MilvusMemoryStoreTests.cs | 204 ------ 14 files changed, 1 insertion(+), 1163 deletions(-) delete mode 100644 dotnet/MEVD.slnx delete mode 100644 dotnet/src/VectorData/Milvus/AssemblyInfo.cs delete mode 100644 dotnet/src/VectorData/Milvus/Milvus.csproj delete mode 100644 dotnet/src/VectorData/Milvus/MilvusMemoryStore.cs delete mode 100644 dotnet/test/VectorData/Milvus.UnitTests/Milvus.UnitTests.csproj delete mode 100644 dotnet/test/VectorData/Milvus.UnitTests/MilvusFilterTests.cs delete mode 100644 dotnet/test/VectorData/Milvus.UnitTests/MilvusFixture.cs delete mode 100644 dotnet/test/VectorData/Milvus.UnitTests/MilvusLegacyServerTests.cs delete mode 100644 dotnet/test/VectorData/Milvus.UnitTests/MilvusMemoryStoreTests.cs diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 534116db67a1..6fedad225c5d 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -182,8 +182,6 @@ - - diff --git a/dotnet/MEVD.slnx b/dotnet/MEVD.slnx deleted file mode 100644 index da53ff8db8d6..000000000000 --- a/dotnet/MEVD.slnx +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/dotnet/docs/EXPERIMENTS.md b/dotnet/docs/EXPERIMENTS.md index 34f61029ee10..6543ca9b0b87 100644 --- a/dotnet/docs/EXPERIMENTS.md +++ b/dotnet/docs/EXPERIMENTS.md @@ -49,7 +49,6 @@ You can use the following diagnostic IDs to ignore warnings or errors for a part | SKEXP0020 | Azure AI Search memory connector | | SKEXP0020 | DuckDB memory connector | | SKEXP0020 | Kusto memory connector | -| SKEXP0020 | Milvus memory connector | | SKEXP0020 | Qdrant memory connector | | SKEXP0020 | Redis memory connector | | SKEXP0020 | Sqlite memory connector | diff --git a/dotnet/src/Experimental/Process.IntegrationTestRunner.Dapr/Process.IntegrationTestRunner.Dapr.csproj b/dotnet/src/Experimental/Process.IntegrationTestRunner.Dapr/Process.IntegrationTestRunner.Dapr.csproj index a410796312af..e9bc973f0831 100644 --- a/dotnet/src/Experimental/Process.IntegrationTestRunner.Dapr/Process.IntegrationTestRunner.Dapr.csproj +++ b/dotnet/src/Experimental/Process.IntegrationTestRunner.Dapr/Process.IntegrationTestRunner.Dapr.csproj @@ -41,7 +41,6 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all - @@ -54,4 +53,3 @@ - diff --git a/dotnet/src/IntegrationTests/IntegrationTests.csproj b/dotnet/src/IntegrationTests/IntegrationTests.csproj index 4e739bf9a96a..8d48956e7143 100644 --- a/dotnet/src/IntegrationTests/IntegrationTests.csproj +++ b/dotnet/src/IntegrationTests/IntegrationTests.csproj @@ -54,7 +54,6 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all - diff --git a/dotnet/src/VectorData/Milvus/AssemblyInfo.cs b/dotnet/src/VectorData/Milvus/AssemblyInfo.cs deleted file mode 100644 index d174fc92303c..000000000000 --- a/dotnet/src/VectorData/Milvus/AssemblyInfo.cs +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Diagnostics.CodeAnalysis; - -// This assembly is currently experimental. -[assembly: Experimental("SKEXP0020")] diff --git a/dotnet/src/VectorData/Milvus/Milvus.csproj b/dotnet/src/VectorData/Milvus/Milvus.csproj deleted file mode 100644 index 9364509c6b20..000000000000 --- a/dotnet/src/VectorData/Milvus/Milvus.csproj +++ /dev/null @@ -1,34 +0,0 @@ - - - - - Microsoft.SemanticKernel.Connectors.Milvus - $(AssemblyName) - net10.0;net8.0;netstandard2.0;net462 - enable - alpha - - $(NoWarn);NU5104 - - - - - - - - - Semantic Kernel - Milvus Connector - Milvus connector for Semantic Kernel plugins and semantic memory - - - - - - - - - - - - - diff --git a/dotnet/src/VectorData/Milvus/MilvusMemoryStore.cs b/dotnet/src/VectorData/Milvus/MilvusMemoryStore.cs deleted file mode 100644 index b5ab785736af..000000000000 --- a/dotnet/src/VectorData/Milvus/MilvusMemoryStore.cs +++ /dev/null @@ -1,615 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; -using System.Runtime.CompilerServices; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Memory; -using Milvus.Client; - -namespace Microsoft.SemanticKernel.Connectors.Milvus; - -/// -/// An implementation of for the Milvus vector database. -/// -public class MilvusMemoryStore : IMemoryStore, IDisposable -{ - private readonly int _vectorSize; - private readonly SimilarityMetricType _metricType; - private readonly ConsistencyLevel _consistencyLevel; - private readonly bool _ownsMilvusClient; - private readonly string _indexName; - - private const string DefaultIndexName = "default"; - private const string IsReferenceFieldName = "is_reference"; - private const string ExternalSourceNameFieldName = "external_source_name"; - private const string IdFieldName = "id"; - private const string DescriptionFieldName = "description"; - private const string TextFieldName = "text"; - private const string AdditionalMetadataFieldName = "additional_metadata"; - private const string EmbeddingFieldName = "embedding"; - private const string KeyFieldName = "key"; - private const string TimestampFieldName = "timestamp"; - - private const int DefaultMilvusPort = 19530; - private const int DefaultVarcharLength = 65_535; - - private readonly QueryParameters _queryParametersWithEmbedding; - private readonly QueryParameters _queryParametersWithoutEmbedding; - - private readonly SearchParameters _searchParameters = new() - { - OutputFields = { IsReferenceFieldName, ExternalSourceNameFieldName, IdFieldName, DescriptionFieldName, TextFieldName, AdditionalMetadataFieldName, KeyFieldName, TimestampFieldName } - }; - - /// - /// Exposes the underlying used to communicate with Milvus. Can be used to execute operations not supported by the abstraction. - /// - public MilvusClient Client { get; } - - #region Constructors - - /// - /// Creates a new , connecting to the given hostname on the default Milvus port of 19530. - /// For more advanced configuration opens, construct a instance and pass it to - /// . - /// - /// The hostname or IP address to connect to. - /// The port to connect to. Defaults to 19530. - /// Whether to use TLS/SSL. Defaults to false. - /// The database to connect to. Defaults to the default Milvus database. - /// The name of the index to use. Defaults to . - /// The size of the vectors used in Milvus. Defaults to 1536. - /// The metric used to measure similarity between vectors. Defaults to . - /// The consistency level to be used in the search. Defaults to . - /// An optional logger factory through which the Milvus client will log. - public MilvusMemoryStore( - string host, - int port = DefaultMilvusPort, - bool ssl = false, - string? database = null, - string? indexName = null, - int vectorSize = 1536, - SimilarityMetricType metricType = SimilarityMetricType.Ip, - ConsistencyLevel consistencyLevel = ConsistencyLevel.Session, - ILoggerFactory? loggerFactory = null) - : this( - new MilvusClient(host, port, ssl, database, callOptions: default, loggerFactory), - indexName, vectorSize, metricType, consistencyLevel) - { - this._ownsMilvusClient = true; - } - - /// - /// Creates a new , connecting to the given hostname on the default Milvus port of 19530. - /// For more advanced configuration opens, construct a instance and pass it to - /// . - /// - /// The hostname or IP address to connect to. - /// The username to use for authentication. - /// The password to use for authentication. - /// The port to connect to. Defaults to 19530. - /// Whether to use TLS/SSL. Defaults to false. - /// The database to connect to. Defaults to the default Milvus database. - /// The name of the index to use. Defaults to . - /// The size of the vectors used in Milvus. Defaults to 1536. - /// The metric used to measure similarity between vectors. Defaults to . - /// The consistency level to be used in the search. Defaults to . - /// An optional logger factory through which the Milvus client will log. - public MilvusMemoryStore( - string host, - string username, - string password, - int port = DefaultMilvusPort, - bool ssl = false, - string? database = null, - string? indexName = null, - int vectorSize = 1536, - SimilarityMetricType metricType = SimilarityMetricType.Ip, - ConsistencyLevel consistencyLevel = ConsistencyLevel.Session, - ILoggerFactory? loggerFactory = null) - : this( - new MilvusClient(host, username, password, port, ssl, database, callOptions: default, loggerFactory), - indexName, vectorSize, metricType, consistencyLevel) - { - this._ownsMilvusClient = true; - } - - /// - /// Creates a new , connecting to the given hostname on the default Milvus port of 19530. - /// For more advanced configuration opens, construct a instance and pass it to - /// . - /// - /// The hostname or IP address to connect to. - /// An API key to be used for authentication, instead of a username and password. - /// The port to connect to. Defaults to 19530. - /// Whether to use TLS/SSL. Defaults to false. - /// The database to connect to. Defaults to the default Milvus database. - /// The name of the index to use. Defaults to . - /// The size of the vectors used in Milvus. Defaults to 1536. - /// The metric used to measure similarity between vectors. Defaults to . - /// The consistency level to be used in the search. Defaults to . - /// An optional logger factory through which the Milvus client will log. - public MilvusMemoryStore( - string host, - string apiKey, - int port = DefaultMilvusPort, - bool ssl = false, - string? database = null, - string? indexName = null, - int vectorSize = 1536, - SimilarityMetricType metricType = SimilarityMetricType.Ip, - ConsistencyLevel consistencyLevel = ConsistencyLevel.Session, - ILoggerFactory? loggerFactory = null) - : this( - new MilvusClient(host, apiKey, port, ssl, database, callOptions: default, loggerFactory), - indexName, vectorSize, metricType, consistencyLevel) - { - this._ownsMilvusClient = true; - } - - /// - /// Initializes a new instance of over the given . - /// - /// A configured with the necessary endpoint and authentication information. - /// The name of the index to use. Defaults to . - /// The size of the vectors used in Milvus. Defaults to 1536. - /// The metric used to measure similarity between vectors. Defaults to . - /// The consistency level to be used in the search. Defaults to . - public MilvusMemoryStore( - MilvusClient client, - string? indexName = null, - int vectorSize = 1536, - SimilarityMetricType metricType = SimilarityMetricType.Ip, - ConsistencyLevel consistencyLevel = ConsistencyLevel.Session) - : this(client, ownsMilvusClient: false, indexName, vectorSize, metricType, consistencyLevel) - { - } - - private MilvusMemoryStore( - MilvusClient client, - bool ownsMilvusClient, - string? indexName, - int vectorSize, - SimilarityMetricType metricType, - ConsistencyLevel consistencyLevel) - { - this.Client = client; - this._indexName = indexName ?? DefaultIndexName; - this._vectorSize = vectorSize; - this._metricType = metricType; - this._ownsMilvusClient = ownsMilvusClient; - this._consistencyLevel = consistencyLevel; - - this._queryParametersWithEmbedding = new() - { - OutputFields = { IsReferenceFieldName, ExternalSourceNameFieldName, IdFieldName, DescriptionFieldName, TextFieldName, AdditionalMetadataFieldName, EmbeddingFieldName, KeyFieldName, TimestampFieldName }, - ConsistencyLevel = this._consistencyLevel - }; - - this._queryParametersWithoutEmbedding = new() - { - OutputFields = { IsReferenceFieldName, ExternalSourceNameFieldName, IdFieldName, DescriptionFieldName, TextFieldName, AdditionalMetadataFieldName, KeyFieldName, TimestampFieldName }, - ConsistencyLevel = this._consistencyLevel - }; - } - - #endregion Constructors - - /// - public async Task CreateCollectionAsync(string collectionName, CancellationToken cancellationToken = default) - { - var exists = await this.Client.HasCollectionAsync(collectionName, cancellationToken: cancellationToken).ConfigureAwait(false); - if (!exists) - { - CollectionSchema schema = new() - { - Fields = - { - FieldSchema.CreateVarchar(IdFieldName, maxLength: DefaultVarcharLength, isPrimaryKey: true, autoId: false), - FieldSchema.CreateFloatVector(EmbeddingFieldName, this._vectorSize) - }, - EnableDynamicFields = true - }; - - MilvusCollection collection = await this.Client.CreateCollectionAsync(collectionName, schema, this._consistencyLevel, cancellationToken: cancellationToken).ConfigureAwait(false); - - await collection.CreateIndexAsync(EmbeddingFieldName, metricType: this._metricType, indexName: this._indexName, cancellationToken: cancellationToken).ConfigureAwait(false); - await collection.WaitForIndexBuildAsync("float_vector", this._indexName, cancellationToken: cancellationToken).ConfigureAwait(false); - - await collection.LoadAsync(cancellationToken: cancellationToken).ConfigureAwait(false); - await collection.WaitForCollectionLoadAsync(waitingInterval: TimeSpan.FromMilliseconds(100), timeout: TimeSpan.FromMinutes(1), cancellationToken: cancellationToken).ConfigureAwait(false); - } - } - - /// - public async IAsyncEnumerable GetCollectionsAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) - { - foreach (MilvusCollectionInfo collection in await this.Client.ListCollectionsAsync(cancellationToken: cancellationToken).ConfigureAwait(false)) - { - yield return collection.Name; - } - } - - /// - public Task DoesCollectionExistAsync(string collectionName, CancellationToken cancellationToken = default) - => this.Client.HasCollectionAsync(collectionName, cancellationToken: cancellationToken); - - /// - public Task DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default) - => this.Client.GetCollection(collectionName).DropAsync(cancellationToken); - - /// - public async Task UpsertAsync(string collectionName, MemoryRecord record, CancellationToken cancellationToken = default) - { - await this.EnsureSupportedServerAsync(cancellationToken).ConfigureAwait(false); - MilvusCollection collection = this.Client.GetCollection(collectionName); - - var metadata = record.Metadata; - - List fieldData = - [ - FieldData.Create(IdFieldName, [metadata.Id]), - FieldData.CreateFloatVector(EmbeddingFieldName, [record.Embedding]), - - FieldData.Create(IsReferenceFieldName, [metadata.IsReference], isDynamic: true), - FieldData.Create(ExternalSourceNameFieldName, [metadata.ExternalSourceName], isDynamic: true), - FieldData.Create(DescriptionFieldName, [metadata.Description], isDynamic: true), - FieldData.Create(TextFieldName, [metadata.Text], isDynamic: true), - FieldData.Create(AdditionalMetadataFieldName, [metadata.AdditionalMetadata], isDynamic: true), - FieldData.Create(KeyFieldName, [record.Key], isDynamic: true), - FieldData.Create(TimestampFieldName, [record.Timestamp?.ToString(CultureInfo.InvariantCulture) ?? string.Empty], isDynamic: true) - ]; - - MutationResult result = await collection.UpsertAsync(fieldData, cancellationToken: cancellationToken).ConfigureAwait(false); - - return result.Ids.StringIds![0]; - } - - /// - public async IAsyncEnumerable UpsertBatchAsync( - string collectionName, - IEnumerable records, - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - await this.EnsureSupportedServerAsync(cancellationToken).ConfigureAwait(false); - List isReferenceData = []; - List externalSourceNameData = []; - List idData = []; - List descriptionData = []; - List textData = []; - List additionalMetadataData = []; - List> embeddingData = []; - List keyData = []; - List timestampData = []; - - foreach (MemoryRecord record in records) - { - var metadata = record.Metadata; - - isReferenceData.Add(metadata.IsReference); - externalSourceNameData.Add(metadata.ExternalSourceName); - idData.Add(record.Metadata.Id); - descriptionData.Add(metadata.Description); - textData.Add(metadata.Text); - additionalMetadataData.Add(metadata.AdditionalMetadata); - embeddingData.Add(record.Embedding); - keyData.Add(record.Key); - timestampData.Add(record.Timestamp?.ToString(CultureInfo.InvariantCulture) ?? string.Empty); - } - - MilvusCollection collection = this.Client.GetCollection(collectionName); - - FieldData[] fieldData = - [ - FieldData.Create(IdFieldName, idData), - FieldData.CreateFloatVector(EmbeddingFieldName, embeddingData), - - FieldData.Create(IsReferenceFieldName, isReferenceData, isDynamic: true), - FieldData.Create(ExternalSourceNameFieldName, externalSourceNameData, isDynamic: true), - FieldData.Create(DescriptionFieldName, descriptionData, isDynamic: true), - FieldData.Create(TextFieldName, textData, isDynamic: true), - FieldData.Create(AdditionalMetadataFieldName, additionalMetadataData, isDynamic: true), - FieldData.Create(KeyFieldName, keyData, isDynamic: true), - FieldData.Create(TimestampFieldName, timestampData, isDynamic: true) - ]; - - MutationResult result = await collection.UpsertAsync(fieldData, cancellationToken: cancellationToken).ConfigureAwait(false); - - foreach (var id in result.Ids.StringIds!) - { - yield return id; - } - } - - /// - public async Task GetAsync( - string collectionName, - string key, - bool withEmbedding = false, - CancellationToken cancellationToken = default) - { - await foreach (MemoryRecord record in this.GetBatchAsync(collectionName, [key], withEmbedding, cancellationToken).ConfigureAwait(false)) - { - return record; - } - - return null; - } - - /// - public async IAsyncEnumerable GetBatchAsync( - string collectionName, - IEnumerable keys, - bool withEmbeddings = false, - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - await this.EnsureSupportedServerAsync(cancellationToken).ConfigureAwait(false); - IReadOnlyList fields = await this.Client - .GetCollection(collectionName) - .QueryAsync( - BuildIdFilter(keys), - withEmbeddings ? this._queryParametersWithEmbedding : this._queryParametersWithoutEmbedding, - cancellationToken: cancellationToken) - .ConfigureAwait(false); - - var rowCount = fields[0].RowCount; - - for (int rowNum = 0; rowNum < rowCount; rowNum++) - { - yield return this.ReadMemoryRecord(fields, rowNum); - } - } - - /// - public Task RemoveAsync(string collectionName, string key, CancellationToken cancellationToken = default) - => this.RemoveBatchAsync(collectionName, [key], cancellationToken); - - /// - public async Task RemoveBatchAsync(string collectionName, IEnumerable keys, CancellationToken cancellationToken = default) - { - await this.EnsureSupportedServerAsync(cancellationToken).ConfigureAwait(false); - await this.Client - .GetCollection(collectionName) - .DeleteAsync(BuildIdFilter(keys), cancellationToken: cancellationToken) - .ConfigureAwait(false); - } - - /// - public async Task<(MemoryRecord, double)?> GetNearestMatchAsync( - string collectionName, - ReadOnlyMemory embedding, - double minRelevanceScore = 0, - bool withEmbedding = false, - CancellationToken cancellationToken = default) - { - await foreach ((MemoryRecord, double) result in this.GetNearestMatchesAsync(collectionName, embedding, limit: 1, minRelevanceScore, withEmbedding, cancellationToken).ConfigureAwait(false)) - { - return result; - } - - return null; - } - - /// - public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync( - string collectionName, - ReadOnlyMemory embedding, - int limit, - double minRelevanceScore = 0, - bool withEmbeddings = false, - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - await this.EnsureSupportedServerAsync(cancellationToken).ConfigureAwait(false); - MilvusCollection collection = this.Client.GetCollection(collectionName); - - SearchResults results = await collection - .SearchAsync(EmbeddingFieldName, [embedding], this._metricType, limit, this._searchParameters, cancellationToken) - .ConfigureAwait(false); - - IReadOnlyList ids = results.Ids.StringIds!; - int rowCount = ids.Count; - IReadOnlyList data = results.FieldsData; - - // Since Milvus does not support fetching vectors via the Search API, we do an extra call to fetch the ids and embeddings using the Query API, - // using the IDs returned from the Search above, populating a map from the IDs to the embedding. - // TODO: There's some support for fetching vectors from Search in Milvus 2.3, check that out. - Dictionary>? embeddingMap = null; - if (withEmbeddings) - { - IReadOnlyList fieldData = await collection.QueryAsync( - BuildIdFilter(ids), - new() { OutputFields = { EmbeddingFieldName } }, - cancellationToken: cancellationToken) - .ConfigureAwait(false); - - IReadOnlyList idData = (fieldData[0] as FieldData ?? fieldData[1] as FieldData)!.Data; - IReadOnlyList> embeddingData = (fieldData[0] as FloatVectorFieldData ?? fieldData[1] as FloatVectorFieldData)!.Data; - - embeddingMap = new Dictionary>(ids.Count); - for (int rowNum = 0; rowNum < ids.Count; rowNum++) - { - embeddingMap[idData[rowNum]] = embeddingData[rowNum]; - } - } - - for (int rowNum = 0; rowNum < rowCount; rowNum++) - { - // TODO: Milvus 2.3 has range search, which will move this to the server. - if (results.Scores[rowNum] >= minRelevanceScore) - { - yield return ( - this.ReadMemoryRecord(data, rowNum, withEmbeddings ? embeddingMap![ids[rowNum]] : null), - results.Scores[rowNum]); - } - } - } - - private async Task EnsureSupportedServerAsync(CancellationToken cancellationToken) - { - string version = await this.Client.GetVersionAsync(cancellationToken).ConfigureAwait(false); - ValidateServerVersion(version); - } - - internal static void ValidateServerVersion(string version) - { - string numericVersion = version.StartsWith("v", StringComparison.Ordinal) ? version.Substring(1) : version; - // The official 2.3.0 release image reports "v2.3.0-dev". - if (numericVersion.EndsWith("-dev", StringComparison.Ordinal)) - { - numericVersion = numericVersion.Substring(0, numericVersion.Length - 4); - } - - // Older servers accept escaped strings but retain the escapes as part of the ID. - if (!Version.TryParse(numericVersion, out Version? parsedVersion) || - parsedVersion < new Version(2, 3, 0)) - { - throw new NotSupportedException($"Milvus 2.3.0 or newer is required to handle record IDs safely. The server reported version '{version}'."); - } - } - - internal static string BuildIdFilter(IEnumerable keys) - { - StringBuilder filter = new(); - filter.Append(IdFieldName).Append(" in ["); - - bool first = true; - foreach (string key in keys) - { - if (!first) - { - filter.Append(','); - } - - first = false; - filter.Append('"'); - foreach (char character in key) - { - switch (character) - { - case '\\': - filter.Append(@"\\"); - break; - case '"': - filter.Append("\\\""); - break; - case '\r': - filter.Append(@"\r"); - break; - case '\n': - filter.Append(@"\n"); - break; - case '\t': - filter.Append(@"\t"); - break; - default: - if (char.IsControl(character)) - { - filter.Append(@"\u").Append(((int)character).ToString("x4", CultureInfo.InvariantCulture)); - } - else - { - filter.Append(character); - } - - break; - } - } - - filter.Append('"'); - } - - return filter.Append(']').ToString(); - } - - private MemoryRecord ReadMemoryRecord(IReadOnlyList data, int rowNum, ReadOnlyMemory? externalEmbedding = null) - { - bool isReference = false; - string externalSourceName = string.Empty; - string id = string.Empty; - string description = string.Empty; - string text = string.Empty; - string additionalMetadata = string.Empty; - ReadOnlyMemory? embedding = null; - string key = string.Empty; - DateTimeOffset? timestamp = null; - - foreach (FieldData field in data) - { - switch (field.FieldName) - { - case IsReferenceFieldName when field is FieldData isReferenceField: - isReference = isReferenceField.Data[rowNum]; - break; - - case ExternalSourceNameFieldName when field is FieldData externalSourceNameField: - externalSourceName = externalSourceNameField.Data[rowNum]; - break; - - case IdFieldName when field is FieldData idField: - id = idField.Data[rowNum]; - break; - - case DescriptionFieldName when field is FieldData descriptionField: - description = descriptionField.Data[rowNum]; - break; - - case TextFieldName when field is FieldData textField: - text = textField.Data[rowNum]; - break; - - case AdditionalMetadataFieldName when field is FieldData additionalMetadataField: - additionalMetadata = additionalMetadataField.Data[rowNum]; - break; - - case EmbeddingFieldName when field is FloatVectorFieldData embeddingField: - Debug.Assert(externalEmbedding is null); - embedding = embeddingField.Data[rowNum]; - break; - - case KeyFieldName when field is FieldData keyField: - key = keyField.Data[rowNum]; - break; - - case TimestampFieldName when field is FieldData timestampField: - string timestampString = timestampField.Data[rowNum]; - timestamp = timestampString is { Length: > 0 } - ? DateTimeOffset.Parse(timestampString, CultureInfo.InvariantCulture) - : null; - break; - - default: - continue; // Unknown field - ignore - } - } - - return new MemoryRecord( - new MemoryRecordMetadata(isReference, id, text, description, externalSourceName, additionalMetadata), - embedding ?? externalEmbedding ?? Array.Empty(), - key, - timestamp); - } - - /// - /// Implements the dispose pattern. - /// - protected virtual void Dispose(bool disposing) - { - if (disposing && this._ownsMilvusClient) - { - this.Client.Dispose(); - } - } - - /// - public void Dispose() - { - this.Dispose(true); - GC.SuppressFinalize(this); - } -} diff --git a/dotnet/src/VectorData/Milvus/README.md b/dotnet/src/VectorData/Milvus/README.md index f1548b6a97ae..741d48915b33 100644 --- a/dotnet/src/VectorData/Milvus/README.md +++ b/dotnet/src/VectorData/Milvus/README.md @@ -1,35 +1 @@ -# Microsoft.SemanticKernel.Connectors.Milvus - -This is an implementation of the Semantic Kernel Memory Store abstraction for the [Milvus vector database](https://milvus.io). - -**Server requirement:** Record operations require Milvus v2.3.0 or newer. The connector checks the server version before saving, reading, removing, or searching records and throws `NotSupportedException` for older or unrecognized versions. Collection management and direct calls through `Client` are not covered by this check. - -Record IDs are treated as literal values, including quotes, backslashes, brackets, and line breaks. IDs are not renamed or normalized. Milvus v2.2.14 does not decode escaped IDs correctly and could otherwise target a different record. Upgrade the server before using this connector with existing collections. - -## Quickstart using a standalone Milvus installation - -1. Download the Milvus docker-compose.yml: - -```bash -wget https://github.com/milvus-io/milvus/releases/download/v2.3.0/milvus-standalone-docker-compose.yml -O docker-compose.yml -``` - -2. Start Milvus: - -```bash -docker-compose up -d -``` - -3. Use Semantic Kernel with Milvus, connecting to `localhost` with the default (gRPC) port of 1536: - -```csharp -using MilvusMemoryStore memoryStore = new("localhost"); - -var embeddingGenerator = new OpenAITextEmbeddingGenerationService("text-embedding-ada-002", apiKey); - -SemanticTextMemory textMemory = new(memoryStore, embeddingGenerator); - -var memoryPlugin = kernel.ImportPluginFromObject(new TextMemoryPlugin(textMemory)); -``` - -More information on setting up Milvus can be found [here](https://milvus.io/docs/v2.3.x/install_standalone-docker.md). The `MilvusMemoryStore` constructor provides additional configuration options, such as the vector size, the similarity metric type, etc. +Microsoft.SemanticKernel.Connectors.Milvus and the MemoryStore abstraction have been deprecated. Please use the [CommunityToolkit.VectorData.Milvus package](https://www.nuget.org/packages/CommunityToolkit.VectorData.Milvus), which implements the Microsoft.Extensions.VectorData abstractions and can be found in the [CommunityToolkit/AI repository](https://github.com/CommunityToolkit/AI/tree/main/MEVD/src/Milvus). diff --git a/dotnet/test/VectorData/Milvus.UnitTests/Milvus.UnitTests.csproj b/dotnet/test/VectorData/Milvus.UnitTests/Milvus.UnitTests.csproj deleted file mode 100644 index 7bc362edf5bb..000000000000 --- a/dotnet/test/VectorData/Milvus.UnitTests/Milvus.UnitTests.csproj +++ /dev/null @@ -1,24 +0,0 @@ - - - SemanticKernel.Connectors.Milvus.UnitTests - SemanticKernel.Connectors.Milvus.UnitTests - net10.0 - true - false - $(NoWarn);SKEXP0001;SKEXP0020 - - - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - - - - diff --git a/dotnet/test/VectorData/Milvus.UnitTests/MilvusFilterTests.cs b/dotnet/test/VectorData/Milvus.UnitTests/MilvusFilterTests.cs deleted file mode 100644 index 954e1bb45165..000000000000 --- a/dotnet/test/VectorData/Milvus.UnitTests/MilvusFilterTests.cs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using Microsoft.SemanticKernel.Connectors.Milvus; -using Xunit; - -namespace SemanticKernel.Connectors.Milvus.UnitTests; - -public sealed class MilvusFilterTests -{ - [Theory] - [InlineData("v2.3.0")] - [InlineData("v2.3.0-dev")] - [InlineData("2.3.0")] - [InlineData("v2.6.1")] - [InlineData("3.0.0")] - public void AcceptsSupportedServerVersions(string version) - { - MilvusMemoryStore.ValidateServerVersion(version); - } - - [Theory] - [InlineData("v2.2.14")] - [InlineData("v2.2.14-dev")] - [InlineData("v2.2.16")] - [InlineData("v2.3.0-beta")] - [InlineData("unknown")] - [InlineData("")] - public void RejectsOlderOrUnrecognizedServerVersions(string version) - { - Assert.Throws(() => MilvusMemoryStore.ValidateServerVersion(version)); - } - - [Theory] - [InlineData("", "")] - [InlineData("ordinary-id", "ordinary-id")] - [InlineData("[],'", "[],'")] - [InlineData("\"", "\\\"")] - [InlineData("\\", "\\\\")] - [InlineData("\\\"", "\\\\\\\"")] - [InlineData("trailing\\", "trailing\\\\")] - [InlineData("\r\n\t", "\\r\\n\\t")] - [InlineData("\\r\\n\\t", "\\\\r\\\\n\\\\t")] - [InlineData("\0\b\f\v\u001f\u007f\u0085", "\\u0000\\u0008\\u000c\\u000b\\u001f\\u007f\\u0085")] - [InlineData("\u00e9\u4e2d\ud83d\ude00", "\u00e9\u4e2d\ud83d\ude00")] - [InlineData("a\"] || true || id in [\"", "a\\\"] || true || id in [\\\"")] - [InlineData("z\"] && text like \"%secret%\" || id in [\"", "z\\\"] && text like \\\"%secret%\\\" || id in [\\\"")] - [InlineData("x\" || id != \"", "x\\\" || id != \\\"")] - [InlineData("a\",\"victim", "a\\\",\\\"victim")] - public void EncodesEachIdAsOneString(string key, string expected) - { - Assert.Equal("id in [\"" + expected + "\"]", MilvusMemoryStore.BuildIdFilter([key])); - } - - [Fact] - public void PreservesEmptyLists() - { - Assert.Equal("id in []", MilvusMemoryStore.BuildIdFilter([])); - } - - [Fact] - public void PreservesOrderDuplicatesAndEmptyIds() - { - Assert.Equal("id in [\"a\",\"\",\"a\",\"b\"]", MilvusMemoryStore.BuildIdFilter(["a", "", "a", "b"])); - } - - [Fact] - public void EnumeratesKeysOnlyOnce() - { - bool enumerated = false; - - IEnumerable Keys() - { - Assert.False(enumerated); - enumerated = true; - yield return "first"; - yield return "second"; - } - - Assert.Equal("id in [\"first\",\"second\"]", MilvusMemoryStore.BuildIdFilter(Keys())); - Assert.True(enumerated); - } -} diff --git a/dotnet/test/VectorData/Milvus.UnitTests/MilvusFixture.cs b/dotnet/test/VectorData/Milvus.UnitTests/MilvusFixture.cs deleted file mode 100644 index 5d70386a2025..000000000000 --- a/dotnet/test/VectorData/Milvus.UnitTests/MilvusFixture.cs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using DotNet.Testcontainers.Builders; -using DotNet.Testcontainers.Containers; -using DotNet.Testcontainers.Networks; -using Testcontainers.Milvus; -using Xunit; - -namespace SemanticKernel.Connectors.Milvus.UnitTests; - -public class MilvusFixture : IAsyncLifetime -{ - private readonly INetwork _network = new NetworkBuilder().Build(); - private readonly IContainer _storage; - - public MilvusFixture() : this("milvusdb/milvus:v2.3.0") - { - } - - protected MilvusFixture(string image) - { - string password = Guid.NewGuid().ToString("N"); - this._storage = new ContainerBuilder("milvusdb/minio:RELEASE.2024-12-18T13-15-44Z") - .WithNetwork(this._network) - .WithNetworkAliases("minio") - .WithEnvironment("MINIO_ROOT_USER", "test-user") - .WithEnvironment("MINIO_ROOT_PASSWORD", password) - .WithCommand("server", "/data") - .WithPortBinding(9000, true) - .WithWaitStrategy(Wait.ForUnixContainer().UntilHttpRequestIsSucceeded( - request => request.ForPort(9000).ForPath("/minio/health/live"))) - .Build(); - - // Milvus 2.2 needs object storage; the module's local-storage default requires a newer server. - this.Container = new MilvusBuilder(image) - .WithNetwork(this._network) - .WithEnvironment("COMMON_STORAGETYPE", "minio") - .WithEnvironment("MINIO_ADDRESS", "minio:9000") - .WithEnvironment("MINIO_ACCESS_KEY_ID", "test-user") - .WithEnvironment("MINIO_SECRET_ACCESS_KEY", password) - .WithWaitStrategy(Wait.ForUnixContainer().UntilHttpRequestIsSucceeded( - request => request.ForPort(MilvusBuilder.MilvusManagementPort).ForPath("/healthz"), - strategy => strategy.WithTimeout(TimeSpan.FromMinutes(2)))) - .Build(); - } - - public MilvusContainer Container { get; } - - public async Task InitializeAsync() - { - await this._network.CreateAsync(); - await this._storage.StartAsync(); - await this.Container.StartAsync(); - } - - public async Task DisposeAsync() - { - try - { - await this.Container.DisposeAsync(); - } - finally - { - try - { - await this._storage.DisposeAsync(); - } - finally - { - await this._network.DisposeAsync(); - } - } - } -} diff --git a/dotnet/test/VectorData/Milvus.UnitTests/MilvusLegacyServerTests.cs b/dotnet/test/VectorData/Milvus.UnitTests/MilvusLegacyServerTests.cs deleted file mode 100644 index 3587723564af..000000000000 --- a/dotnet/test/VectorData/Milvus.UnitTests/MilvusLegacyServerTests.cs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Connectors.Milvus; -using Microsoft.SemanticKernel.Memory; -using Testcontainers.Milvus; -using Xunit; - -namespace SemanticKernel.Connectors.Milvus.UnitTests; - -public sealed class MilvusLegacyFixture : MilvusFixture -{ - public MilvusLegacyFixture() : base("milvusdb/milvus:v2.2.14") - { - } -} - -[Trait("Category", "MilvusIntegration")] -public sealed class MilvusLegacyServerTests : IAsyncLifetime -{ - private readonly MilvusLegacyFixture _fixture = new(); - - public Task InitializeAsync() => this._fixture.InitializeAsync(); - - public Task DisposeAsync() => this._fixture.DisposeAsync(); - - // For manual testing with Docker only. Remove Skip locally to enable this test. - [Fact(Skip = "For manual testing only; requires Docker.")] - public async Task RejectsOldServerBeforeReadingOrChangingRecordsAsync() - { - using MilvusMemoryStore store = new( - this._fixture.Container.Hostname, - port: this._fixture.Container.GetMappedPublicPort(MilvusBuilder.MilvusGrpcPort)); - const string CollectionName = "does_not_exist"; - const string Key = "a\",\"victim"; - MemoryRecord record = MemoryRecord.LocalRecord(Key, "synthetic", null, new float[] { 1, 0 }); - - await Assert.ThrowsAsync(() => store.GetAsync(CollectionName, Key)); - await Assert.ThrowsAsync(() => store.GetBatchAsync(CollectionName, [Key]).ToListAsync().AsTask()); - await Assert.ThrowsAsync(() => store.RemoveAsync(CollectionName, Key)); - await Assert.ThrowsAsync(() => store.RemoveBatchAsync(CollectionName, [Key])); - await Assert.ThrowsAsync(() => store.UpsertAsync(CollectionName, record)); - await Assert.ThrowsAsync(() => store.UpsertBatchAsync(CollectionName, [record]).ToListAsync().AsTask()); - await Assert.ThrowsAsync(() => store.GetNearestMatchAsync(CollectionName, new float[] { 1, 0 }, withEmbedding: true)); - await Assert.ThrowsAsync(() => store.GetNearestMatchesAsync( - CollectionName, new float[] { 1, 0 }, limit: 1, withEmbeddings: true).ToListAsync().AsTask()); - } -} diff --git a/dotnet/test/VectorData/Milvus.UnitTests/MilvusMemoryStoreTests.cs b/dotnet/test/VectorData/Milvus.UnitTests/MilvusMemoryStoreTests.cs deleted file mode 100644 index e67dfc83e4fc..000000000000 --- a/dotnet/test/VectorData/Milvus.UnitTests/MilvusMemoryStoreTests.cs +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.Connectors.Milvus; -using Microsoft.SemanticKernel.Memory; -using Milvus.Client; -using Testcontainers.Milvus; -using Xunit; - -namespace SemanticKernel.Connectors.Milvus.UnitTests; - -[Trait("Category", "MilvusIntegration")] -public sealed class MilvusMemoryStoreTests : IAsyncLifetime -{ - // For manual testing with Docker only. Set to null locally to enable these tests. - private const string? SkipReason = "For manual testing only; requires Docker."; - - private readonly MilvusFixture _fixture = new(); - private readonly string _collectionName = "id_regression_" + Guid.NewGuid().ToString("N"); - private MilvusMemoryStore _store = null!; - - public static TheoryData UnusualIds => new() - { - "ordinary", - "quotes\"and'single", - "brackets[],commas", - "backslash\\and\\\"quote\\", - "line\r\nbreak\ttab", - "literal\\r\\n\\t", - "controls\0\b\f\v\u001f\u007f\u0085", - "\u00e9\u4e2d\ud83d\ude00", - "a\"] || true || id in [\"", - "z\"] && text like \"%secret%\" || id in [\"", - "x\" || id != \"", - "a\",\"victim" - }; - - public async Task InitializeAsync() - { - await this._fixture.InitializeAsync(); - this._store = new MilvusMemoryStore( - this._fixture.Container.Hostname, - port: this._fixture.Container.GetMappedPublicPort(MilvusBuilder.MilvusGrpcPort), - vectorSize: 2, - consistencyLevel: ConsistencyLevel.Strong); - - // Create the fixture directly so these tests only depend on the ID operations under test. - MilvusCollection collection = await this._store.Client.CreateCollectionAsync( - this._collectionName, - new CollectionSchema - { - Fields = - { - FieldSchema.CreateVarchar("id", maxLength: 1024, isPrimaryKey: true, autoId: false), - FieldSchema.CreateFloatVector("embedding", 2) - }, - EnableDynamicFields = true - }, - ConsistencyLevel.Strong); - await collection.CreateIndexAsync("embedding", IndexType.Flat, SimilarityMetricType.Ip); - await collection.LoadAsync(); - await collection.WaitForCollectionLoadAsync( - waitingInterval: TimeSpan.FromMilliseconds(100), timeout: TimeSpan.FromMinutes(1)); - } - - public async Task DisposeAsync() - { - try - { - if (this._store is not null) - { - await this._store.DeleteCollectionAsync(this._collectionName); - } - } - finally - { - this._store?.Dispose(); - await this._fixture.DisposeAsync(); - } - } - - [Theory(Skip = SkipReason)] - [MemberData(nameof(UnusualIds))] - public async Task ReadsAndDeletesOnlyTheExactIdsAsync(string key) - { - MemoryRecord record = CreateRecord(key, [1, 0]); - MemoryRecord victim = CreateRecord("victim", [0, 1]); - await this.InsertRecordsAsync(record, victim); - - foreach (bool withEmbeddings in new[] { false, true }) - { - MemoryRecord? single = await this._store.GetAsync(this._collectionName, key, withEmbeddings); - AssertRecord(record, single, withEmbeddings); - - List batch = await this._store.GetBatchAsync(this._collectionName, [key], withEmbeddings).ToListAsync(); - AssertRecord(record, Assert.Single(batch), withEmbeddings); - } - - await this._store.RemoveAsync(this._collectionName, key); - Assert.Null(await this._store.GetAsync(this._collectionName, key)); - AssertRecord(victim, await this._store.GetAsync(this._collectionName, "victim"), false); - - await this.InsertRecordsAsync(record, CreateRecord("second", [0.5f, 0.5f])); - - List restored = await this._store.GetBatchAsync( - this._collectionName, [key, "second"], withEmbeddings: true).ToListAsync(); - Assert.Equal(2, restored.Count); - AssertRecord(record, Assert.Single(restored, item => item.Metadata.Id == key), true); - - await this._store.RemoveBatchAsync(this._collectionName, [key, "second"]); - Assert.Empty(await this._store.GetBatchAsync(this._collectionName, [key, "second"]).ToListAsync()); - AssertRecord(victim, await this._store.GetAsync(this._collectionName, "victim"), false); - } - - [Theory(Skip = SkipReason)] - [MemberData(nameof(UnusualIds))] - public async Task SearchUsesStoredIdsLiterallyAsync(string key) - { - MemoryRecord record = CreateRecord(key, [1, 0]); - MemoryRecord victim = CreateRecord("victim", [0, 1]); - await this.InsertRecordsAsync(record, victim); - await this._store.Client.GetCollection(this._collectionName).FlushAsync(); - - foreach (bool withEmbeddings in new[] { false, true }) - { - List<(MemoryRecord, double)> matches = await this._store.GetNearestMatchesAsync( - this._collectionName, new float[] { 1, 0 }, limit: 2, withEmbeddings: withEmbeddings).ToListAsync(); - Assert.Equal(2, matches.Count); - AssertRecord(record, matches[0].Item1, withEmbeddings); - AssertRecord(victim, matches[1].Item1, withEmbeddings); - - (MemoryRecord, double)? nearest = await this._store.GetNearestMatchAsync( - this._collectionName, new float[] { 1, 0 }, withEmbedding: withEmbeddings); - Assert.NotNull(nearest); - AssertRecord(record, nearest.Value.Item1, withEmbeddings); - } - } - - private static MemoryRecord CreateRecord(string key, float[] embedding) - => MemoryRecord.LocalRecord(key, "text for " + key, description: null, embedding, key: key); - - [Theory(Skip = SkipReason)] - [InlineData("quotes\"", "quotes\\\"")] - [InlineData("line\nbreak", "line\\nbreak")] - [InlineData("slash\\", "slash\\\\")] - public async Task DoesNotConfuseAnIdWithItsEscapedSpellingAsync(string key, string otherKey) - { - MemoryRecord expected = CreateRecord(key, [1, 0]); - MemoryRecord other = CreateRecord(otherKey, [0, 1]); - await this.InsertRecordsAsync(expected, other); - - AssertRecord(expected, await this._store.GetAsync(this._collectionName, key), false); - await this._store.RemoveAsync(this._collectionName, key); - Assert.Null(await this._store.GetAsync(this._collectionName, key)); - AssertRecord(other, await this._store.GetAsync(this._collectionName, otherKey), false); - } - - [Fact(Skip = SkipReason)] - public async Task MissingHostileIdsDoNotReadOrDeleteOtherRecordsAsync() - { - MemoryRecord victim = CreateRecord("victim", [1, 0]); - await this.InsertRecordsAsync(victim); - string[] keys = ["a\",\"victim", "a\"] || true || id in [\""]; - - foreach (string key in keys) - { - Assert.Null(await this._store.GetAsync(this._collectionName, key)); - await this._store.RemoveAsync(this._collectionName, key); - } - - Assert.Empty(await this._store.GetBatchAsync(this._collectionName, keys).ToListAsync()); - await this._store.RemoveBatchAsync(this._collectionName, keys); - AssertRecord(victim, await this._store.GetAsync(this._collectionName, "victim"), false); - } - - private async Task InsertRecordsAsync(params MemoryRecord[] records) - { - // Seed typed records independently of the legacy client's Upsert response handling. - FieldData[] fields = - [ - FieldData.Create("id", records.Select(record => record.Metadata.Id).ToArray()), - FieldData.CreateFloatVector("embedding", records.Select(record => record.Embedding).ToArray()), - FieldData.Create("text", records.Select(record => record.Metadata.Text).ToArray(), isDynamic: true), - FieldData.Create("is_reference", records.Select(record => record.Metadata.IsReference).ToArray(), isDynamic: true), - FieldData.Create("external_source_name", records.Select(record => record.Metadata.ExternalSourceName).ToArray(), isDynamic: true), - FieldData.Create("description", records.Select(record => record.Metadata.Description).ToArray(), isDynamic: true), - FieldData.Create("additional_metadata", records.Select(record => record.Metadata.AdditionalMetadata).ToArray(), isDynamic: true), - FieldData.Create("key", records.Select(record => record.Key).ToArray(), isDynamic: true), - FieldData.Create("timestamp", records.Select(_ => string.Empty).ToArray(), isDynamic: true) - ]; - await this._store.Client.GetCollection(this._collectionName).InsertAsync(fields); - } - - private static void AssertRecord(MemoryRecord expected, MemoryRecord? actual, bool withEmbedding) - { - Assert.NotNull(actual); - Assert.Equal(expected.Metadata.Id, actual.Metadata.Id); - Assert.Equal(expected.Metadata.Text, actual.Metadata.Text); - Assert.Equal(withEmbedding ? expected.Embedding.ToArray() : [], actual.Embedding.ToArray()); - } -}