Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 10 additions & 7 deletions MEVD/src/SqliteVec/SqliteCollection.cs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ public class SqliteCollection<TKey, TRecord> : VectorStoreCollection<TKey, TReco
/// <summary>The connection string for the SQLite database represented by this <see cref="SqliteVectorStore"/>.</summary>
private readonly string _connectionString;

/// <summary>Optional callback that runs on each connection after it is opened and the sqlite-vec extension is loaded.</summary>
private readonly Action<SqliteConnection>? _connectionInitializer;

/// <summary>The mapper to use when mapping between the consumer data model and the SQLite record.</summary>
private readonly SqliteMapper<TRecord> _mapper;

Expand Down Expand Up @@ -102,6 +105,7 @@ internal SqliteCollection(string connectionString, string name, Func<SqliteColle

// Assign.
_connectionString = connectionString;
_connectionInitializer = options.ConnectionInitializer;
Name = name;
_model = modelFactory(options);

Expand Down Expand Up @@ -389,13 +393,12 @@ public override async Task DeleteAsync(IEnumerable<TKey> keys, CancellationToken

#region private

private async ValueTask<SqliteConnection> GetConnectionAsync(CancellationToken cancellationToken = default)
{
var connection = new SqliteConnection(_connectionString);
await connection.OpenAsync(cancellationToken).ConfigureAwait(false);
connection.LoadVector();
return connection;
}
private ValueTask<SqliteConnection> GetConnectionAsync(CancellationToken cancellationToken = default)
=> SqliteConnectionFactory.OpenAsync(
_connectionString,
_connectionInitializer,
loadVector: true,
cancellationToken: cancellationToken);

private async IAsyncEnumerable<VectorSearchResult<TRecord>> EnumerateAndMapSearchResultsAsync(
List<SqliteWhereCondition> conditions,
Expand Down
26 changes: 26 additions & 0 deletions MEVD/src/SqliteVec/SqliteCollectionOptions.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using Microsoft.Data.Sqlite;
using Microsoft.Extensions.VectorData;

namespace CommunityToolkit.VectorData.SqliteVec;
Expand All @@ -22,6 +23,7 @@ public SqliteCollectionOptions()
internal SqliteCollectionOptions(SqliteCollectionOptions? source) : base(source)
{
VectorVirtualTableName = source?.VectorVirtualTableName;
ConnectionInitializer = source?.ConnectionInitializer;
}

/// <summary>
Expand All @@ -31,4 +33,28 @@ internal SqliteCollectionOptions(SqliteCollectionOptions? source) : base(source)
/// If not provided, collection name with prefix will be used as virtual table name.
/// </remarks>
public string? VectorVirtualTableName { get; set; }

/// <summary>
/// Gets or sets a callback that runs on every connection this collection opens.
/// </summary>
/// <remarks>
/// <para>
/// If defined, the callback runs after the connection is opened and the sqlite-vec extension is loaded.
/// </para>
/// <para>
/// Use it for per-connection settings that have no connection string equivalent, such as
/// <c>PRAGMA synchronous=NORMAL</c>, <c>PRAGMA cache_size</c> or <c>PRAGMA temp_store=MEMORY</c>, or for loading additional extensions.
/// </para>
/// <para>
/// When the collection is created from a <see cref="SqliteVectorStore"/>, the callback configured on the vector store options

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you for documenting this important information 👍

/// is used; this property applies to collections constructed directly.
/// </para>
/// <para>
/// The collection owns the connection and disposes it when the operation completes. The callback must not close or dispose
/// the connection, and must not leave a transaction or a data reader open. If the callback throws, the operation fails and
/// the exception is surfaced to the caller. The callback can run on several connections at the same time, so it must be
/// thread-safe.
/// </para>
/// </remarks>
public Action<SqliteConnection>? ConnectionInitializer { get; set; }

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I looked for some "prior art" in ASP.NET and EF Core and BCL and found "ConfigureHttpClient", "ConfigureEndpointDefaults", "ConfigureDataSource" and I think that "ConfigureConnection" would be better.

Suggested change
public Action<SqliteConnection>? ConnectionInitializer { get; set; }
public Action<SqliteConnection>? ConfigureConnection { get; set; }

}
46 changes: 46 additions & 0 deletions MEVD/src/SqliteVec/SqliteConnectionFactory.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using Microsoft.Data.Sqlite;

namespace CommunityToolkit.VectorData.SqliteVec;

/// <summary>
/// Creates the connections used by the vector store and its collections.
/// </summary>
internal static class SqliteConnectionFactory
{
/// <summary>
/// Opens a connection, runs the shared setup steps in order, and hands it to the caller.
/// </summary>
/// <remarks>
/// The caller owns the returned connection. If any setup step fails, the connection is disposed here
/// because the caller never receives it.
/// </remarks>
public static async ValueTask<SqliteConnection> OpenAsync(
string connectionString,
Action<SqliteConnection>? initializer,
bool loadVector,
CancellationToken cancellationToken)
{
var connection = new SqliteConnection(connectionString);

try
{
await connection.OpenAsync(cancellationToken).ConfigureAwait(false);

if (loadVector)
{
connection.LoadVector();
}

initializer?.Invoke(connection);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The callback is synchronous. Are there any SqliteConnection properties that require calling async methods to configure? I just want to be 100% sure the callback does not need to return a Task/ValueTask in order to avoid calling async methods from sync callback.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With pooling on (the Microsoft.Data.Sqlite default), this runs again every time a pooled physical connection is reused, and what it sets stays on that connection. The tests use Pooling=false, so they don't cover it. Two cases that fail on this branch with the default connection string and pass with Pooling=false:

  • an ATTACH DATABASE initializer fails on the second operation: SQLite Error 1: 'database aux is already in use';
  • PRAGMA query_only=ON on one collection leaks to another collection with the same connection string: SQLite Error 8: 'attempt to write a readonly database'.

Maybe document that the callback must be safe to run more than once on the same connection, and add a test with pooling on?

Repro tests
[Fact]
public async Task AttachInitializerWorksOnEveryOperationWithPooling()
{
    var options = new SqliteCollectionOptions
    {
        ConnectionInitializer = connection =>
        {
            using var command = connection.CreateCommand();
            command.CommandText = $"ATTACH DATABASE '{_auxPath}' AS aux";
            command.ExecuteNonQuery();
        }
    };

    var collection = new SqliteCollection<string, ProbeRecord>(ConnectionString, "AttachCollection", options);

    await collection.EnsureCollectionExistsAsync();
    await collection.UpsertAsync(new ProbeRecord { Key = "k1", Text = "hello", Vector = new([1f, 2f, 3f, 4f]) });
}

[Fact]
public async Task QueryOnlyFromOneCollectionDoesNotLeakToAnotherWithPooling()
{
    var readOnly = new SqliteCollection<string, ProbeRecord>(ConnectionString, "ReadOnlyCollection", new SqliteCollectionOptions
    {
        ConnectionInitializer = connection =>
        {
            using var command = connection.CreateCommand();
            command.CommandText = "PRAGMA query_only=ON";
            command.ExecuteNonQuery();
        }
    });

    var writable = new SqliteCollection<string, ProbeRecord>(ConnectionString, "WritableCollection");
    await writable.EnsureCollectionExistsAsync();

    Assert.False(await readOnly.CollectionExistsAsync());

    await writable.UpsertAsync(new ProbeRecord { Key = "k1", Text = "hello", Vector = new([1f, 2f, 3f, 4f]) });
}

ConnectionString is $"Data Source={_databasePath}" (pooling on), and _auxPath is a second temp file. ProbeRecord has a string key, a Text data property and a 4-dimension vector.

return connection;
}
catch
{
await connection.DisposeAsync().ConfigureAwait(false);
throw;
}
}
}
2 changes: 1 addition & 1 deletion MEVD/src/SqliteVec/SqliteVec.csproj
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<Version>1.0.3-preview</Version>
<Version>1.0.4-preview</Version>
<AssemblyName>CommunityToolkit.VectorData.SqliteVec</AssemblyName>
<RootNamespace>$(AssemblyName)</RootNamespace>
<TargetFrameworks>net10.0;net8.0</TargetFrameworks>
Expand Down
17 changes: 13 additions & 4 deletions MEVD/src/SqliteVec/SqliteVectorStore.cs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ public sealed class SqliteVectorStore : VectorStore

private readonly IEmbeddingGenerator? _embeddingGenerator;

/// <summary>Optional callback that runs on each connection after it is opened.</summary>
private readonly Action<SqliteConnection>? _connectionInitializer;

/// <summary>
/// Initializes a new instance of the <see cref="SqliteVectorStore"/> class.
/// </summary>
Expand All @@ -47,6 +50,7 @@ public SqliteVectorStore(string connectionString, SqliteVectorStoreOptions? opti
options ??= SqliteVectorStoreOptions.Default;
_vectorVirtualTableName = options.VectorVirtualTableName;
_embeddingGenerator = options.EmbeddingGenerator;
_connectionInitializer = options.ConnectionInitializer;

var connectionStringBuilder = new SqliteConnectionStringBuilder(connectionString);

Expand All @@ -71,7 +75,8 @@ public override SqliteCollection<TKey, TRecord> GetCollection<TKey, TRecord>(str
{
Definition = definition,
VectorVirtualTableName = _vectorVirtualTableName,
EmbeddingGenerator = _embeddingGenerator
EmbeddingGenerator = _embeddingGenerator,
ConnectionInitializer = _connectionInitializer
});

/// <inheritdoc />
Expand All @@ -83,7 +88,8 @@ public override SqliteDynamicCollection GetDynamicCollection(string name, Vector
{
Definition = definition,
VectorVirtualTableName = _vectorVirtualTableName,
EmbeddingGenerator = _embeddingGenerator
EmbeddingGenerator = _embeddingGenerator,
ConnectionInitializer = _connectionInitializer
}
);
#pragma warning restore IDE0090
Expand All @@ -95,8 +101,11 @@ public override async IAsyncEnumerable<string> ListCollectionNamesAsync([Enumera
const string TablePropertyName = "name";
const string Query = $"SELECT {TablePropertyName} FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%';";

using var connection = new SqliteConnection(_connectionString);
await connection.OpenAsync(cancellationToken).ConfigureAwait(false);
using var connection = await SqliteConnectionFactory.OpenAsync(
_connectionString,
_connectionInitializer,
loadVector: false,
cancellationToken: cancellationToken).ConfigureAwait(false);
using var command = connection.CreateCommand();

command.CommandText = Query;
Expand Down
23 changes: 23 additions & 0 deletions MEVD/src/SqliteVec/SqliteVectorStoreOptions.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using Microsoft.Data.Sqlite;
using Microsoft.Extensions.AI;

namespace CommunityToolkit.VectorData.SqliteVec;
Expand All @@ -23,6 +24,7 @@ internal SqliteVectorStoreOptions(SqliteVectorStoreOptions? source)
{
VectorVirtualTableName = source?.VectorVirtualTableName;
EmbeddingGenerator = source?.EmbeddingGenerator;
ConnectionInitializer = source?.ConnectionInitializer;
}

/// <summary>
Expand All @@ -37,4 +39,25 @@ internal SqliteVectorStoreOptions(SqliteVectorStoreOptions? source)
/// Gets or sets the default embedding generator to use when generating vectors embeddings with this vector store.
/// </summary>
public IEmbeddingGenerator? EmbeddingGenerator { get; set; }

/// <summary>
/// Gets or sets a callback that runs on every connection this vector store and the collections created from it open.
/// </summary>
/// <remarks>
/// <para>
/// If defined, the callback runs after the connection is opened and, on connections used for collection operations, after the sqlite-vec
/// extension is loaded.
/// </para>
/// <para>
/// Use it for per-connection settings that have no connection string equivalent, such as
/// <c>PRAGMA synchronous=NORMAL</c>, <c>PRAGMA cache_size</c> or <c>PRAGMA temp_store=MEMORY</c>, or for loading additional extensions.
/// </para>
/// <para>
/// The vector store owns the connection and disposes it when the operation completes. The callback must not close or dispose
/// the connection, and must not leave a transaction or a data reader open. If the callback throws, the operation fails and
/// the exception is surfaced to the caller. The callback can run on several connections at the same time, so it must be
/// thread-safe.
/// </para>
/// </remarks>
public Action<SqliteConnection>? ConnectionInitializer { get; set; }
}
Loading
Loading