Repository navigation
SqliteVec: support per-connection initialization callbacks #81
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -1,6 +1,7 @@ | ||||||
| // Licensed to the .NET Foundation under one or more agreements. | ||||||
| // The .NET Foundation licenses this file to you under the MIT license. | ||||||
|
|
||||||
| using Microsoft.Data.Sqlite; | ||||||
| using Microsoft.Extensions.VectorData; | ||||||
|
|
||||||
| namespace CommunityToolkit.VectorData.SqliteVec; | ||||||
|
|
@@ -22,6 +23,7 @@ public SqliteCollectionOptions() | |||||
| internal SqliteCollectionOptions(SqliteCollectionOptions? source) : base(source) | ||||||
| { | ||||||
| VectorVirtualTableName = source?.VectorVirtualTableName; | ||||||
| ConnectionInitializer = source?.ConnectionInitializer; | ||||||
| } | ||||||
|
|
||||||
| /// <summary> | ||||||
|
|
@@ -31,4 +33,28 @@ internal SqliteCollectionOptions(SqliteCollectionOptions? source) : base(source) | |||||
| /// If not provided, collection name with prefix will be used as virtual table name. | ||||||
| /// </remarks> | ||||||
| public string? VectorVirtualTableName { get; set; } | ||||||
|
|
||||||
| /// <summary> | ||||||
| /// Gets or sets a callback that runs on every connection this collection opens. | ||||||
| /// </summary> | ||||||
| /// <remarks> | ||||||
| /// <para> | ||||||
| /// If defined, the callback runs after the connection is opened and the sqlite-vec extension is loaded. | ||||||
| /// </para> | ||||||
| /// <para> | ||||||
| /// Use it for per-connection settings that have no connection string equivalent, such as | ||||||
| /// <c>PRAGMA synchronous=NORMAL</c>, <c>PRAGMA cache_size</c> or <c>PRAGMA temp_store=MEMORY</c>, or for loading additional extensions. | ||||||
| /// </para> | ||||||
| /// <para> | ||||||
| /// When the collection is created from a <see cref="SqliteVectorStore"/>, the callback configured on the vector store options | ||||||
| /// is used; this property applies to collections constructed directly. | ||||||
| /// </para> | ||||||
| /// <para> | ||||||
| /// The collection owns the connection and disposes it when the operation completes. The callback must not close or dispose | ||||||
| /// the connection, and must not leave a transaction or a data reader open. If the callback throws, the operation fails and | ||||||
| /// the exception is surfaced to the caller. The callback can run on several connections at the same time, so it must be | ||||||
| /// thread-safe. | ||||||
| /// </para> | ||||||
| /// </remarks> | ||||||
| public Action<SqliteConnection>? ConnectionInitializer { get; set; } | ||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I looked for some "prior art" in ASP.NET and EF Core and BCL and found "ConfigureHttpClient", "ConfigureEndpointDefaults", "ConfigureDataSource" and I think that "ConfigureConnection" would be better.
Suggested change
|
||||||
| } | ||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
|
|
||
| using Microsoft.Data.Sqlite; | ||
|
|
||
| namespace CommunityToolkit.VectorData.SqliteVec; | ||
|
|
||
| /// <summary> | ||
| /// Creates the connections used by the vector store and its collections. | ||
| /// </summary> | ||
| internal static class SqliteConnectionFactory | ||
| { | ||
| /// <summary> | ||
| /// Opens a connection, runs the shared setup steps in order, and hands it to the caller. | ||
| /// </summary> | ||
| /// <remarks> | ||
| /// The caller owns the returned connection. If any setup step fails, the connection is disposed here | ||
| /// because the caller never receives it. | ||
| /// </remarks> | ||
| public static async ValueTask<SqliteConnection> OpenAsync( | ||
| string connectionString, | ||
| Action<SqliteConnection>? initializer, | ||
| bool loadVector, | ||
| CancellationToken cancellationToken) | ||
| { | ||
| var connection = new SqliteConnection(connectionString); | ||
|
|
||
| try | ||
| { | ||
| await connection.OpenAsync(cancellationToken).ConfigureAwait(false); | ||
|
|
||
| if (loadVector) | ||
| { | ||
| connection.LoadVector(); | ||
| } | ||
|
|
||
| initializer?.Invoke(connection); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The callback is synchronous. Are there any
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. With pooling on (the
Maybe document that the callback must be safe to run more than once on the same connection, and add a test with pooling on? Repro tests[Fact]
public async Task AttachInitializerWorksOnEveryOperationWithPooling()
{
var options = new SqliteCollectionOptions
{
ConnectionInitializer = connection =>
{
using var command = connection.CreateCommand();
command.CommandText = $"ATTACH DATABASE '{_auxPath}' AS aux";
command.ExecuteNonQuery();
}
};
var collection = new SqliteCollection<string, ProbeRecord>(ConnectionString, "AttachCollection", options);
await collection.EnsureCollectionExistsAsync();
await collection.UpsertAsync(new ProbeRecord { Key = "k1", Text = "hello", Vector = new([1f, 2f, 3f, 4f]) });
}
[Fact]
public async Task QueryOnlyFromOneCollectionDoesNotLeakToAnotherWithPooling()
{
var readOnly = new SqliteCollection<string, ProbeRecord>(ConnectionString, "ReadOnlyCollection", new SqliteCollectionOptions
{
ConnectionInitializer = connection =>
{
using var command = connection.CreateCommand();
command.CommandText = "PRAGMA query_only=ON";
command.ExecuteNonQuery();
}
});
var writable = new SqliteCollection<string, ProbeRecord>(ConnectionString, "WritableCollection");
await writable.EnsureCollectionExistsAsync();
Assert.False(await readOnly.CollectionExistsAsync());
await writable.UpsertAsync(new ProbeRecord { Key = "k1", Text = "hello", Vector = new([1f, 2f, 3f, 4f]) });
}
|
||
| return connection; | ||
| } | ||
| catch | ||
| { | ||
| await connection.DisposeAsync().ConfigureAwait(false); | ||
| throw; | ||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thank you for documenting this important information 👍