diff --git a/src/contracts/src/Telescope/TelescopeTag.php b/src/contracts/src/Telescope/TelescopeTag.php index ae6cad18a7..39b78359ed 100644 --- a/src/contracts/src/Telescope/TelescopeTag.php +++ b/src/contracts/src/Telescope/TelescopeTag.php @@ -23,6 +23,7 @@ enum TelescopeTag: string case Algolia = 'algolia'; case Meilisearch = 'meilisearch'; case Saloon = 'saloon'; + case Turbopuffer = 'turbopuffer'; case Typesense = 'typesense'; /** @@ -38,6 +39,7 @@ public function description(): string self::Algolia => 'Scout Algolia driver.', self::Meilisearch => 'Scout Meilisearch driver.', self::Saloon => 'Hypervel Saloon — API integration package.', + self::Turbopuffer => 'Scout Turbopuffer driver.', self::Typesense => 'Scout Typesense driver.', }; } diff --git a/src/docs/porting-from-laravel.md b/src/docs/porting-from-laravel.md index 61d5fd1881..9d17fe5edd 100644 --- a/src/docs/porting-from-laravel.md +++ b/src/docs/porting-from-laravel.md @@ -541,6 +541,8 @@ Fortify ignores Laravel's `fortify.passwords` setting. Declare the password rese Hypervel compiles integer and float values passed to Scout's Algolia `where`, `whereIn`, and `whereNotIn` methods as numeric comparisons. Numeric-looking strings remain facet values. When porting an Algolia index, ensure the indexed attribute type matches the PHP value type used by these filters. +Scout cannot generate embeddings with the Laravel AI SDK, and its database engine does not support semantic or hybrid search. Models whose `toSearchableEmbedding` method returns text must return precomputed embedding arrays or switch to the engine's native embeddings. See [semantic search](/docs/{{version}}/scout#semantic-search). + ### Socialite diff --git a/src/docs/scout.md b/src/docs/scout.md index 0be98230c0..9289a8786b 100644 --- a/src/docs/scout.md +++ b/src/docs/scout.md @@ -18,6 +18,7 @@ - [Meilisearch](#meilisearch-configuration) - [Tenant Tokens](#meilisearch-tenant-tokens) - [Typesense](#typesense-configuration) + - [Turbopuffer](#turbopuffer-configuration) - [Third-Party Engine Indexing](#indexing) - [Batch Import](#batch-import) - [Adding Records](#adding-records) @@ -28,6 +29,7 @@ - [Searching](#searching) - [Where Clauses](#where-clauses) - [Combining Raw and Builder Filters](#combining-raw-and-builder-filters) + - [Semantic Search](#semantic-search) - [Pagination](#pagination) - [Soft Deleting](#soft-deleting) - [Customizing Engine Searches](#customizing-engine-searches) @@ -40,7 +42,7 @@ Scout ships with a built-in `database` engine that uses MySQL / PostgreSQL full-text indexes and `LIKE` clauses to search your existing database — no external service required. For most applications, this is all you need. For an overview of all search options available in Hypervel, consult the [search documentation](/docs/{{version}}/search). -Scout also includes drivers for [Algolia](https://www.algolia.com/), [Meilisearch](https://www.meilisearch.com), and [Typesense](https://typesense.org) when you need features like typo tolerance, faceted filtering, or geo-search at massive scale. A "collection" driver is also available for local development, and you are free to write [custom engines](#custom-engines) as well. +Scout also includes drivers for [Algolia](https://www.algolia.com/), [Meilisearch](https://www.meilisearch.com), [Typesense](https://typesense.org), and [Turbopuffer](https://turbopuffer.com) when you need features like typo tolerance, faceted filtering, vector search, or geo-search at massive scale. A "collection" driver is also available for local development, and you are free to write [custom engines](#custom-engines) as well. ## Installation @@ -209,6 +211,19 @@ TYPESENSE_PROTOCOL=http Additional settings and schema definitions for your Typesense collections can be found within your application's `config/scout.php` configuration file. For more information regarding Typesense, please consult the [Typesense documentation](https://typesense.org/docs/guide/#quick-start). + +### Turbopuffer + +[Turbopuffer](https://turbopuffer.com) is a search engine that supports full-text, semantic, and hybrid search. To use the Turbopuffer driver, set the `SCOUT_DRIVER` environment variable and provide your Turbopuffer API key: + +```ini +SCOUT_DRIVER=turbopuffer +TURBOPUFFER_API_KEY=tpuf_... +TURBOPUFFER_REGION=gcp-us-central1 +``` + +The `TURBOPUFFER_REGION` environment variable is optional and defaults to `gcp-us-central1`. + ## Configuration @@ -321,7 +336,7 @@ Scout::guardModelFlushUsing(function (Model $model, Engine $engine, bool $force) }); ``` -Registering a callback replaces the previously registered callback for that lifecycle. Builder preparation runs once before each terminal Scout search and before filtered deletion through `DeletesByFilter`. Document preparation receives the final document, including Scout metadata and its engine key. Settings preparation runs for `scout:index`, `scout:sync-index-settings`, and lazy Typesense collection creation. +Registering a callback replaces the previously registered callback for that lifecycle. Builder preparation runs once before each terminal Scout search and before filtered deletion through `DeletesByFilter`. Document preparation receives the final document, including Scout metadata, its engine key, and any embedding Scout adds. Settings preparation runs for `scout:index`, `scout:sync-index-settings`, lazy Typesense collection creation, and the `schema` and `distance_metric` settings Scout sends with each Turbopuffer write. > [!WARNING] > Lifecycle callbacks persist for the worker lifetime. Register them only during application boot and do not capture request-scoped state. Arbitrary direct engine and search SDK calls remain low-level operations and do not invoke these callbacks automatically; `DeletesByFilter::deleteByFilter` is an explicit prepared Builder terminal. @@ -620,6 +635,80 @@ After configuring your application's index settings, you must invoke the `scout: php artisan scout:sync-index-settings ``` + +#### Semantic and Hybrid Search + +To use semantic or hybrid search with Meilisearch, configure an embedder in the index settings and embedding settings for each searchable model: + +```php +'meilisearch' => [ + // ... + 'index-settings' => [ + Article::class => [ + 'embedders' => [ + 'default' => [ + 'source' => 'userProvided', + 'dimensions' => 1536, + ], + ], + ], + ], + 'model-settings' => [ + Article::class => [ + 'embedding' => [ + 'embedder' => 'default', + 'dimensions' => 1536, + ], + ], + ], +], +``` + +The model's `toSearchableEmbedding` method should return the record's precomputed embedding, which Scout adds to the indexed document. After updating the configuration, run the `scout:sync-index-settings` command: + +```php +/** + * Get the embedding for the model. + * + * @return array + */ +public function toSearchableEmbedding(): array +{ + return $this->embedding; +} +``` + +Scout does not generate embeddings, so provide each semantic or hybrid search's [query embedding](#semantic-search) using the `vector` search option. + +Alternatively, you may use Meilisearch's native embeddings by setting the embedding `driver` to `meilisearch`. In this mode, Meilisearch generates document and query embeddings using the configured embedder, so the `dimensions` option and `toSearchableEmbedding` method are not required: + +```php +'meilisearch' => [ + 'index-settings' => [ + Article::class => [ + 'embedders' => [ + 'default' => [ + 'source' => 'openAi', + 'apiKey' => env('OPENAI_API_KEY'), + 'model' => 'text-embedding-3-small', + 'documentTemplate' => 'An article titled {{ doc.title }}: {{ doc.body }}', + ], + ], + ], + ], + 'model-settings' => [ + Article::class => [ + 'embedding' => [ + 'embedder' => 'default', + 'driver' => 'meilisearch', + ], + ], + ], +], +``` + +When using native embeddings, Scout will not add vectors to indexed documents. You may still provide a precomputed query vector using the `vector` search option. + #### Searchable Data Types @@ -709,6 +798,65 @@ User::class => [ ], ``` + +#### Embeddings + +To enable semantic and hybrid search, define an `embedding` setting and vector field in the model's Typesense configuration: + +```php +use App\Models\Article; + +'model-settings' => [ + Article::class => [ + 'collection-schema' => [ + 'fields' => [ + ['name' => 'title', 'type' => 'string'], + ['name' => 'embedding', 'type' => 'float[]', 'num_dim' => 1536], + ], + ], + 'search-parameters' => ['query_by' => 'title'], + 'embedding' => [ + 'attribute' => 'embedding', + 'dimensions' => 1536, + ], + ], +], +``` + +Your model's `toSearchableEmbedding` method should return the record's precomputed embedding array. Scout does not generate embeddings, so provide each semantic or hybrid search's [query embedding](#semantic-search) using the `vector` search option: + +```php +public function toSearchableEmbedding(): array +{ + return $this->embedding; +} +``` + +Alternatively, Typesense may generate document and query embeddings itself using an [auto-embedding field](https://typesense.org/docs/latest/api/vector-search.html). Set the embedding `driver` to `typesense` and define the field's `embed` configuration in the collection schema. Native embeddings do not require a `toSearchableEmbedding` method: + +```php +'collection-schema' => [ + 'fields' => [ + ['name' => 'title', 'type' => 'string'], + [ + 'name' => 'embedding', + 'type' => 'float[]', + 'embed' => [ + 'from' => ['title'], + 'model_config' => ['model_name' => 'ts/all-MiniLM-L12-v2'], + ], + ], + ], +], +'search-parameters' => ['query_by' => 'title'], +'embedding' => [ + 'driver' => 'typesense', + 'attribute' => 'embedding', +], +``` + +Hybrid searches require at least one keyword field in the `query_by` search parameter. + #### Dynamic Search Parameters @@ -726,11 +874,86 @@ Scout owns the `page` and `per_page` parameters used by Typesense. Choose the re Typesense accepts between 1 and 250 results per paginator page. Scout rejects values outside that range before sending the search request. + +### Turbopuffer + +Turbopuffer requires a schema and searchable attributes for each model. Define them in the `model-settings` array of your `turbopuffer` configuration within the `scout` configuration file: + +```php +use App\Models\Article; + +'turbopuffer' => [ + // ... + 'model-settings' => [ + Article::class => [ + 'searchable-attributes' => [ + 'title' => 3, + 'body' => 1, + ], + 'schema' => [ + 'title' => ['type' => 'string', 'full_text_search' => true], + 'body' => ['type' => 'string', 'full_text_search' => true], + 'status' => ['type' => 'string'], + ], + ], + ], +], +``` + +The numeric values assigned to `searchable-attributes` are relative BM25 weights. In the example above, matches in the article title contribute three times the score of matches in the body. + +To enable semantic and hybrid search, use Turbopuffer's native embeddings. Set the embedding `driver` to `turbopuffer` and configure an `embed` schema on the searchable source attribute: + +```php +'embedding' => [ + 'driver' => 'turbopuffer', + 'attribute' => 'embedding_text', +], + +'schema' => [ + // ... + 'embedding_text' => [ + 'type' => 'string', + 'embed' => [ + 'model' => 'voyage/voyage-4', + 'dimensions' => 1024, + 'attribute' => 'embedding', + ], + ], +], +``` + +The source attribute must be included in the model's `toSearchableArray` output. + +You may also index precomputed embeddings by adding an `embedding` setting and vector schema to the model's configuration, then returning the embedding array from the model's `toSearchableEmbedding` method: + +```php +'embedding' => [ + 'attribute' => 'embedding', + 'dimensions' => 1536, +], + +'schema' => [ + // ... + 'embedding' => ['type' => '[1536]f32', 'ann' => true], +], +``` + +Since Scout does not generate query embeddings, search precomputed embeddings by passing a Turbopuffer `rank_by` expression to the `options` method rather than using the `semantic` and `hybrid` methods: + +```php +$articles = Article::search() + ->options(['rank_by' => ['embedding', 'ANN', $queryEmbedding]]) + ->get(); +``` + +Scout pages through Turbopuffer results by fetching the requested window, so paginated searches may not go beyond 10,000 results. + ## Third-Party Engine Indexing > [!NOTE] -> The indexing features described in this section are primarily relevant when using a third-party engine (Algolia, Meilisearch, or Typesense). The database engine searches your database tables directly, so it does not require manual index management. +> The indexing features described in this section are primarily relevant when using a third-party engine (Algolia, Meilisearch, Typesense, or Turbopuffer). The database engine searches your database tables directly, so it does not require manual index management. ### Batch Import @@ -946,7 +1169,7 @@ The optional `force` argument is passed to any registered [model-flush guard](#c Order::removeAllFromSearch(force: true); ``` -Algolia, Meilisearch, and Typesense can also delete only the documents matched by a Scout Builder: +Algolia, Meilisearch, Typesense, and Turbopuffer can also delete only the documents matched by a Scout Builder: ```php use App\Models\Order; @@ -965,7 +1188,7 @@ $engine->deleteByFilter($builder); Filtered deletion refuses an empty filter. It uses an explicit `within` index when provided and otherwise targets the model's writable `indexableAs` index. A missing target index is a successful no-op. The method returns only after the engine reports completion; SDK timeout and transport exceptions are not hidden. -Since Algolia and Meilisearch perform these deletions asynchronously, Scout waits for them to finish before returning. Large deletions may take several minutes; Meilisearch waits up to 500 seconds and checks every five seconds, so long deletions should run in a queued job or console command instead of a web request. Typesense deletes synchronously. +Since Algolia and Meilisearch perform these deletions asynchronously, Scout waits for them to finish before returning. Large deletions may take several minutes; Meilisearch waits up to 500 seconds and checks every five seconds, so long deletions should run in a queued job or console command instead of a web request. Typesense deletes synchronously. Turbopuffer limits how many documents each request may delete, so Scout repeats the request until no matching documents remain. ### Pausing Indexing @@ -1135,7 +1358,7 @@ When using Algolia, Scout preserves the type of each filter value. Pass integers ### Combining Raw and Builder Filters -Algolia, Meilisearch, and Typesense preserve a raw application filter supplied through `options` when you also add Scout `where` clauses. Scout groups the application expression with its compiled Builder expression so neither side changes the other's precedence: +Algolia, Meilisearch, Typesense, and Turbopuffer preserve a raw application filter supplied through `options` when you also add Scout `where` clauses. Scout groups the application expression with its compiled Builder expression so neither side changes the other's precedence: ```php Order::search('Star Trek') @@ -1144,7 +1367,7 @@ Order::search('Star Trek') ->get(); ``` -Use `filter` for Meilisearch and `filter_by` for Typesense. Meilisearch array filters retain their documented nested OR and outer AND structure; Scout appends its Builder expression as one additional outer AND term. Engine search callbacks receive the final composed options. +Use `filter` for Meilisearch, `filter_by` for Typesense, and `filters` for Turbopuffer. Meilisearch array filters retain their documented nested OR and outer AND structure; Scout appends its Builder expression as one additional outer AND term. Engine search callbacks receive the final composed options. #### Customizing the Eloquent Results Query @@ -1162,6 +1385,46 @@ $orders = Order::search('Star Trek') When using a third-party engine, this callback is invoked after the relevant models have already been retrieved from the search engine, so it should not be used for "filtering" results — use [Scout where clauses](#where-clauses) instead. However, when using the database engine, the `query` method's constraints are applied directly to the database query, so you may use it for filtering as well. + +### Semantic Search + +The Meilisearch, Typesense, and Turbopuffer engines support semantic search, which matches records based on the meaning of a query. Scout does not generate embeddings, so semantic and hybrid searches use [Meilisearch's](#meilisearch-semantic-and-hybrid-search), [Typesense's](#typesense-embeddings), or [Turbopuffer's](#turbopuffer-configuration) native embeddings, or a precomputed query vector on Meilisearch and Typesense. + +After configuring embeddings for the selected engine, invoke the `semantic` method on a search query: + +```php +$articles = Article::search('staying cool in the summer') + ->semantic() + ->get(); +``` + +When the engine searches precomputed embeddings, provide the query's embedding using the `vector` option: + +```php +$articles = Article::search('staying cool in the summer') + ->options(['vector' => $queryEmbedding]) + ->semantic() + ->get(); +``` + +When using Meilisearch or Typesense, you may provide a minimum similarity threshold between `0` and `1`. Turbopuffer does not support similarity thresholds and ignores this argument: + +```php +$articles = Article::search('renewable energy storage') + ->semantic(minSimilarity: 0.6) + ->get(); +``` + +To combine full-text and semantic search, use the `hybrid` method. Its first two arguments control the relative weights of text and semantic results: + +```php +$articles = Article::search('renewable energy storage') + ->hybrid(textWeight: 1, semanticWeight: 2) + ->get(); +``` + +Engines without semantic search support throw a `NotSupportedException` for semantic searches and perform a normal full-text search for hybrid searches. + ### Pagination diff --git a/src/docs/search.md b/src/docs/search.md index bc174ad86a..12015cd1a7 100644 --- a/src/docs/search.md +++ b/src/docs/search.md @@ -35,7 +35,7 @@ For semantic search that matches results by *meaning* rather than exact keywords #### Hypervel Scout Search -For applications that want a `Searchable` trait that automatically keeps search indexes in sync with Eloquent models, [Hypervel Scout](/docs/{{version}}/scout) offers both a built-in database engine and drivers for third-party services like Algolia, Meilisearch, and Typesense. +For applications that want a `Searchable` trait that automatically keeps search indexes in sync with Eloquent models, [Hypervel Scout](/docs/{{version}}/scout) offers both a built-in database engine and drivers for third-party services like Algolia, Meilisearch, Typesense, and Turbopuffer. ## Full-Text Search diff --git a/src/scout/README.md b/src/scout/README.md index 1efaf15c09..be564c0b53 100644 --- a/src/scout/README.md +++ b/src/scout/README.md @@ -13,5 +13,6 @@ Documentation: https://hypervel.org/docs/scout - Pausing search syncing with `withoutSyncingToSearch()` or `disableSearchSyncing()` applies only to the current coroutine, so other requests keep indexing. See [Pausing Indexing](https://hypervel.org/docs/scout#pausing-indexing). - `MeilisearchEngine::generateTenantToken()` takes the search rules, the parent key's UID, the key itself and an optional expiry. Laravel's engine forwards the call to the Meilisearch client, whose method takes the UID, the search rules and an options array. See [Tenant Tokens](https://hypervel.org/docs/scout#meilisearch-tenant-tokens). - `scout:delete-all-indexes` refuses to run without a configured Scout prefix unless you pass `--force`. +- Scout cannot generate embeddings with the Laravel AI SDK, so semantic and hybrid search use engine-native embeddings or precomputed vectors, and the database engine does not support them. See [Semantic Search](https://hypervel.org/docs/scout#semantic-search). Ported from: https://github.com/laravel/scout diff --git a/src/scout/composer.json b/src/scout/composer.json index 8fb089934b..949e5e7969 100644 --- a/src/scout/composer.json +++ b/src/scout/composer.json @@ -12,6 +12,7 @@ "algolia", "meilisearch", "typesense", + "turbopuffer", "database" ], "authors": [ @@ -47,6 +48,7 @@ "hypervel/coroutine": "^0.4", "hypervel/database": "^0.4", "hypervel/foundation": "^0.4", + "hypervel/http": "^0.4", "hypervel/macroable": "^0.4", "hypervel/pagination": "^0.4", "hypervel/queue": "^0.4", diff --git a/src/scout/config/scout.php b/src/scout/config/scout.php index f30e6af391..caf4fca3fa 100644 --- a/src/scout/config/scout.php +++ b/src/scout/config/scout.php @@ -12,7 +12,8 @@ | using Scout. This connection is used when syncing all models to the | search service. You should adjust this based on your needs. | - | Supported: "algolia", "meilisearch", "typesense", "database", "collection", "null" + | Supported: "algolia", "meilisearch", "typesense", "turbopuffer", + | "database", "collection", "null" | */ @@ -197,6 +198,21 @@ // 'users' => [ // 'filterableAttributes' => ['id', 'name', 'email'], // 'sortableAttributes' => ['created_at'], + // 'embedders' => [ + // 'default' => [ + // 'source' => 'userProvided', + // 'dimensions' => 1536, + // ], + // ], + // ], + ], + 'model-settings' => [ + // Per-model settings can be defined here: + // App\Models\User::class => [ + // 'embedding' => [ + // 'embedder' => 'default', + // 'dimensions' => 1536, + // ], // ], ], ], @@ -250,8 +266,52 @@ // 'search-parameters' => [ // 'query_by' => 'name', // ], + // 'embedding' => [ + // 'attribute' => 'embedding', + // 'dimensions' => 1536, + // ], // ], ], 'import_action' => env('TYPESENSE_IMPORT_ACTION', 'upsert'), ], + + /* + |-------------------------------------------------------------------------- + | Turbopuffer Configuration + |-------------------------------------------------------------------------- + | + | Here you may configure your Turbopuffer connection and the schema and + | searchable attributes defined by each of your application's models. + | Turbopuffer is a scalable engine with full-text and vector search. + | Timeouts are measured in seconds. Omitted region, timeout, and retry + | members use the values shown below. + | + */ + + 'turbopuffer' => [ + 'api_key' => env('TURBOPUFFER_API_KEY'), + 'region' => env('TURBOPUFFER_REGION', 'gcp-us-central1'), + 'base_url' => env('TURBOPUFFER_BASE_URL'), + 'timeout' => (int) env('TURBOPUFFER_TIMEOUT', 60), + 'connect_timeout' => (int) env('TURBOPUFFER_CONNECT_TIMEOUT', 5), + 'retries' => (int) env('TURBOPUFFER_RETRIES', 3), + 'model-settings' => [ + // Per-model settings can be defined here: + // App\Models\User::class => [ + // 'searchable-attributes' => [ + // 'name' => 2, + // 'email' => 1, + // ], + // 'embedding' => [ + // 'attribute' => 'embedding', + // 'dimensions' => 1536, + // ], + // 'schema' => [ + // 'name' => ['type' => 'string', 'full_text_search' => true], + // 'email' => ['type' => 'string', 'full_text_search' => true], + // 'embedding' => ['type' => '[1536]f32', 'ann' => true], + // ], + // ], + ], + ], ]; diff --git a/src/scout/src/Builder.php b/src/scout/src/Builder.php index 27fa256379..12ff109f62 100644 --- a/src/scout/src/Builder.php +++ b/src/scout/src/Builder.php @@ -17,7 +17,10 @@ use Hypervel\Scout\Contracts\PaginatesEloquentModels; use Hypervel\Scout\Contracts\PaginatesEloquentModelsUsingDatabase; use Hypervel\Scout\Contracts\SearchableInterface; +use Hypervel\Scout\Contracts\SupportsSemanticSearch; use Hypervel\Scout\Engines\Engine; +use Hypervel\Scout\Exceptions\NotSupportedException; +use Hypervel\Scout\Exceptions\ScoutException; use Hypervel\Support\Collection; use Hypervel\Support\LazyCollection; use Hypervel\Support\Traits\Conditionable; @@ -100,6 +103,23 @@ class Builder */ public array $orders = []; + /** + * Indicates that the query should use semantic search. + */ + public bool $semanticSearch = false; + + /** + * The minimum similarity for semantic search results. + */ + public float|int|null $minimumSimilarity = null; + + /** + * The hybrid search ranking weights. + * + * @var null|array{text_weight: float|int, semantic_weight: float|int} + */ + public ?array $hybridSearch = null; + /** * Extra options that should be applied to the search. * @@ -289,6 +309,53 @@ public function oldest(?string $column = null): static return $this->orderBy($column, 'asc'); } + /** + * Perform a semantic search for the query expression. + * + * @return $this + */ + public function semantic(float|int|null $minSimilarity = null): static + { + if (trim($this->query) === '') { + throw new ScoutException('Semantic searches require a non-empty query.'); + } + + $this->semanticSearch = true; + $this->minimumSimilarity = $minSimilarity; + $this->hybridSearch = null; + + return $this; + } + + /** + * Perform a hybrid full-text and semantic search for the query expression. + * + * @return $this + */ + public function hybrid( + float|int $textWeight = 1, + float|int $semanticWeight = 1, + float|int|null $minSimilarity = null + ): static { + if (trim($this->query) === '') { + throw new ScoutException('Hybrid searches require a non-empty query.'); + } + + if ($textWeight <= 0 || $semanticWeight <= 0) { + throw new ScoutException('Hybrid search weights must be positive numbers.'); + } + + $this->semanticSearch = false; + $this->minimumSimilarity = $minSimilarity; + + $this->hybridSearch = [ + 'text_weight' => $textWeight, + 'semantic_weight' => $semanticWeight, + ]; + + return $this; + } + /** * Set extra options for the search query. * @@ -620,6 +687,11 @@ protected function preparedEngine(): Engine Scout::prepareBuilder($this, $engine); + // Check after preparation because the callback may enable semantic search. + if ($this->semanticSearch && ! $engine instanceof SupportsSemanticSearch) { + throw new NotSupportedException('The configured Scout engine does not support semantic search.'); + } + return $engine; } diff --git a/src/scout/src/Contracts/SupportsSemanticSearch.php b/src/scout/src/Contracts/SupportsSemanticSearch.php new file mode 100644 index 0000000000..b65d3eee89 --- /dev/null +++ b/src/scout/src/Contracts/SupportsSemanticSearch.php @@ -0,0 +1,15 @@ +container->make(MeilisearchClient::class), - $config->boolean('scout.soft_delete') + $config->boolean('scout.soft_delete'), + $config->array('scout.meilisearch', []) ); } @@ -173,7 +176,8 @@ public function createTypesenseDriver(): TypesenseEngine return new TypesenseEngine( $this->container->make(TypesenseClient::class), - $config->integer('scout.typesense.max_total_results', 1000) + $config->integer('scout.typesense.max_total_results', 1000), + $config->array('scout.typesense', []) ); } @@ -193,6 +197,21 @@ protected function ensureTypesenseClientIsInstalled(): void ); } + /** + * Create a Turbopuffer engine instance. + */ + public function createTurbopufferDriver(): TurbopufferEngine + { + /** @var Repository $config */ + $config = $this->container->make('config'); + + return new TurbopufferEngine( + $this->container->make(TurbopufferClient::class), + $config->array('scout.turbopuffer', []), + $config->boolean('scout.soft_delete'), + ); + } + /** * Create a collection engine instance. */ diff --git a/src/scout/src/Engines/MeilisearchEngine.php b/src/scout/src/Engines/MeilisearchEngine.php index 9d1c3aa64a..8b97cc99c7 100644 --- a/src/scout/src/Engines/MeilisearchEngine.php +++ b/src/scout/src/Engines/MeilisearchEngine.php @@ -12,6 +12,7 @@ use Hypervel\Scout\Builder; use Hypervel\Scout\Contracts\DeletesByFilter; use Hypervel\Scout\Contracts\SearchableInterface; +use Hypervel\Scout\Contracts\SupportsSemanticSearch; use Hypervel\Scout\Contracts\UpdatesIndexSettings; use Hypervel\Scout\Exceptions\ScoutException; use Hypervel\Scout\Jobs\RemoveableScoutCollection; @@ -28,9 +29,9 @@ /** * Meilisearch search engine implementation. * - * Provides full-text search using Meilisearch as the backend. + * Provides full-text, semantic, and hybrid search using Meilisearch as the backend. */ -class MeilisearchEngine extends Engine implements DeletesByFilter, UpdatesIndexSettings +class MeilisearchEngine extends Engine implements DeletesByFilter, SupportsSemanticSearch, UpdatesIndexSettings { /** * The maximum time to wait for a filtered deletion task. @@ -44,10 +45,13 @@ class MeilisearchEngine extends Engine implements DeletesByFilter, UpdatesIndexS /** * Create a new MeilisearchEngine instance. + * + * @param array $config */ public function __construct( protected MeilisearchClient $meilisearch, - protected bool $softDelete = false + protected bool $softDelete = false, + protected array $config = [] ) { } @@ -71,28 +75,97 @@ public function update(EloquentCollection $models): void $models->each->pushSoftDeleteMetadata(); } - $objects = $models->map(function (Model $model) { + $records = $models->map(function (Model $model): ?array { $searchableData = $model->toSearchableArray(); if (empty($searchableData)) { return null; } - $document = array_merge( - $searchableData, - $model->scoutMetadata(), - [$model->getScoutKeyName() => $model->getScoutKey()], - ); - - return Scout::prepareSearchableDocument($document, $model, $this); + return [ + 'model' => $model, + 'object' => array_merge( + $searchableData, + $model->scoutMetadata(), + [$model->getScoutKeyName() => $model->getScoutKey()], + ), + ]; }) ->filter() ->values() ->all(); - if (! empty($objects)) { - $index->addDocuments($objects, $firstModel->getScoutKeyName()); + if (empty($records)) { + return; + } + + $embedding = isset($this->modelSettings($firstModel)['embedding']) + ? $this->embeddingSettings($firstModel) + : null; + + if ($embedding !== null && ! $this->usesNativeEmbeddings($embedding)) { + $records = $this->addEmbeddingsToRecords($records, $embedding); } + + $objects = array_map( + fn (array $record): array => Scout::prepareSearchableDocument($record['object'], $record['model'], $this), + $records, + ); + + $index->addDocuments($objects, $firstModel->getScoutKeyName()); + } + + /** + * Add embeddings to the given searchable records. + * + * @param array}> $records + * @param array $settings + * @return array}> + */ + protected function addEmbeddingsToRecords(array $records, array $settings): array + { + foreach (array_chunk($records, 100, preserve_keys: true) as $batch) { + $inputs = []; + $vectors = []; + + foreach ($batch as $index => $record) { + if (! method_exists($record['model'], 'toSearchableEmbedding')) { + throw new ScoutException('Searchable models using generated embeddings must define a [toSearchableEmbedding] method.'); + } + + $input = $record['model']->toSearchableEmbedding(); + + if (is_array($input)) { + $vectors[$index] = $input; + + continue; + } + + if (! is_string($input) || trim($input) === '') { + throw new ScoutException('The [toSearchableEmbedding] method must return a non-empty string or an embedding array.'); + } + + $inputs[$index] = $input; + } + + if (! empty($inputs)) { + $generatedVectors = $this->generateEmbeddings(array_values($inputs), $settings); + + foreach (array_keys($inputs) as $position => $index) { + $vectors[$index] = $generatedVectors[$position]; + } + } + + foreach (array_keys($batch) as $index) { + if (isset($records[$index]['object']['_vectors']) && ! is_array($records[$index]['object']['_vectors'])) { + throw new ScoutException('The Meilisearch [_vectors] attribute must be an array.'); + } + + $records[$index]['object']['_vectors'][$settings['embedder']] = $vectors[$index]; + } + } + + return $records; } /** @@ -122,10 +195,10 @@ public function delete(EloquentCollection $models): void */ public function search(Builder $builder): mixed { - return $this->performSearch($builder, array_filter([ + return $this->performSearch($builder, array_merge(array_filter([ 'hitsPerPage' => $builder->limit, 'sort' => $this->buildSortFromOrderByClauses($builder), - ])); + ]), $this->semanticSearchParameters($builder))); } /** @@ -133,11 +206,62 @@ public function search(Builder $builder): mixed */ public function paginate(Builder $builder, int $perPage, int $page): mixed { - return $this->performSearch($builder, array_filter([ + return $this->performSearch($builder, array_merge(array_filter([ 'hitsPerPage' => $perPage, 'page' => $page, 'sort' => $this->buildSortFromOrderByClauses($builder), - ])); + ]), $this->semanticSearchParameters($builder))); + } + + /** + * Build semantic and hybrid search parameters. + * + * @return array + */ + protected function semanticSearchParameters(Builder $builder): array + { + if (! $builder->semanticSearch && $builder->hybridSearch === null) { + return []; + } + + if (array_key_exists('hybrid', $builder->options)) { + throw new ScoutException('Meilisearch semantic and hybrid searches cannot be combined with a custom [hybrid] option.'); + } + + $settings = $this->embeddingSettings($builder->model); + + $vector = $builder->options['vector'] ?? null; + + if (! $this->usesNativeEmbeddings($settings)) { + $vector ??= $this->generateEmbeddings([$builder->query], $settings)[0]; + } + + $semanticRatio = $builder->hybridSearch === null + ? 1.0 + : $builder->hybridSearch['semantic_weight'] / array_sum($builder->hybridSearch); + + $parameters = [ + 'hybrid' => [ + 'embedder' => $settings['embedder'], + 'semanticRatio' => $semanticRatio, + ], + ]; + + if ($vector !== null) { + if (! is_array($vector) || $vector === []) { + throw new ScoutException('The Meilisearch query [vector] must be a non-empty embedding array.'); + } + + $parameters = [ + 'vector' => $vector, + ] + $parameters; + } + + if ($builder->minimumSimilarity !== null) { + $parameters['rankingScoreThreshold'] = $builder->minimumSimilarity; + } + + return $parameters; } /** @@ -585,6 +709,78 @@ public function generateTenantToken( ); } + /** + * Get the configured settings for a model. + * + * @return array + */ + protected function modelSettings(Model $model): array + { + return $this->config['model-settings'][$model::class] ?? []; + } + + /** + * Get the validated embedding settings for a model. + * + * @return array + */ + protected function embeddingSettings(Model $model): array + { + $settings = $this->modelSettings($model)['embedding'] ?? null; + + if (! is_array($settings)) { + throw new ScoutException('No Meilisearch embedding settings have been configured for [' . $model::class . '].'); + } + + if (! isset($settings['embedder']) || ! is_string($settings['embedder']) || trim($settings['embedder']) === '') { + throw new ScoutException('Meilisearch embedding settings must contain an [embedder].'); + } + + $driver = $settings['driver'] ?? 'hypervel-ai'; + + if (! in_array($driver, ['hypervel-ai', 'meilisearch'], true)) { + throw new ScoutException("The [{$driver}] Meilisearch embedding driver is not supported."); + } + + $settings['driver'] = $driver; + + if ($this->usesNativeEmbeddings($settings)) { + return $settings; + } + + if (! isset($settings['dimensions']) + || filter_var($settings['dimensions'], FILTER_VALIDATE_INT) === false + || $settings['dimensions'] < 1) { + throw new ScoutException('Meilisearch embedding settings must contain positive [dimensions].'); + } + + $settings['dimensions'] = (int) $settings['dimensions']; + + return $settings; + } + + /** + * Determine if Meilisearch should generate embeddings natively. + * + * @param array $settings + */ + protected function usesNativeEmbeddings(array $settings): bool + { + return ($settings['driver'] ?? null) === 'meilisearch'; + } + + /** + * Generate embeddings for the given inputs. + * + * @param array $inputs + * @param array $settings + * @return array> + */ + protected function generateEmbeddings(array $inputs, array $settings): array + { + throw new ScoutException('AI-generated embeddings are not available in Hypervel. Use native or precomputed embeddings instead.'); + } + /** * Determine if the given model uses soft deletes. */ diff --git a/src/scout/src/Engines/TurbopufferEngine.php b/src/scout/src/Engines/TurbopufferEngine.php new file mode 100644 index 0000000000..7e8efe20e2 --- /dev/null +++ b/src/scout/src/Engines/TurbopufferEngine.php @@ -0,0 +1,914 @@ + $config + */ + public function __construct( + protected TurbopufferClient $turbopuffer, + protected array $config = [], + protected bool $softDelete = false + ) { + } + + /** + * Update the given models in the search index. + * + * @param EloquentCollection $models + */ + public function update(EloquentCollection $models): void + { + if ($models->isEmpty()) { + return; + } + + /** @var EloquentCollection $models */ + $model = $models->first(); + + if ($this->usesSoftDelete($model) && $this->softDelete) { + $models->each->pushSoftDeleteMetadata(); + } + + $records = $models->map(function (Model $model): ?array { + /** @var Model&SearchableInterface $model */ + $searchableData = $model->toSearchableArray(); + + if (empty($searchableData)) { + return null; + } + + return [ + 'model' => $model, + 'row' => array_merge( + $searchableData, + $model->scoutMetadata(), + ['id' => $model->getScoutKey()], + ), + ]; + })->filter()->values()->all(); + + if (empty($records)) { + return; + } + + $settings = $this->modelSettings($model); + + $embeddingSettings = isset($settings['embedding']) + ? $this->embeddingSettings($model) + : null; + + if ($embeddingSettings !== null && ! $this->usesNativeEmbeddings($embeddingSettings)) { + $records = $this->addEmbeddingsToRecords($records, $embeddingSettings); + } + + $rows = array_map(function (array $record) use ($embeddingSettings): array { + if ($embeddingSettings !== null && $this->usesNativeEmbeddings($embeddingSettings)) { + unset($record['row'][$embeddingSettings['generated_attribute']]); + } + + return Scout::prepareSearchableDocument($record['row'], $record['model'], $this); + }, $records); + + $indexSettings = []; + + foreach (['schema', 'distance_metric'] as $option) { + if (isset($settings[$option])) { + $indexSettings[$option] = $settings[$option]; + } + } + + if ($embeddingSettings !== null && $this->usesNativeEmbeddings($embeddingSettings)) { + $embed = &$indexSettings['schema'][$embeddingSettings['attribute']]['embed']; + + if (is_array($embed) && isset($embed['dimensions'])) { + $embed['dims'] = (int) $embed['dimensions']; + + unset($embed['dimensions']); + } + + unset($embed); + } + + if ($embeddingSettings !== null && ! isset($indexSettings['distance_metric'])) { + $indexSettings['distance_metric'] = 'cosine_distance'; + } + + $namespace = $model->indexableAs(); + + $this->turbopuffer->namespace($namespace)->write([ + 'upsert_rows' => $rows, + ...Scout::prepareIndexSettings($indexSettings, $model, $this, $namespace), + ]); + } + + /** + * Add embeddings to the given searchable records. + * + * @param array}> $records + * @param array $settings + * @return array}> + */ + protected function addEmbeddingsToRecords(array $records, array $settings): array + { + foreach (array_chunk($records, 100, preserve_keys: true) as $batch) { + $inputs = []; + $vectors = []; + + foreach ($batch as $index => $record) { + if (! method_exists($record['model'], 'toSearchableEmbedding')) { + throw new ScoutException('Searchable models using generated embeddings must define a [toSearchableEmbedding] method.'); + } + + $input = $record['model']->toSearchableEmbedding(); + + if (is_array($input)) { + $vectors[$index] = $input; + + continue; + } + + if (! is_string($input) || trim($input) === '') { + throw new ScoutException('The [toSearchableEmbedding] method must return a non-empty string or an embedding array.'); + } + + $inputs[$index] = $input; + } + + if (! empty($inputs)) { + $generatedVectors = $this->generateEmbeddings(array_values($inputs), $settings); + + foreach (array_keys($inputs) as $position => $index) { + $vectors[$index] = $generatedVectors[$position]; + } + } + + foreach (array_keys($batch) as $index) { + $records[$index]['row'][$settings['attribute']] = $vectors[$index]; + } + } + + return $records; + } + + /** + * Remove the given models from the search index. + * + * @param EloquentCollection $models + */ + public function delete(EloquentCollection $models): void + { + if ($models->isEmpty()) { + return; + } + + /** @var EloquentCollection $models */ + $model = $models->first(); + + $keys = $models instanceof RemoveableScoutCollection + ? $models->pluck($model->getScoutKeyName()) + : $models->map(fn (Model $model): mixed => $model->getScoutKey()); + + $namespace = $this->turbopuffer->namespace($model->indexableAs()); + + $this->ignoringMissingNamespace( + fn (): array => $namespace->write(['deletes' => $keys->values()->all()]) + ); + } + + /** + * Delete every document matching the prepared Builder filters. + */ + public function deleteByFilter(Builder $builder): void + { + Scout::prepareBuilder($builder, $this); + + $filters = $this->combineFilters($builder->options['filters'] ?? null, $this->filters($builder)); + + if ($filters === null) { + throw new InvalidArgumentException('Turbopuffer filter deletion requires a non-empty filter.'); + } + + $index = $builder->index ?? $builder->model->indexableAs(); + $namespace = $this->turbopuffer->namespace($index); + + $this->runOperation( + 'delete_by_filter', + $builder, + function () use ($namespace, $filters): void { + $this->ignoringMissingNamespace(function () use ($namespace, $filters): void { + // Each request deletes up to Turbopuffer's per-request limit and reports whether matches remain. + do { + $result = $namespace->write([ + 'delete_by_filter' => $filters, + 'delete_by_filter_allow_partial' => true, + ]); + } while ($result['rows_remaining'] ?? false); + }); + }, + index: $index, + ); + } + + /** + * Perform a search against the engine. + */ + public function search(Builder $builder): mixed + { + return $this->performSearch( + $builder, + $builder->limit ?? $builder->model->getPerPage() + ); + } + + /** + * Perform a paginated search against the engine. + */ + public function paginate(Builder $builder, int $perPage, int $page): mixed + { + $page = max(1, $page); + $perPage = max(1, $perPage); + $maximum = min($builder->limit ?? 10000, 10000); + $window = $page * $perPage; + $offset = $window - $perPage; + + // The last page may extend past 10,000 records, so only reject pages that start beyond them. + if ($offset >= 10000) { + throw new ScoutException('Turbopuffer search results may not be paginated beyond 10,000 records.'); + } + + $results = $this->performSearch($builder, min($window, $maximum)); + + $results['rows'] = array_slice( + $results['rows'] ?? [], + $offset, + $perPage + ); + + $count = $this->ignoringMissingNamespace( + fn (): array => $this->namespace($builder)->query(array_filter([ + 'aggregate_by' => ['count' => ['Count']], + 'filters' => $this->countFilters($builder), + 'consistency' => $builder->options['consistency'] ?? null, + ], fn (mixed $value): bool => $value !== null)), + [], + ); + + $results['total'] = min((int) ($count['aggregations']['count'] ?? 0), $maximum); + + return $results; + } + + /** + * Get the filters that count a paginated search's matches. + * + * Full-text rankings exclude documents without a query token, so the count + * matches any token in the weighted searchable attributes as BM25 does. + * + * @return null|array + */ + protected function countFilters(Builder $builder): ?array + { + $filters = $this->combineFilters($builder->options['filters'] ?? null, $this->filters($builder)); + + if ($builder->semanticSearch + || $builder->hybridSearch !== null + || isset($builder->options['rank_by']) + || $builder->query === '' + || $builder->query === '*') { + return $filters; + } + + $tokenFilters = []; + + foreach ($this->searchableAttributeWeights($builder) as $attribute => $weight) { + if ($weight > 0) { + $tokenFilters[] = [$attribute, 'ContainsAnyToken', $builder->query]; + } + } + + $tokenFilter = count($tokenFilters) === 1 ? $tokenFilters[0] : ['Or', $tokenFilters]; + + return $this->combineFilters($filters, $tokenFilter); + } + + /** + * Perform a search against Turbopuffer. + * + * @return array + */ + protected function performSearch(Builder $builder, int $limit): array + { + $namespace = $this->namespace($builder); + + $parameters = $this->buildSearchParameters($builder, $limit); + + $results = $this->ignoringMissingNamespace( + fn (): mixed => $builder->callback !== null + ? call_user_func($builder->callback, $namespace, $builder->query, $parameters) + : $namespace->query($parameters), + ['rows' => []], + ); + + if ($builder->hybridSearch !== null) { + $results['rows'] = array_slice($results['results'][0]['rows'] ?? [], 0, $limit); + } + + $results['total'] = count($results['rows'] ?? []); + + return $results; + } + + /** + * Run a namespace request, treating a missing namespace as empty. + * + * Turbopuffer creates namespaces on their first write and reports a 404 for + * namespaces that were never written or have been flushed. + * + * @template TResult + * @param Closure(): TResult $callback + * @return null|TResult + */ + protected function ignoringMissingNamespace(Closure $callback, mixed $missing = null): mixed + { + try { + return $callback(); + } catch (RequestException $exception) { + if ($exception->response->status() !== 404) { + throw $exception; + } + + return $missing; + } + } + + /** + * Build Turbopuffer search parameters for the query. + * + * @return array + */ + public function buildSearchParameters(Builder $builder, int $limit): array + { + if (isset($builder->options['queries'])) { + throw new ScoutException('Turbopuffer multi-query searches are not supported by this Scout engine.'); + } + + if ($builder->hybridSearch !== null) { + return $this->buildHybridSearchParameters($builder, $limit); + } + + $parameters = $builder->options; + $nativeFilters = $parameters['filters'] ?? null; + $scoutFilters = $this->filters($builder); + + unset($parameters['filters']); + + if ($builder->semanticSearch && isset($parameters['rank_by'])) { + throw new ScoutException('Turbopuffer semantic searches cannot be combined with a custom ranking expression.'); + } + + if (! isset($parameters['rank_by'])) { + $parameters['rank_by'] = $this->rankBy($builder); + } elseif (! empty($builder->orders)) { + throw new ScoutException('Turbopuffer order clauses cannot be combined with a custom ranking expression.'); + } + + if (($filters = $this->combineFilters($nativeFilters, $scoutFilters)) !== null) { + $parameters['filters'] = $filters; + } + + $parameters = $this->ensureIdIsReturned($parameters); + $parameters['limit'] = min(max(1, $limit), 10000); + + return $parameters; + } + + /** + * Build a hybrid full-text and semantic query. + * + * @return array + */ + protected function buildHybridSearchParameters(Builder $builder, int $limit): array + { + if (! empty($builder->orders)) { + throw new ScoutException('Turbopuffer order clauses cannot be combined with hybrid search.'); + } + + foreach (['rank_by', 'rerank_by'] as $option) { + if (isset($builder->options[$option])) { + throw new ScoutException("Turbopuffer hybrid searches cannot be combined with a custom [{$option}] option."); + } + } + + /** @var array{text_weight: float|int, semantic_weight: float|int} $weights */ + $weights = $builder->hybridSearch; + $parameters = $builder->options; + $nativeFilters = $parameters['filters'] ?? null; + $rootParameters = array_intersect_key($parameters, array_flip(['consistency', 'vector_encoding'])); + + unset($parameters['consistency'], $parameters['filters'], $parameters['vector_encoding']); + + if (($filters = $this->combineFilters($nativeFilters, $this->filters($builder))) !== null) { + $parameters['filters'] = $filters; + } + + $parameters = $this->ensureIdIsReturned($parameters); + $parameters['limit'] = min(max(1, $limit), 10000); + + return array_merge($rootParameters, [ + 'queries' => [ + array_merge($parameters, ['rank_by' => $this->fullTextRankBy($builder)]), + array_merge($parameters, ['rank_by' => $this->semanticRankBy($builder)]), + ], + 'rerank_by' => ['RRF', [ + 'weights' => [ + $weights['text_weight'], + $weights['semantic_weight'], + ], + ]], + ]); + } + + /** + * Build the ranking expression for the query. + * + * @return array + */ + protected function rankBy(Builder $builder): array + { + if ($builder->semanticSearch) { + if (! empty($builder->orders)) { + throw new ScoutException('Turbopuffer order clauses cannot be combined with semantic search.'); + } + + return $this->semanticRankBy($builder); + } + + if ($builder->query === '' || $builder->query === '*') { + if (count($builder->orders) > 1) { + throw new ScoutException('Turbopuffer supports one order clause per search.'); + } + + $order = $builder->orders[0] ?? ['column' => 'id', 'direction' => 'asc']; + + return [$this->field($builder, $order['column']), $order['direction']]; + } + + if (! empty($builder->orders)) { + throw new ScoutException('Turbopuffer order clauses cannot be combined with full-text search.'); + } + + return $this->fullTextRankBy($builder); + } + + /** + * Build the full-text ranking expression for a query. + * + * @return array + */ + protected function fullTextRankBy(Builder $builder): array + { + $expressions = []; + + foreach ($this->searchableAttributeWeights($builder) as $attribute => $weight) { + $expression = [$attribute, 'BM25', $builder->query]; + + $expressions[] = (float) $weight === 1.0 + ? $expression + : ['Product', $weight, $expression]; + } + + return count($expressions) === 1 + ? $expressions[0] + : ['Sum', $expressions]; + } + + /** + * Get the validated BM25 weights of the model's searchable attributes. + * + * @return array + */ + protected function searchableAttributeWeights(Builder $builder): array + { + $attributes = $this->modelSettings($builder->model)['searchable-attributes'] ?? []; + + if (empty($attributes)) { + throw new ScoutException('No Turbopuffer searchable attributes have been configured for [' . $builder->model::class . '].'); + } + + $weights = []; + + foreach ($attributes as $key => $value) { + [$attribute, $weight] = is_int($key) ? [$value, 1] : [$key, $value]; + + if (! is_string($attribute) || ! is_numeric($weight) || $weight < 0) { + throw new ScoutException('Turbopuffer searchable attributes must contain attribute names with non-negative numeric weights.'); + } + + $weights[$attribute] = $weight; + } + + return $weights; + } + + /** + * Build the semantic ranking expression for a query. + * + * @return array + */ + protected function semanticRankBy(Builder $builder): array + { + $settings = $this->embeddingSettings($builder->model); + + return [ + $settings['attribute'], + 'ANN', + $this->usesNativeEmbeddings($settings) + ? ['Embed', $builder->query] + : $this->generateEmbeddings([$builder->query], $settings)[0], + ]; + } + + /** + * Build filters for the query. + * + * @return null|array + */ + protected function filters(Builder $builder): ?array + { + $filters = []; + + $operators = [ + '=' => 'Eq', + '!=' => 'NotEq', + '<' => 'Lt', + '<=' => 'Lte', + '>' => 'Gt', + '>=' => 'Gte', + ]; + + foreach ($builder->wheres as $where) { + if (! isset($operators[$where['operator']])) { + throw new ScoutException("The [{$where['operator']}] operator is not supported by the Turbopuffer engine."); + } + + $filters[] = [ + $this->field($builder, $where['field']), + $operators[$where['operator']], + $this->filterValue($where['value']), + ]; + } + + foreach ($builder->whereIns as $field => $values) { + $filters[] = [$this->field($builder, $field), 'In', array_map([$this, 'filterValue'], $values)]; + } + + foreach ($builder->whereNotIns as $field => $values) { + $filters[] = [$this->field($builder, $field), 'NotIn', array_map([$this, 'filterValue'], $values)]; + } + + return match (count($filters)) { + 0 => null, + 1 => $filters[0], + default => ['And', $filters], + }; + } + + /** + * Combine native and Scout filters. + * + * @param null|array $nativeFilters + * @param null|array $scoutFilters + * @return null|array + */ + protected function combineFilters(?array $nativeFilters, ?array $scoutFilters): ?array + { + if ($nativeFilters === null || $nativeFilters === []) { + return $scoutFilters; + } + + if ($scoutFilters === null) { + return $nativeFilters; + } + + return ['And', [$nativeFilters, $scoutFilters]]; + } + + /** + * Normalize a filter value. + */ + protected function filterValue(mixed $value): mixed + { + return $value instanceof BackedEnum ? $value->value : $value; + } + + /** + * Ensure the Scout key is returned for model hydration. + * + * @param array $parameters + * @return array + */ + protected function ensureIdIsReturned(array $parameters): array + { + if (isset($parameters['include_attributes']) && is_array($parameters['include_attributes'])) { + $parameters['include_attributes'] = array_values(array_unique([ + ...$parameters['include_attributes'], + 'id', + ])); + } + + if (isset($parameters['exclude_attributes']) && is_array($parameters['exclude_attributes'])) { + $parameters['exclude_attributes'] = array_values(array_diff($parameters['exclude_attributes'], ['id'])); + } + + return $parameters; + } + + /** + * Resolve a Scout field name to a Turbopuffer field name. + */ + protected function field(Builder $builder, string $field): string + { + return $field === $builder->model->getScoutKeyName() ? 'id' : $field; + } + + /** + * Pluck and return the primary keys of the given results. + */ + public function mapIds(mixed $results): Collection + { + return collect($results['rows'] ?? [])->pluck('id')->values(); + } + + /** + * Map the given results to instances of the given model. + */ + public function map(Builder $builder, mixed $results, Model $model): EloquentCollection + { + /** @var Model&SearchableInterface $model */ + if (empty($results['rows'])) { + return $model->newCollection(); + } + + $rows = collect($results['rows']); + $objectIds = $rows->pluck('id')->values()->all(); + + /** @var array $objectIds */ + $objectIdPositions = array_flip($objectIds); + + $mapped = $model->getScoutModelsByIds($builder, $objectIds) + ->filter(fn (Model $model): bool => in_array($model->getScoutKey(), $objectIds, false)) + ->map(function (Model $model) use ($rows, $objectIdPositions): Model { + $row = $rows[$objectIdPositions[$model->getScoutKey()]]; + + if (array_key_exists('$dist', $row)) { + $model->withScoutMetadata('_turbopuffer_dist', $row['$dist']); + } + + return $model; + }) + ->sortBy(fn (Model $model): int => $objectIdPositions[$model->getScoutKey()]) + ->values(); + + return $model->newCollection($mapped->all()); + } + + /** + * Map the given results to instances of the given model via a lazy collection. + */ + public function lazyMap(Builder $builder, mixed $results, Model $model): LazyCollection + { + /** @var Model&SearchableInterface $model */ + if (empty($results['rows'])) { + return LazyCollection::empty(); + } + + $rows = collect($results['rows']); + $objectIds = $rows->pluck('id')->values()->all(); + + /** @var array $objectIds */ + $objectIdPositions = array_flip($objectIds); + + return $model->queryScoutModelsByIds($builder, $objectIds) + ->cursor() + ->filter(fn (Model $model): bool => in_array($model->getScoutKey(), $objectIds, false)) + ->map(function (Model $model) use ($rows, $objectIdPositions): Model { + $row = $rows[$objectIdPositions[$model->getScoutKey()]]; + + if (array_key_exists('$dist', $row)) { + $model->withScoutMetadata('_turbopuffer_dist', $row['$dist']); + } + + return $model; + }) + ->sortBy(fn (Model $model): int => $objectIdPositions[$model->getScoutKey()]) + ->values(); + } + + /** + * Get the total count from a raw result returned by the engine. + */ + public function getTotalCount(mixed $results): int + { + return (int) ($results['total'] ?? count($results['rows'] ?? [])); + } + + /** + * Flush all of the model's records from the engine. + */ + public function flush(Model $model): void + { + /** @var Model&SearchableInterface $model */ + $this->ignoringMissingNamespace(fn (): array => $this->deleteIndex($model->indexableAs())); + } + + /** + * Create a search index. + * + * @throws NotSupportedException + */ + public function createIndex(string $name, array $options = []): mixed + { + throw new NotSupportedException('Turbopuffer namespaces are created automatically upon adding documents.'); + } + + /** + * Delete a search index. + * + * @return array + */ + public function deleteIndex(string $name): array + { + return $this->turbopuffer->namespace($name)->delete(); + } + + /** + * Get the configured settings for a model. + * + * @return array + */ + protected function modelSettings(Model $model): array + { + return $this->config['model-settings'][$model::class] ?? []; + } + + /** + * Get the validated embedding settings for a model. + * + * @return array + */ + protected function embeddingSettings(Model $model): array + { + $modelSettings = $this->modelSettings($model); + + $settings = $modelSettings['embedding'] ?? null; + + if (! is_array($settings)) { + throw new ScoutException('No Turbopuffer embedding settings have been configured for [' . $model::class . '].'); + } + + $settings = $this->validateEmbeddingSettings($settings); + + if ($this->usesNativeEmbeddings($settings)) { + $schema = $modelSettings['schema'][$settings['attribute']] ?? null; + + if (! is_array($schema) || ($schema['type'] ?? null) !== 'string') { + throw new ScoutException("Turbopuffer native embeddings require a string schema configuration for the [{$settings['attribute']}] attribute."); + } + + $settings['generated_attribute'] = $this->validateNativeEmbeddingSchema($settings['attribute'], $schema['embed'] ?? null); + } + + return $settings; + } + + /** + * Validate embedding configuration shared by indexing and querying. + * + * @param array $settings + * @return array + */ + protected function validateEmbeddingSettings(array $settings): array + { + if (! isset($settings['attribute']) || ! is_string($settings['attribute']) || trim($settings['attribute']) === '') { + throw new ScoutException('Turbopuffer embedding settings must contain an [attribute].'); + } + + $driver = $settings['driver'] ?? 'hypervel-ai'; + + if (! in_array($driver, ['hypervel-ai', 'turbopuffer'], true)) { + throw new ScoutException("The [{$driver}] Turbopuffer embedding driver is not supported."); + } + + $settings['driver'] = $driver; + + if ($this->usesNativeEmbeddings($settings)) { + return $settings; + } + + if (! isset($settings['dimensions']) + || filter_var($settings['dimensions'], FILTER_VALIDATE_INT) === false + || $settings['dimensions'] < 1) { + throw new ScoutException('Turbopuffer embedding settings must contain positive [dimensions].'); + } + + $settings['dimensions'] = (int) $settings['dimensions']; + + return $settings; + } + + /** + * Determine if Turbopuffer should generate embeddings natively. + * + * @param array $settings + */ + protected function usesNativeEmbeddings(array $settings): bool + { + return ($settings['driver'] ?? null) === 'turbopuffer'; + } + + /** + * Validate a native embedding schema and return its vector attribute. + */ + protected function validateNativeEmbeddingSchema(string $attribute, mixed $embed): string + { + if (is_string($embed) && trim($embed) !== '') { + return 'embed_' . $attribute; + } + + if (! is_array($embed) || ! isset($embed['model']) || ! is_string($embed['model']) || trim($embed['model']) === '') { + throw new ScoutException("Turbopuffer native embeddings require a valid [embed] schema configuration for the [{$attribute}] attribute."); + } + + if (isset($embed['dimensions']) && (filter_var($embed['dimensions'], FILTER_VALIDATE_INT) === false || $embed['dimensions'] < 1)) { + throw new ScoutException('Turbopuffer native embedding [dimensions] must be a positive integer.'); + } + + if (isset($embed['attribute']) && (! is_string($embed['attribute']) || trim($embed['attribute']) === '')) { + throw new ScoutException('Turbopuffer native embedding [attribute] must be a non-empty string.'); + } + + return $embed['attribute'] ?? 'embed_' . $attribute; + } + + /** + * Generate embeddings for the given inputs. + * + * @param array $inputs + * @param array $settings + * @return array> + */ + protected function generateEmbeddings(array $inputs, array $settings): array + { + throw new ScoutException('AI-generated embeddings are not available in Hypervel. Use native or precomputed embeddings instead.'); + } + + /** + * Get the Turbopuffer namespace for a search. + */ + protected function namespace(Builder $builder): TurbopufferNamespace + { + return $this->turbopuffer->namespace( + $builder->index ?? $builder->model->searchableAs() + ); + } + + /** + * Determine if the given model uses soft deletes. + */ + protected function usesSoftDelete(Model $model): bool + { + return in_array(SoftDeletes::class, class_uses_recursive($model), true); + } +} diff --git a/src/scout/src/Engines/TypesenseEngine.php b/src/scout/src/Engines/TypesenseEngine.php index 91b21a9725..8055bf5075 100644 --- a/src/scout/src/Engines/TypesenseEngine.php +++ b/src/scout/src/Engines/TypesenseEngine.php @@ -12,7 +12,9 @@ use Hypervel\Scout\Builder; use Hypervel\Scout\Contracts\DeletesByFilter; use Hypervel\Scout\Contracts\SearchableInterface; +use Hypervel\Scout\Contracts\SupportsSemanticSearch; use Hypervel\Scout\Exceptions\NotSupportedException; +use Hypervel\Scout\Exceptions\ScoutException; use Hypervel\Scout\Jobs\RemoveableScoutCollection; use Hypervel\Scout\Scout; use Hypervel\Support\Collection; @@ -21,16 +23,21 @@ use stdClass; use Typesense\Client as Typesense; use Typesense\Collection as TypesenseCollection; +use Typesense\Documents; use Typesense\Exceptions\ObjectAlreadyExists; use Typesense\Exceptions\ObjectNotFound; +use Typesense\Exceptions\ObjectUnprocessable; +use Typesense\Exceptions\RequestMalformed; +use Typesense\Exceptions\RequestUnauthorized; +use Typesense\Exceptions\ServiceUnavailable; use Typesense\Exceptions\TypesenseClientError; /** * Typesense search engine implementation. * - * Provides full-text search using Typesense as the backend. + * Provides full-text, semantic, and hybrid search using Typesense as the backend. */ -class TypesenseEngine extends Engine implements DeletesByFilter +class TypesenseEngine extends Engine implements DeletesByFilter, SupportsSemanticSearch { /** * The maximum number of results that can be fetched per page. @@ -39,10 +46,13 @@ class TypesenseEngine extends Engine implements DeletesByFilter /** * Create a new TypesenseEngine instance. + * + * @param array $config */ public function __construct( protected Typesense $typesense, - protected int $maxTotalResults + protected int $maxTotalResults, + protected array $config = [] ) { } @@ -65,28 +75,42 @@ public function update(EloquentCollection $models): void $models->each->pushSoftDeleteMetadata(); } - $objects = $models->map(function (Model $model): ?array { + $records = $models->map(function (Model $model): ?array { $searchableData = $model->toSearchableArray(); if (empty($searchableData)) { return null; } - $document = array_merge( - $searchableData, - $model->scoutMetadata(), - ); - - return Scout::prepareSearchableDocument($document, $model, $this); + return [ + 'model' => $model, + 'object' => array_merge( + $searchableData, + $model->scoutMetadata(), + ), + ]; }) ->filter() ->values() ->all(); - if (empty($objects)) { + if (empty($records)) { return; } + $embedding = isset($this->modelSettings($firstModel)['embedding']) + ? $this->embeddingSettings($firstModel) + : null; + + if ($embedding !== null && ! $this->usesNativeEmbeddings($embedding)) { + $records = $this->addEmbeddingsToRecords($records, $embedding); + } + + $objects = array_map( + fn (array $record): array => Scout::prepareSearchableDocument($record['object'], $record['model'], $this), + $records, + ); + $collectionName = $firstModel->indexableAs(); $collection = $this->collection($collectionName); @@ -98,6 +122,55 @@ public function update(EloquentCollection $models): void } } + /** + * Add embeddings to the given searchable records. + * + * @param array}> $records + * @param array $settings + * @return array}> + */ + protected function addEmbeddingsToRecords(array $records, array $settings): array + { + foreach (array_chunk($records, 100, preserve_keys: true) as $batch) { + $inputs = []; + $vectors = []; + + foreach ($batch as $index => $record) { + if (! method_exists($record['model'], 'toSearchableEmbedding')) { + throw new ScoutException('Searchable models using generated embeddings must define a [toSearchableEmbedding] method.'); + } + + $input = $record['model']->toSearchableEmbedding(); + + if (is_array($input)) { + $vectors[$index] = $input; + + continue; + } + + if (! is_string($input) || trim($input) === '') { + throw new ScoutException('The [toSearchableEmbedding] method must return a non-empty string or an embedding array.'); + } + + $inputs[$index] = $input; + } + + if (! empty($inputs)) { + $generatedVectors = $this->generateEmbeddings(array_values($inputs), $settings); + + foreach (array_keys($inputs) as $position => $index) { + $vectors[$index] = $generatedVectors[$position]; + } + } + + foreach (array_keys($batch) as $index) { + $records[$index]['object'][$settings['attribute']] = $vectors[$index]; + } + } + + return $records; + } + /** * Import the given documents into the index. * @@ -244,12 +317,62 @@ protected function performSearch(Builder $builder, array $options = []): mixed } try { - return $documents->search($options); + return $this->executeSearch($builder, $documents, $options); } catch (ObjectNotFound) { $this->createCollectionFromModel($builder->model, $collectionName); + return $this->executeSearch($builder, $documents, $options); + } + } + + /** + * Execute the given search using the appropriate Typesense endpoint. + * + * @param array $options + * @throws TypesenseClientError + */ + protected function executeSearch(Builder $builder, Documents $documents, array $options): mixed + { + if (! isset($options['vector_query'])) { return $documents->search($options); } + + // Serialized embeddings may exceed Typesense's query string length, so send them in a multi-search body. + $results = $this->typesense->getMultiSearch()->perform([ + 'searches' => [ + array_merge($options, [ + 'collection' => $builder->index ?? $builder->model->searchableAs(), + ]), + ], + ]); + + $result = $results['results'][0] ?? []; + + if (isset($result['error'])) { + throw $this->marshalMultiSearchException($result); + } + + return $result; + } + + /** + * Convert a multi-search error result into a Typesense exception. + * + * @param array{code?: int, error: string} $result + */ + protected function marshalMultiSearchException(array $result): TypesenseClientError + { + $exception = match ((int) ($result['code'] ?? 500)) { + 400 => new RequestMalformed, + 401 => new RequestUnauthorized, + 404 => new ObjectNotFound, + 409 => new ObjectAlreadyExists, + 422 => new ObjectUnprocessable, + 503 => new ServiceUnavailable, + default => new TypesenseClientError, + }; + + return $exception->setMessage($result['error']); } /** @@ -356,9 +479,185 @@ public function buildSearchParameters(Builder $builder, int $page, ?int $perPage $parameters['page'] = $page; $parameters['per_page'] = $perPage; + return $this->applySemanticSearchParameters($builder, $parameters); + } + + /** + * Apply semantic and hybrid search parameters to the search parameters. + * + * @param array $parameters + * @return array + */ + protected function applySemanticSearchParameters(Builder $builder, array $parameters): array + { + if (! $builder->semanticSearch && $builder->hybridSearch === null) { + return $parameters; + } + + if (array_key_exists('vector_query', $builder->options)) { + throw new ScoutException('Typesense semantic and hybrid searches cannot be combined with a custom [vector_query] option.'); + } + + $settings = $this->embeddingSettings($builder->model); + + unset($parameters['vector']); + + if ($builder->semanticSearch) { + $parameters = $this->usesNativeEmbeddings($settings) + ? $this->applyNativeSemanticQueryBy($parameters, $settings['attribute']) + : array_merge($parameters, ['q' => '*']); + } else { + $parameters = $this->applyHybridQueryBy($parameters, $settings); + } + + if (($vectorQuery = $this->buildVectorQueryParameter($builder, $settings)) !== null) { + $parameters['vector_query'] = $vectorQuery; + } + + $parameters['exclude_fields'] = $this->appendField($parameters['exclude_fields'] ?? '', $settings['attribute']); + + return $parameters; + } + + /** + * Target the embedding field for a semantic search using native embeddings. + * + * @param array $parameters + * @return array + */ + protected function applyNativeSemanticQueryBy(array $parameters, string $attribute): array + { + $parameters['query_by'] = $attribute; + + // Remote embedders reject prefix searches on embedding fields. + $parameters['prefix'] = false; + + unset($parameters['query_by_weights']); + + foreach (['num_typos', 'infix'] as $parameter) { + if (isset($parameters[$parameter]) && str_contains((string) $parameters[$parameter], ',')) { + unset($parameters[$parameter]); + } + } + + return $parameters; + } + + /** + * Prepare the "query_by" fields and their per-field parameters for a hybrid search. + * + * @param array $parameters + * @param array $settings + * @return array + */ + protected function applyHybridQueryBy(array $parameters, array $settings): array + { + $fields = array_filter(array_map('trim', explode(',', (string) ($parameters['query_by'] ?? '')))); + + if (empty(array_diff($fields, [$settings['attribute']]))) { + throw new ScoutException('Typesense hybrid searches require at least one keyword field in the [query_by] search parameter.'); + } + + if (! $this->usesNativeEmbeddings($settings)) { + return $parameters; + } + + if (! in_array($settings['attribute'], $fields, true)) { + $fields[] = $settings['attribute']; + $parameters['query_by'] = implode(',', $fields); + + if (isset($parameters['query_by_weights']) && $parameters['query_by_weights'] !== '') { + $parameters['query_by_weights'] .= ',0'; + } + + foreach (['num_typos' => '0', 'infix' => 'off'] as $parameter => $placeholder) { + if (isset($parameters[$parameter]) && str_contains((string) $parameters[$parameter], ',')) { + $parameters[$parameter] .= ',' . $placeholder; + } + } + } + + // Remote embedders reject prefix searches on embedding fields, so prefix search must be disabled for that field. + if ($parameters['prefix'] === true || $parameters['prefix'] === 'true') { + $prefixes = array_fill(0, count($fields), 'true'); + } elseif (is_string($parameters['prefix']) && str_contains($parameters['prefix'], ',')) { + $prefixes = explode(',', $parameters['prefix']); + } else { + return $parameters; + } + + $prefixes[array_search($settings['attribute'], array_values($fields), true)] = 'false'; + $parameters['prefix'] = implode(',', $prefixes); + return $parameters; } + /** + * Build the "vector_query" parameter for the search. + * + * @param array $settings + */ + protected function buildVectorQueryParameter(Builder $builder, array $settings): ?string + { + if ($this->usesNativeEmbeddings($settings)) { + $vector = []; + } else { + $vector = $builder->options['vector'] + ?? $this->generateEmbeddings([$builder->query], $settings)[0]; + + if (! is_array($vector) || empty($vector)) { + throw new ScoutException('The Typesense query [vector] must be a non-empty embedding array.'); + } + } + + $options = []; + + if ($builder->hybridSearch !== null) { + $options[] = 'alpha: ' . ($builder->hybridSearch['semantic_weight'] / array_sum($builder->hybridSearch)); + } + + if ($builder->minimumSimilarity !== null) { + $options[] = 'distance_threshold: ' . $this->distanceThreshold($builder->minimumSimilarity); + } + + if (empty($vector) && empty($options)) { + return null; + } + + return sprintf( + '%s:([%s]%s)', + $settings['attribute'], + implode(', ', $vector), + empty($options) ? '' : ', ' . implode(', ', $options) + ); + } + + /** + * Get the maximum vector distance for the given minimum similarity. + */ + protected function distanceThreshold(float|int $similarity): float|int + { + if ($similarity < 0 || $similarity > 1) { + throw new ScoutException('The minimum similarity must be between 0 and 1.'); + } + + return 1 - $similarity; + } + + /** + * Append a field to a comma-separated field list if not already present. + */ + protected function appendField(string $fields, string $field): string + { + $fields = array_filter(array_map('trim', explode(',', $fields))); + + if (! in_array($field, $fields, true)) { + $fields[] = $field; + } + + return implode(',', $fields); + } + /** * Combine application and Builder filters without changing their precedence. */ @@ -672,6 +971,78 @@ protected function createCollectionFromModel(Model $model, string $collectionNam } } + /** + * Get the configured settings for a model. + * + * @return array + */ + protected function modelSettings(Model $model): array + { + return $this->config['model-settings'][$model::class] ?? []; + } + + /** + * Get the validated embedding settings for a model. + * + * @return array + */ + protected function embeddingSettings(Model $model): array + { + $settings = $this->modelSettings($model)['embedding'] ?? null; + + if (! is_array($settings)) { + throw new ScoutException('No Typesense embedding settings have been configured for [' . $model::class . '].'); + } + + if (! isset($settings['attribute']) || ! is_string($settings['attribute']) || trim($settings['attribute']) === '') { + throw new ScoutException('Typesense embedding settings must contain an [attribute].'); + } + + $driver = $settings['driver'] ?? 'hypervel-ai'; + + if (! in_array($driver, ['hypervel-ai', 'typesense'], true)) { + throw new ScoutException("The [{$driver}] Typesense embedding driver is not supported."); + } + + $settings['driver'] = $driver; + + if ($this->usesNativeEmbeddings($settings)) { + return $settings; + } + + if (! isset($settings['dimensions']) + || filter_var($settings['dimensions'], FILTER_VALIDATE_INT) === false + || $settings['dimensions'] < 1) { + throw new ScoutException('Typesense embedding settings must contain positive [dimensions].'); + } + + $settings['dimensions'] = (int) $settings['dimensions']; + + return $settings; + } + + /** + * Determine if Typesense should generate embeddings natively. + * + * @param array $settings + */ + protected function usesNativeEmbeddings(array $settings): bool + { + return ($settings['driver'] ?? null) === 'typesense'; + } + + /** + * Generate embeddings for the given inputs. + * + * @param array $inputs + * @param array $settings + * @return array> + */ + protected function generateEmbeddings(array $inputs, array $settings): array + { + throw new ScoutException('AI-generated embeddings are not available in Hypervel. Use native or precomputed embeddings instead.'); + } + /** * Get a detached Typesense collection handle. */ diff --git a/src/scout/src/ScoutServiceProvider.php b/src/scout/src/ScoutServiceProvider.php index 369f06367f..4c78dbd008 100644 --- a/src/scout/src/ScoutServiceProvider.php +++ b/src/scout/src/ScoutServiceProvider.php @@ -13,6 +13,7 @@ use GuzzleHttp\HandlerStack; use Hypervel\Contracts\Telescope\TelescopeTag; use Hypervel\Foundation\Application as HypervelApplication; +use Hypervel\Http\Client\Factory as HttpFactory; use Hypervel\Scout\Console\DeleteAllIndexesCommand; use Hypervel\Scout\Console\DeleteIndexCommand; use Hypervel\Scout\Console\FlushCommand; @@ -21,6 +22,7 @@ use Hypervel\Scout\Console\QueueImportCommand; use Hypervel\Scout\Console\SyncIndexSettingsCommand; use Hypervel\Scout\Engines\MeilisearchRetryPolicy; +use Hypervel\Scout\Services\Turbopuffer\TurbopufferClient; use Hypervel\Support\ServiceProvider; use Meilisearch\Client as MeilisearchClient; use Typesense\Client as TypesenseClient; @@ -40,6 +42,7 @@ public function register(): void $this->registerAlgoliaClient(); $this->registerMeilisearchClient(); $this->registerTypesenseClient(); + $this->registerTurbopufferClient(); } /** @@ -161,6 +164,19 @@ protected function registerTypesenseClient(): void }); } + /** + * Register the Turbopuffer client. + */ + protected function registerTurbopufferClient(): void + { + $this->app->singleton(TurbopufferClient::class, function (): TurbopufferClient { + return new TurbopufferClient( + $this->app->make(HttpFactory::class), + $this->app->make('config')->array('scout.turbopuffer', []), + ); + }); + } + /** * Register the package's publishable resources. */ diff --git a/src/scout/src/Services/Turbopuffer/TurbopufferClient.php b/src/scout/src/Services/Turbopuffer/TurbopufferClient.php new file mode 100644 index 0000000000..f10b677d41 --- /dev/null +++ b/src/scout/src/Services/Turbopuffer/TurbopufferClient.php @@ -0,0 +1,94 @@ + $config + */ + public function __construct( + protected Factory $http, + protected array $config + ) { + } + + /** + * Send a request to Turbopuffer. + * + * @param array $options + * @return array + */ + public function request(string $method, string $uri, array $options = []): array + { + $response = $this->pendingRequest()->send($method, $uri, $options); + + if ($response->status() === 202) { + throw new ScoutException('The Turbopuffer index required by this operation is still building.'); + } + + return $response->throw()->json() ?? []; + } + + /** + * Create a pending HTTP request. + */ + protected function pendingRequest(): PendingRequest + { + $baseUrl = $this->config['base_url'] ?? null; + + if (empty($baseUrl)) { + $baseUrl = sprintf('https://%s.turbopuffer.com', $this->config['region'] ?? 'gcp-us-central1'); + } + + $request = $this->http + ->baseUrl(rtrim($baseUrl, '/')) + ->withToken($this->config['api_key'] ?? '') + ->withTelescopeTags([TelescopeTag::Scout, TelescopeTag::Turbopuffer]) + ->acceptJson() + ->asJson() + ->timeout($this->config['timeout'] ?? 60) + ->connectTimeout($this->config['connect_timeout'] ?? 5); + + $retries = $this->config['retries'] ?? 3; + + if ($retries > 0) { + $request->retry($retries + 1, 250, function (?Throwable $exception): bool { + return $exception instanceof ConnectionException + || ($exception instanceof RequestException && in_array($exception->response->status(), [408, 409, 429, 500, 502, 503, 504], true)); + }); + } + + return $request; + } + + /** + * Get a Turbopuffer namespace instance. + */ + public function namespace(string $name): TurbopufferNamespace + { + if (! preg_match('/^[A-Za-z0-9_.-]{1,128}$/', $name)) { + throw new ScoutException("Invalid Turbopuffer namespace [{$name}]."); + } + + return new TurbopufferNamespace($this, $name); + } +} diff --git a/src/scout/src/Services/Turbopuffer/TurbopufferNamespace.php b/src/scout/src/Services/Turbopuffer/TurbopufferNamespace.php new file mode 100644 index 0000000000..45a0a6ba4e --- /dev/null +++ b/src/scout/src/Services/Turbopuffer/TurbopufferNamespace.php @@ -0,0 +1,57 @@ + $parameters + * @return array + */ + public function query(array $parameters): array + { + return $this->client->request('POST', $this->uri() . '/query', ['json' => $parameters]); + } + + /** + * Write documents to the namespace. + * + * @param array $parameters + * @return array + */ + public function write(array $parameters): array + { + return $this->client->request('POST', $this->uri(), ['json' => $parameters]); + } + + /** + * Delete the namespace. + * + * @return array + */ + public function delete(): array + { + return $this->client->request('DELETE', $this->uri()); + } + + /** + * Get the namespace API URI. + */ + protected function uri(): string + { + return '/v2/namespaces/' . rawurlencode($this->name); + } +} diff --git a/tests/Integration/Scout/Meilisearch/MeilisearchEngineIntegrationTest.php b/tests/Integration/Scout/Meilisearch/MeilisearchEngineIntegrationTest.php index bdd8802073..dc2effc36f 100644 --- a/tests/Integration/Scout/Meilisearch/MeilisearchEngineIntegrationTest.php +++ b/tests/Integration/Scout/Meilisearch/MeilisearchEngineIntegrationTest.php @@ -5,7 +5,11 @@ namespace Hypervel\Tests\Integration\Scout\Meilisearch; use Hypervel\Database\Eloquent\Collection as EloquentCollection; +use Hypervel\Database\Eloquent\Model; +use Hypervel\Scout\Builder; +use Hypervel\Scout\Engines\MeilisearchEngine; use Hypervel\Scout\Jobs\RemoveFromSearch; +use Hypervel\Scout\Searchable; use Hypervel\Tests\Scout\Fixtures\Models\CustomScoutKeyModel; use Hypervel\Tests\Scout\Fixtures\Models\SearchableModel; use Meilisearch\Client; @@ -191,6 +195,47 @@ public function testPaginateReturnsCorrectPage(): void $this->assertSame(10, $page1->total()); } + public function testItCanUseUserProvidedEmbeddingsForSemanticSearch(): void + { + $model = new MeilisearchEmbeddingModel; + + $task = $this->meilisearch->index($model->indexableAs())->updateEmbedders([ + 'default' => [ + 'source' => 'userProvided', + 'dimensions' => 2, + ], + ]); + $this->meilisearch->waitForTask($task['taskUid']); + + $engine = new MeilisearchEngine($this->meilisearch, false, [ + 'model-settings' => [ + MeilisearchEmbeddingModel::class => [ + 'embedding' => [ + 'embedder' => 'default', + 'dimensions' => 2, + ], + ], + ], + ]); + + $cat = new MeilisearchEmbeddingModel(['id' => 1, 'name' => 'A sleeping cat']); + $cat->setAttribute('embedding', [1, 0]); + + $rocket = new MeilisearchEmbeddingModel(['id' => 2, 'name' => 'A rocket launch']); + $rocket->setAttribute('embedding', [0, 1]); + + $engine->update($model->newCollection([$cat, $rocket])); + $this->waitForMeilisearchTasks(); + + $results = $engine->search( + (new Builder($model, 'a relaxed pet')) + ->options(['vector' => [1, 0]]) + ->semantic() + ); + + $this->assertSame(1, $results['hits'][0]['id']); + } + public function testFlushRemovesAllDocumentsFromIndex(): void { $models = new EloquentCollection([ @@ -289,3 +334,30 @@ public function testKeysReturnsScoutKeys(): void $this->assertCount(2, $keys); } } + +class MeilisearchEmbeddingModel extends Model +{ + use Searchable; + + protected array $fillable = ['id', 'name']; + + public bool $timestamps = false; + + /** + * Get the index name for the model when searching. + */ + public function searchableAs(): string + { + return config()->string('scout.prefix') . 'semantic'; + } + + /** + * Get the precomputed vector to index. + * + * @return array + */ + public function toSearchableEmbedding(): array + { + return $this->embedding; + } +} diff --git a/tests/Integration/Scout/Typesense/TypesenseEngineIntegrationTest.php b/tests/Integration/Scout/Typesense/TypesenseEngineIntegrationTest.php index 4b1c68abdb..0d263a4c22 100644 --- a/tests/Integration/Scout/Typesense/TypesenseEngineIntegrationTest.php +++ b/tests/Integration/Scout/Typesense/TypesenseEngineIntegrationTest.php @@ -5,7 +5,11 @@ namespace Hypervel\Tests\Integration\Scout\Typesense; use Hypervel\Database\Eloquent\Collection as EloquentCollection; +use Hypervel\Database\Eloquent\Model; +use Hypervel\Scout\Builder; +use Hypervel\Scout\Engines\TypesenseEngine; use Hypervel\Scout\Jobs\RemoveFromSearch; +use Hypervel\Scout\Searchable; use Hypervel\Tests\Integration\Scout\Typesense\Fixtures\Models\TypesenseSearchableModel; /** @@ -228,6 +232,107 @@ public function testQueuedRemovalDeletesStoredCustomScoutKeyAfterModelIsGone(): ->documents[(string) $model->getScoutKey()] ->retrieve(); } + + public function testItCanUseUserProvidedEmbeddingsForSemanticSearch(): void + { + $model = new TypesenseEmbeddingModel; + $dimensions = 384; + + config()->set('scout.typesense.model-settings.' . TypesenseEmbeddingModel::class, [ + 'collection-schema' => [ + 'fields' => [ + ['name' => 'id', 'type' => 'string'], + ['name' => 'name', 'type' => 'string'], + ['name' => 'embedding', 'type' => 'float[]', 'num_dim' => $dimensions], + ], + ], + 'search-parameters' => [ + 'query_by' => 'name', + ], + ]); + + $engine = new TypesenseEngine($this->typesense, 1000, [ + 'model-settings' => [ + TypesenseEmbeddingModel::class => [ + 'embedding' => [ + 'attribute' => 'embedding', + 'dimensions' => $dimensions, + ], + ], + ], + ]); + + $catVector = array_fill(0, $dimensions, 0.001953125); + $catVector[0] = 1.0; + + $rocketVector = array_fill(0, $dimensions, 0.001953125); + $rocketVector[$dimensions - 1] = 1.0; + + $cat = new TypesenseEmbeddingModel(['id' => 1, 'name' => 'A sleeping cat']); + $cat->setAttribute('embedding', $catVector); + + $rocket = new TypesenseEmbeddingModel(['id' => 2, 'name' => 'A rocket launch']); + $rocket->setAttribute('embedding', $rocketVector); + + $engine->update($model->newCollection([$cat, $rocket])); + + // The serialized query vector exceeds Typesense's 4,000 character query string limit. + $this->assertGreaterThan(4000, strlen(implode(', ', $catVector))); + + $results = $engine->search( + (new Builder($model, 'a relaxed pet')) + ->options(['vector' => $catVector]) + ->semantic() + ); + + $this->assertSame('1', $results['hits'][0]['document']['id']); + + $results = $engine->search( + (new Builder($model, 'rocket')) + ->options(['vector' => $rocketVector]) + ->hybrid() + ); + + $this->assertSame('2', $results['hits'][0]['document']['id']); + } +} + +class TypesenseEmbeddingModel extends Model +{ + use Searchable; + + protected array $fillable = ['id', 'name']; + + public bool $timestamps = false; + + /** + * Get the index name for the model when searching. + */ + public function searchableAs(): string + { + return config()->string('scout.prefix') . 'semantic'; + } + + /** + * Get the indexable data array for the model. + */ + public function toSearchableArray(): array + { + return [ + 'id' => (string) $this->id, + 'name' => $this->name, + ]; + } + + /** + * Get the precomputed vector to index. + * + * @return array + */ + public function toSearchableEmbedding(): array + { + return $this->embedding; + } } class TypesenseReadWriteSearchableModel extends TypesenseSearchableModel diff --git a/tests/Scout/Feature/DatabaseEngineTest.php b/tests/Scout/Feature/DatabaseEngineTest.php index 02b7377b83..04e9c23e6b 100644 --- a/tests/Scout/Feature/DatabaseEngineTest.php +++ b/tests/Scout/Feature/DatabaseEngineTest.php @@ -47,6 +47,16 @@ public function testSearchReturnsAllModelsWithEmptyQuery(): void $this->assertCount(3, $results); } + public function testHybridSearchFallsBackToNormalTextSearchWhenVectorsAreNotSupported(): void + { + $this->createAbigailAndTaylor(); + + $models = SearchableModel::search('Taylor')->hybrid()->get(); + + $this->assertCount(1, $models); + $this->assertSame('Taylor Otwell', $models->first()->title); + } + public function testSearchWithWhereClause(): void { $model1 = SearchableModel::create(['title' => 'Test A', 'body' => 'Body']); diff --git a/tests/Scout/Feature/Engines/TurbopufferEngineTest.php b/tests/Scout/Feature/Engines/TurbopufferEngineTest.php new file mode 100644 index 0000000000..7e29f26752 --- /dev/null +++ b/tests/Scout/Feature/Engines/TurbopufferEngineTest.php @@ -0,0 +1,788 @@ +make('config'); + + $config->set('scout.driver', 'turbopuffer'); + $config->set('scout.prefix', ''); + $config->set('scout.soft_delete', true); + $config->set('scout.turbopuffer', [ + 'api_key' => 'tpuf-test-key', + 'region' => 'gcp-us-central1', + 'base_url' => 'https://turbopuffer.test', + 'retries' => 0, + 'model-settings' => [ + SearchableModelWithPrecomputedEmbedding::class => [ + 'searchable-attributes' => [ + 'name' => 3, + 'description' => 1, + ], + 'schema' => [ + 'name' => ['type' => 'string', 'full_text_search' => true], + 'description' => ['type' => 'string', 'full_text_search' => true], + ], + ], + ], + ]); + } + + public function testDriverIsRegistered(): void + { + $this->assertInstanceOf( + TurbopufferEngine::class, + $this->app->make(EngineManager::class)->engine() + ); + } + + public function testUpdateUpsertsModelsWithSchemaAndScoutIds(): void + { + Http::fake(['*' => Http::response(['rows_affected' => 1])]); + + $model = new SearchableModelWithPrecomputedEmbedding(['id' => 10, 'name' => 'Taylor']); + + $this->engine()->update($model->newCollection([$model])); + + Http::assertSent(function (Request $request): bool { + return $request->method() === 'POST' + && $request->url() === 'https://turbopuffer.test/v2/namespaces/table' + && $request->hasHeader('Authorization', 'Bearer tpuf-test-key') + && $request['upsert_rows'] === [['id' => 10, 'name' => 'Taylor']] + && $request['schema'] === [ + 'name' => ['type' => 'string', 'full_text_search' => true], + 'description' => ['type' => 'string', 'full_text_search' => true], + ]; + }); + } + + public function testUpdateAddsSoftDeleteMetadata(): void + { + Http::fake(['*' => Http::response(['rows_affected' => 1])]); + + $model = new SoftDeletableSearchableModel; + $model->setAttribute('id', 10); + + $this->engine()->update($model->newCollection([$model])); + + Http::assertSent(fn (Request $request): bool => $request['upsert_rows'][0] === [ + 'id' => 10, + 'title' => null, + 'body' => null, + '__soft_deleted' => 0, + ]); + } + + public function testUpdatePreparesRowsAndIndexSettingsBeforeWriting(): void + { + Http::fake(['*' => Http::response(['rows_affected' => 1])]); + + $model = new SearchableModelWithPrecomputedEmbedding(['id' => 10, 'name' => 'Taylor']); + $engine = $this->engine(); + + Scout::prepareSearchableDocumentUsing( + fn (array $document, Model $givenModel, Engine $givenEngine): array => [...$document, 'account_id' => 42] + ); + Scout::prepareIndexSettingsUsing(function (array $settings, ?Model $givenModel, Engine $givenEngine, string $index) use ($model, $engine): array { + $this->assertSame($model, $givenModel); + $this->assertSame($engine, $givenEngine); + $this->assertSame('table', $index); + + $settings['schema']['account_id'] = ['type' => 'uint', 'filterable' => true]; + + return $settings; + }); + + $engine->update($model->newCollection([$model])); + + Http::assertSent(fn (Request $request): bool => $request['upsert_rows'] === [['id' => 10, 'name' => 'Taylor', 'account_id' => 42]] + && $request['schema']['account_id'] === ['type' => 'uint', 'filterable' => true]); + } + + public function testSemanticSearchReportsThatGeneratedEmbeddingsAreUnavailable(): void + { + $this->configureEmbeddings(); + Http::preventStrayRequests(); + + $this->expectException(ScoutException::class); + $this->expectExceptionMessage('AI-generated embeddings are not available in Hypervel.'); + + $this->engine()->search( + (new Builder(new SearchableModelWithPrecomputedEmbedding, 'conceptual query'))->semantic() + ); + } + + public function testUpdateUpsertsPrecomputedEmbeddingsBeforePreparingRows(): void + { + $this->configureEmbeddings(['attribute' => 'vector']); + Http::fake(['*' => Http::response(['rows_affected' => 2])]); + + $first = new SearchableModelWithPrecomputedEmbedding(['id' => 10, 'name' => 'First']); + $first->setAttribute('embedding', [0.1, 0.2]); + + $second = new SearchableModelWithPrecomputedEmbedding(['id' => 20, 'name' => 'Second']); + $second->setAttribute('embedding', [0.3, 0.4]); + + $preparedVectors = []; + + Scout::prepareSearchableDocumentUsing(function (array $document) use (&$preparedVectors): array { + $preparedVectors[] = $document['vector']; + + return $document; + }); + + $this->engine()->update($first->newCollection([$first, $second])); + + $this->assertSame([[0.1, 0.2], [0.3, 0.4]], $preparedVectors); + + Http::assertSent(fn (Request $request): bool => $request['upsert_rows'] === [ + ['id' => 10, 'name' => 'First', 'embedding' => [0.1, 0.2], 'vector' => [0.1, 0.2]], + ['id' => 20, 'name' => 'Second', 'embedding' => [0.3, 0.4], 'vector' => [0.3, 0.4]], + ] + && $request['schema']['vector'] === ['type' => '[2]f32', 'ann' => true] + && $request['distance_metric'] === 'cosine_distance'); + } + + public function testUpdateCanUseTurbopufferNativeEmbeddings(): void + { + $this->configureNativeEmbeddings(); + Http::fake(['*' => Http::response(['rows_affected' => 1])]); + + $model = new SearchableModelWithNativeEmbedding([ + 'id' => 10, + 'name' => 'Native embedding source', + 'embedding' => [0.1, 0.2], + ]); + + $this->engine()->update($model->newCollection([$model])); + + Http::assertSent(function (Request $request): bool { + $this->assertSame([[ + 'id' => 10, + 'name' => 'Native embedding source', + ]], $request['upsert_rows']); + $this->assertEquals([ + 'type' => 'string', + 'full_text_search' => true, + 'embed' => [ + 'model' => 'turbopuffer/native-test', + 'dims' => 2, + 'attribute' => 'embedding', + ], + ], $request['schema']['name']); + $this->assertSame('cosine_distance', $request['distance_metric']); + + return true; + }); + } + + public function testDeleteSendsOneBatchedRequest(): void + { + Http::fake(['*' => Http::response(['rows_affected' => 2])]); + + $models = (new SearchableModelWithPrecomputedEmbedding)->newCollection([ + new SearchableModelWithPrecomputedEmbedding(['id' => 10]), + new SearchableModelWithPrecomputedEmbedding(['id' => 20]), + ]); + + $this->engine()->delete($models); + + Http::assertSent(fn (Request $request): bool => $request['deletes'] === [10, 20]); + } + + public function testDeleteByFilterPreparesTheBuilderAndContinuesUntilNoMatchesRemain(): void + { + Http::fake(['*' => Http::sequence() + ->push(['rows_affected' => 3, 'rows_remaining' => true]) + ->push(['rows_affected' => 1, 'rows_remaining' => false])]); + + Scout::prepareBuilderUsing(function (Builder $builder): void { + $builder->where('account_id', 42); + }); + + $observer = $this->observeOperations(); + + $this->engine()->deleteByFilter( + (new Builder(new SearchableModelWithPrecomputedEmbedding, '')) + ->options(['filters' => ['status', 'Eq', 'archived']]) + ); + + Http::assertSentCount(2); + Http::assertSent(fn (Request $request): bool => $request->url() === 'https://turbopuffer.test/v2/namespaces/table' + && $request['delete_by_filter'] === ['And', [['status', 'Eq', 'archived'], ['account_id', 'Eq', 42]]] + && $request['delete_by_filter_allow_partial'] === true); + $this->assertSame([['delete_by_filter', 'table']], $observer->operations); + $this->assertSame([null], $observer->exceptions); + } + + public function testDeleteByFilterUsesAnExplicitNamespace(): void + { + Http::fake(['*' => Http::response(['rows_affected' => 1])]); + + $this->engine()->deleteByFilter( + (new Builder(new SearchableModelWithPrecomputedEmbedding, ''))->within('archive')->where('status', 'archived') + ); + + Http::assertSent(fn (Request $request): bool => $request->url() === 'https://turbopuffer.test/v2/namespaces/archive'); + } + + /** + * @param array $options + */ + #[DataProvider('emptyDeletionFilters')] + public function testDeleteByFilterRejectsAnEmptyFilterBeforeIo(array $options): void + { + Http::preventStrayRequests(); + $observer = $this->observeOperations(); + + try { + $this->engine()->deleteByFilter((new Builder(new SearchableModelWithPrecomputedEmbedding, ''))->options($options)); + + $this->fail('Expected filter deletion to reject an empty filter.'); + } catch (InvalidArgumentException $exception) { + $this->assertSame('Turbopuffer filter deletion requires a non-empty filter.', $exception->getMessage()); + } + + $this->assertSame([], $observer->operations); + } + + /** + * Get the deletion options that contain no filter. + * + * @return array}> + */ + public static function emptyDeletionFilters(): array + { + return [ + 'no filters' => [[]], + 'empty native filters' => [['filters' => []]], + ]; + } + + public function testDeleteByFilterReportsAFailureAfterPartialCompletion(): void + { + Http::fake(['*' => Http::sequence() + ->push(['rows_affected' => 3, 'rows_remaining' => true]) + ->push(['status' => 'error'], 500)]); + + $observer = $this->observeOperations(); + + try { + $this->engine()->deleteByFilter( + (new Builder(new SearchableModelWithPrecomputedEmbedding, ''))->where('status', 'archived') + ); + + $this->fail('Expected the failed continuation to be reported.'); + } catch (RequestException $exception) { + $this->assertSame(500, $exception->response->status()); + } + + Http::assertSentCount(2); + $this->assertSame([['delete_by_filter', 'table']], $observer->operations); + $this->assertSame([RequestException::class], $observer->exceptions); + } + + public function testMissingNamespacesAreTreatedAsEmpty(): void + { + Http::fake(['*' => Http::response(['status' => 'error', 'error' => 'namespace not found'], 404)]); + + $engine = $this->engine(); + $model = new SearchableModelWithPrecomputedEmbedding(['id' => 10]); + + $engine->delete($model->newCollection([$model])); + $engine->deleteByFilter((new Builder($model, ''))->where('status', 'archived')); + $engine->flush($model); + + $this->assertSame(['rows' => [], 'total' => 0], $engine->search(new Builder($model, 'hypervel'))); + $this->assertSame(['rows' => [], 'total' => 0], $engine->paginate(new Builder($model, 'hypervel'), 15, 1)); + Http::assertSentCount(6); + } + + public function testFailuresOtherThanAMissingNamespaceAreThrown(): void + { + Http::fake(['*' => Http::response(['status' => 'error'], 401)]); + + $this->expectException(RequestException::class); + + $this->engine()->flush(new SearchableModelWithPrecomputedEmbedding); + } + + public function testSearchBuildsWeightedBm25AndScoutFilters(): void + { + Http::fake(['*' => Http::response([ + 'rows' => [['id' => 10, '$dist' => 1.25]], + 'billing' => ['billable_logical_bytes_queried' => 100], + ])]); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, 'hypervel')) + ->where('status', 'published') + ->where('age', '>=', 18) + ->whereIn('language', ['en', 'fr']) + ->whereNotIn('category', ['archived']) + ->take(25); + + $results = $this->engine()->search($builder); + + $this->assertSame(10, $results['rows'][0]['id']); + $this->assertSame(100, $results['billing']['billable_logical_bytes_queried']); + + Http::assertSent(function (Request $request): bool { + return $request->url() === 'https://turbopuffer.test/v2/namespaces/table/query' + && $request['rank_by'] === ['Sum', [ + ['Product', 3, ['name', 'BM25', 'hypervel']], + ['description', 'BM25', 'hypervel'], + ]] + && $request['filters'] === ['And', [ + ['status', 'Eq', 'published'], + ['age', 'Gte', 18], + ['language', 'In', ['en', 'fr']], + ['category', 'NotIn', ['archived']], + ]] + && $request['limit'] === 25; + }); + } + + public function testSearchAcceptsANativeVectorRankingExpression(): void + { + Http::fake(['*' => Http::response(['rows' => []])]); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, '')) + ->options([ + 'rank_by' => ['embedding', 'ANN', [0.1, 0.2]], + 'include_attributes' => ['name'], + 'consistency' => ['level' => 'eventual'], + ]); + + $this->engine()->search($builder); + + Http::assertSent(fn (Request $request): bool => $request['rank_by'] === ['embedding', 'ANN', [0.1, 0.2]] + && $request['include_attributes'] === ['name', 'id'] + && $request['consistency'] === ['level' => 'eventual']); + } + + public function testSemanticSearchCanUseTurbopufferNativeQueryEmbeddings(): void + { + $this->configureNativeEmbeddings(); + Http::fake(['*' => Http::response(['rows' => [['id' => 10, '$dist' => 0.1]]])]); + + $results = $this->engine()->search( + (new Builder(new SearchableModelWithNativeEmbedding, 'conceptual query')) + ->semantic() + ->where('status', 'published') + ->take(10) + ); + + $this->assertSame(10, $results['rows'][0]['id']); + + Http::assertSent(fn (Request $request): bool => $request['rank_by'] === [ + 'name', + 'ANN', + ['Embed', 'conceptual query'], + ] && $request['filters'] === ['status', 'Eq', 'published'] + && $request['limit'] === 10); + } + + public function testHybridSearchCanUseTurbopufferNativeQueryEmbeddings(): void + { + $this->configureNativeEmbeddings(); + Http::fake(['*' => Http::response(['results' => [['rows' => []]]])]); + + $this->engine()->search( + (new Builder(new SearchableModelWithNativeEmbedding, 'combined query'))->hybrid() + ); + + Http::assertSent(fn (Request $request): bool => $request['queries'][1]['rank_by'] === [ + 'name', + 'ANN', + ['Embed', 'combined query'], + ]); + } + + public function testNativeEmbeddingsAcceptAStringEmbedSchema(): void + { + $this->configureNativeEmbeddings(); + + $config = config('scout.turbopuffer'); + $config['model-settings'][SearchableModelWithNativeEmbedding::class]['schema']['name']['embed'] = 'turbopuffer/native-test'; + config()->set('scout.turbopuffer', $config); + + Http::fake(['*' => Http::response(['rows' => []])]); + + $this->engine()->search( + (new Builder(new SearchableModelWithNativeEmbedding, 'conceptual query'))->semantic() + ); + + Http::assertSent(fn (Request $request): bool => $request['rank_by'] === [ + 'name', + 'ANN', + ['Embed', 'conceptual query'], + ]); + } + + public function testNativeEmbeddingsRequireAnEmbedSchema(): void + { + $this->configureNativeEmbeddings(); + + $config = config('scout.turbopuffer'); + unset($config['model-settings'][SearchableModelWithNativeEmbedding::class]['schema']['name']['embed']); + config()->set('scout.turbopuffer', $config); + + Http::preventStrayRequests(); + + $this->expectException(ScoutException::class); + $this->expectExceptionMessage('require a valid [embed] schema configuration'); + + $this->engine()->search( + (new Builder(new SearchableModelWithNativeEmbedding, 'conceptual query'))->semantic() + ); + } + + public function testHybridSearchUsesWeightedRrfAndNormalizesResults(): void + { + $this->configureNativeEmbeddings(); + Http::fake(['*' => Http::response([ + 'results' => [[ + 'rows' => [ + ['id' => 20, '$dist' => 0.03], + ['id' => 10, '$dist' => 0.02], + ['id' => 30, '$dist' => 0.01], + ], + ]], + ])]); + + $builder = (new Builder(new SearchableModelWithNativeEmbedding, 'combined query')) + ->hybrid(textWeight: 1, semanticWeight: 2) + ->where('status', 'published') + ->options([ + 'include_attributes' => ['name'], + 'consistency' => ['level' => 'eventual'], + ]) + ->take(2); + + $results = $this->engine()->search($builder); + + $this->assertSame([20, 10], array_column($results['rows'], 'id')); + $this->assertSame(2, $results['total']); + + Http::assertSent(function (Request $request): bool { + $queries = $request['queries']; + + return $request['consistency'] === ['level' => 'eventual'] + && $request['rerank_by'] === ['RRF', ['weights' => [1, 2]]] + && $queries[0]['rank_by'] === ['Sum', [ + ['Product', 3, ['name', 'BM25', 'combined query']], + ['description', 'BM25', 'combined query'], + ]] + && $queries[1]['rank_by'] === ['name', 'ANN', ['Embed', 'combined query']] + && $queries[0]['filters'] === ['status', 'Eq', 'published'] + && $queries[1]['filters'] === ['status', 'Eq', 'published'] + && $queries[0]['include_attributes'] === ['name', 'id'] + && $queries[1]['include_attributes'] === ['name', 'id'] + && $queries[0]['limit'] === 2 + && $queries[1]['limit'] === 2; + }); + } + + public function testMatchAllSearchUsesTheRequestedOrder(): void + { + Http::fake(['*' => Http::response(['rows' => []])]); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, '*'))->orderByDesc('created_at'); + + $this->engine()->search($builder); + + Http::assertSent(fn (Request $request): bool => $request['rank_by'] === ['created_at', 'desc']); + } + + public function testPaginateSlicesTheRankedWindowAndReportsACappedCountOfTextMatches(): void + { + Http::fake(function (Request $request): PromiseInterface { + if (isset($request['aggregate_by'])) { + return Http::response(['aggregations' => ['count' => 12000]]); + } + + return Http::response(['rows' => [ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ['id' => 4], + ]]); + }); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, 'hypervel'))->where('status', 'published'); + $results = $this->engine()->paginate($builder, 2, 2); + + $this->assertSame([3, 4], array_column($results['rows'], 'id')); + $this->assertSame(10000, $results['total']); + + Http::assertSent(fn (Request $request): bool => isset($request['aggregate_by']) + && $request['filters'] === ['And', [ + ['status', 'Eq', 'published'], + ['Or', [ + ['name', 'ContainsAnyToken', 'hypervel'], + ['description', 'ContainsAnyToken', 'hypervel'], + ]], + ]]); + } + + public function testPaginateCountsEveryFilteredCandidateForSemanticSearches(): void + { + $this->configureNativeEmbeddings(); + Http::fake(function (Request $request): PromiseInterface { + if (isset($request['aggregate_by'])) { + return Http::response(['aggregations' => ['count' => 2]]); + } + + return Http::response(['rows' => [ + ['id' => 1], + ['id' => 2], + ]]); + }); + + $results = $this->engine()->paginate( + (new Builder(new SearchableModelWithNativeEmbedding, 'semantic query'))->semantic()->where('status', 'published'), + 1, + 2 + ); + + $this->assertSame([2], array_column($results['rows'], 'id')); + $this->assertSame(2, $results['total']); + + Http::assertSent(fn (Request $request): bool => isset($request['aggregate_by']) + && $request['filters'] === ['status', 'Eq', 'published']); + } + + public function testPaginationRejectsWindowsAboveTurbopufferLimit(): void + { + Http::preventStrayRequests(); + + $this->expectException(ScoutException::class); + $this->expectExceptionMessage('10,000'); + + $this->engine()->paginate(new Builder(new SearchableModelWithPrecomputedEmbedding, 'hypervel'), 100, 101); + } + + public function testPaginationReturnsTheFinalPartialPageWithinTurbopufferLimit(): void + { + Http::fake(function (Request $request): PromiseInterface { + if (isset($request['aggregate_by'])) { + return Http::response(['aggregations' => ['count' => 12000]]); + } + + return Http::response(['rows' => array_map(fn (int $id): array => ['id' => $id], range(1, 10000))]); + }); + + $results = $this->engine()->paginate(new Builder(new SearchableModelWithPrecomputedEmbedding, 'hypervel'), 15, 667); + + $this->assertSame(range(9991, 10000), array_column($results['rows'], 'id')); + $this->assertSame(10000, $results['total']); + + Http::assertSent(fn (Request $request): bool => ! isset($request['aggregate_by']) && $request['limit'] === 10000); + } + + public function testFlushDeletesTheNamespace(): void + { + Http::fake(['*' => Http::response([])]); + + $this->engine()->flush(new SearchableModelWithPrecomputedEmbedding); + + Http::assertSent(fn (Request $request): bool => $request->method() === 'DELETE' + && $request->url() === 'https://turbopuffer.test/v2/namespaces/table'); + } + + public function testCreateIndexIsNotSupported(): void + { + $this->expectException(NotSupportedException::class); + + $this->engine()->createIndex('table'); + } + + public function testAnIndexBuildingResponseThrowsAScoutException(): void + { + Http::fake(['*' => Http::response([], 202)]); + + $this->expectException(ScoutException::class); + $this->expectExceptionMessage('still building'); + + $this->engine()->search(new Builder(new SearchableModelWithPrecomputedEmbedding, 'hypervel')); + } + + public function testInvalidNamespacesAreRejectedBeforeARequestIsSent(): void + { + Http::preventStrayRequests(); + + $this->expectException(ScoutException::class); + $this->expectExceptionMessage('Invalid Turbopuffer namespace'); + + $this->engine()->search( + (new Builder(new SearchableModelWithPrecomputedEmbedding, 'hypervel'))->within('invalid namespace') + ); + } + + public function testTransientFailuresAreRetried(): void + { + config()->set('scout.turbopuffer.retries', 1); + Sleep::fake(); + Http::fake(['*' => Http::sequence() + ->push(['status' => 'error'], 503) + ->push(['rows' => [['id' => 10]]])]); + + $results = $this->engine()->search(new Builder(new SearchableModelWithPrecomputedEmbedding, 'hypervel')); + + $this->assertSame([['id' => 10]], $results['rows']); + Http::assertSentCount(2); + } + + public function testRequestsAreTaggedForTelescope(): void + { + $request = (new ClassInvoker($this->app->make(TurbopufferClient::class)))->pendingRequest(); + + $this->assertSame([TelescopeTag::Scout, TelescopeTag::Turbopuffer], $request->getOptions()['telescope_tags']); + } + + /** + * Get the configured Turbopuffer engine. + */ + protected function engine(): TurbopufferEngine + { + return $this->app->make(EngineManager::class)->engine('turbopuffer'); + } + + /** + * Observe the Scout engine operations run during the test. + */ + protected function observeOperations(): TurbopufferOperationObserver + { + $observer = new TurbopufferOperationObserver; + + $this->app->make(EngineOperationRunner::class)->observe($observer); + + return $observer; + } + + /** + * Configure precomputed embeddings for the searchable model fixture. + * + * @param array $overrides + */ + protected function configureEmbeddings(array $overrides = []): void + { + $config = config('scout.turbopuffer'); + $settings = $config['model-settings'][SearchableModelWithPrecomputedEmbedding::class]; + + $settings['embedding'] = array_merge([ + 'attribute' => 'embedding', + 'dimensions' => 2, + ], $overrides); + $settings['schema'][$settings['embedding']['attribute']] = ['type' => '[2]f32', 'ann' => true]; + + $config['model-settings'][SearchableModelWithPrecomputedEmbedding::class] = $settings; + + config()->set('scout.turbopuffer', $config); + } + + /** + * Configure Turbopuffer's native embeddings for the native embedding model fixture. + */ + protected function configureNativeEmbeddings(): void + { + $config = config('scout.turbopuffer'); + $settings = $config['model-settings'][SearchableModelWithPrecomputedEmbedding::class]; + + $settings['embedding'] = [ + 'driver' => 'turbopuffer', + 'attribute' => 'name', + ]; + $settings['schema']['name']['embed'] = [ + 'model' => 'turbopuffer/native-test', + 'dimensions' => '2', + 'attribute' => 'embedding', + ]; + + $config['model-settings'][SearchableModelWithNativeEmbedding::class] = $settings; + + config()->set('scout.turbopuffer', $config); + } +} + +class TurbopufferOperationObserver implements EngineOperationObserver +{ + /** + * The observed operation names and indexes. + * + * @var array + */ + public array $operations = []; + + /** + * The exception class reported for each finished operation. + * + * @var array> + */ + public array $exceptions = []; + + /** + * Start observing an engine operation. + */ + public function starting(EngineOperation $operation): mixed + { + $this->operations[] = [$operation->operation, $operation->index]; + + return null; + } + + /** + * Finish observing an engine operation. + */ + public function finished(EngineOperation $operation, mixed $token, ?Throwable $exception): void + { + $this->exceptions[] = $exception === null ? null : $exception::class; + } +} diff --git a/tests/Scout/Fixtures/Models/SearchableModelWithNativeEmbedding.php b/tests/Scout/Fixtures/Models/SearchableModelWithNativeEmbedding.php new file mode 100644 index 0000000000..6005338c93 --- /dev/null +++ b/tests/Scout/Fixtures/Models/SearchableModelWithNativeEmbedding.php @@ -0,0 +1,36 @@ +|string + */ + public function toSearchableEmbedding(): array|string + { + return $this->embedding ?? $this->name; + } +} diff --git a/tests/Scout/Unit/BuilderTest.php b/tests/Scout/Unit/BuilderTest.php index c4b7de7e21..4e7d31ecd1 100644 --- a/tests/Scout/Unit/BuilderTest.php +++ b/tests/Scout/Unit/BuilderTest.php @@ -19,6 +19,8 @@ use Hypervel\Scout\EngineOperationRunner; use Hypervel\Scout\Engines\DatabaseEngine; use Hypervel\Scout\Engines\Engine; +use Hypervel\Scout\Exceptions\NotSupportedException; +use Hypervel\Scout\Exceptions\ScoutException; use Hypervel\Scout\Scout; use Hypervel\Support\Collection; use Hypervel\Support\LazyCollection; @@ -287,6 +289,88 @@ public function testOnlyTrashedSetsSoftDeleteWhereToOne(): void ]], $builder->wheres); } + public function testSemanticSearchCanBeEnabled(): void + { + $builder = (new Builder(m::mock(Model::class), 'conceptual query'))->semantic(minSimilarity: 0.7); + + $this->assertTrue($builder->semanticSearch); + $this->assertNull($builder->hybridSearch); + $this->assertSame(0.7, $builder->minimumSimilarity); + } + + public function testHybridSearchCanBeEnabledWithWeights(): void + { + $builder = (new Builder(m::mock(Model::class), 'combined query'))->hybrid(2, 3, minSimilarity: 0.8); + + $this->assertFalse($builder->semanticSearch); + $this->assertSame([ + 'text_weight' => 2, + 'semantic_weight' => 3, + ], $builder->hybridSearch); + $this->assertSame(0.8, $builder->minimumSimilarity); + } + + public function testSemanticAndHybridSearchRequireAQuery(): void + { + foreach (['semantic', 'hybrid'] as $method) { + try { + (new Builder(m::mock(Model::class), ''))->{$method}(); + + $this->fail("Expected [{$method}] to reject an empty query."); + } catch (ScoutException $e) { + $this->assertStringContainsString('non-empty query', $e->getMessage()); + } + } + } + + public function testHybridSearchRequiresPositiveWeights(): void + { + $this->expectException(ScoutException::class); + $this->expectExceptionMessage('positive numbers'); + + (new Builder(m::mock(Model::class), 'query'))->hybrid(1, 0); + } + + public function testUnsupportedEnginesRejectSemanticSearch(): void + { + $model = m::mock(Model::class); + $model->shouldReceive('searchableUsing')->andReturn(m::mock(Engine::class)); + + $this->expectException(NotSupportedException::class); + $this->expectExceptionMessage('does not support semantic search'); + + (new Builder($model, 'query'))->semantic()->raw(); + } + + public function testUnsupportedEnginesRejectSemanticSearchEnabledDuringPreparation(): void + { + $model = m::mock(Model::class); + $engine = m::mock(Engine::class); + $model->shouldReceive('searchableUsing')->andReturn($engine); + $engine->shouldNotReceive('runSearch'); + + Scout::prepareBuilderUsing(function (Builder $builder): void { + $builder->semantic(); + }); + + $this->expectException(NotSupportedException::class); + $this->expectExceptionMessage('does not support semantic search'); + + (new Builder($model, 'query'))->raw(); + } + + public function testUnsupportedEnginesTreatHybridSearchAsNormalTextSearch(): void + { + $model = m::mock(Model::class); + $engine = m::mock(Engine::class); + $model->shouldReceive('searchableUsing')->andReturn($engine); + $engine->shouldReceive('runSearch')->once()->andReturn(['results']); + + $results = (new Builder($model, 'query'))->hybrid()->raw(); + + $this->assertSame(['results'], $results); + } + public function testRawCallsEngineSearchEntryPoint(): void { $model = m::mock(Model::class); diff --git a/tests/Scout/Unit/EngineManagerTest.php b/tests/Scout/Unit/EngineManagerTest.php index de71240e31..12c3aee0e7 100644 --- a/tests/Scout/Unit/EngineManagerTest.php +++ b/tests/Scout/Unit/EngineManagerTest.php @@ -18,7 +18,9 @@ use Hypervel\Scout\Engines\Engine; use Hypervel\Scout\Engines\MeilisearchEngine; use Hypervel\Scout\Engines\NullEngine; +use Hypervel\Scout\Engines\TurbopufferEngine; use Hypervel\Scout\Engines\TypesenseEngine; +use Hypervel\Scout\Services\Turbopuffer\TurbopufferClient; use Hypervel\Support\ClassInvoker; use Hypervel\Tests\TestCase; use InvalidArgumentException; @@ -109,9 +111,16 @@ public function testResolveAlgoliaEngineWithIdentify(): void public function testResolveMeilisearchEngine(): void { + $meilisearchConfig = [ + 'model-settings' => [ + Model::class => ['embedding' => ['embedder' => 'default', 'dimensions' => 2]], + ], + ]; + $container = $this->createMockContainer([ 'driver' => 'meilisearch', 'soft_delete' => false, + 'meilisearch' => $meilisearchConfig, ]); $meilisearchClient = m::mock(MeilisearchClient::class); @@ -123,6 +132,7 @@ public function testResolveMeilisearchEngine(): void $engine = $manager->engine('meilisearch'); $this->assertInstanceOf(MeilisearchEngine::class, $engine); + $this->assertSame($meilisearchConfig, (new ClassInvoker($engine))->config); } public function testResolveMeilisearchEngineWithSoftDelete(): void @@ -141,6 +151,7 @@ public function testResolveMeilisearchEngineWithSoftDelete(): void $engine = $manager->engine('meilisearch'); $this->assertInstanceOf(MeilisearchEngine::class, $engine); + $this->assertTrue((new ClassInvoker($engine))->softDelete); } public function testResolveDatabaseEngine(): void @@ -155,9 +166,16 @@ public function testResolveDatabaseEngine(): void public function testResolveTypesenseEngine(): void { + $typesenseConfig = [ + 'model-settings' => [ + Model::class => ['embedding' => ['attribute' => 'embedding', 'dimensions' => 2]], + ], + ]; + $container = $this->createMockContainerWithTypesense([ 'driver' => 'typesense', 'soft_delete' => false, + 'typesense' => $typesenseConfig, ]); $typesenseClient = m::mock(TypesenseClient::class); @@ -169,6 +187,35 @@ public function testResolveTypesenseEngine(): void $engine = $manager->engine('typesense'); $this->assertInstanceOf(TypesenseEngine::class, $engine); + $this->assertSame($typesenseConfig, (new ClassInvoker($engine))->config); + } + + public function testResolveTurbopufferEngine(): void + { + $turbopufferConfig = [ + 'model-settings' => [ + Model::class => ['searchable-attributes' => ['name' => 1]], + ], + ]; + + $container = $this->createMockContainer([ + 'driver' => 'turbopuffer', + 'soft_delete' => true, + 'turbopuffer' => $turbopufferConfig, + ]); + + $turbopufferClient = m::mock(TurbopufferClient::class); + $container->shouldReceive('make') + ->with(TurbopufferClient::class) + ->andReturn($turbopufferClient); + + $manager = $this->createManager($container); + $engine = $manager->engine('turbopuffer'); + + $this->assertInstanceOf(TurbopufferEngine::class, $engine); + $this->assertSame($turbopufferClient, (new ClassInvoker($engine))->turbopuffer); + $this->assertSame($turbopufferConfig, (new ClassInvoker($engine))->config); + $this->assertTrue((new ClassInvoker($engine))->softDelete); } public function testEngineUsesDefaultDriver(): void @@ -379,6 +426,12 @@ protected function createMockContainer(array $config): m\MockInterface&Container $configService->shouldReceive('boolean') ->with('scout.soft_delete') ->andReturn($config['soft_delete'] ?? false); + $configService->shouldReceive('array') + ->with('scout.meilisearch', []) + ->andReturn($config['meilisearch'] ?? []); + $configService->shouldReceive('array') + ->with('scout.turbopuffer', []) + ->andReturn($config['turbopuffer'] ?? []); $container->shouldReceive('make') ->with('config') @@ -401,6 +454,9 @@ protected function createMockContainerWithTypesense(array $config): m\MockInterf $configService->shouldReceive('integer') ->with('scout.typesense.max_total_results', m::any()) ->andReturn($config['max_total_results'] ?? 1000); + $configService->shouldReceive('array') + ->with('scout.typesense', []) + ->andReturn($config['typesense'] ?? []); $container->shouldReceive('make') ->with('config') diff --git a/tests/Scout/Unit/Engines/MeilisearchEngineTest.php b/tests/Scout/Unit/Engines/MeilisearchEngineTest.php index a0e6993625..7b83de107f 100644 --- a/tests/Scout/Unit/Engines/MeilisearchEngineTest.php +++ b/tests/Scout/Unit/Engines/MeilisearchEngineTest.php @@ -22,6 +22,8 @@ use Hypervel\Scout\Scout; use Hypervel\Scout\Searchable; use Hypervel\Support\LazyCollection; +use Hypervel\Tests\Scout\Fixtures\Models\SearchableModelWithNativeEmbedding; +use Hypervel\Tests\Scout\Fixtures\Models\SearchableModelWithPrecomputedEmbedding; use Hypervel\Tests\TestCase; use InvalidArgumentException; use Meilisearch\Client; @@ -144,6 +146,103 @@ public function testUpdatePreparesTheFinalSearchableDocument(): void $engine->update(new EloquentCollection([$model])); } + public function testUpdateAddsPrecomputedEmbeddingsToVectorsBeforePreparingDocuments(): void + { + $client = m::mock(Client::class); + $engine = $this->engineWithEmbedding($client, ['embedder' => 'default', 'dimensions' => 2]); + + $first = new SearchableModelWithPrecomputedEmbedding(['id' => 10, 'name' => 'First']); + $first->setAttribute('embedding', [0.1, 0.2]); + $first->setAttribute('_vectors', ['other' => [0.9, 0.8]]); + + $second = new SearchableModelWithPrecomputedEmbedding(['id' => 20, 'name' => 'Second']); + $second->setAttribute('embedding', [0.3, 0.4]); + + $preparedVectors = []; + + Scout::prepareSearchableDocumentUsing(function (array $document) use (&$preparedVectors): array { + $preparedVectors[] = $document['_vectors']; + + return $document; + }); + + $client->shouldReceive('index')->once()->with('table')->andReturn($index = m::mock(Indexes::class)); + $index->shouldReceive('addDocuments')->once()->with([ + [ + 'id' => 10, + 'name' => 'First', + 'embedding' => [0.1, 0.2], + '_vectors' => ['other' => [0.9, 0.8], 'default' => [0.1, 0.2]], + ], + [ + 'id' => 20, + 'name' => 'Second', + 'embedding' => [0.3, 0.4], + '_vectors' => ['default' => [0.3, 0.4]], + ], + ], 'id'); + + $engine->update($first->newCollection([$first, $second])); + + $this->assertSame([ + ['other' => [0.9, 0.8], 'default' => [0.1, 0.2]], + ['default' => [0.3, 0.4]], + ], $preparedVectors); + } + + public function testUpdateDoesNotAddVectorsWhenUsingNativeEmbeddings(): void + { + $client = m::mock(Client::class); + $engine = $this->engineWithEmbedding( + $client, + ['embedder' => 'default', 'driver' => 'meilisearch'], + SearchableModelWithNativeEmbedding::class, + ); + + $model = new SearchableModelWithNativeEmbedding(['id' => 1, 'name' => 'Model 1']); + + $client->shouldReceive('index')->once()->with('table')->andReturn($index = m::mock(Indexes::class)); + $index->shouldReceive('addDocuments')->once()->with([ + ['id' => 1, 'name' => 'Model 1'], + ], 'id'); + + $engine->update($model->newCollection([$model])); + } + + public function testUpdatePreservesUserProvidedVectorsWhenUsingNativeEmbeddings(): void + { + $client = m::mock(Client::class); + $engine = $this->engineWithEmbedding( + $client, + ['embedder' => 'default', 'driver' => 'meilisearch'], + SearchableModelWithNativeEmbedding::class, + ); + + $model = new SearchableModelWithNativeEmbedding(['id' => 1, 'name' => 'Model 1']); + $model->setAttribute('_vectors', [ + 'default' => [ + 'embeddings' => [0.1, 0.2], + 'regenerate' => false, + ], + ]); + + $client->shouldReceive('index')->once()->with('table')->andReturn($index = m::mock(Indexes::class)); + $index->shouldReceive('addDocuments')->once()->with([ + [ + 'id' => 1, + 'name' => 'Model 1', + '_vectors' => [ + 'default' => [ + 'embeddings' => [0.1, 0.2], + 'regenerate' => false, + ], + ], + ], + ], 'id'); + + $engine->update($model->newCollection([$model])); + } + public function testDeleteRemovesDocumentsFromIndex(): void { $client = m::mock(Client::class); @@ -430,6 +529,150 @@ public function testPaginatePerformsPaginatedSearch(): void $engine->paginate($builder, 15, 2); } + public function testSemanticSearchWithNativeEmbeddingsOmitsTheQueryVector(): void + { + $client = m::mock(Client::class); + $engine = $this->engineWithEmbedding( + $client, + ['embedder' => 'default', 'driver' => 'meilisearch'], + SearchableModelWithNativeEmbedding::class, + ); + + $client->shouldReceive('index')->once()->with('table')->andReturn($index = m::mock(Indexes::class)); + $index->shouldReceive('rawSearch')->once()->with('conceptual query', m::on(fn (array $parameters): bool => $parameters === [ + 'hitsPerPage' => 10, + 'hybrid' => [ + 'embedder' => 'default', + 'semanticRatio' => 1.0, + ], + 'rankingScoreThreshold' => 0, + 'filter' => 'status="published"', + ]))->andReturn(['hits' => []]); + + $builder = (new Builder(new SearchableModelWithNativeEmbedding, 'conceptual query')) + ->semantic(minSimilarity: 0) + ->where('status', 'published') + ->take(10); + + $engine->search($builder); + } + + public function testHybridSearchWithNativeEmbeddingsAcceptsAPrecomputedQueryVector(): void + { + $client = m::mock(Client::class); + $engine = $this->engineWithEmbedding( + $client, + ['embedder' => 'default', 'driver' => 'meilisearch'], + SearchableModelWithNativeEmbedding::class, + ); + + $client->shouldReceive('index')->once()->with('table')->andReturn($index = m::mock(Indexes::class)); + $index->shouldReceive('rawSearch')->once()->with('combined query', [ + 'vector' => [0.4, 0.6], + 'hitsPerPage' => 5, + 'page' => 2, + 'hybrid' => [ + 'embedder' => 'default', + 'semanticRatio' => 2 / 3, + ], + ])->andReturn(['hits' => []]); + + $builder = (new Builder(new SearchableModelWithNativeEmbedding, 'combined query')) + ->options(['vector' => [0.4, 0.6]]) + ->hybrid(textWeight: 1, semanticWeight: 2); + + $engine->paginate($builder, 5, 2); + } + + public function testSemanticSearchRejectsAnUnsupportedEmbeddingDriver(): void + { + $client = m::mock(Client::class); + $client->shouldNotReceive('index'); + $engine = $this->engineWithEmbedding($client, ['embedder' => 'default', 'dimensions' => 2, 'driver' => 'foo']); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, 'conceptual query'))->semantic(); + + $this->expectException(ScoutException::class); + $this->expectExceptionMessage('The [foo] Meilisearch embedding driver is not supported.'); + + $engine->search($builder); + } + + public function testHybridSearchAcceptsAPrecomputedQueryVectorAndNormalizesWeights(): void + { + $client = m::mock(Client::class); + $engine = $this->engineWithEmbedding($client, ['embedder' => 'default', 'dimensions' => 2]); + + $client->shouldReceive('index')->once()->with('table')->andReturn($index = m::mock(Indexes::class)); + $index->shouldReceive('rawSearch')->once()->with('combined query', [ + 'vector' => [0.4, 0.6], + 'hitsPerPage' => 5, + 'page' => 2, + 'hybrid' => [ + 'embedder' => 'default', + 'semanticRatio' => 2 / 3, + ], + 'rankingScoreThreshold' => 0.5, + ])->andReturn(['hits' => []]); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, 'combined query')) + ->options(['vector' => [0.4, 0.6]]) + ->hybrid(textWeight: 1, semanticWeight: 2, minSimilarity: 0.5); + + $engine->paginate($builder, 5, 2); + } + + public function testSemanticSearchWithoutAQueryVectorReportsThatGeneratedEmbeddingsAreUnavailable(): void + { + $client = m::mock(Client::class); + $client->shouldNotReceive('index'); + $engine = $this->engineWithEmbedding($client, ['embedder' => 'default', 'dimensions' => 2]); + + $this->expectException(ScoutException::class); + $this->expectExceptionMessage('AI-generated embeddings are not available in Hypervel.'); + + $engine->search((new Builder(new SearchableModelWithPrecomputedEmbedding, 'conceptual query'))->semantic()); + } + + /** + * @param array $options + */ + #[DataProvider('invalidSemanticSearchOptions')] + public function testSemanticSearchRejectsInvalidOptions(array $options, string $message): void + { + $client = m::mock(Client::class); + $client->shouldNotReceive('index'); + $engine = $this->engineWithEmbedding($client, ['embedder' => 'default', 'dimensions' => 2]); + + $this->expectException(ScoutException::class); + $this->expectExceptionMessage($message); + + $engine->search( + (new Builder(new SearchableModelWithPrecomputedEmbedding, 'conceptual query')) + ->options($options) + ->semantic() + ); + } + + /** + * Provide semantic search options Meilisearch rejects. + * + * @return array, string}> + */ + public static function invalidSemanticSearchOptions(): array + { + return [ + 'custom hybrid option' => [ + ['vector' => [0.1, 0.2], 'hybrid' => ['embedder' => 'other']], + 'Meilisearch semantic and hybrid searches cannot be combined with a custom [hybrid] option.', + ], + 'empty query vector' => [ + ['vector' => []], + 'The Meilisearch query [vector] must be a non-empty embedding array.', + ], + ]; + } + public function testMapIdsReturnsEmptyCollectionIfNoHits(): void { $client = m::mock(Client::class); @@ -1161,6 +1404,24 @@ protected function apiException(int $status, string $code): ApiException ]); } + /** + * Create an engine with embedding settings for the given model. + * + * @param array $embedding + * @param class-string $model + */ + protected function engineWithEmbedding( + Client $client, + array $embedding, + string $model = SearchableModelWithPrecomputedEmbedding::class + ): MeilisearchEngine { + return new MeilisearchEngine($client, false, [ + 'model-settings' => [ + $model => ['embedding' => $embedding], + ], + ]); + } + protected function createSearchableModelMock(): m\MockInterface { return m::mock(Model::class . ', ' . SearchableInterface::class); diff --git a/tests/Scout/Unit/Engines/TypesenseEngineTest.php b/tests/Scout/Unit/Engines/TypesenseEngineTest.php index cf72d089ac..1e3087d0f8 100644 --- a/tests/Scout/Unit/Engines/TypesenseEngineTest.php +++ b/tests/Scout/Unit/Engines/TypesenseEngineTest.php @@ -15,12 +15,16 @@ use Hypervel\Scout\Engines\Engine; use Hypervel\Scout\Engines\TypesenseEngine; use Hypervel\Scout\Exceptions\NotSupportedException; +use Hypervel\Scout\Exceptions\ScoutException; use Hypervel\Scout\Jobs\RemoveableScoutCollection; use Hypervel\Scout\Scout; +use Hypervel\Tests\Scout\Fixtures\Models\SearchableModelWithNativeEmbedding; +use Hypervel\Tests\Scout\Fixtures\Models\SearchableModelWithPrecomputedEmbedding; use Hypervel\Tests\TestCase; use InvalidArgumentException; use Mockery as m; use Mockery\MockInterface; +use PHPUnit\Framework\Attributes\DataProvider; use ReflectionMethod; use Throwable; use Typesense\ApiCall; @@ -31,7 +35,9 @@ use Typesense\Documents; use Typesense\Exceptions\ObjectAlreadyExists; use Typesense\Exceptions\ObjectNotFound; +use Typesense\Exceptions\RequestMalformed; use Typesense\Exceptions\TypesenseClientError; +use Typesense\MultiSearch; class TypesenseEngineTest extends TestCase { @@ -298,6 +304,71 @@ public function testUpdatePreparesTheFinalSearchableDocument(): void $engine->update(new EloquentCollection([$model])); } + public function testUpdateAddsPrecomputedEmbeddingsToDocumentsBeforePreparingThem(): void + { + $client = m::mock(TypesenseClient::class); + $collections = m::mock(Collections::class); + $collection = m::mock(TypesenseCollection::class); + $documents = m::mock(Documents::class); + $client->shouldReceive('getCollections')->once()->andReturn($collections); + $collections->shouldReceive('offsetGet')->with('table')->once()->andReturn($collection); + $collections->shouldReceive('offsetUnset')->with('table')->once(); + $collection->shouldReceive('getDocuments')->once()->andReturn($documents); + $documents->shouldReceive('import') + ->once() + ->with([ + ['id' => 10, 'name' => 'First', 'embedding' => [0.1, 0.2], 'embedding_vector' => [0.1, 0.2]], + ['id' => 20, 'name' => 'Second', 'embedding' => [0.3, 0.4], 'embedding_vector' => [0.3, 0.4]], + ], ['action' => 'upsert']) + ->andReturn([['success' => true], ['success' => true]]); + + $engine = $this->createSemanticEngine(['attribute' => 'embedding_vector', 'dimensions' => 2], client: $client); + + $first = new SearchableModelWithPrecomputedEmbedding(['id' => 10, 'name' => 'First']); + $first->setAttribute('embedding', [0.1, 0.2]); + + $second = new SearchableModelWithPrecomputedEmbedding(['id' => 20, 'name' => 'Second']); + $second->setAttribute('embedding', [0.3, 0.4]); + + $preparedVectors = []; + + Scout::prepareSearchableDocumentUsing(function (array $document) use (&$preparedVectors): array { + $preparedVectors[] = $document['embedding_vector']; + + return $document; + }); + + $engine->update($first->newCollection([$first, $second])); + + $this->assertSame([[0.1, 0.2], [0.3, 0.4]], $preparedVectors); + } + + public function testUpdateDoesNotGenerateEmbeddingsWhenUsingNativeEmbeddings(): void + { + $client = m::mock(TypesenseClient::class); + $collections = m::mock(Collections::class); + $collection = m::mock(TypesenseCollection::class); + $documents = m::mock(Documents::class); + $client->shouldReceive('getCollections')->once()->andReturn($collections); + $collections->shouldReceive('offsetGet')->with('table')->once()->andReturn($collection); + $collections->shouldReceive('offsetUnset')->with('table')->once(); + $collection->shouldReceive('getDocuments')->once()->andReturn($documents); + $documents->shouldReceive('import') + ->once() + ->with([['id' => 1, 'name' => 'Model 1']], ['action' => 'upsert']) + ->andReturn([['success' => true]]); + + $engine = $this->createSemanticEngine( + ['attribute' => 'embedding', 'driver' => 'typesense'], + client: $client, + model: SearchableModelWithNativeEmbedding::class, + ); + + $model = new SearchableModelWithNativeEmbedding(['id' => 1, 'name' => 'Model 1']); + + $engine->update($model->newCollection([$model])); + } + public function testUpdateCreatesMissingCollectionAndRetriesImportOnce(): void { $client = m::mock(TypesenseClient::class); @@ -867,6 +938,86 @@ public function testSearchPropagatesNonMissingFailure(): void $this->createPartialEngineWithConfig($client)->search($builder); } + public function testSearchesWithVectorQueriesUseTheMultiSearchEndpoint(): void + { + $multiSearch = m::mock(MultiSearch::class); + $engine = $this->createMultiSearchEngine($multiSearch); + + $engine->shouldReceive('buildSearchParameters')->once()->andReturn([ + 'q' => '*', + 'query_by' => 'name', + 'vector_query' => 'embedding:([0.1, 0.2])', + ]); + + $multiSearch->shouldReceive('perform') + ->once() + ->with([ + 'searches' => [ + [ + 'q' => '*', + 'query_by' => 'name', + 'vector_query' => 'embedding:([0.1, 0.2])', + 'collection' => 'table', + ], + ], + ]) + ->andReturn(['results' => [['found' => 1, 'hits' => [['document' => ['id' => '1']]]]]]); + + $results = $engine->search(new Builder(new SearchableModelWithPrecomputedEmbedding, 'conceptual query')); + + $this->assertSame(1, $results['found']); + $this->assertSame('1', $results['hits'][0]['document']['id']); + } + + public function testMultiSearchErrorsAreConvertedToTypesenseExceptions(): void + { + $multiSearch = m::mock(MultiSearch::class); + $engine = $this->createMultiSearchEngine($multiSearch); + + $engine->shouldReceive('buildSearchParameters')->andReturn([ + 'q' => '*', + 'query_by' => 'name', + 'vector_query' => 'embedding:([0.1, 0.2])', + ]); + + $multiSearch->shouldReceive('perform')->andReturn([ + 'results' => [['code' => 400, 'error' => 'Query string exceeds max allowed length.']], + ]); + + $this->expectException(RequestMalformed::class); + $this->expectExceptionMessage('Query string exceeds max allowed length.'); + + $engine->search(new Builder(new SearchableModelWithPrecomputedEmbedding, 'conceptual query')); + } + + public function testMultiSearchCreatesMissingCollectionsAndRetries(): void + { + $multiSearch = m::mock(MultiSearch::class); + $engine = $this->createMultiSearchEngine($multiSearch); + $collections = $engine->getTypesenseClient()->getCollections(); + $collections->shouldReceive('create') + ->once() + ->with(['name' => 'table']) + ->andReturn(['name' => 'table']); + + $engine->shouldReceive('buildSearchParameters')->andReturn([ + 'q' => '*', + 'query_by' => 'name', + 'vector_query' => 'embedding:([0.1, 0.2])', + ]); + + $multiSearch->shouldReceive('perform') + ->twice() + ->andReturn( + ['results' => [['code' => 404, 'error' => 'Not found.']]], + ['results' => [['found' => 0, 'hits' => []]]], + ); + + $results = $engine->search(new Builder(new SearchableModelWithPrecomputedEmbedding, 'conceptual query')); + + $this->assertSame(0, $results['found']); + } + public function testLargeTakeUsesFixedPageSizesAndTruthfulMetadata(): void { $engine = $this->createPartialEngineWithConfig(); @@ -1244,20 +1395,247 @@ public function testBuildSearchParametersWithEmptyQuery(): void $this->assertSame('', $params['filter_by']); } + public function testSemanticSearchUsesAPrecomputedQueryVector(): void + { + $engine = $this->createSemanticEngine(['attribute' => 'embedding', 'dimensions' => 2]); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, 'conceptual query')) + ->options(['vector' => [0.25, 0.75]]) + ->semantic(minSimilarity: 0.7); + + $parameters = $engine->buildSearchParameters($builder, 1, 10); + + $this->assertSame('*', $parameters['q']); + $this->assertSame('name', $parameters['query_by']); + $this->assertSame('embedding:([0.25, 0.75], distance_threshold: 0.3)', $parameters['vector_query']); + $this->assertSame('embedding', $parameters['exclude_fields']); + $this->assertArrayNotHasKey('vector', $parameters); + } + + public function testSemanticSearchWithoutAQueryVectorReportsThatGeneratedEmbeddingsAreUnavailable(): void + { + $engine = $this->createSemanticEngine(['attribute' => 'embedding', 'dimensions' => 2]); + + $this->expectException(ScoutException::class); + $this->expectExceptionMessage('AI-generated embeddings are not available in Hypervel.'); + + $engine->buildSearchParameters( + (new Builder(new SearchableModelWithPrecomputedEmbedding, 'conceptual query'))->semantic(), + 1, + 10, + ); + } + + public function testHybridSearchAcceptsAPrecomputedQueryVectorAndNormalizesWeights(): void + { + $engine = $this->createSemanticEngine(['attribute' => 'embedding', 'dimensions' => 2]); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, 'combined query')) + ->options(['vector' => [0.4, 0.6]]) + ->hybrid(textWeight: 1, semanticWeight: 2); + + $parameters = $engine->buildSearchParameters($builder, 2, 5); + + $this->assertSame('combined query', $parameters['q']); + $this->assertSame('name', $parameters['query_by']); + $this->assertSame('embedding:([0.4, 0.6], alpha: ' . (2 / 3) . ')', $parameters['vector_query']); + $this->assertSame('embedding', $parameters['exclude_fields']); + $this->assertArrayNotHasKey('vector', $parameters); + } + + public function testSemanticSearchWithNativeEmbeddingsQueriesTheEmbeddingField(): void + { + $engine = $this->createSemanticEngine(['attribute' => 'embedding', 'driver' => 'typesense']); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, 'conceptual query'))->semantic(); + + $parameters = $engine->buildSearchParameters($builder, 1, 10); + + $this->assertSame('conceptual query', $parameters['q']); + $this->assertSame('embedding', $parameters['query_by']); + $this->assertFalse($parameters['prefix']); + $this->assertArrayNotHasKey('vector_query', $parameters); + $this->assertSame('embedding', $parameters['exclude_fields']); + } + + public function testSemanticSearchWithNativeEmbeddingsAppliesADistanceThreshold(): void + { + $engine = $this->createSemanticEngine(['attribute' => 'embedding', 'driver' => 'typesense']); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, 'conceptual query'))->semantic(minSimilarity: 0.5); + + $parameters = $engine->buildSearchParameters($builder, 1, 10); + + $this->assertSame('embedding', $parameters['query_by']); + $this->assertSame('embedding:([], distance_threshold: 0.5)', $parameters['vector_query']); + } + + public function testSemanticSearchWithNativeEmbeddingsDropsPerFieldParameters(): void + { + $engine = $this->createSemanticEngine(['attribute' => 'embedding', 'driver' => 'typesense'], 'name,description'); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, 'conceptual query')) + ->options(['query_by_weights' => '2,1', 'num_typos' => '2,1', 'infix' => 'off']) + ->semantic(); + + $parameters = $engine->buildSearchParameters($builder, 1, 10); + + $this->assertSame('embedding', $parameters['query_by']); + $this->assertFalse($parameters['prefix']); + $this->assertArrayNotHasKey('query_by_weights', $parameters); + $this->assertArrayNotHasKey('num_typos', $parameters); + $this->assertSame('off', $parameters['infix']); + } + + public function testHybridSearchWithNativeEmbeddingsAppendsTheEmbeddingFieldToQueryBy(): void + { + $engine = $this->createSemanticEngine(['attribute' => 'embedding', 'driver' => 'typesense']); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, 'combined query'))->hybrid(); + + $parameters = $engine->buildSearchParameters($builder, 1, 10); + + $this->assertSame('combined query', $parameters['q']); + $this->assertSame('name,embedding', $parameters['query_by']); + $this->assertSame('true,false', $parameters['prefix']); + $this->assertSame('embedding:([], alpha: 0.5)', $parameters['vector_query']); + $this->assertSame('embedding', $parameters['exclude_fields']); + } + + public function testHybridSearchWithNativeEmbeddingsExtendsPerFieldParameters(): void + { + $engine = $this->createSemanticEngine(['attribute' => 'embedding', 'driver' => 'typesense'], 'name,description'); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, 'combined query')) + ->options(['query_by_weights' => '2,1', 'num_typos' => '2,1', 'prefix' => 'true,false', 'infix' => 'off']) + ->hybrid(); + + $parameters = $engine->buildSearchParameters($builder, 1, 10); + + $this->assertSame('name,description,embedding', $parameters['query_by']); + $this->assertSame('2,1,0', $parameters['query_by_weights']); + $this->assertSame('2,1,0', $parameters['num_typos']); + $this->assertSame('true,false,false', $parameters['prefix']); + $this->assertSame('off', $parameters['infix']); + } + + #[DataProvider('hybridSearchPrefixes')] + public function testHybridSearchWithNativeEmbeddingsDisablesPrefixSearchOnAnEmbeddingFieldAlreadyInQueryBy( + string $queryBy, + bool|string $prefix, + bool|string $expected, + ): void { + $engine = $this->createSemanticEngine(['attribute' => 'embedding', 'driver' => 'typesense'], $queryBy); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, 'combined query')) + ->options(['prefix' => $prefix]) + ->hybrid(); + + $parameters = $engine->buildSearchParameters($builder, 1, 10); + + $this->assertSame($queryBy, $parameters['query_by']); + $this->assertSame($expected, $parameters['prefix']); + } + + /** + * Provide prefix settings for hybrid searches whose "query_by" already includes the embedding field. + * + * @return array + */ + public static function hybridSearchPrefixes(): array + { + return [ + 'global prefix' => ['embedding,name', true, 'false,true'], + 'per-field prefixes' => ['name,embedding,description', 'true,true,false', 'true,false,false'], + 'disabled prefix' => ['name,embedding', false, false], + ]; + } + + public function testHybridSearchRequiresAKeywordFieldInQueryBy(): void + { + $engine = $this->createSemanticEngine(['attribute' => 'embedding', 'driver' => 'typesense'], ''); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, 'combined query'))->hybrid(); + + $this->expectException(ScoutException::class); + $this->expectExceptionMessage('Typesense hybrid searches require at least one keyword field in the [query_by] search parameter.'); + + $engine->buildSearchParameters($builder, 1, 10); + } + + public function testSemanticSearchCannotBeCombinedWithACustomVectorQueryOption(): void + { + $engine = $this->createSemanticEngine(['attribute' => 'embedding', 'dimensions' => 2]); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, 'conceptual query')) + ->options(['vector_query' => 'embedding:([], k: 10)']) + ->semantic(); + + $this->expectException(ScoutException::class); + $this->expectExceptionMessage('Typesense semantic and hybrid searches cannot be combined with a custom [vector_query] option.'); + + $engine->buildSearchParameters($builder, 1, 10); + } + + /** + * @param array $options + */ + #[DataProvider('invalidSemanticSearchVectors')] + public function testSemanticSearchRejectsInvalidVectorQueries(array $options, float $minSimilarity, string $message): void + { + $engine = $this->createSemanticEngine(['attribute' => 'embedding', 'dimensions' => 2]); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, 'conceptual query')) + ->options($options) + ->semantic($minSimilarity); + + $this->expectException(ScoutException::class); + $this->expectExceptionMessage($message); + + $engine->buildSearchParameters($builder, 1, 10); + } + + /** + * Provide semantic search vectors and similarities Typesense rejects. + * + * @return array, float, string}> + */ + public static function invalidSemanticSearchVectors(): array + { + return [ + 'empty query vector' => [['vector' => []], 0.5, 'The Typesense query [vector] must be a non-empty embedding array.'], + 'similarity above one' => [['vector' => [0.1, 0.2]], 1.5, 'The minimum similarity must be between 0 and 1.'], + ]; + } + + public function testSemanticSearchRequiresEmbeddingSettings(): void + { + $engine = $this->createPartialEngineWithConfig(); + + $builder = (new Builder(new SearchableModelWithPrecomputedEmbedding, 'conceptual query'))->semantic(); + + $this->expectException(ScoutException::class); + $this->expectExceptionMessage('No Typesense embedding settings have been configured for [' . SearchableModelWithPrecomputedEmbedding::class . '].'); + + $engine->buildSearchParameters($builder, 1, 10); + } + /** * Create a partial engine mock that stubs getConfig to avoid container dependency. * * @param array $config + * @param array $engineConfig */ protected function createPartialEngineWithConfig( ?MockInterface $client = null, int $maxTotalResults = 1000, array $config = [], + array $engineConfig = [], ): MockInterface&TypesenseEngine { $client = $client ?? m::mock(TypesenseClient::class); /** @var MockInterface&TypesenseEngine */ - $engine = m::mock(TypesenseEngine::class, [$client, $maxTotalResults]) + $engine = m::mock(TypesenseEngine::class, [$client, $maxTotalResults, $engineConfig]) ->shouldAllowMockingProtectedMethods() ->makePartial(); @@ -1266,6 +1644,45 @@ protected function createPartialEngineWithConfig( return $engine; } + + /** + * Create a partial engine with embedding settings and search parameters for the given model. + * + * @param array $embedding + * @param class-string $model + */ + protected function createSemanticEngine( + array $embedding, + string $queryBy = 'name', + ?MockInterface $client = null, + string $model = SearchableModelWithPrecomputedEmbedding::class, + ): MockInterface&TypesenseEngine { + return $this->createPartialEngineWithConfig( + $client, + config: ["typesense.model-settings.{$model}.search-parameters" => ['query_by' => $queryBy]], + engineConfig: ['model-settings' => [$model => ['embedding' => $embedding]]], + ); + } + + /** + * Create a partial engine whose client performs multi-searches against the "table" collection. + */ + protected function createMultiSearchEngine(MultiSearch $multiSearch): MockInterface&TypesenseEngine + { + $client = m::mock(TypesenseClient::class); + $collections = m::mock(Collections::class); + $collection = m::mock(TypesenseCollection::class); + $documents = m::mock(Documents::class); + + $client->shouldReceive('getMultiSearch')->andReturn($multiSearch); + $client->shouldReceive('getCollections')->andReturn($collections); + $collections->shouldReceive('offsetGet')->with('table')->andReturn($collection); + $collections->shouldReceive('offsetUnset')->with('table'); + $collection->shouldReceive('getDocuments')->andReturn($documents); + $documents->shouldNotReceive('search'); + + return $this->createPartialEngineWithConfig($client); + } } class TypesenseEngineOperationObserver implements EngineOperationObserver