Add Scout semantic and hybrid search and the Turbopuffer engine

src/contracts/src/Telescope/TelescopeTag.php

@@ -23,6 +23,7 @@ enum TelescopeTag: string
     case Algolia = 'algolia';
     case Meilisearch = 'meilisearch';
     case Saloon = 'saloon';
+    case Turbopuffer = 'turbopuffer';
     case Typesense = 'typesense';
 
     /**
@@ -38,6 +39,7 @@ public function description(): string
             self::Algolia => 'Scout Algolia driver.',
             self::Meilisearch => 'Scout Meilisearch driver.',
             self::Saloon => 'Hypervel Saloon — API integration package.',
+            self::Turbopuffer => 'Scout Turbopuffer driver.',
             self::Typesense => 'Scout Typesense driver.',
         };
     }

src/docs/porting-from-laravel.md

@@ -541,6 +541,8 @@ Fortify ignores Laravel's `fortify.passwords` setting. Declare the password rese
 
 Hypervel compiles integer and float values passed to Scout's Algolia `where`, `whereIn`, and `whereNotIn` methods as numeric comparisons. Numeric-looking strings remain facet values. When porting an Algolia index, ensure the indexed attribute type matches the PHP value type used by these filters.
 
+Scout cannot generate embeddings with the Laravel AI SDK, and its database engine does not support semantic or hybrid search. Models whose `toSearchableEmbedding` method returns text must return precomputed embedding arrays or switch to the engine's native embeddings. See [semantic search](/docs/{{version}}/scout#semantic-search).
+
 <a name="socialite"></a>
 ### Socialite
 

src/docs/search.md

@@ -35,7 +35,7 @@ For semantic search that matches results by *meaning* rather than exact keywords
 <a name="introduction-scout-search-engines"></a>
 #### Hypervel Scout Search
 
-For applications that want a `Searchable` trait that automatically keeps search indexes in sync with Eloquent models, [Hypervel Scout](/docs/{{version}}/scout) offers both a built-in database engine and drivers for third-party services like Algolia, Meilisearch, and Typesense.
+For applications that want a `Searchable` trait that automatically keeps search indexes in sync with Eloquent models, [Hypervel Scout](/docs/{{version}}/scout) offers both a built-in database engine and drivers for third-party services like Algolia, Meilisearch, Typesense, and Turbopuffer.
 
 <a name="full-text-search"></a>
 ## Full-Text Search

src/scout/README.md

@@ -13,5 +13,6 @@ Documentation: https://hypervel.org/docs/scout
 - Pausing search syncing with `withoutSyncingToSearch()` or `disableSearchSyncing()` applies only to the current coroutine, so other requests keep indexing. See [Pausing Indexing](https://hypervel.org/docs/scout#pausing-indexing).
 - `MeilisearchEngine::generateTenantToken()` takes the search rules, the parent key's UID, the key itself and an optional expiry. Laravel's engine forwards the call to the Meilisearch client, whose method takes the UID, the search rules and an options array. See [Tenant Tokens](https://hypervel.org/docs/scout#meilisearch-tenant-tokens).
 - `scout:delete-all-indexes` refuses to run without a configured Scout prefix unless you pass `--force`.
+- Scout cannot generate embeddings with the Laravel AI SDK, so semantic and hybrid search use engine-native embeddings or precomputed vectors, and the database engine does not support them. See [Semantic Search](https://hypervel.org/docs/scout#semantic-search).
 
 Ported from: https://github.com/laravel/scout

src/scout/composer.json

@@ -12,6 +12,7 @@
         "algolia",
         "meilisearch",
         "typesense",
+        "turbopuffer",
         "database"
     ],
     "authors": [
@@ -47,6 +48,7 @@
         "hypervel/coroutine": "^0.4",
         "hypervel/database": "^0.4",
         "hypervel/foundation": "^0.4",
+        "hypervel/http": "^0.4",
         "hypervel/macroable": "^0.4",
         "hypervel/pagination": "^0.4",
         "hypervel/queue": "^0.4",

src/scout/config/scout.php

@@ -12,7 +12,8 @@
     | using Scout. This connection is used when syncing all models to the
     | search service. You should adjust this based on your needs.
     |
-    | Supported: "algolia", "meilisearch", "typesense", "database", "collection", "null"
+    | Supported: "algolia", "meilisearch", "typesense", "turbopuffer",
+    |            "database", "collection", "null"
     |
     */
 
@@ -197,6 +198,21 @@
             // 'users' => [
             //     'filterableAttributes' => ['id', 'name', 'email'],
             //     'sortableAttributes' => ['created_at'],
+            //     'embedders' => [
+            //         'default' => [
+            //             'source' => 'userProvided',
+            //             'dimensions' => 1536,
+            //         ],
+            //     ],
+            // ],
+        ],
+        'model-settings' => [
+            // Per-model settings can be defined here:
+            // App\Models\User::class => [
+            //     'embedding' => [
+            //         'embedder' => 'default',
+            //         'dimensions' => 1536,
+            //     ],
             // ],
         ],
     ],
@@ -250,8 +266,52 @@
             //     'search-parameters' => [
             //         'query_by' => 'name',
             //     ],
+            //     'embedding' => [
+            //         'attribute' => 'embedding',
+            //         'dimensions' => 1536,
+            //     ],
             // ],
         ],
         'import_action' => env('TYPESENSE_IMPORT_ACTION', 'upsert'),
     ],
+
+    /*
+    |--------------------------------------------------------------------------
+    | Turbopuffer Configuration
+    |--------------------------------------------------------------------------
+    |
+    | Here you may configure your Turbopuffer connection and the schema and
+    | searchable attributes defined by each of your application's models.
+    | Turbopuffer is a scalable engine with full-text and vector search.
+    | Timeouts are measured in seconds. Omitted region, timeout, and retry
+    | members use the values shown below.
+    |
+    */
+
+    'turbopuffer' => [
+        'api_key' => env('TURBOPUFFER_API_KEY'),
+        'region' => env('TURBOPUFFER_REGION', 'gcp-us-central1'),
+        'base_url' => env('TURBOPUFFER_BASE_URL'),
+        'timeout' => (int) env('TURBOPUFFER_TIMEOUT', 60),
+        'connect_timeout' => (int) env('TURBOPUFFER_CONNECT_TIMEOUT', 5),
+        'retries' => (int) env('TURBOPUFFER_RETRIES', 3),
+        'model-settings' => [
+            // Per-model settings can be defined here:
+            // App\Models\User::class => [
+            //     'searchable-attributes' => [
+            //         'name' => 2,
+            //         'email' => 1,
+            //     ],
+            //     'embedding' => [
+            //         'attribute' => 'embedding',
+            //         'dimensions' => 1536,
+            //     ],
+            //     'schema' => [
+            //         'name' => ['type' => 'string', 'full_text_search' => true],
+            //         'email' => ['type' => 'string', 'full_text_search' => true],
+            //         'embedding' => ['type' => '[1536]f32', 'ann' => true],
+            //     ],
+            // ],
+        ],
+    ],
 ];

src/scout/src/Builder.php

@@ -17,7 +17,10 @@
 use Hypervel\Scout\Contracts\PaginatesEloquentModels;
 use Hypervel\Scout\Contracts\PaginatesEloquentModelsUsingDatabase;
 use Hypervel\Scout\Contracts\SearchableInterface;
+use Hypervel\Scout\Contracts\SupportsSemanticSearch;
 use Hypervel\Scout\Engines\Engine;
+use Hypervel\Scout\Exceptions\NotSupportedException;
+use Hypervel\Scout\Exceptions\ScoutException;
 use Hypervel\Support\Collection;
 use Hypervel\Support\LazyCollection;
 use Hypervel\Support\Traits\Conditionable;
@@ -100,6 +103,23 @@ class Builder
      */
     public array $orders = [];
 
+    /**
+     * Indicates that the query should use semantic search.
+     */
+    public bool $semanticSearch = false;
+
+    /**
+     * The minimum similarity for semantic search results.
+     */
+    public float|int|null $minimumSimilarity = null;
+
+    /**
+     * The hybrid search ranking weights.
+     *
+     * @var null|array{text_weight: float|int, semantic_weight: float|int}
+     */
+    public ?array $hybridSearch = null;
+
     /**
      * Extra options that should be applied to the search.
      *
@@ -289,6 +309,53 @@ public function oldest(?string $column = null): static
         return $this->orderBy($column, 'asc');
     }
 
+    /**
+     * Perform a semantic search for the query expression.
+     *
+     * @return $this
+     */
+    public function semantic(float|int|null $minSimilarity = null): static
+    {
+        if (trim($this->query) === '') {
+            throw new ScoutException('Semantic searches require a non-empty query.');
+        }
+
+        $this->semanticSearch = true;
+        $this->minimumSimilarity = $minSimilarity;
+        $this->hybridSearch = null;
+
+        return $this;
+    }
+
+    /**
+     * Perform a hybrid full-text and semantic search for the query expression.
+     *
+     * @return $this
+     */
+    public function hybrid(
+        float|int $textWeight = 1,
+        float|int $semanticWeight = 1,
+        float|int|null $minSimilarity = null
+    ): static {
+        if (trim($this->query) === '') {
+            throw new ScoutException('Hybrid searches require a non-empty query.');
+        }
+
+        if ($textWeight <= 0 || $semanticWeight <= 0) {
+            throw new ScoutException('Hybrid search weights must be positive numbers.');
+        }
+
+        $this->semanticSearch = false;
+        $this->minimumSimilarity = $minSimilarity;
+
+        $this->hybridSearch = [
+            'text_weight' => $textWeight,
+            'semantic_weight' => $semanticWeight,
+        ];
+
+        return $this;
+    }
+
     /**
      * Set extra options for the search query.
      *
@@ -620,6 +687,11 @@ protected function preparedEngine(): Engine
 
         Scout::prepareBuilder($this, $engine);
 
+        // Check after preparation because the callback may enable semantic search.
+        if ($this->semanticSearch && ! $engine instanceof SupportsSemanticSearch) {
+            throw new NotSupportedException('The configured Scout engine does not support semantic search.');
+        }
+
         return $engine;
     }
 

src/scout/src/Contracts/SupportsSemanticSearch.php

@@ -0,0 +1,15 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Hypervel\Scout\Contracts;
+
+/**
+ * Contract for engines that support semantic and hybrid search.
+ *
+ * Semantic searches fail on other engines, while hybrid searches fall back
+ * to their normal full-text search.
+ */
+interface SupportsSemanticSearch
+{
+}

src/scout/src/EngineManager.php

@@ -15,7 +15,9 @@
 use Hypervel\Scout\Engines\Engine;
 use Hypervel\Scout\Engines\MeilisearchEngine;
 use Hypervel\Scout\Engines\NullEngine;
+use Hypervel\Scout\Engines\TurbopufferEngine;
 use Hypervel\Scout\Engines\TypesenseEngine;
+use Hypervel\Scout\Services\Turbopuffer\TurbopufferClient;
 use InvalidArgumentException;
 use Meilisearch\Client as MeilisearchClient;
 use Meilisearch\Meilisearch;
@@ -139,7 +141,8 @@ public function createMeilisearchDriver(): MeilisearchEngine
 
         return new MeilisearchEngine(
             $this->container->make(MeilisearchClient::class),
-            $config->boolean('scout.soft_delete')
+            $config->boolean('scout.soft_delete'),
+            $config->array('scout.meilisearch', [])
         );
     }
 
@@ -173,7 +176,8 @@ public function createTypesenseDriver(): TypesenseEngine
 
         return new TypesenseEngine(
             $this->container->make(TypesenseClient::class),
-            $config->integer('scout.typesense.max_total_results', 1000)
+            $config->integer('scout.typesense.max_total_results', 1000),
+            $config->array('scout.typesense', [])
         );
     }
 
@@ -193,6 +197,21 @@ protected function ensureTypesenseClientIsInstalled(): void
         );
     }
 
+    /**
+     * Create a Turbopuffer engine instance.
+     */
+    public function createTurbopufferDriver(): TurbopufferEngine
+    {
+        /** @var Repository $config */
+        $config = $this->container->make('config');
+
+        return new TurbopufferEngine(
+            $this->container->make(TurbopufferClient::class),
+            $config->array('scout.turbopuffer', []),
+            $config->boolean('scout.soft_delete'),
+        );
+    }
+
     /**
      * Create a collection engine instance.
      */