From 77d8e484cca62dce04a65b4b143b59a00d0f598d Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 31 Oct 2026 22:29:24 +0800 Subject: [PATCH 01/14] [core][python] Unify global index configuration for PK and DE tables --- docs/docs/multimodal-table/global-index.mdx | 22 + .../docs/multimodal-table/global-index/fm.mdx | 10 +- docs/docs/primary-key-table/global-index.mdx | 102 ++-- .../java/org/apache/paimon/CoreOptions.java | 468 ++++++++++++------ .../paimon/globalindex/GlobalIndexer.java | 8 +- .../DataEvolutionGlobalIndexScanner.java | 3 +- .../paimon/globalindex/GlobalIndexQuery.java | 10 +- .../sorted/SortedGlobalIndexWriter.java | 22 +- .../index/pk/PrimaryKeyIndexDefinitions.java | 12 +- .../paimon/schema/SchemaManagerUtils.java | 2 +- .../paimon/schema/SchemaValidation.java | 84 +++- .../AbstractDataEvolutionVectorRead.java | 43 +- .../source/BatchVectorSearchBuilderImpl.java | 3 +- .../source/DataEvolutionFullTextRead.java | 17 +- .../table/source/DataEvolutionVectorScan.java | 24 +- .../table/source/PrimaryKeyVectorRead.java | 5 +- .../table/source/RawFullTextReadImpl.java | 5 +- .../table/source/VectorSearchBuilderImpl.java | 3 +- .../apache/paimon/GlobalIndexOptionsTest.java | 278 +++++++++++ .../globalindex/GlobalIndexQueryTest.java | 42 ++ .../sorted/SortedGlobalIndexWriterTest.java | 22 + .../pk/PrimaryKeyIndexDefinitionsTest.java | 43 +- .../PrimaryKeySortedIndexMaintenanceTest.java | 24 +- .../table/BtreeGlobalIndexTableTest.java | 32 ++ .../table/source/VectorSearchBuilderTest.java | 60 +++ .../globalindex/SortedIndexTopoBuilder.java | 21 +- .../procedure/VectorSearchProcedure.java | 3 +- .../VectorSearchProcedureITCase.java | 5 +- .../pypaimon/common/options/core_options.py | 207 +++++--- .../execution/native/index_options.py | 109 ++++ .../native/read/native_full_text_search.py | 7 +- .../native/read/native_hybrid_search.py | 31 +- .../native/read/native_search_plan.py | 24 + .../native/read/native_vector_search.py | 8 +- .../pypaimon/execution/native/read/plan.py | 5 + .../pypaimon/execution/native/table.py | 3 +- .../execution/native/write/native_commit.py | 3 +- .../native/write/native_index_build.py | 7 +- .../python/globalindex/create_global_index.py | 2 +- .../data_evolution_global_index_scanner.py | 1 + .../python/read/search/full_text_read.py | 11 +- .../python/read/search/vector_search_read.py | 15 +- .../python/read/search/vector_search_scan.py | 15 +- .../globalindex/create_global_index.py | 3 + .../table/source/vector_search_builder.py | 2 +- .../pypaimon/tests/global_index_build_test.py | 129 ++++- .../pypaimon/tests/global_index_test.py | 70 +++ .../tests/lazy_search_execution_test.py | 111 +++++ .../tests/native_index_options_test.py | 176 +++++++ .../primary_key_index_definitions_test.py | 21 +- .../tests/vector_metric_consistency_test.py | 21 + .../tests/vector_search_filter_test.py | 11 +- .../sorted/SortedIndexTopoBuilder.java | 19 +- .../read/SparkDataEvolutionVectorRead.java | 9 +- .../SparkDataEvolutionVectorReadTest.java | 74 ++- ...he.paimon.globalindex.GlobalIndexerFactory | 18 + 56 files changed, 2010 insertions(+), 475 deletions(-) create mode 100644 paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java create mode 100644 paimon-python/pypaimon/execution/native/index_options.py create mode 100644 paimon-python/pypaimon/tests/native_index_options_test.py create mode 100644 paimon-spark/paimon-spark-common/src/test/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory diff --git a/docs/docs/multimodal-table/global-index.mdx b/docs/docs/multimodal-table/global-index.mdx index 4c8d96c10ce3..11f704be512f 100644 --- a/docs/docs/multimodal-table/global-index.mdx +++ b/docs/docs/multimodal-table/global-index.mdx @@ -65,6 +65,28 @@ database exists and that SQL uses that database. Load data before building index `global-index.enabled` defaults to `true`; setting it does not create index files. +Index declarations and field options use the same configuration names as +[primary-key indexes](../primary-key-table/global-index#options): + +```properties +global-index.btree.index-columns=category +global-index.ivf-pq.index-columns=embedding +ivf-pq.dimension=768 +fields.embedding.index-options={"nlist":64,"distance.metric":"cosine"} +``` + +Use the actual implementation identifier, such as `ivf-pq`, in the declaration. +Each listed column declares an independent index. The singular `index-column` +spelling remains an alias; `index-columns` takes precedence. Field JSON options +override implementation-level parameters and support short keys without repeating +the implementation prefix. Existing flat vector field options such as +`fields.embedding.dimension` and `fields.embedding.nlist` remain supported. + +These declarations do not schedule builds in standalone Paimon. Continue to use +the [index build commands](./global-index/manage-indexes#build-an-index), or the +index scheduling service provided by your catalog. Data Evolution indexes address +table-wide row IDs; primary-key indexes retain their bucket-local compaction lifecycle. + The example table below is shared by the BTree, Bitmap, Multivalue, Vector, and Full-Text guides. It contains three-dimensional sample vectors and is partitioned by `dt`, so the partition-scoped build and drop examples also apply. Create and diff --git a/docs/docs/multimodal-table/global-index/fm.mdx b/docs/docs/multimodal-table/global-index/fm.mdx index 79ffeb6e081a..7bfd0880b036 100644 --- a/docs/docs/multimodal-table/global-index/fm.mdx +++ b/docs/docs/multimodal-table/global-index/fm.mdx @@ -72,7 +72,8 @@ choose a suitable [search mode](./manage-indexes#coverage-and-freshness). The [visibility callback](./manage-indexes#wait-for-index-visibility) can wait for a separate index-building job to cover committed data; it does not build indexes. -For a primary-key table, configure `pk-fm.index.columns` instead. Primary-key FM indexes follow +For a primary-key table, declare `global-index.fm.index-columns` to enable index maintenance. +Primary-key FM indexes follow data compaction and scan uncovered files through the ordinary data path, so partial coverage does not make `CONTAINS` results incomplete. See [Primary-Key Indexes](../../primary-key-table/global-index). @@ -88,9 +89,10 @@ CALL sys.drop_global_index( ## Options -Global options can be passed to `create_global_index`. For a primary-key index, place the same -keys in `fields..pk-fm.index.options`; the `fm-index.` prefix may be omitted inside that -JSON object. +Global options can be passed to `create_global_index`. Both Data Evolution and primary-key +indexes accept field overrides in `fields..index-options`; the `fm-index.` prefix may +be omitted inside that JSON object. The legacy `fields..pk-fm.index.options` remains +supported when the shared field option is absent. | Option | Default | Description | |---|---|---| diff --git a/docs/docs/primary-key-table/global-index.mdx b/docs/docs/primary-key-table/global-index.mdx index a5c2a8d986d3..808935a3ff17 100644 --- a/docs/docs/primary-key-table/global-index.mdx +++ b/docs/docs/primary-key-table/global-index.mdx @@ -63,9 +63,9 @@ FM performs exact substring matching without a configured gram size and accepts byte length. Vector and Full Text expose dedicated ranked-search APIs in Spark, Flink, and Java. Full Text uses the fixed `full-text` implementation; Vector uses the configured ANN implementation. -Different columns in one table can use different index families. One column can occur in at most -one of `pk-vector.index.columns`, `pk-full-text.index.columns`, `pk-btree.index.columns`, -`pk-bitmap.index.columns`, `pk-multivalue.index.columns`, and `pk-fm.index.columns`. +Different columns in one table can use different index families. A primary-key table column can +occur in at most one `global-index..index-columns` declaration, including declarations +for different ANN implementations. ## Requirements @@ -76,8 +76,9 @@ All primary-key indexes require: - `pk-clustering-override = false`. - An indexed column supported by the selected index implementation. -Dynamic-bucket tables, append-only tables, compound indexes, and custom index families are not -supported by this configuration. +Primary-key index maintenance does not support dynamic-bucket tables, compound indexes, or +custom index families. Data Evolution append tables share the configuration names and use +their independent [global-index build lifecycle](../multimodal-table/global-index). @@ -142,7 +143,7 @@ CREATE TABLE indexed_orders ( ) WITH ( 'bucket' = '8', 'deletion-vectors.enabled' = 'true', - 'pk-btree.index.columns' = 'amount' + 'global-index.btree.index-columns' = 'amount' ); ``` @@ -158,7 +159,7 @@ TBLPROPERTIES ( 'primary-key' = 'id', 'bucket' = '8', 'deletion-vectors.enabled' = 'true', - 'pk-btree.index.columns' = 'amount' + 'global-index.btree.index-columns' = 'amount' ); ``` @@ -200,20 +201,19 @@ CREATE TABLE items ( ) WITH ( 'bucket' = '8', 'deletion-vectors.enabled' = 'true', - 'pk-vector.index.columns' = 'embedding', - 'fields.embedding.pk-vector.index.type' = 'ivf-flat', - 'fields.embedding.pk-vector.distance.metric' = 'cosine', - 'fields.embedding.pk-vector.index.options' = '{"nlist":"256"}', - 'pk-full-text.index.columns' = 'content', - 'fields.content.pk-full-text.index.options' = '{"tokenizer":"jieba"}', - 'pk-fm.index.columns' = 'raw_text', - 'fields.raw_text.pk-fm.index.options' = '{"partition-row-count":"100000"}', - 'pk-btree.index.columns' = 'amount', - 'fields.amount.pk-btree.index.options' = '{"block-size":"64 kb"}', - 'pk-bitmap.index.columns' = 'status', - 'fields.status.pk-bitmap.index.options' = '{"dictionary-block-size":"16 kb"}', - 'pk-multivalue.index.columns' = 'tags', - 'fields.tags.pk-multivalue.index.options' = '{"dictionary-block-size":"16 kb"}' + 'global-index.ivf-flat.index-columns' = 'embedding', + 'ivf-flat.distance.metric' = 'cosine', + 'fields.embedding.index-options' = '{"nlist":"256"}', + 'global-index.full-text.index-columns' = 'content', + 'fields.content.index-options' = '{"tokenizer":"jieba"}', + 'global-index.fm.index-columns' = 'raw_text', + 'fields.raw_text.index-options' = '{"partition-row-count":"100000"}', + 'global-index.btree.index-columns' = 'amount', + 'fields.amount.index-options' = '{"block-size":"64 kb"}', + 'global-index.bitmap.index-columns' = 'status', + 'fields.status.index-options' = '{"dictionary-block-size":"16 kb"}', + 'global-index.multivalue.index-columns' = 'tags', + 'fields.tags.index-options' = '{"dictionary-block-size":"16 kb"}' ); ``` @@ -239,20 +239,19 @@ TBLPROPERTIES ( 'primary-key' = 'id', 'bucket' = '8', 'deletion-vectors.enabled' = 'true', - 'pk-vector.index.columns' = 'embedding', - 'fields.embedding.pk-vector.index.type' = 'ivf-flat', - 'fields.embedding.pk-vector.distance.metric' = 'cosine', - 'fields.embedding.pk-vector.index.options' = '{"nlist":"256"}', - 'pk-full-text.index.columns' = 'content', - 'fields.content.pk-full-text.index.options' = '{"tokenizer":"jieba"}', - 'pk-fm.index.columns' = 'raw_text', - 'fields.raw_text.pk-fm.index.options' = '{"partition-row-count":"100000"}', - 'pk-btree.index.columns' = 'amount', - 'fields.amount.pk-btree.index.options' = '{"block-size":"64 kb"}', - 'pk-bitmap.index.columns' = 'status', - 'fields.status.pk-bitmap.index.options' = '{"dictionary-block-size":"16 kb"}', - 'pk-multivalue.index.columns' = 'tags', - 'fields.tags.pk-multivalue.index.options' = '{"dictionary-block-size":"16 kb"}' + 'global-index.ivf-flat.index-columns' = 'embedding', + 'ivf-flat.distance.metric' = 'cosine', + 'fields.embedding.index-options' = '{"nlist":"256"}', + 'global-index.full-text.index-columns' = 'content', + 'fields.content.index-options' = '{"tokenizer":"jieba"}', + 'global-index.fm.index-columns' = 'raw_text', + 'fields.raw_text.index-options' = '{"partition-row-count":"100000"}', + 'global-index.btree.index-columns' = 'amount', + 'fields.amount.index-options' = '{"block-size":"64 kb"}', + 'global-index.bitmap.index-columns' = 'status', + 'fields.status.index-options' = '{"dictionary-block-size":"16 kb"}', + 'global-index.multivalue.index-columns' = 'tags', + 'fields.tags.index-options' = '{"dictionary-block-size":"16 kb"}' ); ``` @@ -269,20 +268,10 @@ schema validation. | Option | Default | Description | |---|---|---| -| `pk-vector.index.columns` | Not set | Vector column to index. Exactly one vector column is currently supported. | -| `fields..pk-vector.index.type` | Required | ANN implementation, such as `ivf-flat`, `ivf-pq`, `ivf-sq`, `ivf-rq`, or `diskann`. | -| `fields..pk-vector.distance.metric` | `inner_product` | Distance metric: `l2`, `cosine`, or `inner_product`. | -| `fields..pk-vector.index.options` | Not set | JSON object containing build options for the selected ANN implementation. | -| `pk-full-text.index.columns` | Not set | Character column to index. Exactly one full-text column is currently supported. | -| `fields..pk-full-text.index.options` | Not set | JSON object containing native analyzer options. Unqualified keys are scoped to `full-text`. | -| `pk-fm.index.columns` | Not set | Comma-separated character columns which own independent exact FM indexes. | -| `fields..pk-fm.index.options` | Not set | JSON object containing FM build and read options. Unqualified keys are scoped to `fm-index`. | -| `pk-btree.index.columns` | Not set | Comma-separated columns which own independent BTree indexes. | -| `fields..pk-btree.index.options` | Not set | JSON object containing BTree build options. Unqualified keys are scoped to `btree-index`. | -| `pk-bitmap.index.columns` | Not set | Comma-separated columns which own independent Bitmap indexes. | -| `fields..pk-bitmap.index.options` | Not set | JSON object containing Bitmap build options. Unqualified keys are scoped to `bitmap-index`. | -| `pk-multivalue.index.columns` | Not set | Comma-separated `ARRAY` columns which own independent Multivalue indexes. | -| `fields..pk-multivalue.index.options` | Not set | JSON object containing Multivalue build options. Unqualified keys are scoped to `multivalue-index`. | +| `global-index..index-columns` | Not set | Comma-separated columns, each owning an independent index. Types include `btree`, `bitmap`, `multivalue`, `fm`, `full-text`, and actual ANN implementations such as `ivf-flat`, `ivf-pq`, `ivf-sq`, `ivf-rq`, or `diskann`. | +| `fields..index-options` | Not set | JSON object of field overrides. Short keys are scoped to the selected implementation; for example, `{"nlist":64,"distance.metric":"cosine"}` for IVF-PQ. | +| `.dimension` | Implementation default | Vector dimension for the selected ANN implementation. A fixed-length `VECTOR` column also carries its dimension in the schema. | +| `.distance.metric` | `inner_product` | Vector distance metric: `l2`, `cosine`, or `inner_product`. Field overrides take precedence. | | `multivalue-index.dictionary-block-size` | `16 kb` | Target dictionary block size. | | `multivalue-index.compression` | `none` | Dictionary-block compression algorithm. | | `multivalue-index.compression-level` | `1` | Dictionary-block compression level. | @@ -290,6 +279,21 @@ schema validation. | `vector-index.search-mode` | `fast` | Search mode for primary-key Vector queries. `full` and `detail` search uncovered files exactly. | | `full-text-index.search-mode` | `fast` | Search mode for primary-key Full Text queries. Only `fast` is currently supported. | +The plural `index-columns` spelling takes precedence over the singular +`global-index..index-column` alias. Both take precedence over legacy +`pk-.index.columns` declarations. For legacy Vector configurations, +`pk-vector.index.columns` and `fields..pk-vector.index.type` remain supported; +a declaration such as `global-index.ivf-pq.index-columns` supplies the implementation +directly and takes precedence over the legacy Vector declaration. + +Legacy `fields..pk-.index.options` and +`fields..pk-vector.distance.metric` remain supported. When present, +`fields..index-options` replaces the legacy JSON option object. Its values +override existing field parameters such as `fields.embedding.nlist`, then +implementation-level defaults such as `ivf-pq.nlist`. Short JSON keys use +`btree-index.`, `bitmap-index.`, `multivalue-index.`, `fm-index.`, +`full-text.`, or the actual ANN implementation prefix. + For algorithm-specific options, see the corresponding [BTree](../multimodal-table/global-index/btree), [Bitmap](../multimodal-table/global-index/bitmap), diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index c3202188dc09..36716b0c7720 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -38,8 +38,11 @@ import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.StringUtils; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode; + import javax.annotation.Nullable; +import java.io.IOException; import java.io.Serializable; import java.lang.reflect.Field; import java.time.Duration; @@ -49,6 +52,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -56,7 +60,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.TreeMap; import java.util.UUID; import java.util.stream.Collectors; @@ -3211,6 +3214,46 @@ public String toString() { "The batch size for lateral vector search. Each batch executes vector " + "topK search and table lookup for multiple query vectors."); + public static final ConfigOption GLOBAL_INDEX_BTREE_INDEX_COLUMNS = + key("global-index.btree.index-columns") + .stringType() + .noDefaultValue() + .withFallbackKeys("global-index.btree.index-column", "pk-btree.index.columns") + .withDescription("Comma-separated columns to index with BTree indexes."); + + public static final ConfigOption GLOBAL_INDEX_BITMAP_INDEX_COLUMNS = + key("global-index.bitmap.index-columns") + .stringType() + .noDefaultValue() + .withFallbackKeys("global-index.bitmap.index-column", "pk-bitmap.index.columns") + .withDescription("Comma-separated columns to index with Bitmap indexes."); + + public static final ConfigOption GLOBAL_INDEX_MULTIVALUE_INDEX_COLUMNS = + key("global-index.multivalue.index-columns") + .stringType() + .noDefaultValue() + .withFallbackKeys( + "global-index.multivalue.index-column", "pk-multivalue.index.columns") + .withDescription( + "Comma-separated ARRAY columns to index with Multivalue indexes."); + + public static final ConfigOption GLOBAL_INDEX_FULL_TEXT_INDEX_COLUMNS = + key("global-index.full-text.index-columns") + .stringType() + .noDefaultValue() + .withFallbackKeys( + "global-index.full-text.index-column", "pk-full-text.index.columns") + .withDescription( + "Comma-separated character columns to index with full-text indexes."); + + public static final ConfigOption GLOBAL_INDEX_FM_INDEX_COLUMNS = + key("global-index.fm.index-columns") + .stringType() + .noDefaultValue() + .withFallbackKeys("global-index.fm.index-column", "pk-fm.index.columns") + .withDescription("Comma-separated character columns to index with FM indexes."); + + // Retain these option keys for applications using the primary-key index API. public static final ConfigOption PK_VECTOR_INDEX_COLUMNS = key("pk-vector.index.columns") .stringType() @@ -5048,145 +5091,292 @@ public int vectorSearchLateralJoinBatchSize() { } public boolean primaryKeyVectorIndexEnabled() { - return options.getOptional(PK_VECTOR_INDEX_COLUMNS).isPresent(); + return !primaryKeyVectorIndexColumns().isEmpty(); } public boolean primaryKeyFullTextIndexEnabled() { - return options.getOptional(PK_FULL_TEXT_INDEX_COLUMNS).isPresent(); + return !primaryKeyFullTextIndexColumns().isEmpty(); } public boolean primaryKeyFMIndexEnabled() { - return options.getOptional(PK_FM_INDEX_COLUMNS).isPresent(); + return !primaryKeyFMIndexColumns().isEmpty(); } public List primaryKeyVectorIndexColumns() { - return primaryKeyIndexColumns(PK_VECTOR_INDEX_COLUMNS); + List types = vectorIndexTypes(); + if (!types.isEmpty()) { + return types.stream() + .flatMap(type -> globalIndexColumns(type).stream()) + .collect(Collectors.toList()); + } + return indexColumns(options.get(PK_VECTOR_INDEX_COLUMNS)); } public List primaryKeyBTreeIndexColumns() { - return primaryKeyIndexColumns(PK_BTREE_INDEX_COLUMNS); + return globalIndexColumns("btree"); } public List primaryKeyBitmapIndexColumns() { - return primaryKeyIndexColumns(PK_BITMAP_INDEX_COLUMNS); + return globalIndexColumns("bitmap"); } public List primaryKeyMultiValueIndexColumns() { - return primaryKeyIndexColumns(PK_MULTIVALUE_INDEX_COLUMNS); + return globalIndexColumns("multivalue"); } public List primaryKeyFullTextIndexColumns() { - return primaryKeyIndexColumns(PK_FULL_TEXT_INDEX_COLUMNS); + return globalIndexColumns("full-text"); } public List primaryKeyFMIndexColumns() { - return primaryKeyIndexColumns(PK_FM_INDEX_COLUMNS); + return globalIndexColumns("fm"); + } + + /** Index implementation identifiers declared with {@code global-index..index-columns}. */ + public List globalIndexTypes() { + String prefix = "global-index."; + return options.toMap().keySet().stream() + .filter( + k -> + k.startsWith(prefix) + && k.lastIndexOf('.') >= prefix.length() + && (k.endsWith(".index-columns") + || k.endsWith(".index-column"))) + .map(k -> k.substring(prefix.length(), k.lastIndexOf('.'))) + .distinct() + .sorted() + .collect(Collectors.toList()); } - private List primaryKeyIndexColumns(ConfigOption option) { - String columns = options.get(option); - if (columns == null) { - return Collections.emptyList(); - } - return Arrays.stream(columns.split(",", -1)).map(String::trim).collect(Collectors.toList()); + private static boolean isVectorIndexType(String type) { + return !Arrays.asList("btree", "bitmap", "multivalue", "full-text", "fm").contains(type); } - public Options primaryKeyBTreeIndexOptions(String column) { - return primaryKeySortedIndexOptions(column, "pk-btree", "btree-index."); + private List vectorIndexTypes() { + return globalIndexTypes().stream() + .filter(CoreOptions::isVectorIndexType) + .collect(Collectors.toList()); } - public Options primaryKeyBitmapIndexOptions(String column) { - return primaryKeySortedIndexOptions(column, "pk-bitmap", "bitmap-index."); + public List globalIndexColumns(String type) { + String columns = options.get("global-index." + type + ".index-columns"); + if (columns == null) { + columns = options.get("global-index." + type + ".index-column"); + } + if (columns == null) { + columns = options.get("pk-" + type + ".index.columns"); + } + if (columns != null) { + return indexColumns(columns); + } + return indexColumns(options.get(PK_VECTOR_INDEX_COLUMNS)).stream() + .filter( + column -> + type.equals( + options.get("fields." + column + ".pk-vector.index.type"))) + .collect(Collectors.toList()); } - public Options primaryKeyMultiValueIndexOptions(String column) { - return primaryKeySortedIndexOptions(column, "pk-multivalue", "multivalue-index."); + private static List indexColumns(@Nullable String columns) { + return columns == null + ? Collections.emptyList() + : Arrays.stream(columns.split(",", -1)) + .map(String::trim) + .collect(Collectors.toList()); } - public Options primaryKeyFMIndexOptions(String column) { - return primaryKeySortedIndexOptions(column, "pk-fm", "fm-index."); + public String globalIndexColumnsOptionKey(String type) { + if ("vector".equals(type) && !vectorIndexTypes().isEmpty()) { + return vectorIndexTypes().stream() + .map( + t -> + "global-index." + + t + + (options.get("global-index." + t + ".index-columns") + != null + ? ".index-columns" + : ".index-column")) + .collect(Collectors.joining(", ")); + } + for (String suffix : Arrays.asList(".index-columns", ".index-column")) { + String key = "global-index." + type + suffix; + if (options.get(key) != null) { + return key; + } + } + String legacy = "pk-" + type + ".index.columns"; + return options.get(legacy) != null ? legacy : "global-index." + type + ".index-columns"; } - public Options primaryKeyFullTextIndexOptions(String column) { - String optionKey = "fields." + column + ".pk-full-text.index.options"; - TreeMap resolved = new TreeMap<>(); - for (Map.Entry entry : toConfiguration().toMap().entrySet()) { - if (entry.getKey().startsWith("full-text.")) { - resolved.put(entry.getKey(), entry.getValue()); + /** + * Resolves implementation defaults and field overrides for either table mode. + * + *

The shared field JSON object takes precedence over the legacy JSON object. Short keys are + * qualified with the implementation's existing option prefix. + */ + public Options globalIndexOptions(@Nullable String type, String column) { + return globalIndexOptions(type, column, true); + } + + /** The same resolver, optionally retaining the absence of an explicitly configured metric. */ + public Options globalIndexOptions(@Nullable String type, String column, boolean defaultMetric) { + Options resolved = new Options(options.toMap(), options); + if (type == null) { + type = vectorIndexType(column); + if (type == null) { + return resolved; + } + } + boolean vector = isVectorIndexType(type); + String prefix = vector || "full-text".equals(type) ? type + "." : type + "-index."; + String fieldPrefix = "fields." + column + "."; + String sharedKey = fieldPrefix + "index-options"; + boolean shared = options.get(sharedKey) != null; + String optionKey = + shared + ? sharedKey + : fieldPrefix + "pk-" + (vector ? "vector" : type) + ".index.options"; + String metric = null; + if (vector) { + for (String key : + Arrays.asList( + fieldPrefix + "distance.metric", + fieldPrefix + "metric", + fieldPrefix + "pk-vector.distance.metric", + prefix + "distance.metric", + prefix + "metric", + "distance.metric", + "metric")) { + if (options.get(key) != null) { + metric = options.get(key); + break; + } } } String serialized = options.get(optionKey); - if (serialized == null || serialized.trim().isEmpty()) { - return new Options(resolved); + if (serialized != null && !serialized.trim().isEmpty()) { + for (Map.Entry entry : + parseIndexOptions(serialized, optionKey, type).entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + checkArgument( + key != null && !key.trim().isEmpty(), + "%s contains an empty option key.", + optionKey); + checkArgument( + value != null, "%s value for key %s must not be null.", optionKey, key); + String qualified = + key.startsWith(prefix) || key.startsWith("fields.") ? key : prefix + key; + boolean metricOption = + vector + && (qualified.equals(prefix + "metric") + || qualified.equals(prefix + "distance.metric") + || qualified.equals(fieldPrefix + "metric") + || qualified.equals(fieldPrefix + "distance.metric")); + String previous = resolved.get(qualified); + if (metricOption) { + value = normalizeVectorMetric(value); + previous = previous == null ? null : normalizeVectorMetric(previous); + } + checkArgument( + shared || previous == null || previous.equals(value), + "%s defines conflicting values for %s.", + optionKey, + qualified); + resolved.setString(qualified, value); + if (metricOption) { + metric = value; + } else if (vector && qualified.startsWith(prefix)) { + String shortKey = qualified.substring(prefix.length()); + if (shared) { + // Native vector implementations also accept the flat field syntax. + resolved.setString(fieldPrefix + shortKey, value); + } + } + } + } + if (vector && (metric != null || defaultMetric)) { + metric = normalizeVectorMetric(metric); + // Keep both accepted aliases consistent for implementations using either spelling. + resolved.setString(prefix + "metric", metric); + resolved.setString(prefix + "distance.metric", metric); + resolved.setString(fieldPrefix + "metric", metric); + resolved.setString(fieldPrefix + "distance.metric", metric); } + return resolved; + } - LinkedHashMap parsed; + private static LinkedHashMap parseIndexOptions( + String serialized, String optionKey, String type) { + if (!optionKey.endsWith(".index-options")) { + try { + return JsonSerdeUtil.parseJsonMap(serialized, String.class); + } catch (RuntimeException e) { + throw new IllegalArgumentException( + optionKey + " must be a JSON object of option key-value pairs.", e); + } + } try { - parsed = JsonSerdeUtil.parseJsonMap(serialized, String.class); - } catch (RuntimeException e) { + JsonNode root = JsonSerdeUtil.OBJECT_MAPPER_INSTANCE.readTree(serialized); + checkArgument( + root != null && root.isObject(), + "%s must be a JSON object of option key-value pairs.", + optionKey); + LinkedHashMap parsed = new LinkedHashMap<>(); + Iterator> fields = root.fields(); + while (fields.hasNext()) { + Map.Entry entry = fields.next(); + checkArgument( + !entry.getValue().isNull(), + "%s value for key %s must not be null.", + optionKey, + entry.getKey()); + if ("full-text".equals(type) && entry.getValue().isArray()) { + List values = new ArrayList<>(); + entry.getValue().forEach(value -> values.add(value.asText())); + parsed.put(entry.getKey(), String.join(";", values)); + } else { + parsed.put( + entry.getKey(), + entry.getValue().isValueNode() + ? entry.getValue().asText() + : entry.getValue().toString()); + } + } + return parsed; + } catch (IOException e) { throw new IllegalArgumentException( optionKey + " must be a JSON object of option key-value pairs.", e); } + } - for (Map.Entry entry : parsed.entrySet()) { - String key = entry.getKey(); - String value = entry.getValue(); - checkArgument( - key != null && !key.trim().isEmpty(), - "%s contains an empty option key.", - optionKey); - checkArgument(value != null, "%s value for key %s must not be null.", optionKey, key); - String qualifiedKey = key.startsWith("full-text.") ? key : "full-text." + key; - String previous = resolved.put(qualifiedKey, value); - checkArgument( - previous == null || previous.equals(value), - "%s defines conflicting values for %s.", - optionKey, - qualifiedKey); - } + public Options primaryKeyBTreeIndexOptions(String column) { + return primaryKeySortedIndexOptions(column, "btree"); + } + + public Options primaryKeyBitmapIndexOptions(String column) { + return primaryKeySortedIndexOptions(column, "bitmap"); + } + + public Options primaryKeyMultiValueIndexOptions(String column) { + return primaryKeySortedIndexOptions(column, "multivalue"); + } + + public Options primaryKeyFMIndexOptions(String column) { + return primaryKeySortedIndexOptions(column, "fm"); + } + + public Options primaryKeyFullTextIndexOptions(String column) { + Map resolved = globalIndexOptions("full-text", column).toMap(); + resolved.keySet().removeIf(key -> !key.startsWith("full-text.")); return new Options(resolved); } - private Options primaryKeySortedIndexOptions( - String column, String optionFamily, String algorithmPrefix) { - Options resolved = new Options(toConfiguration().toMap()); + private Options primaryKeySortedIndexOptions(String column, String type) { + Options resolved = globalIndexOptions(type, column); resolved.remove("sorted-index.records-per-file"); resolved.remove("sorted-index.records-per-range"); - String optionKey = "fields." + column + "." + optionFamily + ".index.options"; - String serialized = options.get(optionKey); - if (serialized == null || serialized.trim().isEmpty()) { - return resolved; - } - - LinkedHashMap parsed; - try { - parsed = JsonSerdeUtil.parseJsonMap(serialized, String.class); - } catch (RuntimeException e) { - throw new IllegalArgumentException( - optionKey + " must be a JSON object of option key-value pairs.", e); - } - - for (Map.Entry entry : parsed.entrySet()) { - String key = entry.getKey(); - String value = entry.getValue(); - checkArgument( - key != null && !key.trim().isEmpty(), - "%s contains an empty option key.", - optionKey); - checkArgument(value != null, "%s value for key %s must not be null.", optionKey, key); - String qualifiedKey = - key.startsWith(algorithmPrefix) || key.startsWith("fields.") - ? key - : algorithmPrefix + key; - String previous = resolved.get(qualifiedKey); - checkArgument( - previous == null || previous.equals(value), - "%s defines conflicting values for %s.", - optionKey, - qualifiedKey); - resolved.setString(qualifiedKey, value); - } return resolved; } @@ -5194,87 +5384,59 @@ public String primaryKeyVectorIndexColumn() { List columns = primaryKeyVectorIndexColumns(); checkArgument( columns.size() == 1, - "pk-vector.index.columns must contain exactly one column in the first release, but is %s.", + "%s must contain exactly one column in the first release, but is %s.", + globalIndexColumnsOptionKey("vector"), columns); return columns.get(0); } @Nullable public String primaryKeyVectorIndexType(String column) { - return options.get("fields." + column + ".pk-vector.index.type"); + return vectorIndexType(column); } @Nullable - private String primaryKeyVectorIndexOptionsJson(String column) { - return options.get("fields." + column + ".pk-vector.index.options"); - } - - public Options primaryKeyVectorIndexOptions(String column) { - Options resolved = new Options(toConfiguration().toMap()); - for (Map.Entry option : - primaryKeyVectorAlgorithmOptions(column).entrySet()) { - resolved.setString(option.getKey(), option.getValue()); + public String vectorIndexType(String column) { + List declared = + vectorIndexTypes().stream() + .filter(type -> globalIndexColumns(type).contains(column)) + .collect(Collectors.toList()); + if (!declared.isEmpty()) { + return declared.size() == 1 ? declared.get(0) : null; + } + for (String key : + Arrays.asList( + "fields." + column + ".pk-vector.index.type", + "index_type", + "index-type", + "vector.index-type", + "fields." + column + ".index-type")) { + String type = options.get(key); + if (type != null) { + return type.toLowerCase(Locale.ROOT).trim(); + } } - return resolved; + return null; } - private Map primaryKeyVectorAlgorithmOptions(String column) { - String indexTypeKey = "fields." + column + ".pk-vector.index.type"; - String indexOptionsKey = "fields." + column + ".pk-vector.index.options"; - String algorithm = primaryKeyVectorIndexType(column); + public Options primaryKeyVectorIndexOptions(String column) { + String type = primaryKeyVectorIndexType(column); checkArgument( - algorithm != null && !algorithm.trim().isEmpty(), - "%s must be configured before resolving index options.", - indexTypeKey); - TreeMap algorithmOptions = new TreeMap<>(); - String algorithmPrefix = algorithm + "."; - String fieldPrefix = "fields." + column + "."; - for (Map.Entry entry : toConfiguration().toMap().entrySet()) { - if (entry.getKey().startsWith(algorithmPrefix) - || (entry.getKey().startsWith(fieldPrefix) - && !entry.getKey().startsWith(fieldPrefix + "pk-vector."))) { - algorithmOptions.put(entry.getKey(), entry.getValue()); - } - } - String serialized = primaryKeyVectorIndexOptionsJson(column); - if (serialized != null && !serialized.trim().isEmpty()) { - LinkedHashMap parsed; - try { - parsed = JsonSerdeUtil.parseJsonMap(serialized, String.class); - } catch (RuntimeException e) { - throw new IllegalArgumentException( - indexOptionsKey + " must be a JSON object of option key-value pairs.", e); - } - for (Map.Entry entry : parsed.entrySet()) { - String key = entry.getKey(); - String value = entry.getValue(); - checkArgument( - key != null && !key.trim().isEmpty(), - "%s contains an empty option key.", - indexOptionsKey); - checkArgument( - value != null, - "%s value for key %s must not be null.", - indexOptionsKey, - key); - String qualifiedKey = - key.startsWith(algorithmPrefix) || key.startsWith("fields.") - ? key - : algorithmPrefix + key; - String previous = algorithmOptions.put(qualifiedKey, value); - checkArgument( - previous == null || previous.equals(value), - "%s defines conflicting values for %s.", - indexOptionsKey, - qualifiedKey); - } - } - algorithmOptions.put(algorithmPrefix + "metric", primaryKeyVectorDistanceMetric(column)); - return algorithmOptions; + type != null && !type.trim().isEmpty(), + "fields.%s.pk-vector.index.type must be configured before resolving index options.", + column); + return globalIndexOptions(type, column); } public String primaryKeyVectorDistanceMetric(String column) { - String metric = options.get("fields." + column + ".pk-vector.distance.metric"); + String type = primaryKeyVectorIndexType(column); + return type == null + ? normalizeVectorMetric( + options.get("fields." + column + ".pk-vector.distance.metric")) + : globalIndexOptions(type, column).get(type + ".metric"); + } + + private static String normalizeVectorMetric(String metric) { return (metric == null ? "inner_product" : metric) .toLowerCase(Locale.ROOT) .replace('-', '_'); diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexer.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexer.java index 9792602393dd..795fc8dd164b 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexer.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexer.java @@ -18,6 +18,7 @@ package org.apache.paimon.globalindex; +import org.apache.paimon.CoreOptions; import org.apache.paimon.globalindex.io.GlobalIndexFileReader; import org.apache.paimon.globalindex.io.GlobalIndexFileWriter; import org.apache.paimon.options.Options; @@ -50,6 +51,11 @@ GlobalIndexReader createReader( ExecutorService executor); static GlobalIndexer create(String type, List indexFields, Options options) { - return GlobalIndexerFactoryUtils.load(type).create(indexFields, options); + Options resolved = options; + if (!indexFields.isEmpty()) { + // The leading field owns the index; remaining fields are indexed payload columns. + resolved = new CoreOptions(options).globalIndexOptions(type, indexFields.get(0).name()); + } + return GlobalIndexerFactoryUtils.load(type).create(indexFields, resolved); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexScanner.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexScanner.java index e294cc09d118..34bcabf259dc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexScanner.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexScanner.java @@ -504,8 +504,7 @@ private Collection createReaders( for (Map.Entry>> entry : group.metas.entrySet()) { String indexType = entry.getKey(); Map> metas = entry.getValue(); - GlobalIndexerFactory globalIndexerFactory = GlobalIndexerFactoryUtils.load(indexType); - GlobalIndexer globalIndexer = globalIndexerFactory.create(indexFields, options); + GlobalIndexer globalIndexer = GlobalIndexer.create(indexType, indexFields, options); List> futures = new ArrayList<>(metas.size()); for (Map.Entry> rangeMetas : metas.entrySet()) { diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexQuery.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexQuery.java index af6a1b03dc2c..382d4016b70d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexQuery.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexQuery.java @@ -18,6 +18,7 @@ package org.apache.paimon.globalindex; +import org.apache.paimon.CoreOptions; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.globalindex.DataEvolutionGlobalIndexScanner.IndexMetaFileGroup; @@ -228,7 +229,6 @@ private static CompositeCandidate selectComposite( Predicate predicate, Map> groups, Options options) { CompositeCandidate selected = null; Set> seen = new HashSet<>(); - long budget = options.get(BTreeIndexOptions.BTREE_INDEX_FALLBACK_SCAN_MAX_SIZE).getBytes(); for (List fieldGroups : groups.values()) { for (IndexGroup group : fieldGroups) { if (!group.isCompositeBTree() || !seen.add(group.indexFields())) { @@ -239,6 +239,11 @@ private static CompositeCandidate selectComposite( if (!planned.isPresent()) { continue; } + long budget = + new CoreOptions(options) + .globalIndexOptions(group.type, group.field.name()) + .get(BTreeIndexOptions.BTREE_INDEX_FALLBACK_SCAN_MAX_SIZE) + .getBytes(); CompositeBTreePredicate.Plan plan = planned.get(); List definition = groups.get(group.field.id()).stream() @@ -532,8 +537,7 @@ private Optional evaluateWithExecutor( supported = true; continue; } - GlobalIndexer indexer = - GlobalIndexerFactoryUtils.load(group.type).create(group.indexFields(), options); + GlobalIndexer indexer = GlobalIndexer.create(group.type, group.indexFields(), options); try (GlobalIndexReader reader = indexer.createReader( meta -> fileIO.newInputStream(meta.filePath()), diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java index 9a5f62cb9797..19a8ce457557 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java @@ -18,6 +18,7 @@ package org.apache.paimon.globalindex.sorted; +import org.apache.paimon.CoreOptions; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.InternalRow.FieldGetter; @@ -60,7 +61,7 @@ public class SortedGlobalIndexWriter implements Serializable { private final FileStoreTable table; private final RowType rowType; private final Options options; - private final long recordsPerRange; + private Options indexOptions; private List indexFields; private GlobalIndexKeyExtractor keyExtractor; @@ -74,8 +75,7 @@ public SortedGlobalIndexWriter(Table table, String indexType, Options options) { this.table = (FileStoreTable) table; this.rowType = this.table.rowType(); this.options = options; - this.recordsPerRange = - (long) (options.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE) * FLOATING); + this.indexOptions = options; } public SortedGlobalIndexWriter withIndexFields(List fieldNames) { @@ -89,7 +89,9 @@ public SortedGlobalIndexWriter withIndexFields(List fieldNames) { table.fullName()); indexFields.add(rowType.getField(name)); } - GlobalIndexer indexer = GlobalIndexer.create(indexType, indexFields, options); + this.indexOptions = + new CoreOptions(options).globalIndexOptions(indexType, indexFields.get(0).name()); + GlobalIndexer indexer = GlobalIndexer.create(indexType, indexFields, indexOptions); checkArgument( indexer instanceof SortedGlobalIndexer, "Index algorithm %s does not expose sorted index keys.", @@ -103,7 +105,12 @@ public GlobalIndexKeyExtractor keyExtractor() { } public long recordsPerRange() { - return recordsPerRange; + return (long) + (indexOptions.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE) * FLOATING); + } + + public Options indexOptions() { + return indexOptions; } public List buildForSinglePartition( @@ -128,13 +135,14 @@ public List buildForSinglePartition( /** Creates a task writer which owns file rotation and source-row coverage semantics. */ public SortedSingleColumnIndexWriter createTaskWriter(Range rowRange) throws IOException { if (keyExtractor.isIdentity()) { - return new SortedSingleColumnIndexWriter(recordsPerRange, this::createWriter); + return new SortedSingleColumnIndexWriter(recordsPerRange(), this::createWriter); } return SortedSingleColumnIndexWriter.forSourceRowCount(rowRange.count(), createWriter()); } public GlobalIndexSingleColumnWriter createWriter() throws IOException { - GlobalIndexWriter indexWriter = createIndexWriter(table, indexType, indexFields, options); + GlobalIndexWriter indexWriter = + createIndexWriter(table, indexType, indexFields, indexOptions); if (!(indexWriter instanceof GlobalIndexSingleColumnWriter)) { throw new RuntimeException( "Unexpected implementation, the index writer of " diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java b/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java index f7fef0e9f13d..ce31c5de68f5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java @@ -51,12 +51,12 @@ public static PrimaryKeyIndexDefinitions create(TableSchema schema) { List multiValueColumns = options.primaryKeyMultiValueIndexColumns(); List fullTextColumns = options.primaryKeyFullTextIndexColumns(); List fmColumns = options.primaryKeyFMIndexColumns(); - validateNoDuplicates(vectorColumns, CoreOptions.PK_VECTOR_INDEX_COLUMNS.key()); - validateNoDuplicates(btreeColumns, CoreOptions.PK_BTREE_INDEX_COLUMNS.key()); - validateNoDuplicates(bitmapColumns, CoreOptions.PK_BITMAP_INDEX_COLUMNS.key()); - validateNoDuplicates(multiValueColumns, CoreOptions.PK_MULTIVALUE_INDEX_COLUMNS.key()); - validateNoDuplicates(fullTextColumns, CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key()); - validateNoDuplicates(fmColumns, CoreOptions.PK_FM_INDEX_COLUMNS.key()); + validateNoDuplicates(vectorColumns, options.globalIndexColumnsOptionKey("vector")); + validateNoDuplicates(btreeColumns, options.globalIndexColumnsOptionKey("btree")); + validateNoDuplicates(bitmapColumns, options.globalIndexColumnsOptionKey("bitmap")); + validateNoDuplicates(multiValueColumns, options.globalIndexColumnsOptionKey("multivalue")); + validateNoDuplicates(fullTextColumns, options.globalIndexColumnsOptionKey("full-text")); + validateNoDuplicates(fmColumns, options.globalIndexColumnsOptionKey("fm")); validateOneIndexPerColumn( vectorColumns, btreeColumns, diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManagerUtils.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManagerUtils.java index 52adcf42b325..c7642c44865b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManagerUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManagerUtils.java @@ -804,7 +804,7 @@ static void assertNotRenamingBlobColumn(List fields, String[] fieldNa static void assertNotUpdatingPrimaryKeyIndexColumn( TableSchema schema, String[] fieldNames, String operation) { - if (fieldNames.length > 1) { + if (fieldNames.length > 1 || schema.primaryKeys().isEmpty()) { return; } String fieldName = fieldNames[0]; diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index e279d6479587..71cf03e8ee71 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -403,10 +403,13 @@ public static void validateTableSchema(TableSchema schema, Set dynamicOp fieldNamesSpecifiedAsVector.isEmpty(), "Some of the columns specified as vector-field are unknown."); - validatePrimaryKeyIndexColumns(options); - validatePrimaryKeySortedIndexes(schema, options); - validatePrimaryKeyVectorIndex(schema, options); - validatePrimaryKeyFullTextIndex(schema, options); + validateGlobalIndexColumns(schema, options); + if (!options.dataEvolutionEnabled()) { + validatePrimaryKeyIndexColumns(options); + validatePrimaryKeySortedIndexes(schema, options); + validatePrimaryKeyVectorIndex(schema, options); + validatePrimaryKeyFullTextIndex(schema, options); + } validateMergeFunctionFactory(schema); @@ -1226,6 +1229,33 @@ private static void validateForDeletionVectors(CoreOptions options) { } } + private static void validateGlobalIndexColumns(TableSchema schema, CoreOptions options) { + for (String type : options.globalIndexTypes()) { + checkArgument(!type.trim().isEmpty(), "Global index type must not be empty."); + List columns = options.globalIndexColumns(type); + String optionKey = options.globalIndexColumnsOptionKey(type); + validateNoDuplicatePrimaryKeyIndexColumns(columns, optionKey); + for (String column : columns) { + checkArgument(!column.isEmpty(), "%s must contain non-empty columns.", optionKey); + checkArgument( + schema.nameToFieldMap().containsKey(column), + "%s entry '%s' must reference an existing column.", + optionKey, + column); + options.globalIndexOptions(type, column); + } + } + if (options.dataEvolutionEnabled()) { + for (String type : + Arrays.asList("vector", "btree", "bitmap", "multivalue", "full-text", "fm")) { + checkArgument( + !schema.options().containsKey("pk-" + type + ".index.columns"), + "pk-%s.index.columns requires a primary-key table; use global-index..index-columns for Data Evolution tables.", + type); + } + } + } + private static void validatePrimaryKeyVectorIndex(TableSchema schema, CoreOptions options) { if (!options.primaryKeyVectorIndexEnabled()) { return; @@ -1234,17 +1264,20 @@ private static void validatePrimaryKeyVectorIndex(TableSchema schema, CoreOption List indexColumns = options.primaryKeyVectorIndexColumns(); checkArgument( new HashSet<>(indexColumns).size() == indexColumns.size(), - "pk-vector.index.columns must not contain duplicate columns, but is %s.", + "%s must not contain duplicate columns, but is %s.", + options.globalIndexColumnsOptionKey("vector"), indexColumns); checkArgument( indexColumns.size() == 1, - "pk-vector.index.columns must contain exactly one column in the first release, but is %s.", + "%s must contain exactly one column in the first release, but is %s.", + options.globalIndexColumnsOptionKey("vector"), indexColumns); String indexColumn = indexColumns.get(0); String indexType = options.primaryKeyVectorIndexType(indexColumn); checkArgument( !StringUtils.isNullOrWhitespaceOnly(indexColumn), - "pk-vector.index.columns must contain a non-empty column."); + "%s must contain a non-empty column.", + options.globalIndexColumnsOptionKey("vector")); checkArgument( !StringUtils.isNullOrWhitespaceOnly(indexType), "fields.%s.pk-vector.index.type must be configured when a primary-key vector index is defined.", @@ -1276,12 +1309,14 @@ private static void validatePrimaryKeyVectorIndex(TableSchema schema, CoreOption .orElse(null); checkArgument( vectorField != null && vectorField.type().getTypeRoot() == VECTOR, - "pk-vector.index.columns entry '%s' must reference a VECTOR column.", + "%s entry '%s' must reference a VECTOR column.", + options.globalIndexColumnsOptionKey("vector"), indexColumn); checkArgument( ((VectorType) vectorField.type()).getElementType().getTypeRoot() == DataTypeRoot.FLOAT, - "pk-vector.index.columns entry '%s' must use FLOAT elements.", + "%s entry '%s' must use FLOAT elements.", + options.globalIndexColumnsOptionKey("vector"), indexColumn); checkArgument( Arrays.asList("l2", "cosine", "inner_product") @@ -1300,13 +1335,13 @@ private static void validatePrimaryKeyFullTextIndex(TableSchema schema, CoreOpti checkArgument( indexColumns.size() == 1, "%s must contain exactly one column in the first release, but is %s.", - CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key(), + options.globalIndexColumnsOptionKey("full-text"), indexColumns); String indexColumn = indexColumns.get(0); checkArgument( !StringUtils.isNullOrWhitespaceOnly(indexColumn), "%s must contain a non-empty column.", - CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key()); + options.globalIndexColumnsOptionKey("full-text")); checkArgument( !schema.primaryKeys().isEmpty(), "Primary-key full-text index requires a primary-key table."); @@ -1327,13 +1362,13 @@ private static void validatePrimaryKeyFullTextIndex(TableSchema schema, CoreOpti checkArgument( schema.nameToFieldMap().containsKey(indexColumn), "%s entry '%s' must reference an existing column.", - CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key(), + options.globalIndexColumnsOptionKey("full-text"), indexColumn); DataTypeRoot typeRoot = schema.nameToFieldMap().get(indexColumn).type().getTypeRoot(); checkArgument( typeRoot == DataTypeRoot.CHAR || typeRoot == DataTypeRoot.VARCHAR, "%s entry '%s' must reference a CHAR/VARCHAR/STRING column.", - CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key(), + options.globalIndexColumnsOptionKey("full-text"), indexColumn); options.primaryKeyFullTextIndexOptions(indexColumn); } @@ -1346,16 +1381,17 @@ private static void validatePrimaryKeyIndexColumns(CoreOptions options) { List fullTextColumns = options.primaryKeyFullTextIndexColumns(); List fmColumns = options.primaryKeyFMIndexColumns(); validateNoDuplicatePrimaryKeyIndexColumns( - vectorColumns, CoreOptions.PK_VECTOR_INDEX_COLUMNS.key()); + vectorColumns, options.globalIndexColumnsOptionKey("vector")); + validateNoDuplicatePrimaryKeyIndexColumns( + btreeColumns, options.globalIndexColumnsOptionKey("btree")); validateNoDuplicatePrimaryKeyIndexColumns( - btreeColumns, CoreOptions.PK_BTREE_INDEX_COLUMNS.key()); + bitmapColumns, options.globalIndexColumnsOptionKey("bitmap")); validateNoDuplicatePrimaryKeyIndexColumns( - bitmapColumns, CoreOptions.PK_BITMAP_INDEX_COLUMNS.key()); + multiValueColumns, options.globalIndexColumnsOptionKey("multivalue")); validateNoDuplicatePrimaryKeyIndexColumns( - multiValueColumns, CoreOptions.PK_MULTIVALUE_INDEX_COLUMNS.key()); + fullTextColumns, options.globalIndexColumnsOptionKey("full-text")); validateNoDuplicatePrimaryKeyIndexColumns( - fullTextColumns, CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key()); - validateNoDuplicatePrimaryKeyIndexColumns(fmColumns, CoreOptions.PK_FM_INDEX_COLUMNS.key()); + fmColumns, options.globalIndexColumnsOptionKey("fm")); Set indexedColumns = new HashSet<>(); validateUniquePrimaryKeyIndexColumns(indexedColumns, vectorColumns); @@ -1414,17 +1450,19 @@ private static void validatePrimaryKeySortedIndexes(TableSchema schema, CoreOpti validatePrimaryKeySortedIndexColumns( schema, options.primaryKeyBTreeIndexColumns(), - CoreOptions.PK_BTREE_INDEX_COLUMNS.key()); + options.globalIndexColumnsOptionKey("btree")); validatePrimaryKeySortedIndexColumns( schema, options.primaryKeyBitmapIndexColumns(), - CoreOptions.PK_BITMAP_INDEX_COLUMNS.key()); + options.globalIndexColumnsOptionKey("bitmap")); validatePrimaryKeySortedIndexColumns( schema, options.primaryKeyMultiValueIndexColumns(), - CoreOptions.PK_MULTIVALUE_INDEX_COLUMNS.key()); + options.globalIndexColumnsOptionKey("multivalue")); validatePrimaryKeySortedIndexColumns( - schema, options.primaryKeyFMIndexColumns(), CoreOptions.PK_FM_INDEX_COLUMNS.key()); + schema, + options.primaryKeyFMIndexColumns(), + options.globalIndexColumnsOptionKey("fm")); Map fields = schema.nameToFieldMap(); for (String column : options.primaryKeyBTreeIndexColumns()) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java index 5163bb580ef0..bf7409aa35f5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java @@ -29,7 +29,6 @@ import org.apache.paimon.globalindex.GlobalIndexIOMeta; import org.apache.paimon.globalindex.GlobalIndexReader; import org.apache.paimon.globalindex.GlobalIndexer; -import org.apache.paimon.globalindex.GlobalIndexerFactoryUtils; import org.apache.paimon.globalindex.OffsetGlobalIndexReader; import org.apache.paimon.globalindex.ScoredGlobalIndexResult; import org.apache.paimon.globalindex.VectorGlobalIndexer; @@ -38,6 +37,7 @@ import org.apache.paimon.index.GlobalIndexMeta; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.index.IndexPathFactory; +import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.predicate.BatchVectorSearch; import org.apache.paimon.predicate.Predicate; @@ -129,20 +129,20 @@ protected GlobalIndexer createGlobalIndexer(List splits) } protected GlobalIndexer createGlobalIndexer(String indexType) { - return GlobalIndexerFactoryUtils.load(indexType) - .create( - Collections.singletonList(vectorColumn), - table.coreOptions().toConfiguration()); + return GlobalIndexer.create( + indexType, + Collections.singletonList(vectorColumn), + table.coreOptions().toConfiguration()); } private GlobalIndexer createGlobalIndexer(String indexType, GlobalIndexMeta meta) { if (meta.extraFieldIds() == null) { return createGlobalIndexer(indexType); } - return GlobalIndexerFactoryUtils.load(indexType) - .create( - meta.getIndexedFields(table.rowType()), - table.coreOptions().toConfiguration()); + return GlobalIndexer.create( + indexType, + meta.getIndexedFields(table.rowType()), + table.coreOptions().toConfiguration()); } /** @@ -780,6 +780,20 @@ private String configuredRawSearchMetric() { @Nullable private String configuredRawSearchMetric(Map options) { String fieldPrefix = "fields." + vectorColumn.name() + "."; + if (options.containsKey(fieldPrefix + "index-options")) { + CoreOptions configured = new CoreOptions(new Options(table.options(), options)); + String type = configured.primaryKeyVectorIndexType(vectorColumn.name()); + if (type == null) { + type = "vector"; + } + String configuredMetric = + configured + .globalIndexOptions(type, vectorColumn.name(), false) + .get(type + ".metric"); + if (configuredMetric != null) { + return configuredMetric; + } + } String metric = option(options, fieldPrefix + "distance.metric"); if (metric == null) { metric = option(options, fieldPrefix + "metric"); @@ -826,11 +840,14 @@ private static boolean isRawSearchMetric(String metric) { } protected int configuredRefineFactor(String indexType) { + Options resolved = + new CoreOptions( + new Options( + table == null ? Collections.emptyMap() : table.options(), + options)) + .globalIndexOptions(indexType, vectorColumn.name()); return VectorSearchRefineOptions.resolve( - options, - table == null ? Collections.emptyMap() : table.options(), - vectorColumn.name(), - indexType); + options, resolved.toMap(), vectorColumn.name(), indexType); } private static IndexFileMeta firstVectorIndexFile(List splits) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java index fd02d0011c40..8c3b738985e1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java @@ -157,7 +157,8 @@ public BatchVectorRead newBatchVectorRead() { } protected boolean isPrimaryKeyVectorSearch() { - return vectorColumn != null + return !table.schema().primaryKeys().isEmpty() + && vectorColumn != null && table.coreOptions().primaryKeyVectorIndexColumns().contains(vectorColumn.name()); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java index 1ed32cb51f00..06dd143dd85f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java @@ -28,7 +28,6 @@ import org.apache.paimon.globalindex.GlobalIndexReader; import org.apache.paimon.globalindex.GlobalIndexResult; import org.apache.paimon.globalindex.GlobalIndexer; -import org.apache.paimon.globalindex.GlobalIndexerFactoryUtils; import org.apache.paimon.globalindex.OffsetGlobalIndexReader; import org.apache.paimon.globalindex.ScoredGlobalIndexResult; import org.apache.paimon.globalindex.io.GlobalIndexFileReader; @@ -392,15 +391,15 @@ private GlobalIndexer createIndexer(IndexFileMeta file, DataField textColumn) { String indexType = file.indexType(); GlobalIndexMeta meta = checkNotNull(file.globalIndexMeta()); if (meta.extraFieldIds() != null) { - return GlobalIndexerFactoryUtils.load(indexType) - .create( - meta.getIndexedFields(table.rowType()), - table.coreOptions().toConfiguration()); + return GlobalIndexer.create( + indexType, + meta.getIndexedFields(table.rowType()), + table.coreOptions().toConfiguration()); } - return GlobalIndexerFactoryUtils.load(indexType) - .create( - Collections.singletonList(textColumn), - table.coreOptions().toConfiguration()); + return GlobalIndexer.create( + indexType, + Collections.singletonList(textColumn), + table.coreOptions().toConfiguration()); } private CompletableFuture> eval( diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java index 876f49d8226a..f088fcee1830 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java @@ -26,6 +26,7 @@ import org.apache.paimon.index.IndexFileHandler; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.manifest.IndexManifestEntry; +import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.table.FileStoreTable; @@ -40,7 +41,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -293,26 +293,8 @@ private String vectorIndexType(List indexFiles) { @Nullable private String configuredVectorIndexType() { - String indexType = option("index_type"); - if (indexType == null) { - indexType = option("index-type"); - } - if (indexType == null) { - indexType = option("vector.index-type"); - } - if (indexType == null) { - indexType = option("fields." + vectorColumn.name() + ".index-type"); - } - return indexType; - } - - @Nullable - private String option(String key) { - String value = options.get(key); - if (value == null) { - value = table.options().get(key); - } - return value == null ? null : value.toLowerCase(Locale.ROOT).trim(); + return new CoreOptions(new Options(table.options(), options)) + .vectorIndexType(vectorColumn.name()); } private static boolean containsField(GlobalIndexMeta meta, int fieldId) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java index 6a632ed64607..f4ccd0beb9a8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java @@ -144,7 +144,10 @@ public PrimaryKeyVectorRead( normalize(table.coreOptions().primaryKeyVectorDistanceMetric(vectorField.name())); this.refineFactor = VectorSearchRefineOptions.resolve( - this.searchOptions, table.options(), vectorField.name(), indexType); + this.searchOptions, + this.indexOptions.toMap(), + vectorField.name(), + indexType); this.indexedLimit = VectorSearchRefineOptions.searchLimit(limit, refineFactor); this.filter = filter; } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/RawFullTextReadImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/RawFullTextReadImpl.java index 58cb5a59e9c1..91271ab95d6e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/RawFullTextReadImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/RawFullTextReadImpl.java @@ -27,7 +27,6 @@ import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; import org.apache.paimon.globalindex.GlobalIndexWriter; import org.apache.paimon.globalindex.GlobalIndexer; -import org.apache.paimon.globalindex.GlobalIndexerFactoryUtils; import org.apache.paimon.globalindex.ResultEntry; import org.apache.paimon.globalindex.ScoredGlobalIndexResult; import org.apache.paimon.globalindex.io.GlobalIndexFileReader; @@ -211,8 +210,8 @@ private Map createRawFullTextIndexes( indexType = checkNotNull(fallbackIndexType); } GlobalIndexer globalIndexer = - GlobalIndexerFactoryUtils.load(indexType) - .create(Collections.singletonList(textColumn), rawSearchOptions()); + GlobalIndexer.create( + indexType, Collections.singletonList(textColumn), rawSearchOptions()); try { RawFullTextIndexFileWriter fileWriter = new RawFullTextIndexFileWriter(column); GlobalIndexWriter indexWriter = globalIndexer.createWriter(fileWriter); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java index b5e572b8752b..897cea418531 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java @@ -155,7 +155,8 @@ public VectorRead newVectorRead() { } protected boolean isPrimaryKeyVectorSearch() { - return vectorColumn != null + return !table.schema().primaryKeys().isEmpty() + && vectorColumn != null && table.coreOptions().primaryKeyVectorIndexColumns().contains(vectorColumn.name()); } diff --git a/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java new file mode 100644 index 000000000000..906fa0101e94 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon; + +import org.apache.paimon.globalindex.GlobalIndexer; +import org.apache.paimon.globalindex.SortedGlobalIndexer; +import org.apache.paimon.globalindex.VectorGlobalIndexer; +import org.apache.paimon.index.pk.PrimaryKeyIndexDefinition; +import org.apache.paimon.index.pk.PrimaryKeyIndexDefinitions; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.SchemaValidation; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Configuration contracts shared by primary-key and Data Evolution indexes. */ +class GlobalIndexOptionsTest { + + @ParameterizedTest + @CsvSource({ + "btree,btree", + "bitmap,bitmap", + "multivalue,multivalue", + "fm,fm", + "full-text,full-text", + "ivf-pq,vector" + }) + void testColumnDeclarationsAndLegacyFallback(String type, String legacyFamily) { + Map values = new HashMap<>(); + values.put("pk-" + legacyFamily + ".index.columns", " old_column "); + values.put("fields.old_column.pk-vector.index.type", "ivf-pq"); + CoreOptions options = new CoreOptions(values); + assertThat(options.globalIndexColumns(type)).containsExactly("old_column"); + + values.put("global-index." + type + ".index-column", "singular_column"); + options = new CoreOptions(values); + assertThat(options.globalIndexColumns(type)).containsExactly("singular_column"); + assertThat(options.globalIndexTypes()).containsExactly(type); + + values.put("global-index." + type + ".index-columns", " first, second "); + options = new CoreOptions(values); + assertThat(options.globalIndexColumns(type)).containsExactly("first", "second"); + if ("ivf-pq".equals(type)) { + assertThat(options.primaryKeyVectorIndexColumns()).containsExactly("first", "second"); + assertThat(options.primaryKeyVectorIndexType("first")).isEqualTo("ivf-pq"); + } + } + + @Test + void testSingularColumnOptionIsRecognized() { + CoreOptions options = + new CoreOptions( + Collections.singletonMap("global-index.btree.index-column", "amount")); + assertThat(options.globalIndexTypes()).containsExactly("btree"); + assertThat(options.primaryKeyBTreeIndexColumns()).containsExactly("amount"); + } + + @ParameterizedTest + @CsvSource({ + "btree,btree-index.,block-size,4 kb,8 kb", + "bitmap,bitmap-index.,dictionary-block-size,4 kb,8 kb", + "multivalue,multivalue-index.,dictionary-block-size,4 kb,8 kb", + "fm,fm-index.,sa-sample-rate,16,32", + "full-text,full-text.,tokenizer,default,jieba", + "ivf-pq,ivf-pq.,nlist,32,64" + }) + void testFieldOptionsOverrideTypeDefaults( + String type, String prefix, String key, String tableValue, String fieldValue) { + Map values = new HashMap<>(); + values.put(prefix + key, tableValue); + values.put("fields.payload.index-options", "{\"" + key + "\":\"" + fieldValue + "\"}"); + values.put( + "fields.payload.pk-" + ("ivf-pq".equals(type) ? "vector" : type) + ".index.options", + "invalid legacy JSON"); + values.put("sorted-index.records-per-file", "20"); + Options options = new CoreOptions(values).globalIndexOptions(type, "payload"); + assertThat(options.get(prefix + key)).isEqualTo(fieldValue); + assertThat(options.get("sorted-index.records-per-file")).isEqualTo("20"); + assertThat(values.get(prefix + key)).isEqualTo(tableValue); + assertThat(new CoreOptions(options).globalIndexOptions(type, "payload").toMap()) + .isEqualTo(options.toMap()); + } + + @ParameterizedTest + @ValueSource(strings = {"COSINE", "inner-product", "L2"}) + void testLegacyPrimaryKeyMetricSurvivesFactoryResolution(String metric) { + Map values = new HashMap<>(); + values.put("pk-vector.index.columns", "embedding"); + values.put("fields.embedding.pk-vector.index.type", "test-vector-ann"); + values.put( + "fields.embedding.pk-vector.distance.metric", + metric.toLowerCase().replace('-', '_')); + values.put("fields.embedding.pk-vector.index.options", "{\"metric\":\"" + metric + "\"}"); + TableSchema schema = schema(values, true); + PrimaryKeyIndexDefinition definition = + PrimaryKeyIndexDefinitions.create(schema).definitions().get(0); + + assertThat( + new CoreOptions(definition.options()) + .globalIndexOptions(definition.indexType(), "embedding") + .toMap()) + .isEqualTo(definition.options().toMap()); + assertThat( + GlobalIndexer.create( + definition.indexType(), + Collections.singletonList(schema.nameToFieldMap().get("embedding")), + definition.options())) + .isInstanceOf(VectorGlobalIndexer.class); + assertThat(values.get("fields.embedding.pk-vector.index.options")) + .isEqualTo("{\"metric\":\"" + metric + "\"}"); + } + + @Test + void testCompositeIndexResolvesOptionsForLeadingField() { + Map values = new HashMap<>(); + values.put("btree-index.fallback-scan-max-size", "invalid table default"); + values.put("fields.amount.index-options", "{\"fallback-scan-max-size\":\"1 mb\"}"); + values.put("fields.id.index-options", "invalid options for a payload field"); + TableSchema schema = schema(values, false); + + assertThat( + GlobalIndexer.create( + "btree", + Arrays.asList( + schema.nameToFieldMap().get("amount"), + schema.nameToFieldMap().get("id")), + new Options(values))) + .isInstanceOf(SortedGlobalIndexer.class); + } + + @Test + void testTypedFullTextFieldOptionsUseNativeEncoding() { + Options resolved = + new CoreOptions( + Collections.singletonMap( + "fields.text.index-options", + "{\"lower-case\":false,\"stop-words\":[\"paimon\",\"lake\"]}")) + .globalIndexOptions("full-text", "text"); + assertThat(resolved.get("full-text.lower-case")).isEqualTo("false"); + assertThat(resolved.get("full-text.stop-words")).isEqualTo("paimon;lake"); + } + + @Test + void testUnrelatedFieldOptionsDoNotRequestDefaultMetric() { + CoreOptions options = + new CoreOptions( + Collections.singletonMap( + "fields.embedding.index-options", "{\"nlist\":64}")); + assertThat(options.globalIndexOptions("ivf-pq", "embedding", false).get("ivf-pq.metric")) + .isNull(); + } + + @Test + void testDocumentedVectorOptionsAndFieldOverrides() { + Map values = new HashMap<>(); + values.put("global-index.ivf-pq.index-columns", "embedding"); + values.put("ivf-pq.dimension", "768"); + values.put("ivf-pq.distance.metric", "l2"); + values.put("ivf-pq.nlist", "32"); + values.put("fields.embedding.nlist", "48"); + values.put("fields.embedding.distance.metric", "cosine"); + CoreOptions options = new CoreOptions(values); + assertThat(options.primaryKeyVectorDistanceMetric("embedding")).isEqualTo("cosine"); + assertThat(options.primaryKeyVectorIndexOptions("embedding").get("ivf-pq.dimension")) + .isEqualTo("768"); + + values.put( + "fields.embedding.index-options", + "{\"nlist\":64,\"distance.metric\":\"inner_product\"}"); + options = new CoreOptions(values); + assertThat(options.primaryKeyVectorDistanceMetric("embedding")).isEqualTo("inner_product"); + Options resolved = options.primaryKeyVectorIndexOptions("embedding"); + assertThat(resolved.get("ivf-pq.nlist")).isEqualTo("64"); + assertThat(resolved.get("fields.embedding.nlist")).isEqualTo("64"); + assertThat(resolved.get("ivf-pq.metric")).isEqualTo("inner_product"); + assertThat(resolved.get("ivf-pq.distance.metric")).isEqualTo("inner_product"); + assertThat(resolved.get("fields.embedding.distance.metric")).isEqualTo("inner_product"); + } + + @ParameterizedTest + @ValueSource(strings = {"[]", "{\"nlist\":null}", "{\"\":64}", "invalid"}) + void testRejectsMalformedSharedFieldOptions(String json) { + CoreOptions options = + new CoreOptions(Collections.singletonMap("fields.embedding.index-options", json)); + assertThatThrownBy(() -> options.globalIndexOptions("ivf-pq", "embedding")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("fields.embedding.index-options"); + } + + @Test + void testPrimaryKeyAndDataEvolutionValidationUseSameDeclarations() { + Map values = new HashMap<>(); + values.put("global-index.btree.index-columns", "amount"); + values.put("global-index.ivf-pq.index-columns", "embedding"); + values.put("fields.embedding.index-options", "{\"nlist\":64}"); + values.put("bucket", "1"); + values.put("deletion-vectors.enabled", "true"); + assertThatCode(() -> SchemaValidation.validateTableSchema(schema(values, true))) + .doesNotThrowAnyException(); + + values.put("bucket", "-1"); + values.put("row-tracking.enabled", "true"); + values.put("data-evolution.enabled", "true"); + values.remove("deletion-vectors.enabled"); + values.put("global-index.diskann.index-columns", "embedding"); + assertThatCode(() -> SchemaValidation.validateTableSchema(schema(values, false))) + .doesNotThrowAnyException(); + } + + @Test + void testPrimaryKeyRequirementsStillApply() { + Map values = new HashMap<>(); + values.put("global-index.btree.index-columns", "amount"); + values.put("bucket", "1"); + assertThatThrownBy(() -> SchemaValidation.validateTableSchema(schema(values, true))) + .hasMessageContaining("require deletion-vectors.enabled = true"); + values.put("deletion-vectors.enabled", "true"); + values.put("global-index.bitmap.index-columns", "amount"); + assertThatThrownBy(() -> SchemaValidation.validateTableSchema(schema(values, true))) + .hasMessageContaining("at most one primary-key index"); + } + + @ParameterizedTest + @ValueSource(strings = {"amount,amount", "unknown", "amount,", ""}) + void testInvalidColumnDeclarationsAreRejected(String columns) { + Map values = new HashMap<>(); + values.put("global-index.btree.index-columns", columns); + values.put("bucket", "-1"); + values.put("row-tracking.enabled", "true"); + values.put("data-evolution.enabled", "true"); + assertThatThrownBy(() -> SchemaValidation.validateTableSchema(schema(values, false))) + .hasMessageContaining("global-index.btree.index-columns"); + } + + private static TableSchema schema(Map options, boolean primaryKey) { + return new TableSchema( + 0, + Arrays.asList( + new DataField(0, "id", DataTypes.INT().notNull()), + new DataField(1, "amount", DataTypes.INT()), + new DataField(2, "embedding", DataTypes.VECTOR(768, DataTypes.FLOAT()))), + 2, + Collections.emptyList(), + primaryKey ? Collections.singletonList("id") : Collections.emptyList(), + options, + ""); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexQueryTest.java b/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexQueryTest.java index 844317fc57f0..944fcb0c71f0 100644 --- a/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexQueryTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexQueryTest.java @@ -756,6 +756,48 @@ void testDedicatedLeadingIndexPreferredOnlyWithCompleteCoverage() { assertThat(complete.coveredRanges()).containsExactly(new Range(0, 99)); } + @ParameterizedTest + @CsvSource({ + "index-options,1 gb,true", + "index-options,0 bytes,false", + "pk-btree.index.options,1 gb,true" + }) + void testCompositePlanningUsesResolvedLeadingFieldOptions( + String suffix, String budget, boolean eligible) { + RowType rowType = + RowType.of( + new DataType[] {DataTypes.INT(), DataTypes.INT()}, + new String[] {"amount", "category"}); + IndexFileMeta composite = + new IndexFileMeta( + "btree", + "composite", + 300L * 1024 * 1024, + 100, + new GlobalIndexMeta(0, 99, 0, new int[] {1}, null), + null); + Options options = + new Options( + Collections.singletonMap( + "fields.amount." + suffix, + "{\"fallback-scan-max-size\":\"" + budget + "\"}")); + IndexPathFactory paths = mock(IndexPathFactory.class); + when(paths.toPath(any(IndexFileMeta.class))).thenReturn(new Path("index")); + GlobalIndexQuery query = + GlobalIndexQuery.create( + rowType, + new PredicateBuilder(rowType).equal(0, 7), + Collections.singletonList(composite), + paths, + options); + if (eligible) { + assertThat(query).isNotNull(); + assertThat(query.hasCompositeQuery()).isTrue(); + } else { + assertThat(query).isNull(); + } + } + private IndexFileMeta indexFile( String type, String name, long start, long end, int fieldId, int[] extraFields) { return new IndexFileMeta( diff --git a/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriterTest.java b/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriterTest.java index fb1b90fa6a1a..817eeb02105a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriterTest.java @@ -22,6 +22,7 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; import org.apache.paimon.globalindex.ResultEntry; +import org.apache.paimon.options.Options; import org.apache.paimon.schema.Schema; import org.apache.paimon.table.TableTestBase; import org.apache.paimon.types.DataTypes; @@ -45,6 +46,27 @@ /** Tests for {@link SortedGlobalIndexWriter}. */ public class SortedGlobalIndexWriterTest extends TableTestBase { + @Test + public void testFieldOptionsControlFileRotationAndBuildSettings() throws Exception { + createTableDefault(); + Options options = new Options(); + options.setString( + "fields.f0.index-options", "{\"records-per-range\":2,\"build.max-parallelism\":1}"); + SortedGlobalIndexWriter writer = + new SortedGlobalIndexWriter(getTableDefault(), "btree", options) + .withIndexFields(Collections.singletonList("f0")); + assertThat(writer.recordsPerRange()).isEqualTo(2); + assertThat(writer.indexOptions().get(SortedIndexOptions.SORTED_INDEX_BUILD_MAX_PARALLELISM)) + .isEqualTo(1); + try (SortedSingleColumnIndexWriter task = writer.createTaskWriter(new Range(0, 2))) { + task.write(10, 0); + task.write(20, 1); + task.write(30, 2); + assertThat(task.finish()).hasSize(2); + } + assertThat(options.get("btree-index.records-per-range")).isNull(); + } + @Test public void testSingleColumnWriterRotationPreservesResultGroups() throws Exception { GlobalIndexSingleColumnWriter first = mock(GlobalIndexSingleColumnWriter.class); diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitionsTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitionsTest.java index bdc92049d1ec..6a951f333f11 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitionsTest.java @@ -24,6 +24,8 @@ import org.apache.paimon.types.DataTypes; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.Arrays; import java.util.Collections; @@ -37,14 +39,17 @@ /** Tests for {@link PrimaryKeyIndexDefinitions}. */ class PrimaryKeyIndexDefinitionsTest { - @Test - void testCreatesMixedDefinitionsInSchemaFieldOrder() { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testCreatesMixedDefinitionsInSchemaFieldOrder(boolean sharedOptions) { Map options = new HashMap<>(); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); - options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "name"); - options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "status"); - options.put(CoreOptions.PK_MULTIVALUE_INDEX_COLUMNS.key(), "tags"); + options.put(columnsKey(sharedOptions ? "ivf-pq" : "vector", sharedOptions), "embedding"); + if (!sharedOptions) { + options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); + } + options.put(columnsKey("btree", sharedOptions), "name"); + options.put(columnsKey("bitmap", sharedOptions), "status"); + options.put(columnsKey("multivalue", sharedOptions), "tags"); List definitions = PrimaryKeyIndexDefinitions.create(schema(options)).definitions(); @@ -67,10 +72,11 @@ void testCreatesMixedDefinitionsInSchemaFieldOrder() { .containsExactly("btree", "bitmap", "ivf-pq", "multivalue"); } - @Test - void testCreatesFullTextDefinitionWithoutIndexType() { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testCreatesFullTextDefinitionWithoutIndexType(boolean sharedOptions) { Map options = new HashMap<>(); - options.put("pk-full-text.index.columns", "name"); + options.put(columnsKey("full-text", sharedOptions), "name"); List definitions = PrimaryKeyIndexDefinitions.create(schema(options)).definitions(); @@ -99,12 +105,15 @@ void testResolvesFullTextIndexOptions() { .doesNotContainKey("fields.name.pk-full-text.index.options"); } - @Test - void testCreatesFMDefinitionAndResolvesOptions() { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testCreatesFMDefinitionAndResolvesOptions(boolean sharedOptions) { Map options = new HashMap<>(); - options.put(CoreOptions.PK_FM_INDEX_COLUMNS.key(), "name"); + options.put(columnsKey("fm", sharedOptions), "name"); options.put("fm-index.sa-sample-rate", "16"); - options.put("fields.name.pk-fm.index.options", "{\"partition-row-count\":\"2\"}"); + options.put( + sharedOptions ? "fields.name.index-options" : "fields.name.pk-fm.index.options", + "{\"partition-row-count\":\"2\"}"); PrimaryKeyIndexDefinition definition = PrimaryKeyIndexDefinitions.create(schema(options)).definitions().get(0); @@ -138,6 +147,12 @@ void testRejectsColumnConfiguredByMultipleFamilies() { .hasMessageContaining("at most one primary-key index"); } + private static String columnsKey(String type, boolean sharedOptions) { + return sharedOptions + ? "global-index." + type + ".index-columns" + : "pk-" + type + ".index.columns"; + } + private static TableSchema schema(Map options) { return new TableSchema( 0, diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexMaintenanceTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexMaintenanceTest.java index 796485d7f622..0acb484d3c4e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexMaintenanceTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexMaintenanceTest.java @@ -60,6 +60,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; import java.util.ArrayList; @@ -90,9 +92,10 @@ void after() throws Exception { ioManager.close(); } - @Test - void testBuildRestoreAndReplaceSourceGroups() throws Exception { - TestFileStore store = createStore(); + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testBuildRestoreAndReplaceSourceGroups(boolean sharedOptions) throws Exception { + TestFileStore store = createStore(sharedOptions); TestKeyValueGenerator generator = new TestKeyValueGenerator(); List firstBatch = Arrays.asList( @@ -317,13 +320,21 @@ private static KeyValue withVector(KeyValue record, float[] vector) { return record.replaceValue(extended); } - private TestFileStore createStore() throws Exception { + private TestFileStore createStore(boolean sharedOptions) throws Exception { Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "1"); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); options.put(CoreOptions.COMPACTION_FORCE_REWRITE_ALL_FILES.key(), "true"); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "itemId"); - options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "comment"); + options.put( + sharedOptions + ? "global-index.btree.index-columns" + : CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), + "itemId"); + options.put( + sharedOptions + ? "global-index.bitmap.index-columns" + : CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), + "comment"); SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); TableSchema schema = @@ -362,6 +373,7 @@ private TestFileStore createMixedStore() throws Exception { options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); options.put("fields.embedding.pk-vector.index.type", "test-vector-ann"); options.put("fields.embedding.pk-vector.distance.metric", "l2"); + options.put("fields.embedding.pk-vector.index.options", "{\"metric\":\"L2\"}"); List fields = new ArrayList<>(TestKeyValueGenerator.DEFAULT_ROW_TYPE.getFields()); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java index 29daa35f0c46..76bc39072466 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java @@ -39,6 +39,7 @@ import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.predicate.TopN; +import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.sink.BatchTableCommit; import org.apache.paimon.table.sink.BatchTableWrite; @@ -64,7 +65,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import static org.apache.paimon.predicate.SortValue.NullOrdering.NULLS_LAST; @@ -76,6 +79,35 @@ /** Test for BTree indexed batch scan. */ public class BtreeGlobalIndexTableTest extends DataEvolutionTestBase { + @Test + public void testSharedFieldOptionsUsedByBuildAndScan() throws Exception { + Schema schema = schemaDefault(); + Map options = new HashMap<>(schema.options()); + options.put("global-index.btree.index-columns", "f1"); + options.put("btree-index.block-size", "invalid table default"); + options.put("btree-index.fallback-scan-max-size", "invalid table default"); + options.put( + "fields.f1.index-options", + "{\"block-size\":\"4 kb\",\"fallback-scan-max-size\":\"1 mb\"}"); + catalog.createTable( + identifier(), + new Schema( + schema.fields(), + schema.partitionKeys(), + schema.primaryKeys(), + options, + schema.comment()), + false); + write(2); + createIndex("f1"); + + FileStoreTable table = (FileStoreTable) catalog.getTable(identifier()); + Predicate predicate = + new PredicateBuilder(table.rowType()).equal(1, BinaryString.fromString("a1")); + assertThat(globalIndexScan(table, predicate).toRangeList()) + .containsExactly(new Range(1, 1)); + } + @Test public void testBTreeGlobalIndex() throws Exception { write(100000L); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java index f56d1fc230fd..38a7258fc667 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java @@ -828,6 +828,66 @@ public void testVectorSearchRefineFactorReranksIndexCandidates() throws Exceptio assertThat(batchRefined).hasSize(2); assertThat(batchRefined.get(0).results()).containsExactly(0L); assertThat(batchRefined.get(1).results()).containsExactly(2L); + + FileStoreTable fieldConfigured = + table.copy( + Collections.singletonMap( + "fields." + VECTOR_FIELD_NAME + ".index-options", + "{\"refine_factor\":3}")); + assertThat( + fieldConfigured + .newVectorSearchBuilder() + .withVector(new float[] {0.0f, 0.0f}) + .withLimit(1) + .withVectorColumn(VECTOR_FIELD_NAME) + .executeLocal() + .results()) + .containsExactly(0L); + List fieldBatch = + fieldConfigured + .newBatchVectorSearchBuilder() + .withVectors(new float[][] {{0.0f, 0.0f}, {20.0f, 0.0f}}) + .withLimit(1) + .withVectorColumn(VECTOR_FIELD_NAME) + .executeBatchLocal(); + assertThat(fieldBatch.get(0).results()).containsExactly(0L); + assertThat(fieldBatch.get(1).results()).containsExactly(2L); + } + + @Test + public void testSharedVectorDeclarationKeepsDataEvolutionReads() throws Exception { + catalog.createTable( + identifier("declared_vector"), + vectorSchemaBuilder(VECTOR_FIELD_NAME) + .option( + "global-index." + + TestVectorGlobalIndexerFactory.IDENTIFIER + + ".index-columns", + VECTOR_FIELD_NAME) + .build(), + false); + FileStoreTable table = getTable(identifier("declared_vector")); + float[][] vectors = {{1.0f, 0.0f}, {0.0f, 1.0f}}; + writeVectors(table, vectors); + buildAndCommitIndex(table, vectors); + + assertThat( + table.newVectorSearchBuilder() + .withVector(new float[] {1.0f, 0.0f}) + .withLimit(1) + .withVectorColumn(VECTOR_FIELD_NAME) + .executeLocal() + .results()) + .containsExactly(0L); + assertThat( + table.newBatchVectorSearchBuilder() + .withVectors(new float[][] {{0.0f, 1.0f}}) + .withLimit(1) + .withVectorColumn(VECTOR_FIELD_NAME) + .executeBatchLocal() + .get(0) + .results()) + .containsExactly(1L); } @Test diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/SortedIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/SortedIndexTopoBuilder.java index 11900b0fadc9..7ac4eab99569 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/SortedIndexTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/SortedIndexTopoBuilder.java @@ -39,10 +39,8 @@ import org.apache.paimon.flink.utils.JavaTypeInfo; import org.apache.paimon.flink.utils.StreamExecutionEnvironmentUtils; import org.apache.paimon.globalindex.GlobalIndexKeyExtractor; -import org.apache.paimon.globalindex.GlobalIndexer; import org.apache.paimon.globalindex.ResultEntry; import org.apache.paimon.globalindex.ScanResult; -import org.apache.paimon.globalindex.SortedGlobalIndexer; import org.apache.paimon.globalindex.sorted.SortedGlobalIndexScanner; import org.apache.paimon.globalindex.sorted.SortedGlobalIndexWriter; import org.apache.paimon.globalindex.sorted.SortedIndexOptions; @@ -184,13 +182,11 @@ public static Optional> buildIndexStream( RowType sourceReadType = table.rowType().project(indexColumns); RowType dataReadType = SpecialFields.rowTypeWithRowId(sourceReadType); String buildTaskIdField = buildTaskIdFieldName(dataReadType); - GlobalIndexer indexer = - GlobalIndexer.create(indexType, sourceReadType.getFields(), userOptions); - if (!(indexer instanceof SortedGlobalIndexer)) { - throw new IllegalArgumentException( - "Index algorithm " + indexType + " does not expose sorted index keys."); - } - GlobalIndexKeyExtractor keyExtractor = ((SortedGlobalIndexer) indexer).keyExtractor(); + SortedGlobalIndexWriter indexWriter = + new SortedGlobalIndexWriter(table, indexType, userOptions) + .withIndexFields(indexColumns); + Options indexOptions = indexWriter.indexOptions(); + GlobalIndexKeyExtractor keyExtractor = indexWriter.keyExtractor(); RowType sortReadType = normalizedReadType( dataReadType, buildTaskIdField, indexColumn, keyExtractor.keyType()); @@ -201,9 +197,9 @@ public static Optional> buildIndexStream( // 3. Calculate maximum parallelism bound long recordsPerRange = - userOptions.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE); + indexOptions.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE); int maxParallelism = - userOptions.get(SortedIndexOptions.SORTED_INDEX_BUILD_MAX_PARALLELISM); + indexOptions.get(SortedIndexOptions.SORTED_INDEX_BUILD_MAX_PARALLELISM); // 4. Build one topology for all contiguous row ranges CoreOptions coreOptions = table.coreOptions(); @@ -245,8 +241,7 @@ public static Optional> buildIndexStream( buildTasks, splitTasks, readBuilder, - new SortedGlobalIndexWriter(table, indexType, userOptions) - .withIndexFields(indexColumns), + indexWriter, scanResult.scanSnapshotId(), partitionFieldSize, taskIdPos, diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/VectorSearchProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/VectorSearchProcedure.java index d49810c9dd52..d9370eebf294 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/VectorSearchProcedure.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/VectorSearchProcedure.java @@ -260,7 +260,8 @@ private static PartitionPredicate parsePartitions( private static void validatePrimaryKeyFilter( FileStoreTable table, String vectorColumn, FilterParts filterParts) { - if (!table.coreOptions().primaryKeyVectorIndexColumns().contains(vectorColumn) + if (table.schema().primaryKeys().isEmpty() + || !table.coreOptions().primaryKeyVectorIndexColumns().contains(vectorColumn) || !filterParts.hasDataFilter()) { return; } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java index 544836ee88df..5d98d24fd10d 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java @@ -111,7 +111,10 @@ public void testEmptyTableVectorSearch() { @Test public void testVectorSearchKeepsResolvedSchema() throws Exception { - createVectorTable("SCHEMA_EVOLUTION_T", "'global-index.search-mode' = 'full'"); + createVectorTable( + "SCHEMA_EVOLUTION_T", + "'global-index.search-mode' = 'full', " + + "'global-index.test-vector-ann.index-columns' = 'vec'"); FileStoreTable table = paimonTable("SCHEMA_EVOLUTION_T"); float[][] vectors = {{0.0f, 0.0f}, {1.0f, 0.0f}}; writeVectors(table, vectors); diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py index 8c5214f52a2e..df991cad752a 100644 --- a/paimon-python/pypaimon/common/options/core_options.py +++ b/paimon-python/pypaimon/common/options/core_options.py @@ -1860,111 +1860,156 @@ def global_index_build_parallelism(self) -> int: return self.options.get(CoreOptions.GLOBAL_INDEX_BUILD_PARALLELISM) def primary_key_btree_index_columns(self) -> List[str]: - return self._primary_key_index_columns(CoreOptions.PK_BTREE_INDEX_COLUMNS) + return self.global_index_columns("btree") def primary_key_bitmap_index_columns(self) -> List[str]: - return self._primary_key_index_columns(CoreOptions.PK_BITMAP_INDEX_COLUMNS) + return self.global_index_columns("bitmap") def primary_key_vector_index_columns(self) -> List[str]: - return self._primary_key_index_columns(CoreOptions.PK_VECTOR_INDEX_COLUMNS) + types = self._vector_index_types() + if types: + return [column for index_type in types + for column in self.global_index_columns(index_type)] + return self._index_columns(self.options.get(CoreOptions.PK_VECTOR_INDEX_COLUMNS)) def primary_key_full_text_index_columns(self) -> List[str]: - return self._primary_key_index_columns(CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS) + return self.global_index_columns("full-text") + + def global_index_types(self) -> List[str]: + prefix = "global-index." + return sorted({key[len(prefix):key.rfind('.')] + for key in self.options.to_map() + if key.startswith(prefix) and key.endswith((".index-columns", ".index-column")) + and key.rfind('.') >= len(prefix)}) + + @staticmethod + def _is_vector_index_type(index_type: str) -> bool: + return index_type not in ("btree", "bitmap", "multivalue", "fm", "full-text") + + def _vector_index_types(self) -> List[str]: + return [index_type for index_type in self.global_index_types() + if self._is_vector_index_type(index_type)] + + def global_index_columns(self, index_type: str) -> List[str]: + values = self.options.to_map() + columns = values.get("global-index.%s.index-columns" % index_type) + if columns is None: + columns = values.get("global-index.%s.index-column" % index_type) + if columns is None: + columns = values.get("pk-%s.index.columns" % index_type) + if columns is not None: + return self._index_columns(columns) + return [column for column in self._index_columns(values.get("pk-vector.index.columns")) + if values.get("fields.%s.pk-vector.index.type" % column) == index_type] + + @staticmethod + def _index_columns(columns) -> List[str]: + return [] if columns is None else [column.strip() for column in str(columns).split(',')] + + def global_index_options(self, index_type: Optional[str], column: str, *, default_metric=True) -> Options: + """Resolve the same implementation and field options for either table mode.""" + values = dict(self.options.to_map()) + if index_type is None: + index_type = self.vector_index_type(column) + if index_type is None: + return Options(values) + vector = self._is_vector_index_type(index_type) + prefix = index_type + ("." if vector or index_type == "full-text" else "-index.") + field_prefix = "fields.%s." % column + shared_key = field_prefix + "index-options" + shared = values.get(shared_key) is not None + option_key = shared_key if shared else field_prefix + "pk-" + ( + "vector" if vector else index_type) + ".index.options" + metric = None + if vector: + for key in (field_prefix + "distance.metric", field_prefix + "metric", + field_prefix + "pk-vector.distance.metric", prefix + "distance.metric", + prefix + "metric", "distance.metric", "metric"): + if values.get(key) is not None: + metric = values[key] + break + serialized = values.get(option_key) + if serialized is not None and str(serialized).strip(): + try: + parsed = json.loads(serialized) + except (TypeError, ValueError) as exc: + raise ValueError("%s must be a JSON object of option key-value pairs." % option_key) from exc + if not isinstance(parsed, dict): + raise ValueError("%s must be a JSON object of option key-value pairs." % option_key) + for key, value in parsed.items(): + if not key.strip() or value is None: + raise ValueError("%s contains an invalid option." % option_key) + if isinstance(value, bool): + value = str(value).lower() + elif index_type == 'full-text' and isinstance(value, list): + value = ';'.join(str(item) for item in value) + qualified = key if key.startswith((prefix, "fields.")) else prefix + key + metric_option = vector and qualified in ( + prefix + "metric", prefix + "distance.metric", + field_prefix + "metric", field_prefix + "distance.metric") + previous = values.get(qualified) + if metric_option: + value = str(value).lower().replace('-', '_') + previous = None if previous is None else str(previous).lower().replace('-', '_') + if not shared and previous is not None and previous != str(value): + raise ValueError("%s defines conflicting values for %s." % (option_key, qualified)) + values[qualified] = str(value) + if metric_option: + metric = value + elif vector and qualified.startswith(prefix): + short_key = qualified[len(prefix):] + if shared: + values[field_prefix + short_key] = str(value) + if vector and (metric is not None or default_metric): + metric = str(metric or "inner_product").lower().replace('-', '_') + for key in (prefix + "metric", prefix + "distance.metric", + field_prefix + "metric", field_prefix + "distance.metric"): + values[key] = metric + return Options(values) def primary_key_vector_index_type(self, column: str): - return self.options.to_map().get( - "fields.%s.pk-vector.index.type" % column) + return self.vector_index_type(column) + + def vector_index_type(self, column: str): + declared = [index_type for index_type in self._vector_index_types() + if column in self.global_index_columns(index_type)] + if declared: + return declared[0] if len(declared) == 1 else None + for key in ("fields.%s.pk-vector.index.type" % column, "index_type", "index-type", + "vector.index-type", "fields.%s.index-type" % column): + value = self.options.to_map().get(key) + if value is not None: + return str(value).lower().strip() + return None def primary_key_vector_index_options(self, column: str) -> Options: index_type = self.primary_key_vector_index_type(column) if index_type is None or not str(index_type).strip(): - raise ValueError( - "fields.%s.pk-vector.index.type must be configured before " - "resolving index options." % column) - option_key = "fields.%s.pk-vector.index.options" % column - field_prefix = "fields.%s." % column - resolved = self._primary_key_json_options( - option_key, - lambda key: (key.startswith(str(index_type) + ".") - or (key.startswith(field_prefix) - and not key.startswith(field_prefix + "pk-vector."))), - str(index_type) + ".") - values = dict(resolved.to_map()) - values[str(index_type) + ".metric"] = \ - self.primary_key_vector_distance_metric(column) - return Options(values) + raise ValueError("fields.%s.pk-vector.index.type must be configured before " + "resolving index options." % column) + return self.global_index_options(str(index_type), column) def primary_key_vector_distance_metric(self, column: str) -> str: - value = self.options.to_map().get( - "fields.%s.pk-vector.distance.metric" % column, - "inner_product") + index_type = self.primary_key_vector_index_type(column) + value = (self.global_index_options(index_type, column).to_map()[index_type + ".metric"] + if index_type else self.options.to_map().get( + "fields.%s.pk-vector.distance.metric" % column, "inner_product")) return str(value).lower().replace('-', '_') def primary_key_full_text_index_options(self, column: str) -> Options: - return self._primary_key_json_options( - "fields.%s.pk-full-text.index.options" % column, - lambda key: key.startswith("full-text."), - "full-text.") + return Options({key: value for key, value in self.global_index_options( + "full-text", column).to_map().items() if key.startswith("full-text.")}) def primary_key_btree_index_options(self, column: str) -> Options: - return self._primary_key_sorted_index_options(column, "pk-btree", "btree-index.") + return self._primary_key_sorted_index_options(column, "btree") def primary_key_bitmap_index_options(self, column: str) -> Options: - return self._primary_key_sorted_index_options(column, "pk-bitmap", "bitmap-index.") - - def _primary_key_index_columns(self, option: ConfigOption[str]) -> List[str]: - columns = self.options.get(option) - if columns is None: - return [] - return [column.strip() for column in columns.split(',')] + return self._primary_key_sorted_index_options(column, "bitmap") - def _primary_key_sorted_index_options( - self, column: str, option_family: str, algorithm_prefix: str) -> Options: - resolved = dict(self.options.to_map()) + def _primary_key_sorted_index_options(self, column: str, index_type: str) -> Options: + resolved = self.global_index_options(index_type, column).to_map() resolved.pop("sorted-index.records-per-file", None) resolved.pop("sorted-index.records-per-range", None) - option_key = "fields.%s.%s.index.options" % (column, option_family) - serialized = self.options.to_map().get(option_key) - if serialized is None or not str(serialized).strip(): - return Options(resolved) - try: - parsed = json.loads(serialized) - except (TypeError, ValueError) as exc: - raise ValueError("%s must be a JSON object of option key-value pairs." % option_key) from exc - if not isinstance(parsed, dict): - raise ValueError("%s must be a JSON object of option key-value pairs." % option_key) - for key, value in parsed.items(): - if not key or value is None: - raise ValueError("%s contains an invalid option." % option_key) - qualified = key if key.startswith((algorithm_prefix, "fields.")) \ - else algorithm_prefix + key - previous = resolved.get(qualified) - if previous is not None and previous != value: - raise ValueError("%s defines conflicting values for %s." % (option_key, qualified)) - resolved[qualified] = value - return Options(resolved) - - def _primary_key_json_options(self, option_key, include, prefix): - resolved = {key: value for key, value in self.options.to_map().items() - if include(key) and not key.startswith(option_key)} - serialized = self.options.to_map().get(option_key) - if serialized is None or not str(serialized).strip(): - return Options(resolved) - try: - parsed = json.loads(serialized) - except (TypeError, ValueError) as exc: - raise ValueError("%s must be a JSON object of option key-value pairs." % option_key) from exc - if not isinstance(parsed, dict): - raise ValueError("%s must be a JSON object of option key-value pairs." % option_key) - for key, value in parsed.items(): - if not key or value is None: - raise ValueError("%s contains an invalid option." % option_key) - qualified = key if (prefix is None - or key.startswith((prefix, "fields."))) else prefix + key - previous = resolved.get(qualified) - if previous is not None and previous != str(value): - raise ValueError("%s defines conflicting values for %s." % (option_key, qualified)) - resolved[qualified] = str(value) return Options(resolved) def btree_index_fallback_scan_max_size(self) -> int: diff --git a/paimon-python/pypaimon/execution/native/index_options.py b/paimon-python/pypaimon/execution/native/index_options.py new file mode 100644 index 000000000000..2c93112737a8 --- /dev/null +++ b/paimon-python/pypaimon/execution/native/index_options.py @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Transport resolved index configuration through the existing native schema contract.""" + +import json +from collections.abc import Mapping + +from pypaimon.common.options.core_options import CoreOptions +from pypaimon.common.options.options import Options + + +def native_index_options(schema): + return _resolve(schema)[0] + + +def native_scalar_index_read_supported(table): + if not table.options.global_index_enabled(): + return True + return _resolve(table.table_schema)[1] + + +def _resolve(schema): + if not isinstance(getattr(schema, 'options', None), Mapping): + return {}, True + values = dict(schema.options) + core = CoreOptions(Options(values)) + result = dict(values) + declarations = core.global_index_types() + # Native's older shared declaration keys enable automatic maintenance. Python's + # declarations select implementations and keep the explicit DE build lifecycle. + for key in list(result): + if key.startswith('global-index.') and key.endswith(('.index-columns', '.index-column')): + del result[key] + primary_key = bool(schema.primary_keys) + families = ['btree', 'bitmap', 'multivalue', 'fm', 'full-text'] + definitions = [(family, core.global_index_columns(family)) for family in families] + definitions += [(kind, core.global_index_columns(kind)) for kind in declarations + if kind not in families] + if primary_key and not any(kind not in families for kind in declarations): + definitions += [(core.vector_index_type(column), [column]) + for column in core.primary_key_vector_index_columns()] + known_columns = set() + native_columns = {} + profiles = {} + compatible = True + for kind, columns in definitions: + if kind is None: + continue + vector = kind not in families + family = 'vector' if vector else kind + prefix = kind + ('.' if vector or kind == 'full-text' else '-index.') + if primary_key and columns: + native_columns.setdefault(family, []).extend(columns) + result['pk-%s.index.columns' % family] = ','.join(native_columns[family]) + for column in columns: + known_columns.add(column) + resolved = core.global_index_options(kind, column, default_metric=False).to_map() + profile = {key: value for key, value in resolved.items() + if key.startswith(prefix) and value is not None} + field_prefix = 'fields.%s.' % column + if primary_key: + if vector: + result[field_prefix + 'pk-vector.index.type'] = kind + result[field_prefix + 'pk-vector.distance.metric'] = core.primary_key_vector_distance_metric(column) + profile.update({key: value for key, value in resolved.items() + if key.startswith(field_prefix) and not key.startswith(field_prefix + 'pk-') + and key != field_prefix + 'index-options' and value is not None}) + if vector or kind == 'full-text': + # Native ranked readers consume flat options, unlike scalar definitions. + result.update(profile) + result[field_prefix + 'pk-%s.index.options' % family] = json.dumps(profile) + result.pop(field_prefix + 'index-options', None) + elif vector: + result[field_prefix + 'index-type'] = kind + result.update({key: value for key, value in resolved.items() + if key.startswith(field_prefix) and key != field_prefix + 'index-options'}) + elif kind != 'full-text': + # Native scalar scanners accept one set of read options per family. + if kind in profiles and profiles[kind] != profile: + compatible = False + profiles[kind] = profile + if primary_key and columns and not vector and kind != 'full-text': + for key in list(result): + if key.startswith(prefix): + del result[key] + if not primary_key: + for profile in profiles.values(): + result.update(profile) + for key in values: + if key.startswith('fields.') and key.endswith('.index-options'): + column = key[len('fields.'):-len('.index-options')] + if column not in known_columns: + compatible = False + return result, compatible diff --git a/paimon-python/pypaimon/execution/native/read/native_full_text_search.py b/paimon-python/pypaimon/execution/native/read/native_full_text_search.py index 4568371e0038..37c8fdc51068 100644 --- a/paimon-python/pypaimon/execution/native/read/native_full_text_search.py +++ b/paimon-python/pypaimon/execution/native/read/native_full_text_search.py @@ -24,7 +24,7 @@ from pypaimon.execution.native.read.builder import (_predicate_to_native) from pypaimon.execution.native.runtime import (native_runtime_available) from pypaimon.table.file_store_table import FileStoreTable -from pypaimon.execution.native.read.native_search_plan import _result +from pypaimon.execution.native.read.native_search_plan import _result, _index_search_options from pypaimon.execution.native.write.native_commit import _rest_catalog_supported @@ -48,8 +48,11 @@ def _native_full_text_builder(builder, require_limit=True): raise ValueError('Limit must be positive, set via with_limit()') # The full-text adapter accepts typed lists and booleans. Reuse its mapping # before the generic table bridge stringifies options for Rust. + resolved = _index_search_options(table, builder._field_name, 'full-text', scalar_filter=builder._filter) + if resolved is None: + return None options = {'full-text.' + key: value for key, value in NativeFullTextIndexOptions.from_options( - table.options.options.to_map()).to_native_options().items()} + resolved).to_native_options().items()} native = _native_table(table.copy(options)).new_full_text_search_builder() if builder._field_name is not None and builder._query is not None: native.with_query(builder._field_name, builder._query) diff --git a/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py b/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py index 56230dd9cf13..2325ec6c9cce 100644 --- a/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py +++ b/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py @@ -25,6 +25,7 @@ from pypaimon.execution.native.runtime import (native_runtime_available) from pypaimon.table.file_store_table import FileStoreTable from pypaimon.execution.native.write.native_commit import _rest_catalog_supported +from pypaimon.execution.native.read.native_search_plan import _index_search_options def _native_builder(builder, executor=None): @@ -43,19 +44,33 @@ def _native_builder(builder, executor=None): builder._validate_search() if executor is not None and not table.options.data_evolution_enabled(): raise ValueError('Distributed search supports only data-evolution tables') - if any(route.is_full_text() for route in builder._routes): - # Preserve the same typed analyzer normalization used by native full-text. - options = {'full-text.' + key: value for key, value in NativeFullTextIndexOptions.from_options( - table.options.options.to_map()).to_native_options().items()} - table = table.copy(options) + route_options = [] + full_text_options = None + for route in builder._routes: + options = _index_search_options(table, route.field_name, + 'full-text' if route.is_full_text() else None, route.options, + scalar_filter=builder._filter) + if options is None: + return None + if route.is_full_text(): + resolved = {'full-text.' + key: value for key, value in + NativeFullTextIndexOptions.from_options(options).to_native_options().items()} + # Core accepts analyzer options only on the table, not on individual routes. + if full_text_options is not None and full_text_options != resolved: + return None + full_text_options = resolved + options = {} + route_options.append(options) + if full_text_options is not None: + table = table.copy(full_text_options) native = _native_table(table).new_hybrid_search_builder() native.with_limit(builder._limit).with_ranker(builder._ranker) - for route in builder._routes: + for route, options in zip(builder._routes, route_options): if route.is_vector(): - native.add_vector_route(route.field_name, route.vector, route.limit, route.weight, route.options) + native.add_vector_route(route.field_name, route.vector, route.limit, route.weight, options) else: native.add_full_text_route( - route.field_name, route.full_text_query, route.limit, route.weight, route.options) + route.field_name, route.full_text_query, route.limit, route.weight, options) try: if builder._partition_filter is not None: native.with_partition_filter(_predicate_to_native(builder._partition_filter)) diff --git a/paimon-python/pypaimon/execution/native/read/native_search_plan.py b/paimon-python/pypaimon/execution/native/read/native_search_plan.py index 56d29ce63cbe..3f06109bf7f2 100644 --- a/paimon-python/pypaimon/execution/native/read/native_search_plan.py +++ b/paimon-python/pypaimon/execution/native/read/native_search_plan.py @@ -20,6 +20,9 @@ from copy import copy, deepcopy from pypaimon.common.json_util import JSON +from pypaimon.common.options.core_options import CoreOptions +from pypaimon.common.options.options import Options +from pypaimon.execution.native.index_options import native_scalar_index_read_supported from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult from pypaimon.read.split_serializer import deserialize_split_v1, serialize_split_v1 from pypaimon.execution.native.table import (_partition_fields) @@ -34,6 +37,27 @@ from pypaimon.write.commit_message_serializer import _index_from_row, _index_row +def _index_search_options(table, column, index_type=None, overrides=None, scalar_filter=None): + """Use the common resolver before transferring search options to Core.""" + values = dict(table.options.options.to_map()) + values.update(overrides or {}) + if scalar_filter is not None and not native_scalar_index_read_supported(table.copy(values)): + return None + if column is None: + return values + core = CoreOptions(Options(values)) + if index_type is None: + index_type = core.vector_index_type(column) + if index_type is None: + # The Python reader can resolve field options once persisted index metadata is known. + if any(values.get(key) is not None for key in ( + 'fields.%s.index-options' % column, 'fields.%s.pk-vector.index.options' % column)): + return None + return values + # Keep an unspecified metric absent so readers can use persisted index metadata. + return core.global_index_options(index_type, column, default_metric=False).to_map() + + def _range(value): return Range(*value) diff --git a/paimon-python/pypaimon/execution/native/read/native_vector_search.py b/paimon-python/pypaimon/execution/native/read/native_vector_search.py index a4b0bbe7e377..32c492c0d9f6 100644 --- a/paimon-python/pypaimon/execution/native/read/native_vector_search.py +++ b/paimon-python/pypaimon/execution/native/read/native_vector_search.py @@ -22,7 +22,7 @@ from pypaimon.execution.native.table import (_catalog_metastore, _native_table) from pypaimon.execution.native.read.builder import (_predicate_to_native) from pypaimon.execution.native.runtime import (native_runtime_available) -from pypaimon.execution.native.read.native_search_plan import _result +from pypaimon.execution.native.read.native_search_plan import _result, _index_search_options def _native_vector_builder(builder, batch=False): @@ -44,7 +44,11 @@ def _native_vector_builder(builder, batch=False): return None if not native_runtime_available(): return None - native_table = _native_table(table) + options = _index_search_options(table, builder._vector_column.name if builder._vector_column else None, + overrides=builder._options, scalar_filter=builder._filter) + if options is None: + return None + native_table = _native_table(table.copy(options)) native = (native_table.new_batch_vector_search_builder() if batch else native_table.new_vector_search_builder()) if builder._vector_column is not None: diff --git a/paimon-python/pypaimon/execution/native/read/plan.py b/paimon-python/pypaimon/execution/native/read/plan.py index 357de9d4f28d..a3ac9ebb3c1f 100644 --- a/paimon-python/pypaimon/execution/native/read/plan.py +++ b/paimon-python/pypaimon/execution/native/read/plan.py @@ -26,6 +26,7 @@ from pypaimon.common.predicate import Predicate from pypaimon.execution.native import runtime from pypaimon.execution.native import table as native_table +from pypaimon.execution.native.index_options import native_scalar_index_read_supported from pypaimon.execution.native.read import builder as native_builder from pypaimon.read.plan import Plan from pypaimon.read.query_auth_split import QueryAuthSplit @@ -69,6 +70,8 @@ def native_plan( if not runtime.native_runtime_available(): raise RuntimeError( "scan.native-plan.enabled needs the pypaimon-rust extension") + if predicate is not None and not native_scalar_index_read_supported(table): + raise NotImplementedError('Native scalar reads cannot represent these field index options') builder = native_builder._configure_native_read_builder( native_builder._native_read_builder(table), predicate, limit, projection) if partition_predicate is not None: @@ -128,6 +131,8 @@ def native_plan( def native_stream_scan(table, predicate=None, read_type=None, bucket_filter=None, consumer_id=None): """Create Rust's stateful StreamTableScan with the canonical reader type.""" + if predicate is not None and not native_scalar_index_read_supported(table): + raise NotImplementedError('Native scalar reads cannot represent these field index options') builder = native_builder._configure_native_read_builder( native_builder._native_read_builder(table), predicate, None, None, read_type=read_type) scan = builder.new_stream_scan() diff --git a/paimon-python/pypaimon/execution/native/table.py b/paimon-python/pypaimon/execution/native/table.py index 251d4cc6f69d..b185ea2f8c14 100644 --- a/paimon-python/pypaimon/execution/native/table.py +++ b/paimon-python/pypaimon/execution/native/table.py @@ -137,8 +137,9 @@ def _resolved_schema_file_io_options(table) -> Optional[dict]: def _resolved_schema_json(table) -> str: """Preserve all effective table options as strings for Rust.""" from pypaimon.common.json_util import JSON + from pypaimon.execution.native.index_options import native_index_options options = {str(key): _option_value_to_string(value) - for key, value in table.table_schema.options.items() if value is not None} + for key, value in native_index_options(table.table_schema).items() if value is not None} if 'query-auth.enabled' in options: # Preserve Python's effective policy. Forwarding an enabled spelling # such as "on" unchanged would disable authorization in Rust. diff --git a/paimon-python/pypaimon/execution/native/write/native_commit.py b/paimon-python/pypaimon/execution/native/write/native_commit.py index 9cf739cf59eb..289e65c60c1d 100644 --- a/paimon-python/pypaimon/execution/native/write/native_commit.py +++ b/paimon-python/pypaimon/execution/native/write/native_commit.py @@ -20,6 +20,7 @@ import json from pypaimon.common.json_util import JSON +from pypaimon.execution.native.index_options import native_index_options from pypaimon.execution.native.runtime import native_runtime_available from pypaimon.execution.native.table import ( _catalog_context_options, _catalog_metastore, _option_value_to_string, @@ -154,7 +155,7 @@ def create_native_write_table(table): # Preserve the resolved schema and all effective table options, including # copy() overrides. Do not inject scan options or reload catalog schemas. options = {str(key): _option_value_to_string(value) - for key, value in table.table_schema.options.items() if value is not None} + for key, value in native_index_options(table.table_schema).items() if value is not None} # Python accepts boolean spellings such as "off"; pass the parsed value. options['dynamic-partition-overwrite'] = _option_value_to_string( table.options.dynamic_partition_overwrite()) diff --git a/paimon-python/pypaimon/execution/native/write/native_index_build.py b/paimon-python/pypaimon/execution/native/write/native_index_build.py index 7c0735b0bca1..265af7892180 100644 --- a/paimon-python/pypaimon/execution/native/write/native_index_build.py +++ b/paimon-python/pypaimon/execution/native/write/native_index_build.py @@ -45,8 +45,9 @@ def build_native_global_index(builder, partition_filter, *, execution="local", c return None native = _native_table(table).new_global_index_build_builder() native.with_index_column(builder._index_columns[0]).with_index_type(builder._index_type) + resolved_options = builder._options.to_map() options = {str(key): _option_value_to_string(value) - for key, value in builder._user_options.items() if value is not None} + for key, value in resolved_options.items() if value is not None} if builder._index_type == 'full-text': from pypaimon.globalindex.full_text_options import NativeFullTextIndexOptions options.update({'full-text.' + key: value for key, value in @@ -56,7 +57,7 @@ def build_native_global_index(builder, partition_filter, *, execution="local", c column = builder._index_columns[0] # Reuse the Java-compatible option mapping; Rust's low-level builder also # accepts native aliases which Python's public API deliberately ignores. - mapped = native_options(table.field_dict[column].type, table.options.options.to_map(), + mapped = native_options(table.field_dict[column].type, resolved_options, builder._index_type, column, builder._user_options) options = {key: value for key, value in options.items() if key in ('global-index.row-count-per-shard', 'global-index.build.parallelism', @@ -65,7 +66,7 @@ def build_native_global_index(builder, partition_filter, *, execution="local", c options.update({'fields.' + column + '.' + key: value for key, value in mapped.items() if key != 'index.type'}) options['fields.' + column + '.train.sample-ratio'] = str(train_sample_ratio( - table.options.options.to_map(), builder._index_type, column, builder._user_options)) + resolved_options, builder._index_type, column, builder._user_options)) if builder._index_type == 'full-text' or builder._index_type in VINDEX_IDENTIFIERS: # Resolve typed values and explicit default overrides through the same # ConfigOption conversion as the local builder before crossing the FFI. diff --git a/paimon-python/pypaimon/execution/python/globalindex/create_global_index.py b/paimon-python/pypaimon/execution/python/globalindex/create_global_index.py index 804b698ce34d..dd6faa5296c3 100644 --- a/paimon-python/pypaimon/execution/python/globalindex/create_global_index.py +++ b/paimon-python/pypaimon/execution/python/globalindex/create_global_index.py @@ -389,7 +389,7 @@ def _create_generic_index_writer(self, index_path: str, index_field): index_path, index_field.type, self._index_type, - self._table.options.options.to_map(), + self._options.to_map(), index_field.name, self._user_options, ) diff --git a/paimon-python/pypaimon/execution/python/globalindex/data_evolution_global_index_scanner.py b/paimon-python/pypaimon/execution/python/globalindex/data_evolution_global_index_scanner.py index eb8627f5d8c7..3ae8540a5d3d 100644 --- a/paimon-python/pypaimon/execution/python/globalindex/data_evolution_global_index_scanner.py +++ b/paimon-python/pypaimon/execution/python/globalindex/data_evolution_global_index_scanner.py @@ -612,6 +612,7 @@ def _create_inner_readers( index_type, file_io, index_path, field, io_metas, executor=None, options=None): """Build per-file (or per-shard) readers for a single indexType/range.""" core_options = options or CoreOptions(Options.from_none()) + core_options = CoreOptions(core_options.global_index_options(index_type, field.name)) if index_type == 'btree': from pypaimon.execution.python.globalindex.btree.lazy_filtered_btree_reader import LazyFilteredBTreeReader from pypaimon.execution.python.globalindex.key_serializer import create_serializer diff --git a/paimon-python/pypaimon/execution/python/read/search/full_text_read.py b/paimon-python/pypaimon/execution/python/read/search/full_text_read.py index e4bac13cf3b2..530642a16a93 100644 --- a/paimon-python/pypaimon/execution/python/read/search/full_text_read.py +++ b/paimon-python/pypaimon/execution/python/read/search/full_text_read.py @@ -24,6 +24,8 @@ from io import BytesIO from typing import Dict, List +from pypaimon.common.options.core_options import CoreOptions +from pypaimon.common.options.options import Options from pypaimon.globalindex.full_text_search import FullTextSearch from pypaimon.globalindex.global_index_meta import GlobalIndexIOMeta from pypaimon.globalindex.global_index_result import GlobalIndexResult @@ -309,7 +311,7 @@ def _build_raw_index(self, row_ids, texts, row_range_start): writer = FullTextIndexWriter( NativeFullTextIndexOptions.from_options( - _raw_search_options(self._table)).to_native_options()) + _raw_search_options(self._table, self._text_columns[0].name)).to_native_options()) indexed = False try: for row_id, text in zip(row_ids, texts): @@ -404,14 +406,15 @@ def _search_raw_full_text(index_bytes, row_range_start, query, limit, include_ro reader.close() -def _raw_search_options(table): +def _raw_search_options(table, column): table_options = getattr(getattr(table, "options", None), "options", None) if table_options is not None: options = dict(table_options.to_map()) else: options = dict(getattr(getattr(table, "table_schema", None), "options", {}) or {}) - options["full-text.searcher-pool.max-size"] = "0" - return options + resolved = CoreOptions(Options(options)).global_index_options("full-text", column).to_map() + resolved["full-text.searcher-pool.max-size"] = "0" + return resolved def _materialize_text(value) -> str: diff --git a/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py b/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py index bd4fe301a6f1..7696c469547a 100644 --- a/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py +++ b/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py @@ -313,7 +313,7 @@ def _open_offset_reader(self, vector_index_files, row_range_start, row_range_end self._table.file_io, self._table.path_factory().global_index_path_factory().index_path(), index_io_meta_list, - self._table.table_schema.options, + self._resolved_index_options(vector_index_files[0].index_type), ) try: self._record_index_metric(reader, vector_index_files[0].index_type, metric_lock) @@ -655,12 +655,17 @@ def _score_refine_splits(self, table_read, splits, queries_by_row, query_vectors del batch, row_ids, vectors return [_scored_result(heap) for heap in heaps] + def _resolved_index_options(self, index_type): + values = dict(_table_options_map(self._table)) + values.update(self._options) + return CoreOptions(Options(values)).global_index_options(index_type, self._vector_column.name).to_map() + def _configured_refine_factor(self, index_type): value = _configured_refine_factor( self._options, self._vector_column.name, index_type) if value is None: value = _configured_refine_factor( - _table_options_map(self._table), self._vector_column.name, index_type) + self._resolved_index_options(index_type), self._vector_column.name, index_type) if value is None: return 0 try: @@ -986,6 +991,12 @@ def _table_options_map(table): def _configured_vector_metric(options, vector_column, index_type=None): field_prefix = "fields.%s." % vector_column.name + if field_prefix + "index-options" in options: + core = CoreOptions(Options(options)) + selected_type = index_type or core.primary_key_vector_index_type(vector_column.name) or "vector" + metric = core.global_index_options(selected_type, vector_column.name, default_metric=False).to_map().get(selected_type + ".metric") + if metric is not None: + return metric index_prefix = "%s." % index_type if index_type else None keys = [field_prefix + "pk-vector.distance.metric", field_prefix + "distance.metric", field_prefix + "metric"] diff --git a/paimon-python/pypaimon/execution/python/read/search/vector_search_scan.py b/paimon-python/pypaimon/execution/python/read/search/vector_search_scan.py index e70d825d4e07..2c50e5f5b9b6 100644 --- a/paimon-python/pypaimon/execution/python/read/search/vector_search_scan.py +++ b/paimon-python/pypaimon/execution/python/read/search/vector_search_scan.py @@ -241,16 +241,7 @@ def _vector_index_type(vector_column, index_files): def _configured_vector_index_type(table, vector_column, options): - keys = [ - "index_type", - "index-type", - "vector.index-type", - "fields.%s.index-type" % vector_column.name, - ] table_options = getattr(getattr(table, "options", None), "options", None) - table_map = table_options.to_map() if table_options is not None else {} - for key in keys: - value = options.get(key) or table_map.get(key) - if value is not None: - return str(value).lower().strip() - return None + values = dict(table_options.to_map()) if table_options is not None else {} + values.update(options) + return CoreOptions(Options(values)).vector_index_type(vector_column.name) diff --git a/paimon-python/pypaimon/globalindex/create_global_index.py b/paimon-python/pypaimon/globalindex/create_global_index.py index a01412e67787..6cc6ef2ea74f 100644 --- a/paimon-python/pypaimon/globalindex/create_global_index.py +++ b/paimon-python/pypaimon/globalindex/create_global_index.py @@ -110,6 +110,9 @@ def __init__( "Python global index build currently supports one column, got %s." % self._index_columns ) + self._options = self._core_options.global_index_options( + self._index_type, self._index_columns[0]) + self._core_options = CoreOptions(self._options) if not self._table.options.row_tracking_enabled(): raise ValueError( "Table '%s' must enable 'row-tracking.enabled=true' before " diff --git a/paimon-python/pypaimon/table/source/vector_search_builder.py b/paimon-python/pypaimon/table/source/vector_search_builder.py index f4dc16c201fc..0bd98a2a8c0b 100644 --- a/paimon-python/pypaimon/table/source/vector_search_builder.py +++ b/paimon-python/pypaimon/table/source/vector_search_builder.py @@ -226,7 +226,7 @@ def new_vector_search_scan(self): return create_vector_scan(self, execution, self._native_batch_search) def _is_primary_key_vector_search(self): - if self._vector_column is None: + if self._vector_column is None or not getattr(self._table.table_schema, 'primary_keys', None): return False core = CoreOptions(Options(dict(self._table.table_schema.options))) return self._vector_column.name in core.primary_key_vector_index_columns() diff --git a/paimon-python/pypaimon/tests/global_index_build_test.py b/paimon-python/pypaimon/tests/global_index_build_test.py index fd3163dab980..cc9a4d3e649d 100644 --- a/paimon-python/pypaimon/tests/global_index_build_test.py +++ b/paimon-python/pypaimon/tests/global_index_build_test.py @@ -226,6 +226,116 @@ class GlobalIndexBuildTest( 'file.format': 'parquet', } + def test_native_vector_build_uses_the_same_resolved_field_options(self): + schema = pa.schema([('id', pa.int32()), ('embedding', pa.list_(pa.float32()))]) + configurations = { + 'shared': {'fields.embedding.index-options': + '{"dimension":768,"nlist":64,"distance.metric":"cosine","train.sample-ratio":0.5}'}, + 'legacy': {'fields.embedding.pk-vector.index.options': + '{"dimension":768,"nlist":64,"train.sample-ratio":0.5}', + 'fields.embedding.pk-vector.distance.metric': 'cosine'}, + 'dlf': {'fields.embedding.dimension': '768', 'fields.embedding.nlist': '64', + 'fields.embedding.distance.metric': 'cosine', + 'fields.embedding.train.sample-ratio': '0.5'}, + } + for name, field_options in configurations.items(): + with self.subTest(configuration=name): + options = dict(self.table_options, **{ + 'write.native.enabled': 'true', 'global-index.ivf-pq.index-columns': 'embedding', + 'ivf-pq.dimension': '128', 'ivf-pq.nlist': '32'}) + if name == 'legacy': + options.update({'ivf-pq.dimension': '768', 'ivf-pq.nlist': '64'}) + options.update(field_options) + table = self._create_table(pa_schema=schema, options=options) + request = GlobalIndexBuilder(table, 'embedding', 'ivf-pq') + actual = self._native_build_options(request) + self.assertEqual('768', actual['fields.embedding.dimension']) + self.assertEqual('64', actual['fields.embedding.nlist']) + self.assertEqual('cosine', actual['fields.embedding.metric']) + self.assertEqual('0.5', actual['fields.embedding.train.sample-ratio']) + + def test_native_scalar_and_full_text_builds_use_resolved_field_options(self): + schema = pa.schema([('id', pa.int32()), ('name', pa.string())]) + for kind, key, value in (('btree', 'btree-index.block-size', '4 kb'), + ('bitmap', 'bitmap-index.dictionary-block-size', '8 kb'), + ('full-text', 'full-text.tokenizer', 'raw')): + with self.subTest(index_type=kind): + short_key = key.split('.', 1)[1] + options = dict(self.table_options, **{'write.native.enabled': 'true', + 'global-index.%s.index-columns' % kind: 'name', + 'fields.name.index-options': '{"%s":"%s"}' % (short_key, value)}) + table = self._create_table(pa_schema=schema, options=options) + actual = self._native_build_options(GlobalIndexBuilder(table, 'name', kind)) + self.assertEqual(value, actual[key]) + + def _native_build_options(self, request): + from unittest.mock import Mock + from pypaimon.execution.native.write import native_index_build + + native = Mock() + native.build.return_value = [] + sdk = types.ModuleType('pypaimon_rust.datafusion') + sdk.GlobalIndexBuildBuilder = types.SimpleNamespace(supports_index_type=lambda kind: True) + proxy = types.SimpleNamespace(new_global_index_build_builder=lambda: native) + with patch.dict(sys.modules, {'pypaimon_rust.datafusion': sdk}), \ + patch.object(native_index_build, 'native_runtime_available', return_value=True), \ + patch.object(native_index_build, '_rest_catalog_supported', return_value=True), \ + patch.object(native_index_build, '_native_table', return_value=proxy), \ + patch.object(native_index_build, 'from_native_commit_messages', + side_effect=lambda table, messages: messages): + self.assertEqual([], request.build()) + return native.with_options.call_args[0][0] + + def test_full_text_build_and_raw_fallback_share_field_options(self): + from pypaimon.execution.python.read.search.full_text_read import DataEvolutionFullTextRead + from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult + from pypaimon.table.source.full_text_search_split import RawFullTextSearchSplit + + schema = pa.schema([('id', pa.int32()), ('content', pa.string())]) + configurations = { + 'shared': {'fields.content.index-options': '{"tokenizer":"raw","lower-case":false,"stop-words":["paimon","lake"]}'}, + 'legacy': {'fields.content.pk-full-text.index.options': '{"tokenizer":"raw"}'}, + 'dlf': {'full-text.tokenizer': 'raw'}, + } + for name, field_options in configurations.items(): + with self.subTest(configuration=name): + options = dict(self.table_options, **{'global-index.full-text.index-column': 'content'}) + options.update(field_options) + table = self._create_table(pa_schema=schema, options=options) + self._write_arrow(table, pa.table({'id': [1], 'content': ['Raw Text']}, schema=schema)) + ftindex = _FakeFullTextForBuild() + with patch.dict(sys.modules, {'paimon_ftindex': ftindex}): + self.assertEqual(1, table.create_global_index('content', index_type=FULL_TEXT_IDENTIFIER)) + self._write_arrow(table, pa.table({'id': [2], 'content': ['Raw Text']}, schema=schema)) + read = DataEvolutionFullTextRead(table, 10, table.field_dict['content'], '{}') + with patch('pypaimon.execution.python.read.search.full_text_read._search_raw_full_text', + return_value=DictBasedScoredIndexResult({1: 1.0})) as search: + result = read.read([RawFullTextSearchSplit([Range(1, 1)])]) + self.assertEqual([1], list(result.results())) + self.assertEqual(b'fake-ftindex', search.call_args[0][0]) + expected = {'tokenizer': 'raw'} + if name == 'shared': + expected['lower-case'] = 'false' + expected['stop-words'] = 'paimon;lake' + self.assertEqual(expected, ftindex.writers[0].options) + self.assertEqual(dict(expected, **{'searcher-pool.max-size': '0'}), ftindex.writers[1].options) + self.assertTrue(all(writer.closed for writer in ftindex.writers)) + + def test_declared_vector_index_keeps_data_evolution_single_and_batch_reads(self): + schema = pa.schema([('id', pa.int32()), ('embedding', pa.list_(pa.float32()))]) + options = dict(self.table_options, **{ + 'global-index.ivf-pq.index-columns': 'embedding', 'ivf-pq.dimension': '2', + 'ivf-pq.distance.metric': 'l2', 'vector-index.search-mode': 'full', + 'read.native.enabled': 'false', 'scan.native-plan.enabled': 'false'}) + table = self._create_table(pa_schema=schema, options=options) + self._write_arrow(table, pa.table({'id': [1, 2], 'embedding': [[1.0, 0.0], [0.0, 1.0]]}, schema=schema)) + single = table.new_vector_search_builder().with_vector_column('embedding').with_limit(1).with_query_vector([1.0, 0.0]) + batch = table.new_batch_vector_search_builder().with_vector_column('embedding').with_limit(1).with_query_vectors([[0.0, 1.0]]) + self.assertFalse(single._is_primary_key_vector_search()) + self.assertFalse(batch._is_primary_key_vector_search()) + self.assertEqual([0], list(single.execute_local().results())) + self.assertEqual([1], list(batch.execute_batch_local()[0].results())) + def test_sorted_build_indexes_deleted_physical_rows_like_java(self): for kind in (BTREE_IDENTIFIER, BITMAP_IDENTIFIER): with self.subTest(kind=kind): @@ -622,12 +732,25 @@ def test_create_btree_global_index_for_java_scalar_types(self): self.assertEqual(1, table.create_global_index(column)) def test_create_vindex_global_index_from_python(self): + self._check_create_vindex_global_index(False) + + def test_create_vindex_global_index_with_shared_field_options(self): + self._check_create_vindex_global_index(True) + + def _check_create_vindex_global_index(self, shared_options): schema = pa.schema([ ('id', pa.int32()), ('embedding', pa.list_(pa.float32())), ]) - table = self._create_table(pa_schema=schema, options=dict( - self.table_options, **{'read.batch-size': '1'})) + table_options = dict(self.table_options, **{'read.batch-size': '1'}) + if shared_options: + table_options.update({ + 'global-index.ivf-flat.index-columns': 'embedding', + 'ivf-flat.dimension': '2', + 'ivf-flat.nlist': '32', + 'fields.embedding.index-options': '{"nlist":1,"distance.metric":"l2"}', + }) + table = self._create_table(pa_schema=schema, options=table_options) vectors = pa.array( [[1.0, 0.0], [0.0, 1.0], None], type=pa.list_(pa.float32()), @@ -646,7 +769,7 @@ def test_create_vindex_global_index_from_python(self): added = table.create_global_index( 'embedding', index_type='ivf-flat', - options={ + options=None if shared_options else { 'ivf-flat.dimension': '2', 'ivf-flat.distance.metric': 'l2', 'ivf-flat.nlist': '1', diff --git a/paimon-python/pypaimon/tests/global_index_test.py b/paimon-python/pypaimon/tests/global_index_test.py index f1532a291e0e..3b24945777a5 100644 --- a/paimon-python/pypaimon/tests/global_index_test.py +++ b/paimon-python/pypaimon/tests/global_index_test.py @@ -45,6 +45,76 @@ class GlobalIndexTest(unittest.TestCase): + def test_legacy_primary_key_metric_resolution_is_idempotent(self): + for metric in ('COSINE', 'inner-product', 'L2'): + with self.subTest(metric=metric): + values = {'pk-vector.index.columns': 'embedding', + 'fields.embedding.pk-vector.index.type': 'ivf-pq', + 'fields.embedding.pk-vector.distance.metric': metric.lower().replace('-', '_'), + 'fields.embedding.pk-vector.index.options': '{"metric":"%s"}' % metric} + first = CoreOptions(Options(values)).primary_key_vector_index_options('embedding') + second = CoreOptions(first).global_index_options('ivf-pq', 'embedding') + self.assertEqual(first.to_map(), second.to_map()) + self.assertEqual('{"metric":"%s"}' % metric, + values['fields.embedding.pk-vector.index.options']) + + def test_shared_index_column_precedence_and_legacy_fallback(self): + for index_type, legacy in (("btree", "btree"), ("bitmap", "bitmap"), + ("full-text", "full-text"), ("ivf-pq", "vector"), + ("fm", "fm"), ("multivalue", "multivalue")): + with self.subTest(index_type=index_type): + values = {"pk-%s.index.columns" % legacy: "old", + "fields.old.pk-vector.index.type": "ivf-pq"} + self.assertEqual(["old"], CoreOptions(Options(values)).global_index_columns(index_type)) + values["global-index.%s.index-column" % index_type] = "single" + self.assertEqual(["single"], CoreOptions(Options(values)).global_index_columns(index_type)) + values["global-index.%s.index-columns" % index_type] = " first, second " + core = CoreOptions(Options(values)) + self.assertEqual(["first", "second"], core.global_index_columns(index_type)) + self.assertEqual([index_type], core.global_index_types()) + + def test_shared_field_options_override_type_defaults(self): + for index_type, prefix, key in (("btree", "btree-index.", "block-size"), + ("bitmap", "bitmap-index.", "dictionary-block-size"), + ("full-text", "full-text.", "tokenizer"), + ("ivf-pq", "ivf-pq.", "nlist")): + with self.subTest(index_type=index_type): + values = {prefix + key: "old", + "fields.payload.index-options": '{"%s":"new"}' % key, + "sorted-index.records-per-file": "20"} + core = CoreOptions(Options(values)) + resolved = core.global_index_options(index_type, "payload").to_map() + self.assertEqual("new", resolved[prefix + key]) + self.assertEqual("20", resolved["sorted-index.records-per-file"]) + self.assertEqual("old", values[prefix + key]) + + def test_shared_vector_type_and_documented_metric(self): + values = {"global-index.ivf-pq.index-columns": "embedding", + "pk-vector.index.columns": "old", + "fields.embedding.pk-vector.index.type": "diskann", + "ivf-pq.distance.metric": "l2", + "ivf-pq.dimension": "768", + "fields.embedding.nlist": "48", + "fields.embedding.distance.metric": "cosine"} + core = CoreOptions(Options(values)) + self.assertEqual(["embedding"], core.primary_key_vector_index_columns()) + self.assertEqual("ivf-pq", core.primary_key_vector_index_type("embedding")) + self.assertEqual("cosine", core.primary_key_vector_distance_metric("embedding")) + values["fields.embedding.index-options"] = '{"nlist":64,"distance.metric":"inner_product"}' + core = CoreOptions(Options(values)) + self.assertEqual("inner_product", core.primary_key_vector_distance_metric("embedding")) + resolved = core.primary_key_vector_index_options("embedding").to_map() + self.assertEqual("768", resolved["ivf-pq.dimension"]) + self.assertEqual("64", resolved["fields.embedding.nlist"]) + self.assertEqual("inner_product", resolved["ivf-pq.distance.metric"]) + + def test_shared_field_options_reject_invalid_values(self): + for serialized in ('[]', '{"nlist":null}', '{"":64}', 'invalid'): + with self.subTest(serialized=serialized): + core = CoreOptions(Options({"fields.embedding.index-options": serialized})) + with self.assertRaisesRegex(ValueError, 'fields.embedding.index-options'): + core.global_index_options("ivf-pq", "embedding") + def test_primary_key_sorted_index_columns_preserve_java_split_semantics(self): options = CoreOptions(Options({ "pk-btree.index.columns": "name, id", diff --git a/paimon-python/pypaimon/tests/lazy_search_execution_test.py b/paimon-python/pypaimon/tests/lazy_search_execution_test.py index 36313fd9e671..fc5942c49790 100644 --- a/paimon-python/pypaimon/tests/lazy_search_execution_test.py +++ b/paimon-python/pypaimon/tests/lazy_search_execution_test.py @@ -51,6 +51,117 @@ def _builder(table, family): return table.new_vector_search_builder().with_vector_column('embedding').with_limit(2).with_query_vector([1.0, 0.0]) +@pytest.mark.parametrize('family', ['vector', 'batch', 'text', 'hybrid']) +@pytest.mark.parametrize('configure_metric', [True, False]) +def test_native_search_routes_use_shared_index_options(search_table, family, configure_metric): + from pypaimon.execution.native.read import native_vector_search, native_full_text_search, native_hybrid_search + from pypaimon.table.source.hybrid_search_builder import HybridSearchRoute + + table = search_table.copy({ + 'global-index.ivf-pq.index-columns': 'embedding', + 'global-index.full-text.index-column': 'body', + 'ivf-pq.dimension': '2', + 'fields.embedding.index-options': json.dumps( + {'distance.metric': 'COSINE', 'refine_factor': 3} if configure_metric else {'refine_factor': 3}), + 'fields.body.index-options': '{"tokenizer":"raw","lower-case":false}', + }) + core = Mock() + core.with_limit.return_value = core + core.add_full_text_route.side_effect = HybridSearchRoute.full_text_route + proxy = SimpleNamespace(new_vector_search_builder=lambda: core, + new_batch_vector_search_builder=lambda: core, + new_full_text_search_builder=lambda: core, + new_hybrid_search_builder=lambda: core) + module = native_hybrid_search if family == 'hybrid' else ( + native_full_text_search if family == 'text' else native_vector_search) + with ExitStack() as stack: + stack.enter_context(patch.object(module, 'native_runtime_available', return_value=True)) + bridge = stack.enter_context(patch.object(module, '_native_table', return_value=proxy)) + if family in ('vector', 'batch'): + stack.enter_context(patch.object(module, '_catalog_metastore', return_value='rest')) + module._native_vector_builder(_builder(table, family), family == 'batch') + else: + stack.enter_context(patch.object(module, '_rest_catalog_supported', return_value=True)) + if family == 'text': + module._native_full_text_builder(_builder(table, family)) + else: + builder = (table.new_hybrid_search_builder().with_limit(2) + .add_vector_route('embedding', [1.0, 0.0], 2) + .add_full_text_route('body', '{}', 2)) + module._native_builder(builder) + if family == 'hybrid': + vector = core.add_vector_route.call_args[0][-1] + assert core.add_full_text_route.call_args[0][-1] == {} + text = bridge.call_args[0][0].options.options.to_map() + if configure_metric: + assert vector['fields.embedding.metric'] == 'cosine' + else: + assert 'fields.embedding.metric' not in vector + assert 'ivf-pq.metric' not in vector + assert vector['fields.embedding.refine_factor'] == '3' + assert text['full-text.tokenizer'] == 'raw' + assert text['full-text.lower-case'] == 'false' + else: + options = bridge.call_args[0][0].options.options.to_map() + if family == 'text': + assert options['full-text.tokenizer'] == 'raw' + assert options['full-text.lower-case'] == 'false' + else: + if configure_metric: + assert options['fields.embedding.metric'] == 'cosine' + else: + assert 'fields.embedding.metric' not in options + assert 'ivf-pq.metric' not in options + assert options['fields.embedding.refine_factor'] == '3' + + +@pytest.mark.parametrize('same_analyzer', [True, False]) +def test_native_hybrid_full_text_routes_preserve_each_field_analyzer(tmp_path, same_analyzer): + from pypaimon.execution.native.read import native_hybrid_search + from pypaimon.table.source.hybrid_search_builder import HybridSearchRoute + + catalog = CatalogFactory.create({'warehouse': str(tmp_path / 'warehouse')}) + catalog.create_database('default', True) + catalog.create_table('default.search', Schema.from_pyarrow_schema( + pa.schema([('body', pa.string()), ('title', pa.string())]), + options={'row-tracking.enabled': 'true', 'data-evolution.enabled': 'true', + 'read.native.enabled': 'true', + 'fields.body.index-options': '{"tokenizer":"raw"}', + 'fields.title.index-options': '{"tokenizer":"%s"}' % ('raw' if same_analyzer else 'jieba')}), + False) + table = catalog.get_table('default.search') + builder = (table.new_hybrid_search_builder().with_limit(2) + .add_full_text_route('body', '{}', 2).add_full_text_route('title', '{}', 2)) + core = Mock() + core.with_limit.return_value = core + core.add_full_text_route.side_effect = HybridSearchRoute.full_text_route + proxy = SimpleNamespace(new_hybrid_search_builder=lambda: core) + with patch.object(native_hybrid_search, 'native_runtime_available', return_value=True), \ + patch.object(native_hybrid_search, '_rest_catalog_supported', return_value=True), \ + patch.object(native_hybrid_search, '_native_table', return_value=proxy) as bridge: + configured = native_hybrid_search._native_builder(builder) + if same_analyzer: + assert configured is not None + assert bridge.call_args[0][0].options.options.to_map()['full-text.tokenizer'] == 'raw' + assert core.add_full_text_route.call_count == 2 + assert all(call.args[-1] == {} for call in core.add_full_text_route.call_args_list) + else: + assert configured is None + bridge.assert_not_called() + core.add_full_text_route.assert_not_called() + + +def test_native_vector_defers_field_options_until_index_type_is_known(search_table): + from pypaimon.execution.native.read import native_vector_search + + table = search_table.copy({'fields.embedding.index-options': '{"nlist":64}'}) + with patch.object(native_vector_search, 'native_runtime_available', return_value=True), \ + patch.object(native_vector_search, '_catalog_metastore', return_value='rest'), \ + patch.object(native_vector_search, '_native_table') as bridge: + assert native_vector_search._native_vector_builder(_builder(table, 'vector')) is None + bridge.assert_not_called() + + def _factories(builder, family): if family == 'text': return builder.new_full_text_scan, builder.new_full_text_read diff --git a/paimon-python/pypaimon/tests/native_index_options_test.py b/paimon-python/pypaimon/tests/native_index_options_test.py new file mode 100644 index 000000000000..190db364ef97 --- /dev/null +++ b/paimon-python/pypaimon/tests/native_index_options_test.py @@ -0,0 +1,176 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +from types import ModuleType +from unittest.mock import Mock, patch + +import pytest + +from pypaimon import CatalogFactory, Schema +from pypaimon.execution.native.index_options import native_scalar_index_read_supported +from pypaimon.execution.native.read.builder import _native_read_builder +from pypaimon.execution.native.read.plan import native_plan +from pypaimon.execution.native.read.native_search_plan import _index_search_options +from pypaimon.execution.native.table import _resolved_schema_json +from pypaimon.execution.native.write.native_commit import create_native_write_table +from pypaimon.index.pk.primary_key_index_definitions import PrimaryKeyIndexDefinitions +from pypaimon.schema.data_types import AtomicType, DataField, VectorType + + +def _table(tmp_path, primary_key, extra=None): + options = {'row-tracking.enabled': 'false' if primary_key else 'true', + 'bucket': '2' if primary_key else '-1', + 'deletion-vectors.enabled': 'true' if primary_key else 'false', + 'data-evolution.enabled': 'false' if primary_key else 'true'} + options.update(extra or {}) + catalog = CatalogFactory.create({'warehouse': str(tmp_path / 'warehouse')}) + catalog.create_database('default', True) + fields = [DataField(0, 'id', AtomicType('INT', False)), + DataField(1, 'amount', AtomicType('INT')), + DataField(2, 'other', AtomicType('INT')), + DataField(3, 'embedding', VectorType(True, AtomicType('FLOAT'), 2)), + DataField(4, 'body', AtomicType('STRING'))] + catalog.create_table('default.search', Schema(fields=fields, + primary_keys=['id'] if primary_key else [], + options=options), False) + return catalog.get_table('default.search') + + +@pytest.mark.parametrize('primary_key', [False, True]) +@pytest.mark.parametrize('suffix', ['index-column', 'index-columns']) +def test_native_read_and_write_schemas_share_resolved_index_contract(tmp_path, primary_key, suffix): + table = _table(tmp_path, primary_key, { + 'global-index.btree.' + suffix: 'amount', + 'fields.amount.index-options': '{"fallback-scan-max-size":"1 kb"}', + 'global-index.ivf-pq.' + suffix: 'embedding', + 'ivf-pq.dimension': '2', + 'ivf-pq.refine_factor': '3', + 'fields.embedding.index-options': '{"nlist":64,"metric":"COSINE"}', + 'global-index.full-text.' + suffix: 'body', + 'fields.body.index-options': '{"tokenizer":"raw","lower-case":false}', + }) + original = dict(table.table_schema.options) + read_options = json.loads(_resolved_schema_json(table))['options'] + module = ModuleType('pypaimon_rust.datafusion') + module.Table = Mock() + with patch.dict('sys.modules', {'pypaimon_rust.datafusion': module}): + assert create_native_write_table(table) is module.Table.from_resolved_schema.return_value + write_options = json.loads(module.Table.from_resolved_schema.call_args[0][1])['options'] + for options in (read_options, write_options): + assert not any(key.startswith('global-index.') and key.endswith(('.index-column', '.index-columns')) + for key in options) + if primary_key: + assert options['pk-btree.index.columns'] == 'amount' + assert options['pk-vector.index.columns'] == 'embedding' + assert options['fields.embedding.pk-vector.index.type'] == 'ivf-pq' + assert options['fields.embedding.pk-vector.distance.metric'] == 'cosine' + assert options['ivf-pq.dimension'] == '2' + assert options['ivf-pq.refine_factor'] == '3' + assert options['full-text.tokenizer'] == 'raw' + assert options['full-text.lower-case'] == 'false' + definitions = PrimaryKeyIndexDefinitions.create(table.table_schema.copy(new_options=options)).definitions + by_type = {definition.index_type: definition for definition in definitions} + assert by_type['btree'].options.to_map()['btree-index.fallback-scan-max-size'] == '1 kb' + assert by_type['ivf-pq'].options.to_map()['ivf-pq.nlist'] == '64' + assert by_type['ivf-pq'].options.to_map()['ivf-pq.dimension'] == '2' + assert by_type['ivf-pq'].options.to_map()['ivf-pq.metric'] == 'cosine' + assert by_type['full-text'].options.to_map()['full-text.lower-case'] == 'false' + else: + assert not any(key.startswith('pk-') for key in options) + assert options['fields.embedding.index-type'] == 'ivf-pq' + assert options['fields.embedding.nlist'] == '64' + assert options['fields.embedding.metric'] == 'cosine' + assert options['btree-index.fallback-scan-max-size'] == '1 kb' + assert table.table_schema.options == original + + +def test_native_primary_key_fields_keep_independent_overrides(tmp_path): + table = _table(tmp_path, True, { + 'global-index.btree.index-columns': 'amount,other', + 'btree-index.block-size': '4 kb', + 'fields.amount.index-options': '{"block-size":"8 kb"}', + 'fields.other.index-options': '{"block-size":"16 kb"}', + }) + options = json.loads(_resolved_schema_json(table))['options'] + assert 'btree-index.block-size' not in options + assert json.loads(options['fields.amount.pk-btree.index.options'])['btree-index.block-size'] == '8 kb' + assert json.loads(options['fields.other.pk-btree.index.options'])['btree-index.block-size'] == '16 kb' + + +@pytest.mark.parametrize('field_options', [ + {}, + {'fields.embedding.pk-vector.index.options': '{"nlist":64}'}, + {'fields.embedding.index-options': '{"nlist":64}'}, +]) +def test_native_legacy_primary_key_keeps_flat_vector_read_parameters(tmp_path, field_options): + options = {'pk-vector.index.columns': 'embedding', + 'fields.embedding.pk-vector.index.type': 'ivf-pq', + 'fields.embedding.pk-vector.distance.metric': 'COSINE', + 'ivf-pq.dimension': '2', 'ivf-pq.refine_factor': '3'} + options.update(field_options) + table = _table(tmp_path, True, options) + wire = json.loads(_resolved_schema_json(table))['options'] + assert wire['pk-vector.index.columns'] == 'embedding' + assert wire['fields.embedding.pk-vector.index.type'] == 'ivf-pq' + assert wire['fields.embedding.pk-vector.distance.metric'] == 'cosine' + assert wire['ivf-pq.dimension'] == '2' + assert wire['ivf-pq.refine_factor'] == '3' + assert wire['ivf-pq.metric'] == 'cosine' + if field_options: + assert wire['ivf-pq.nlist'] == '64' + + +def test_native_data_evolution_does_not_invent_a_vector_metric(tmp_path): + table = _table(tmp_path, False, { + 'global-index.ivf-pq.index-columns': 'embedding', + 'fields.embedding.index-options': '{"nlist":64}', + }) + options = json.loads(_resolved_schema_json(table))['options'] + assert options['fields.embedding.nlist'] == '64' + assert options['fields.embedding.index-type'] == 'ivf-pq' + assert 'fields.embedding.metric' not in options + assert 'ivf-pq.metric' not in options + + +def test_native_qualifies_distinct_scalar_field_options_before_reading(tmp_path): + table = _table(tmp_path, False, { + 'global-index.btree.index-columns': 'amount,other', + 'fields.amount.index-options': '{"fallback-scan-max-size":"1 kb"}', + 'fields.other.index-options': '{"fallback-scan-max-size":"2 kb"}', + }) + assert not native_scalar_index_read_supported(table) + predicate = table.new_read_builder().new_predicate_builder().equal('amount', 1) + with patch('pypaimon.execution.native.table._native_table') as bridge, \ + patch('pypaimon.execution.native.read.plan.runtime.native_runtime_available', return_value=True): + with pytest.raises(NotImplementedError, match='field index options'): + native_plan(table, predicate=predicate) + assert _index_search_options(table, 'body', 'full-text', scalar_filter=predicate) is None + bridge.assert_not_called() + + +@pytest.mark.parametrize('global_index', [True, False]) +def test_index_field_options_do_not_disable_native_physical_reads(tmp_path, global_index): + table = _table(tmp_path, False, { + 'global-index.enabled': str(global_index).lower(), + 'fields.embedding.index-options': '{"nlist":64}', + }) + if not global_index: + assert native_scalar_index_read_supported(table) + with patch('pypaimon.execution.native.table._native_table') as bridge: + assert _native_read_builder(table) is bridge.return_value.new_read_builder.return_value + bridge.assert_called_once_with(table) diff --git a/paimon-python/pypaimon/tests/primary_key_index_definitions_test.py b/paimon-python/pypaimon/tests/primary_key_index_definitions_test.py index 3541bf940777..33c37b52f86e 100644 --- a/paimon-python/pypaimon/tests/primary_key_index_definitions_test.py +++ b/paimon-python/pypaimon/tests/primary_key_index_definitions_test.py @@ -53,6 +53,23 @@ class PrimaryKeyIndexDefinitionsTest(unittest.TestCase): + def test_shared_vector_and_full_text_definitions(self): + schema = TableSchema( + fields=[DataField(0, "embedding", AtomicType("VECTOR(3)")), + DataField(1, "content", AtomicType("STRING"))], + options={ + "global-index.ivf-pq.index-columns": "embedding", + "ivf-pq.dimension": "3", + "fields.embedding.index-options": '{"nlist":64,"metric":"cosine"}', + "global-index.full-text.index-column": "content", + "fields.content.index-options": '{"tokenizer":"jieba"}', + }) + definitions = PrimaryKeyIndexDefinitions.create(schema).definitions + self.assertEqual(["ivf-pq", "full-text"], [d.index_type for d in definitions]) + self.assertEqual("64", definitions[0].options.to_map()["ivf-pq.nlist"]) + self.assertEqual("cosine", definitions[0].options.to_map()["ivf-pq.metric"]) + self.assertEqual("jieba", definitions[1].options.to_map()["full-text.tokenizer"]) + def test_bucket_planners_preserve_capture_and_physical_metadata(self): from pypaimon.execution.python.read.search.primary_key_full_text_scan import _full_text_bucket_splits from pypaimon.table.source.deletion_file import DeletionFile @@ -133,7 +150,9 @@ def test_vector_and_full_text_definitions(self): def test_builders_route_primary_key_searches(self): schema = TableSchema( fields=[DataField(0, "embedding", AtomicType("VECTOR(3)")), - DataField(1, "content", AtomicType("STRING"))], + DataField(1, "content", AtomicType("STRING")), + DataField(2, "id", AtomicType("INT"))], + primary_keys=["id"], options={ "pk-vector.index.columns": "embedding", "fields.embedding.pk-vector.index.type": "vindex", diff --git a/paimon-python/pypaimon/tests/vector_metric_consistency_test.py b/paimon-python/pypaimon/tests/vector_metric_consistency_test.py index 1ccd60d01be1..58994188e4d6 100644 --- a/paimon-python/pypaimon/tests/vector_metric_consistency_test.py +++ b/paimon-python/pypaimon/tests/vector_metric_consistency_test.py @@ -177,6 +177,27 @@ def test_raw_only_defaults_match_index_writers(self): self.table, self.column, {'metric': 'cosine'}, kind)) self.assertEqual('l2', _raw_search_metric(self.table, self.column, {})) + def test_shared_field_metric_is_used_by_raw_and_indexed_search(self): + options = {'global-index.ivf-flat.index-columns': 'embedding', + 'ivf-flat.distance.metric': 'l2', + 'fields.embedding.index-options': '{"distance.metric":"cosine"}'} + self.assertEqual('cosine', _raw_search_metric(self.table, self.column, options, 'ivf-flat')) + native = Mock(spec=['vector_metric']) + native.vector_metric.return_value = 'cosine' + reader = self._reader(options) + reader._record_index_metric(native, 'ivf-flat') + self.assertEqual('cosine', reader._search_metric('ivf-flat')) + native.vector_metric.return_value = 'inner_product' + with self.assertRaisesRegex(ValueError, 'does not match index metric'): + self._reader(options)._record_index_metric(native, 'ivf-flat') + + def test_unrelated_field_options_do_not_override_persisted_metric(self): + native = Mock(spec=['vector_metric']) + native.vector_metric.return_value = 'cosine' + reader = self._reader({'fields.embedding.index-options': '{"nlist":64}'}) + reader._record_index_metric(native, 'ivf-flat') + self.assertEqual('cosine', reader._search_metric('ivf-flat')) + def test_known_families_ignore_other_columns_metric(self): for kind in ('ivf-flat', 'lumina', 'lumina-vector-ann'): with self.subTest(index_type=kind): diff --git a/paimon-python/pypaimon/tests/vector_search_filter_test.py b/paimon-python/pypaimon/tests/vector_search_filter_test.py index 02cc80ac2d7b..1cf307ef8a4f 100644 --- a/paimon-python/pypaimon/tests/vector_search_filter_test.py +++ b/paimon-python/pypaimon/tests/vector_search_filter_test.py @@ -1336,6 +1336,15 @@ def close(self_inner): self.assertEqual(["embedding", "_ROW_ID"], raw_calls["projection"]) self.assertEqual([0], sorted(list(result.results()))) + table.options = CoreOptions(Options({'fields.embedding.index-options': '{"refine_factor":3,"metric":"l2"}'})) + captured_limits.clear() + with mock.patch('pypaimon.execution.python.read.search.vector_search_read._create_vector_reader', + side_effect=_fake_create): + result = (VectorSearchBuilderImpl(table).with_vector_column('embedding') + .with_query_vector([0.0]).with_limit(1).execute_local()) + self.assertEqual([3], captured_limits) + self.assertEqual([0], sorted(list(result.results()))) + def test_refine_factor_one_reranks_without_expanding_candidates(self): from pypaimon.globalindex.vector_search_result import ( DictBasedScoredIndexResult, @@ -3588,6 +3597,7 @@ def test_batch_refine_factor_reranks_each_query(self): file_name="vec.index", row_range_start=0, row_range_end=2) table = _StubTable(fields=[embedding_field], entries=[entry]) + table.options = CoreOptions(Options({'fields.embedding.index-options': '{"refine_factor":2,"metric":"l2"}'})) _patch_snapshot(self, [entry]) raw_calls = _install_raw_vector_read_builder( table, "embedding", {0: [10.0], 1: [0.0], 2: [5.0]}) @@ -3619,7 +3629,6 @@ def close(self_inner): .with_vector_column("embedding") .with_query_vectors([[0.0], [20.0]]) .with_limit(1) - .with_option("ivf.refine_factor", "2") .execute_batch_local() ) diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/sorted/SortedIndexTopoBuilder.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/sorted/SortedIndexTopoBuilder.java index 54a361f0023c..d93f3bcc6972 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/sorted/SortedIndexTopoBuilder.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/sorted/SortedIndexTopoBuilder.java @@ -23,9 +23,7 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.serializer.BinaryRowSerializer; import org.apache.paimon.globalindex.GlobalIndexKeyExtractor; -import org.apache.paimon.globalindex.GlobalIndexer; import org.apache.paimon.globalindex.ScanResult; -import org.apache.paimon.globalindex.SortedGlobalIndexer; import org.apache.paimon.globalindex.sorted.SortedGlobalIndexScanner; import org.apache.paimon.globalindex.sorted.SortedGlobalIndexWriter; import org.apache.paimon.globalindex.sorted.SortedIndexOptions; @@ -157,24 +155,21 @@ public List buildIndex( return Collections.emptyList(); } + SortedGlobalIndexWriter indexWriter = + new SortedGlobalIndexWriter(table, indexType, options).withIndexFields(indexNames); + Options indexOptions = indexWriter.indexOptions(); // Calculate maximum parallelism bound - long recordsPerRange = options.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE); - int maxParallelism = options.get(SortedIndexOptions.SORTED_INDEX_BUILD_MAX_PARALLELISM); + long recordsPerRange = indexOptions.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE); + int maxParallelism = + indexOptions.get(SortedIndexOptions.SORTED_INDEX_BUILD_MAX_PARALLELISM); List allMessages = new ArrayList<>(); - GlobalIndexer indexer = GlobalIndexer.create(indexType, indexFields, options); - if (!(indexer instanceof SortedGlobalIndexer)) { - throw new IllegalArgumentException( - "Index algorithm " + indexType + " does not expose sorted index keys."); - } - GlobalIndexKeyExtractor keyExtractor = ((SortedGlobalIndexer) indexer).keyExtractor(); + GlobalIndexKeyExtractor keyExtractor = indexWriter.keyExtractor(); String taskIdField = buildTaskIdFieldName(readType); RowType normalizedReadType = normalizedReadType(readType, taskIdField, indexField, keyExtractor.keyType()); final int partitionKeyNum = table.partitionKeys().size(); BinaryRowSerializer binaryRowSerializer = new BinaryRowSerializer(partitionKeyNum); - SortedGlobalIndexWriter indexWriter = - new SortedGlobalIndexWriter(table, indexType, options).withIndexFields(indexNames); final byte[] serializedWriter = InstantiationUtil.serializeObject(indexWriter); if (keyExtractor.isIdentity()) { List buildTasks = new ArrayList<>(); diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorRead.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorRead.java index 5b0896a50c5b..1233c70468f9 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorRead.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorRead.java @@ -22,7 +22,6 @@ import org.apache.paimon.globalindex.GlobalIndexResult; import org.apache.paimon.globalindex.GlobalIndexResultSerializer; import org.apache.paimon.globalindex.GlobalIndexer; -import org.apache.paimon.globalindex.GlobalIndexerFactoryUtils; import org.apache.paimon.globalindex.ScoredGlobalIndexResult; import org.apache.paimon.index.IndexPathFactory; import org.apache.paimon.partition.PartitionPredicate; @@ -126,10 +125,10 @@ protected ScoredGlobalIndexResult readIndexSplitsInSpark( SerializableFunction, byte[]> task = group -> { GlobalIndexer taskGlobalIndexer = - GlobalIndexerFactoryUtils.load(indexType) - .create( - Collections.singletonList(vectorColumn), - table.coreOptions().toConfiguration()); + GlobalIndexer.create( + indexType, + Collections.singletonList(vectorColumn), + table.coreOptions().toConfiguration()); IndexPathFactory indexPathFactory = table.store().pathFactory().globalIndexFileFactory(); diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorReadTest.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorReadTest.java index 58ae40d4ac34..ce87c4e9a48f 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorReadTest.java +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorReadTest.java @@ -28,6 +28,7 @@ import org.apache.paimon.globalindex.GlobalIndexResultSerializer; import org.apache.paimon.globalindex.GlobalIndexWriter; import org.apache.paimon.globalindex.GlobalIndexer; +import org.apache.paimon.globalindex.GlobalIndexerFactory; import org.apache.paimon.globalindex.ScoredGlobalIndexResult; import org.apache.paimon.globalindex.VectorGlobalIndexer; import org.apache.paimon.globalindex.io.GlobalIndexFileReader; @@ -56,6 +57,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import javax.annotation.Nullable; @@ -63,13 +66,16 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static org.apache.paimon.utils.Preconditions.checkArgument; import static org.assertj.core.api.Assertions.assertThat; /** Tests for {@link SparkDataEvolutionVectorRead}. */ @@ -181,10 +187,36 @@ public void testDistributedIndexMaterializesManySplitsInSingleTask() throws Exce assertThat(result.results()).contains(splitCount - 1L); } + @ParameterizedTest + @ValueSource(ints = {1, 2}) + public void testSharedFieldOptionsReachLocalAndDistributedIndexReads(int splitCount) + throws Exception { + Map options = new HashMap<>(); + options.put("row-tracking.enabled", "true"); + options.put("data-evolution.enabled", "true"); + options.put("global-index.test-shared-vector.index-columns", "vec"); + options.put("test-shared-vector.train.sample-ratio", "0"); + options.put( + "fields.vec.index-options", "{\"train.sample-ratio\":0.5,\"metric\":\"COSINE\"}"); + SingleTaskSparkVectorRead read = + new SingleTaskSparkVectorRead(createTable(options), "cosine"); + + ScoredGlobalIndexResult result = + read.readIndexSplitsInSpark(indexSplits("test-shared-vector", splitCount), null); + + assertThat(read.evaluatedSplits).hasValue(splitCount); + assertThat(result.results()).contains(splitCount - 1L); + } + private FileStoreTable createTable() throws Exception { + return createTable(Collections.emptyMap()); + } + + private FileStoreTable createTable(Map additionalOptions) throws Exception { Path tablePath = new Path(tempDir.toUri()); - Options options = new Options(); + Options options = new Options(additionalOptions); options.set(CoreOptions.PATH, tablePath.toString()); + options.set(CoreOptions.FILE_FORMAT, "avro"); options.set(CoreOptions.GLOBAL_INDEX_THREAD_NUM, 1); Schema schema = Schema.newBuilder() @@ -329,9 +361,15 @@ private static ScoredGlobalIndexResult scoredResult(long rowId, float score) { private static class SingleTaskSparkVectorRead extends SparkDataEvolutionVectorRead { private final AtomicInteger scoreCalls = new AtomicInteger(); + private final AtomicInteger evaluatedSplits = new AtomicInteger(); + private final String expectedMetric; private int sparkParallelism; private SingleTaskSparkVectorRead(FileStoreTable table) { + this(table, null); + } + + private SingleTaskSparkVectorRead(FileStoreTable table, String expectedMetric) { super( table, null, @@ -340,6 +378,7 @@ private SingleTaskSparkVectorRead(FileStoreTable table) { new DataField(0, "vec", new ArrayType(DataTypes.FLOAT())), new float[] {0.0f}, null); + this.expectedMetric = expectedMetric; } @Override @@ -366,6 +405,11 @@ protected CompletableFuture> eval( int searchLimit, @Nullable RoaringNavigableMap64 includeRowIds, ExecutorService executor) { + evaluatedSplits.incrementAndGet(); + if (expectedMetric != null) { + assertThat(((VectorGlobalIndexer) globalIndexer).metric()) + .isEqualTo(expectedMetric); + } RoaringNavigableMap64 rows = new RoaringNavigableMap64(); rows.add(rowRangeStart); return CompletableFuture.completedFuture( @@ -431,8 +475,34 @@ protected ScoredGlobalIndexResult readRawSearch( } } + /** Factory with construction-time validation, mirroring native ANN factories. */ + public static class OptionsVectorIndexerFactory implements GlobalIndexerFactory { + + @Override + public String identifier() { + return "test-shared-vector"; + } + + @Override + public GlobalIndexer create(List fields, Options options) { + double ratio = Double.parseDouble(options.get("test-shared-vector.train.sample-ratio")); + checkArgument(ratio > 0 && ratio <= 1, "Invalid train.sample-ratio: %s", ratio); + return new L2Indexer(options.get("test-shared-vector.metric")); + } + } + private static class L2Indexer implements VectorGlobalIndexer { + private final String metric; + + private L2Indexer() { + this("l2"); + } + + private L2Indexer(String metric) { + this.metric = metric; + } + @Override public GlobalIndexWriter createWriter(GlobalIndexFileWriter fileWriter) { throw new UnsupportedOperationException(); @@ -450,7 +520,7 @@ public GlobalIndexReader createReader( @Override public String metric() { - return "l2"; + return metric; } } } diff --git a/paimon-spark/paimon-spark-common/src/test/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory b/paimon-spark/paimon-spark-common/src/test/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory new file mode 100644 index 000000000000..0c1038da41dc --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/test/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.paimon.globalindex.testvector.TestVectorGlobalIndexerFactory + +org.apache.paimon.spark.read.SparkDataEvolutionVectorReadTest$OptionsVectorIndexerFactory From 4869dea610ff9e733336ef5ec60bbe71d4f944e7 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 31 Oct 2026 23:03:36 +0800 Subject: [PATCH 02/14] [core][python] Separate shared vector parameter resolution --- .../java/org/apache/paimon/CoreOptions.java | 64 ++--------- .../paimon/options/VectorIndexOptions.java | 102 +++++++++++++++++ .../paimon/globalindex/GlobalIndexer.java | 7 +- .../pk/BucketedPrimaryKeyIndexMaintainer.java | 5 +- .../index/pk/PrimaryKeyIndexDefinitions.java | 6 +- .../paimon/schema/SchemaValidation.java | 15 ++- .../AbstractDataEvolutionVectorRead.java | 15 +-- .../table/source/PrimaryKeyVectorRead.java | 7 +- .../apache/paimon/GlobalIndexOptionsTest.java | 24 +++- .../options/VectorIndexOptionsTest.java | 103 ++++++++++++++++++ .../pypaimon/common/options/core_options.py | 35 +----- .../common/options/vector_index_options.py | 62 +++++++++++ .../execution/native/index_options.py | 6 +- .../native/read/native_search_plan.py | 5 +- .../python/read/search/vector_search_read.py | 6 +- .../globalindex/create_global_index.py | 6 +- .../pypaimon/tests/global_index_test.py | 11 +- .../tests/vector_index_options_test.py | 76 +++++++++++++ 18 files changed, 442 insertions(+), 113 deletions(-) create mode 100644 paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java create mode 100644 paimon-python/pypaimon/common/options/vector_index_options.py create mode 100644 paimon-python/pypaimon/tests/vector_index_options_test.py diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 36716b0c7720..1dc222b57c84 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -30,6 +30,7 @@ import org.apache.paimon.options.ExpireConfig; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; +import org.apache.paimon.options.VectorIndexOptions; import org.apache.paimon.options.description.DescribedEnum; import org.apache.paimon.options.description.Description; import org.apache.paimon.options.description.InlineElement; @@ -5148,7 +5149,7 @@ public List globalIndexTypes() { .collect(Collectors.toList()); } - private static boolean isVectorIndexType(String type) { + public static boolean isVectorIndexType(String type) { return !Arrays.asList("btree", "bitmap", "multivalue", "full-text", "fm").contains(type); } @@ -5215,11 +5216,6 @@ public String globalIndexColumnsOptionKey(String type) { * qualified with the implementation's existing option prefix. */ public Options globalIndexOptions(@Nullable String type, String column) { - return globalIndexOptions(type, column, true); - } - - /** The same resolver, optionally retaining the absence of an explicitly configured metric. */ - public Options globalIndexOptions(@Nullable String type, String column, boolean defaultMetric) { Options resolved = new Options(options.toMap(), options); if (type == null) { type = vectorIndexType(column); @@ -5236,24 +5232,6 @@ public Options globalIndexOptions(@Nullable String type, String column, boolean shared ? sharedKey : fieldPrefix + "pk-" + (vector ? "vector" : type) + ".index.options"; - String metric = null; - if (vector) { - for (String key : - Arrays.asList( - fieldPrefix + "distance.metric", - fieldPrefix + "metric", - fieldPrefix + "pk-vector.distance.metric", - prefix + "distance.metric", - prefix + "metric", - "distance.metric", - "metric")) { - if (options.get(key) != null) { - metric = options.get(key); - break; - } - } - } - String serialized = options.get(optionKey); if (serialized != null && !serialized.trim().isEmpty()) { for (Map.Entry entry : @@ -5268,26 +5246,14 @@ public Options globalIndexOptions(@Nullable String type, String column, boolean value != null, "%s value for key %s must not be null.", optionKey, key); String qualified = key.startsWith(prefix) || key.startsWith("fields.") ? key : prefix + key; - boolean metricOption = - vector - && (qualified.equals(prefix + "metric") - || qualified.equals(prefix + "distance.metric") - || qualified.equals(fieldPrefix + "metric") - || qualified.equals(fieldPrefix + "distance.metric")); String previous = resolved.get(qualified); - if (metricOption) { - value = normalizeVectorMetric(value); - previous = previous == null ? null : normalizeVectorMetric(previous); - } checkArgument( shared || previous == null || previous.equals(value), "%s defines conflicting values for %s.", optionKey, qualified); resolved.setString(qualified, value); - if (metricOption) { - metric = value; - } else if (vector && qualified.startsWith(prefix)) { + if (vector && qualified.startsWith(prefix)) { String shortKey = qualified.substring(prefix.length()); if (shared) { // Native vector implementations also accept the flat field syntax. @@ -5296,14 +5262,6 @@ public Options globalIndexOptions(@Nullable String type, String column, boolean } } } - if (vector && (metric != null || defaultMetric)) { - metric = normalizeVectorMetric(metric); - // Keep both accepted aliases consistent for implementations using either spelling. - resolved.setString(prefix + "metric", metric); - resolved.setString(prefix + "distance.metric", metric); - resolved.setString(fieldPrefix + "metric", metric); - resolved.setString(fieldPrefix + "distance.metric", metric); - } return resolved; } @@ -5419,27 +5377,19 @@ public String vectorIndexType(String column) { return null; } + /** Compatibility delegate to the vector resolver shared by both table modes. */ public Options primaryKeyVectorIndexOptions(String column) { String type = primaryKeyVectorIndexType(column); checkArgument( type != null && !type.trim().isEmpty(), "fields.%s.pk-vector.index.type must be configured before resolving index options.", column); - return globalIndexOptions(type, column); + return VectorIndexOptions.resolve(options, type, column); } + /** Compatibility delegate to the vector resolver shared by both table modes. */ public String primaryKeyVectorDistanceMetric(String column) { - String type = primaryKeyVectorIndexType(column); - return type == null - ? normalizeVectorMetric( - options.get("fields." + column + ".pk-vector.distance.metric")) - : globalIndexOptions(type, column).get(type + ".metric"); - } - - private static String normalizeVectorMetric(String metric) { - return (metric == null ? "inner_product" : metric) - .toLowerCase(Locale.ROOT) - .replace('-', '_'); + return VectorIndexOptions.metric(options, primaryKeyVectorIndexType(column), column); } /** Specifies the merge engine for table with primary key. */ diff --git a/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java b/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java new file mode 100644 index 000000000000..a8a69fa8a3c8 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.options; + +import org.apache.paimon.CoreOptions; + +import javax.annotation.Nullable; + +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +/** Vector parameter resolution shared by primary-key and Data Evolution indexes. */ +public final class VectorIndexOptions { + + private VectorIndexOptions() {} + + public static Options resolve(Options options, @Nullable String type, String column) { + return resolve(options, type, column, true); + } + + /** Retains an unspecified metric when readers must obtain it from persisted index metadata. */ + public static Options resolve( + Options options, @Nullable String type, String column, boolean defaultMetric) { + if (type == null) { + type = new CoreOptions(options).vectorIndexType(column); + if (type == null) { + return new Options(options.toMap(), options); + } + } + String prefix = type + "."; + String fieldPrefix = "fields." + column + "."; + List aliases = + Arrays.asList( + fieldPrefix + "distance.metric", + fieldPrefix + "metric", + prefix + "distance.metric", + prefix + "metric"); + String metric = null; + for (String key : + Arrays.asList( + fieldPrefix + "distance.metric", + fieldPrefix + "metric", + fieldPrefix + "pk-vector.distance.metric", + prefix + "distance.metric", + prefix + "metric", + "distance.metric", + "metric")) { + if (options.get(key) != null) { + metric = options.get(key); + break; + } + } + Options defaults = new Options(options.toMap(), options); + // Metric aliases are one parameter. Merge field JSON before applying that parameter, + // so generic string conflict checks cannot reject aliases such as COSINE and cosine. + aliases.forEach(defaults::remove); + Options resolved = new CoreOptions(defaults).globalIndexOptions(type, column); + for (String key : aliases) { + if (resolved.get(key) != null) { + metric = resolved.get(key); + break; + } + } + if (metric != null || defaultMetric) { + String normalized = normalizeMetric(metric); + aliases.forEach(key -> resolved.setString(key, normalized)); + } + return resolved; + } + + public static String metric(Options options, @Nullable String type, String column) { + if (type == null) { + type = new CoreOptions(options).vectorIndexType(column); + } + return type == null + ? normalizeMetric(options.get("fields." + column + ".pk-vector.distance.metric")) + : resolve(options, type, column).get(type + ".metric"); + } + + private static String normalizeMetric(@Nullable String metric) { + return (metric == null ? "inner_product" : metric) + .toLowerCase(Locale.ROOT) + .replace('-', '_'); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexer.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexer.java index 795fc8dd164b..b9d84693fd83 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexer.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexer.java @@ -22,6 +22,7 @@ import org.apache.paimon.globalindex.io.GlobalIndexFileReader; import org.apache.paimon.globalindex.io.GlobalIndexFileWriter; import org.apache.paimon.options.Options; +import org.apache.paimon.options.VectorIndexOptions; import org.apache.paimon.types.DataField; import org.apache.paimon.utils.Range; @@ -54,7 +55,11 @@ static GlobalIndexer create(String type, List indexFields, Options op Options resolved = options; if (!indexFields.isEmpty()) { // The leading field owns the index; remaining fields are indexed payload columns. - resolved = new CoreOptions(options).globalIndexOptions(type, indexFields.get(0).name()); + String column = indexFields.get(0).name(); + resolved = + CoreOptions.isVectorIndexType(type) + ? VectorIndexOptions.resolve(options, type, column) + : new CoreOptions(options).globalIndexOptions(type, column); } return GlobalIndexerFactoryUtils.load(type).create(indexFields, resolved); } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainer.java b/paimon-core/src/main/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainer.java index 585c9a7b6c8a..92a9513a91cd 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainer.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainer.java @@ -38,6 +38,7 @@ import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataIncrement; import org.apache.paimon.io.KeyValueFileReaderFactory; +import org.apache.paimon.options.VectorIndexOptions; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.types.DataField; @@ -370,7 +371,9 @@ public static Factory create( handler, readerFactoryBuilder, field, - coreOptions.primaryKeyVectorDistanceMetric( + VectorIndexOptions.metric( + definition.options(), + definition.indexType(), definition.column()), definition.indexType()) .withIndexOptions(definition.options()); diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java b/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java index ce31c5de68f5..8dbaeff52cb0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java @@ -23,6 +23,7 @@ import org.apache.paimon.globalindex.bitmap.MultiValueGlobalIndexerFactory; import org.apache.paimon.globalindex.btree.BTreeGlobalIndexerFactory; import org.apache.paimon.globalindex.fmindex.FMGlobalIndexerFactory; +import org.apache.paimon.options.VectorIndexOptions; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.types.DataField; @@ -98,7 +99,10 @@ public static PrimaryKeyIndexDefinitions create(TableSchema schema) { column, field.id(), options.primaryKeyVectorIndexType(column), - options.primaryKeyVectorIndexOptions(column), + VectorIndexOptions.resolve( + options.toConfiguration(), + options.primaryKeyVectorIndexType(column), + column), PrimaryKeyIndexDefinition.Family.VECTOR)); } else if (fullTextColumns.contains(column)) { definitions.add( diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index 71cf03e8ee71..576022cdaee3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -41,6 +41,7 @@ import org.apache.paimon.mergetree.compact.aggregate.factory.FieldLastValueAggFactory; import org.apache.paimon.options.ConfigOption; import org.apache.paimon.options.Options; +import org.apache.paimon.options.VectorIndexOptions; import org.apache.paimon.table.BucketMode; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; @@ -1242,7 +1243,11 @@ private static void validateGlobalIndexColumns(TableSchema schema, CoreOptions o "%s entry '%s' must reference an existing column.", optionKey, column); - options.globalIndexOptions(type, column); + if (CoreOptions.isVectorIndexType(type)) { + VectorIndexOptions.resolve(options.toConfiguration(), type, column); + } else { + options.globalIndexOptions(type, column); + } } } if (options.dataEvolutionEnabled()) { @@ -1300,7 +1305,7 @@ private static void validatePrimaryKeyVectorIndex(TableSchema schema, CoreOption checkArgument( !options.pkClusteringOverride(), "Primary-key vector index does not support pk-clustering-override."); - options.primaryKeyVectorIndexOptions(indexColumn); + VectorIndexOptions.resolve(options.toConfiguration(), indexType, indexColumn); DataField vectorField = schema.fields().stream() @@ -1320,10 +1325,12 @@ private static void validatePrimaryKeyVectorIndex(TableSchema schema, CoreOption indexColumn); checkArgument( Arrays.asList("l2", "cosine", "inner_product") - .contains(options.primaryKeyVectorDistanceMetric(indexColumn)), + .contains( + VectorIndexOptions.metric( + options.toConfiguration(), indexType, indexColumn)), "fields.%s.pk-vector.distance.metric must be one of l2, cosine, inner_product, but is %s.", indexColumn, - options.primaryKeyVectorDistanceMetric(indexColumn)); + VectorIndexOptions.metric(options.toConfiguration(), indexType, indexColumn)); } private static void validatePrimaryKeyFullTextIndex(TableSchema schema, CoreOptions options) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java index bf7409aa35f5..c68ae0d0b359 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java @@ -38,6 +38,7 @@ import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.index.IndexPathFactory; import org.apache.paimon.options.Options; +import org.apache.paimon.options.VectorIndexOptions; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.predicate.BatchVectorSearch; import org.apache.paimon.predicate.Predicate; @@ -787,8 +788,8 @@ private String configuredRawSearchMetric(Map options) { type = "vector"; } String configuredMetric = - configured - .globalIndexOptions(type, vectorColumn.name(), false) + VectorIndexOptions.resolve( + configured.toConfiguration(), type, vectorColumn.name(), false) .get(type + ".metric"); if (configuredMetric != null) { return configuredMetric; @@ -841,11 +842,11 @@ private static boolean isRawSearchMetric(String metric) { protected int configuredRefineFactor(String indexType) { Options resolved = - new CoreOptions( - new Options( - table == null ? Collections.emptyMap() : table.options(), - options)) - .globalIndexOptions(indexType, vectorColumn.name()); + VectorIndexOptions.resolve( + new Options( + table == null ? Collections.emptyMap() : table.options(), options), + indexType, + vectorColumn.name()); return VectorSearchRefineOptions.resolve( options, resolved.toMap(), vectorColumn.name(), indexType); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java index f4ccd0beb9a8..e709cd734d20 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java @@ -37,6 +37,7 @@ import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.KeyValueFileReaderFactory; import org.apache.paimon.options.Options; +import org.apache.paimon.options.VectorIndexOptions; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateVisitor; import org.apache.paimon.reader.RecordReader; @@ -136,12 +137,14 @@ public PrimaryKeyVectorRead( this.table = table; this.vectorField = vectorField; this.indexType = table.coreOptions().primaryKeyVectorIndexType(vectorField.name()); - this.indexOptions = table.coreOptions().primaryKeyVectorIndexOptions(vectorField.name()); + this.indexOptions = + VectorIndexOptions.resolve( + table.coreOptions().toConfiguration(), indexType, vectorField.name()); this.searchOptions = Collections.unmodifiableMap(new HashMap<>(searchOptions)); this.query = query.clone(); this.limit = limit; this.metric = - normalize(table.coreOptions().primaryKeyVectorDistanceMetric(vectorField.name())); + normalize(VectorIndexOptions.metric(indexOptions, indexType, vectorField.name())); this.refineFactor = VectorSearchRefineOptions.resolve( this.searchOptions, diff --git a/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java index 906fa0101e94..d1bec9fc9e6b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java @@ -24,6 +24,7 @@ import org.apache.paimon.index.pk.PrimaryKeyIndexDefinition; import org.apache.paimon.index.pk.PrimaryKeyIndexDefinitions; import org.apache.paimon.options.Options; +import org.apache.paimon.options.VectorIndexOptions; import org.apache.paimon.schema.SchemaValidation; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.types.DataField; @@ -46,6 +47,20 @@ /** Configuration contracts shared by primary-key and Data Evolution indexes. */ class GlobalIndexOptionsTest { + @Test + void testGenericMergeDoesNotInterpretVectorParameters() { + Map values = new HashMap<>(); + values.put("fields.embedding.index-options", "{\"metric\":\"INNER-PRODUCT\",\"nlist\":64}"); + Options resolved = new CoreOptions(values).globalIndexOptions("ivf-pq", "embedding"); + assertThat(resolved.get("ivf-pq.metric")).isEqualTo("INNER-PRODUCT"); + assertThat(resolved.get("ivf-pq.distance.metric")).isNull(); + assertThat( + new CoreOptions(Collections.emptyMap()) + .globalIndexOptions("ivf-pq", "embedding") + .get("ivf-pq.metric")) + .isNull(); + } + @ParameterizedTest @CsvSource({ "btree,btree", @@ -126,8 +141,8 @@ void testLegacyPrimaryKeyMetricSurvivesFactoryResolution(String metric) { PrimaryKeyIndexDefinitions.create(schema).definitions().get(0); assertThat( - new CoreOptions(definition.options()) - .globalIndexOptions(definition.indexType(), "embedding") + VectorIndexOptions.resolve( + definition.options(), definition.indexType(), "embedding") .toMap()) .isEqualTo(definition.options().toMap()); assertThat( @@ -176,7 +191,10 @@ void testUnrelatedFieldOptionsDoNotRequestDefaultMetric() { new CoreOptions( Collections.singletonMap( "fields.embedding.index-options", "{\"nlist\":64}")); - assertThat(options.globalIndexOptions("ivf-pq", "embedding", false).get("ivf-pq.metric")) + assertThat( + VectorIndexOptions.resolve( + options.toConfiguration(), "ivf-pq", "embedding", false) + .get("ivf-pq.metric")) .isNull(); } diff --git a/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java new file mode 100644 index 000000000000..7ad519f641b5 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.options; + +import org.apache.paimon.CoreOptions; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for vector parameter resolution shared by both table modes. */ +class VectorIndexOptionsTest { + + @ParameterizedTest + @CsvSource({ + "fields.embedding.pk-vector.distance.metric,COSINE,cosine", + "fields.embedding.distance.metric,COSINE,cosine", + "fields.embedding.metric,INNER-PRODUCT,inner_product", + "ivf-pq.distance.metric,COSINE,cosine", + "ivf-pq.metric,L2,l2", + "distance.metric,INNER-PRODUCT,inner_product", + "metric,COSINE,cosine" + }) + void testLegacyAndDlfMetricAliases(String key, String value, String expected) { + Map values = new HashMap<>(); + values.put("global-index.ivf-pq.index-column", "embedding"); + values.put("ivf-pq.dimension", "768"); + values.put("fields.embedding.nlist", "48"); + values.put(key, value); + Options resolved = VectorIndexOptions.resolve(new Options(values), null, "embedding"); + assertMetric(resolved, expected); + assertThat(resolved.get("ivf-pq.dimension")).isEqualTo("768"); + assertThat(resolved.get("fields.embedding.nlist")).isEqualTo("48"); + assertThat(VectorIndexOptions.resolve(resolved, "ivf-pq", "embedding").toMap()) + .isEqualTo(resolved.toMap()); + assertThat(values.get(key)).isEqualTo(value); + } + + @ParameterizedTest + @CsvSource({ + "index-options,metric", + "index-options,distance.metric", + "pk-vector.index.options,metric" + }) + void testFieldJsonAliasesRemainIdempotent(String suffix, String key) { + Map values = new HashMap<>(); + values.put("pk-vector.index.columns", "embedding"); + values.put("fields.embedding.pk-vector.index.type", "ivf-pq"); + values.put("fields.embedding.pk-vector.distance.metric", "cosine"); + values.put("fields.embedding." + suffix, "{\"" + key + "\":\"COSINE\",\"nlist\":64}"); + Options resolved = VectorIndexOptions.resolve(new Options(values), null, "embedding"); + assertMetric(resolved, "cosine"); + assertThat(resolved.get("ivf-pq.nlist")).isEqualTo("64"); + assertThat(VectorIndexOptions.resolve(resolved, "ivf-pq", "embedding").toMap()) + .isEqualTo(resolved.toMap()); + assertThat(new CoreOptions(values).primaryKeyVectorIndexOptions("embedding").toMap()) + .isEqualTo(resolved.toMap()); + } + + @Test + void testReaderKeepsUnspecifiedMetricAndWriterUsesDefault() { + Options input = + new Options( + Collections.singletonMap( + "fields.embedding.index-options", "{\"nlist\":64}")); + Options read = VectorIndexOptions.resolve(input, "ivf-pq", "embedding", false); + assertThat(read.get("ivf-pq.nlist")).isEqualTo("64"); + assertThat(read.get("ivf-pq.metric")).isNull(); + assertThat(read.get("fields.embedding.distance.metric")).isNull(); + assertMetric(VectorIndexOptions.resolve(input, "ivf-pq", "embedding"), "inner_product"); + assertThat(VectorIndexOptions.resolve(input, null, "embedding").toMap()) + .isEqualTo(input.toMap()); + } + + private static void assertMetric(Options options, String expected) { + assertThat(options.get("ivf-pq.metric")).isEqualTo(expected); + assertThat(options.get("ivf-pq.distance.metric")).isEqualTo(expected); + assertThat(options.get("fields.embedding.metric")).isEqualTo(expected); + assertThat(options.get("fields.embedding.distance.metric")).isEqualTo(expected); + } +} diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py index df991cad752a..a1dc1e4fbf32 100644 --- a/paimon-python/pypaimon/common/options/core_options.py +++ b/paimon-python/pypaimon/common/options/core_options.py @@ -1906,7 +1906,7 @@ def global_index_columns(self, index_type: str) -> List[str]: def _index_columns(columns) -> List[str]: return [] if columns is None else [column.strip() for column in str(columns).split(',')] - def global_index_options(self, index_type: Optional[str], column: str, *, default_metric=True) -> Options: + def global_index_options(self, index_type: Optional[str], column: str) -> Options: """Resolve the same implementation and field options for either table mode.""" values = dict(self.options.to_map()) if index_type is None: @@ -1920,14 +1920,6 @@ def global_index_options(self, index_type: Optional[str], column: str, *, defaul shared = values.get(shared_key) is not None option_key = shared_key if shared else field_prefix + "pk-" + ( "vector" if vector else index_type) + ".index.options" - metric = None - if vector: - for key in (field_prefix + "distance.metric", field_prefix + "metric", - field_prefix + "pk-vector.distance.metric", prefix + "distance.metric", - prefix + "metric", "distance.metric", "metric"): - if values.get(key) is not None: - metric = values[key] - break serialized = values.get(option_key) if serialized is not None and str(serialized).strip(): try: @@ -1944,27 +1936,14 @@ def global_index_options(self, index_type: Optional[str], column: str, *, defaul elif index_type == 'full-text' and isinstance(value, list): value = ';'.join(str(item) for item in value) qualified = key if key.startswith((prefix, "fields.")) else prefix + key - metric_option = vector and qualified in ( - prefix + "metric", prefix + "distance.metric", - field_prefix + "metric", field_prefix + "distance.metric") previous = values.get(qualified) - if metric_option: - value = str(value).lower().replace('-', '_') - previous = None if previous is None else str(previous).lower().replace('-', '_') if not shared and previous is not None and previous != str(value): raise ValueError("%s defines conflicting values for %s." % (option_key, qualified)) values[qualified] = str(value) - if metric_option: - metric = value - elif vector and qualified.startswith(prefix): + if vector and qualified.startswith(prefix): short_key = qualified[len(prefix):] if shared: values[field_prefix + short_key] = str(value) - if vector and (metric is not None or default_metric): - metric = str(metric or "inner_product").lower().replace('-', '_') - for key in (prefix + "metric", prefix + "distance.metric", - field_prefix + "metric", field_prefix + "distance.metric"): - values[key] = metric return Options(values) def primary_key_vector_index_type(self, column: str): @@ -1983,18 +1962,16 @@ def vector_index_type(self, column: str): return None def primary_key_vector_index_options(self, column: str) -> Options: + from pypaimon.common.options.vector_index_options import VectorIndexOptions index_type = self.primary_key_vector_index_type(column) if index_type is None or not str(index_type).strip(): raise ValueError("fields.%s.pk-vector.index.type must be configured before " "resolving index options." % column) - return self.global_index_options(str(index_type), column) + return VectorIndexOptions.resolve(self.options, str(index_type), column) def primary_key_vector_distance_metric(self, column: str) -> str: - index_type = self.primary_key_vector_index_type(column) - value = (self.global_index_options(index_type, column).to_map()[index_type + ".metric"] - if index_type else self.options.to_map().get( - "fields.%s.pk-vector.distance.metric" % column, "inner_product")) - return str(value).lower().replace('-', '_') + from pypaimon.common.options.vector_index_options import VectorIndexOptions + return VectorIndexOptions.metric(self.options, self.primary_key_vector_index_type(column), column) def primary_key_full_text_index_options(self, column: str) -> Options: return Options({key: value for key, value in self.global_index_options( diff --git a/paimon-python/pypaimon/common/options/vector_index_options.py b/paimon-python/pypaimon/common/options/vector_index_options.py new file mode 100644 index 000000000000..6442f2c04e40 --- /dev/null +++ b/paimon-python/pypaimon/common/options/vector_index_options.py @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Vector parameter resolution shared by primary-key and Data Evolution indexes.""" + +from pypaimon.common.options.core_options import CoreOptions +from pypaimon.common.options.options import Options + + +class VectorIndexOptions: + @staticmethod + def resolve(options, index_type, column, *, default_metric=True): + values = dict(options.to_map()) + if index_type is None: + index_type = CoreOptions(options).vector_index_type(column) + if index_type is None: + return Options(values) + prefix = index_type + '.' + field_prefix = 'fields.%s.' % column + aliases = (field_prefix + 'distance.metric', field_prefix + 'metric', + prefix + 'distance.metric', prefix + 'metric') + metric = next((values[key] for key in ( + field_prefix + 'distance.metric', field_prefix + 'metric', + field_prefix + 'pk-vector.distance.metric', prefix + 'distance.metric', + prefix + 'metric', 'distance.metric', 'metric') if values.get(key) is not None), None) + # Resolve aliases after generic merging, so COSINE/cosine cannot be + # mistaken for conflicting generic strings when options are resolved twice. + for key in aliases: + values.pop(key, None) + resolved = CoreOptions(Options(values)).global_index_options(index_type, column).to_map() + metric = next((resolved[key] for key in aliases if resolved.get(key) is not None), metric) + if metric is not None or default_metric: + normalized = _normalize_metric(metric) + for key in aliases: + resolved[key] = normalized + return Options(resolved) + + @staticmethod + def metric(options, index_type, column): + if index_type is None: + index_type = CoreOptions(options).vector_index_type(column) + if index_type is None: + return _normalize_metric(options.to_map().get('fields.%s.pk-vector.distance.metric' % column)) + return VectorIndexOptions.resolve(options, index_type, column).to_map()[index_type + '.metric'] + + +def _normalize_metric(metric): + return str('inner_product' if metric is None else metric).lower().replace('-', '_') diff --git a/paimon-python/pypaimon/execution/native/index_options.py b/paimon-python/pypaimon/execution/native/index_options.py index 2c93112737a8..49190300c18c 100644 --- a/paimon-python/pypaimon/execution/native/index_options.py +++ b/paimon-python/pypaimon/execution/native/index_options.py @@ -22,6 +22,7 @@ from pypaimon.common.options.core_options import CoreOptions from pypaimon.common.options.options import Options +from pypaimon.common.options.vector_index_options import VectorIndexOptions def native_index_options(schema): @@ -69,14 +70,15 @@ def _resolve(schema): result['pk-%s.index.columns' % family] = ','.join(native_columns[family]) for column in columns: known_columns.add(column) - resolved = core.global_index_options(kind, column, default_metric=False).to_map() + resolved = (VectorIndexOptions.resolve(core.options, kind, column, default_metric=False).to_map() + if vector else core.global_index_options(kind, column).to_map()) profile = {key: value for key, value in resolved.items() if key.startswith(prefix) and value is not None} field_prefix = 'fields.%s.' % column if primary_key: if vector: result[field_prefix + 'pk-vector.index.type'] = kind - result[field_prefix + 'pk-vector.distance.metric'] = core.primary_key_vector_distance_metric(column) + result[field_prefix + 'pk-vector.distance.metric'] = VectorIndexOptions.metric(core.options, kind, column) profile.update({key: value for key, value in resolved.items() if key.startswith(field_prefix) and not key.startswith(field_prefix + 'pk-') and key != field_prefix + 'index-options' and value is not None}) diff --git a/paimon-python/pypaimon/execution/native/read/native_search_plan.py b/paimon-python/pypaimon/execution/native/read/native_search_plan.py index 3f06109bf7f2..a1ba3fa2507d 100644 --- a/paimon-python/pypaimon/execution/native/read/native_search_plan.py +++ b/paimon-python/pypaimon/execution/native/read/native_search_plan.py @@ -22,6 +22,7 @@ from pypaimon.common.json_util import JSON from pypaimon.common.options.core_options import CoreOptions from pypaimon.common.options.options import Options +from pypaimon.common.options.vector_index_options import VectorIndexOptions from pypaimon.execution.native.index_options import native_scalar_index_read_supported from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult from pypaimon.read.split_serializer import deserialize_split_v1, serialize_split_v1 @@ -55,7 +56,9 @@ def _index_search_options(table, column, index_type=None, overrides=None, scalar return None return values # Keep an unspecified metric absent so readers can use persisted index metadata. - return core.global_index_options(index_type, column, default_metric=False).to_map() + return (VectorIndexOptions.resolve(Options(values), index_type, column, default_metric=False).to_map() + if CoreOptions._is_vector_index_type(index_type) else + core.global_index_options(index_type, column).to_map()) def _range(value): diff --git a/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py b/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py index 7696c469547a..a1a099bd9f5f 100644 --- a/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py +++ b/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py @@ -27,6 +27,7 @@ from pypaimon.common.options.core_options import CoreOptions from pypaimon.common.options.options import Options +from pypaimon.common.options.vector_index_options import VectorIndexOptions from pypaimon.globalindex.batch_vector_search import BatchVectorSearch from pypaimon.globalindex.global_index_meta import GlobalIndexIOMeta from pypaimon.globalindex.global_index_result import GlobalIndexResult @@ -658,7 +659,7 @@ def _score_refine_splits(self, table_read, splits, queries_by_row, query_vectors def _resolved_index_options(self, index_type): values = dict(_table_options_map(self._table)) values.update(self._options) - return CoreOptions(Options(values)).global_index_options(index_type, self._vector_column.name).to_map() + return VectorIndexOptions.resolve(Options(values), index_type, self._vector_column.name).to_map() def _configured_refine_factor(self, index_type): value = _configured_refine_factor( @@ -994,7 +995,8 @@ def _configured_vector_metric(options, vector_column, index_type=None): if field_prefix + "index-options" in options: core = CoreOptions(Options(options)) selected_type = index_type or core.primary_key_vector_index_type(vector_column.name) or "vector" - metric = core.global_index_options(selected_type, vector_column.name, default_metric=False).to_map().get(selected_type + ".metric") + metric = VectorIndexOptions.resolve(Options(options), selected_type, vector_column.name, + default_metric=False).to_map().get(selected_type + ".metric") if metric is not None: return metric index_prefix = "%s." % index_type if index_type else None diff --git a/paimon-python/pypaimon/globalindex/create_global_index.py b/paimon-python/pypaimon/globalindex/create_global_index.py index 6cc6ef2ea74f..516cd631206c 100644 --- a/paimon-python/pypaimon/globalindex/create_global_index.py +++ b/paimon-python/pypaimon/globalindex/create_global_index.py @@ -21,6 +21,7 @@ from pypaimon.common.options.core_options import CoreOptions from pypaimon.common.options.options import Options +from pypaimon.common.options.vector_index_options import VectorIndexOptions from pypaimon.common.predicate import Predicate from pypaimon.common.predicate_builder import PredicateBuilder from pypaimon.globalindex.index_types import BTREE_IDENTIFIER, BITMAP_IDENTIFIER @@ -110,8 +111,9 @@ def __init__( "Python global index build currently supports one column, got %s." % self._index_columns ) - self._options = self._core_options.global_index_options( - self._index_type, self._index_columns[0]) + self._options = (VectorIndexOptions.resolve(self._options, self._index_type, self._index_columns[0]) + if CoreOptions._is_vector_index_type(self._index_type) else + self._core_options.global_index_options(self._index_type, self._index_columns[0])) self._core_options = CoreOptions(self._options) if not self._table.options.row_tracking_enabled(): raise ValueError( diff --git a/paimon-python/pypaimon/tests/global_index_test.py b/paimon-python/pypaimon/tests/global_index_test.py index 3b24945777a5..bbd5620303f4 100644 --- a/paimon-python/pypaimon/tests/global_index_test.py +++ b/paimon-python/pypaimon/tests/global_index_test.py @@ -25,6 +25,7 @@ from pypaimon import CatalogFactory, Schema from pypaimon.common.options.core_options import CoreOptions, GlobalIndexSearchMode from pypaimon.common.options.options import Options +from pypaimon.common.options.vector_index_options import VectorIndexOptions from pypaimon.common.predicate import Predicate from pypaimon.common.predicate_builder import PredicateBuilder from pypaimon.globalindex.global_index_meta import GlobalIndexMeta @@ -45,6 +46,14 @@ class GlobalIndexTest(unittest.TestCase): + def test_generic_merge_does_not_interpret_vector_parameters(self): + values = {'fields.embedding.index-options': '{"metric":"INNER-PRODUCT","nlist":64}'} + resolved = CoreOptions(Options(values)).global_index_options('ivf-pq', 'embedding').to_map() + self.assertEqual('INNER-PRODUCT', resolved['ivf-pq.metric']) + self.assertNotIn('ivf-pq.distance.metric', resolved) + self.assertNotIn('ivf-pq.metric', CoreOptions(Options({})).global_index_options( + 'ivf-pq', 'embedding').to_map()) + def test_legacy_primary_key_metric_resolution_is_idempotent(self): for metric in ('COSINE', 'inner-product', 'L2'): with self.subTest(metric=metric): @@ -53,7 +62,7 @@ def test_legacy_primary_key_metric_resolution_is_idempotent(self): 'fields.embedding.pk-vector.distance.metric': metric.lower().replace('-', '_'), 'fields.embedding.pk-vector.index.options': '{"metric":"%s"}' % metric} first = CoreOptions(Options(values)).primary_key_vector_index_options('embedding') - second = CoreOptions(first).global_index_options('ivf-pq', 'embedding') + second = VectorIndexOptions.resolve(first, 'ivf-pq', 'embedding') self.assertEqual(first.to_map(), second.to_map()) self.assertEqual('{"metric":"%s"}' % metric, values['fields.embedding.pk-vector.index.options']) diff --git a/paimon-python/pypaimon/tests/vector_index_options_test.py b/paimon-python/pypaimon/tests/vector_index_options_test.py new file mode 100644 index 000000000000..89d18dbc2727 --- /dev/null +++ b/paimon-python/pypaimon/tests/vector_index_options_test.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +import pytest + +from pypaimon.common.options.core_options import CoreOptions +from pypaimon.common.options.options import Options +from pypaimon.common.options.vector_index_options import VectorIndexOptions + + +def _assert_metric(options, expected): + for key in ('ivf-pq.metric', 'ivf-pq.distance.metric', + 'fields.embedding.metric', 'fields.embedding.distance.metric'): + assert options.to_map()[key] == expected + + +@pytest.mark.parametrize('key,value,expected', [ + ('fields.embedding.pk-vector.distance.metric', 'COSINE', 'cosine'), + ('fields.embedding.distance.metric', 'COSINE', 'cosine'), + ('fields.embedding.metric', 'INNER-PRODUCT', 'inner_product'), + ('ivf-pq.distance.metric', 'COSINE', 'cosine'), + ('ivf-pq.metric', 'L2', 'l2'), + ('distance.metric', 'INNER-PRODUCT', 'inner_product'), + ('metric', 'COSINE', 'cosine'), +]) +def test_legacy_and_dlf_metric_aliases(key, value, expected): + values = {'global-index.ivf-pq.index-column': 'embedding', 'ivf-pq.dimension': '768', + 'fields.embedding.nlist': '48', key: value} + resolved = VectorIndexOptions.resolve(Options(values), None, 'embedding') + _assert_metric(resolved, expected) + assert resolved.to_map()['ivf-pq.dimension'] == '768' + assert resolved.to_map()['fields.embedding.nlist'] == '48' + assert VectorIndexOptions.resolve(resolved, 'ivf-pq', 'embedding').to_map() == resolved.to_map() + assert values[key] == value + + +@pytest.mark.parametrize('suffix,key', [ + ('index-options', 'metric'), ('index-options', 'distance.metric'), + ('pk-vector.index.options', 'metric'), +]) +def test_field_json_aliases_remain_idempotent(suffix, key): + values = {'pk-vector.index.columns': 'embedding', + 'fields.embedding.pk-vector.index.type': 'ivf-pq', + 'fields.embedding.pk-vector.distance.metric': 'cosine', + 'fields.embedding.' + suffix: json.dumps({key: 'COSINE', 'nlist': 64})} + resolved = VectorIndexOptions.resolve(Options(values), None, 'embedding') + _assert_metric(resolved, 'cosine') + assert resolved.to_map()['ivf-pq.nlist'] == '64' + assert VectorIndexOptions.resolve(resolved, 'ivf-pq', 'embedding').to_map() == resolved.to_map() + assert CoreOptions(Options(values)).primary_key_vector_index_options('embedding').to_map() == resolved.to_map() + + +def test_reader_keeps_unspecified_metric_and_writer_uses_default(): + values = Options({'fields.embedding.index-options': '{"nlist":64}'}) + resolved = VectorIndexOptions.resolve(values, 'ivf-pq', 'embedding', default_metric=False).to_map() + assert resolved['ivf-pq.nlist'] == '64' + assert 'ivf-pq.metric' not in resolved + assert 'fields.embedding.distance.metric' not in resolved + _assert_metric(VectorIndexOptions.resolve(values, 'ivf-pq', 'embedding'), 'inner_product') + assert VectorIndexOptions.resolve(values, None, 'embedding').to_map() == values.to_map() From 28b27739ace9c7e8f7540fffb16fbf08592a9706 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 31 Oct 2026 23:20:59 +0800 Subject: [PATCH 03/14] [core] Remove schema-level global index validation --- docs/docs/primary-key-table/global-index.mdx | 3 - .../paimon/schema/SchemaValidation.java | 293 ------------------ .../source/FullTextSearchBuilderImpl.java | 2 +- .../apache/paimon/GlobalIndexOptionsTest.java | 37 ++- .../PrimaryKeyFMIndexValidationTest.java | 147 --------- ...PrimaryKeyFullTextIndexValidationTest.java | 234 -------------- .../PrimaryKeySortedIndexValidationTest.java | 232 -------------- .../PrimaryKeyVectorIndexValidationTest.java | 272 ---------------- .../source/FullTextSearchBuilderTest.java | 25 ++ .../table/source/full_text_search_builder.py | 3 +- .../primary_key_index_definitions_test.py | 12 + 11 files changed, 65 insertions(+), 1195 deletions(-) delete mode 100644 paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyFMIndexValidationTest.java delete mode 100644 paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyFullTextIndexValidationTest.java delete mode 100644 paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeySortedIndexValidationTest.java delete mode 100644 paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java diff --git a/docs/docs/primary-key-table/global-index.mdx b/docs/docs/primary-key-table/global-index.mdx index 808935a3ff17..1de4b8ca0ccd 100644 --- a/docs/docs/primary-key-table/global-index.mdx +++ b/docs/docs/primary-key-table/global-index.mdx @@ -259,9 +259,6 @@ TBLPROPERTIES ( -Duplicate, empty, unknown, unsupported, or cross-family duplicate columns are rejected during -schema validation. - ### Options diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index 576022cdaee3..0a6bf8bb5742 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -30,18 +30,12 @@ import org.apache.paimon.fileindex.FileIndexerFactory; import org.apache.paimon.fileindex.FileIndexerFactoryUtils; import org.apache.paimon.format.FileFormat; -import org.apache.paimon.globalindex.GlobalIndexer; -import org.apache.paimon.globalindex.bitmap.BitmapGlobalIndexerFactory; -import org.apache.paimon.globalindex.bitmap.MultiValueGlobalIndexerFactory; -import org.apache.paimon.globalindex.btree.BTreeGlobalIndexerFactory; -import org.apache.paimon.globalindex.fmindex.FMGlobalIndexerFactory; import org.apache.paimon.iceberg.IcebergOptions; import org.apache.paimon.mergetree.compact.aggregate.FieldAggregator; import org.apache.paimon.mergetree.compact.aggregate.factory.FieldAggregatorFactory; import org.apache.paimon.mergetree.compact.aggregate.factory.FieldLastValueAggFactory; import org.apache.paimon.options.ConfigOption; import org.apache.paimon.options.Options; -import org.apache.paimon.options.VectorIndexOptions; import org.apache.paimon.table.BucketMode; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; @@ -404,14 +398,6 @@ public static void validateTableSchema(TableSchema schema, Set dynamicOp fieldNamesSpecifiedAsVector.isEmpty(), "Some of the columns specified as vector-field are unknown."); - validateGlobalIndexColumns(schema, options); - if (!options.dataEvolutionEnabled()) { - validatePrimaryKeyIndexColumns(options); - validatePrimaryKeySortedIndexes(schema, options); - validatePrimaryKeyVectorIndex(schema, options); - validatePrimaryKeyFullTextIndex(schema, options); - } - validateMergeFunctionFactory(schema); validateMapStorageLayout(schema, options); @@ -1230,285 +1216,6 @@ private static void validateForDeletionVectors(CoreOptions options) { } } - private static void validateGlobalIndexColumns(TableSchema schema, CoreOptions options) { - for (String type : options.globalIndexTypes()) { - checkArgument(!type.trim().isEmpty(), "Global index type must not be empty."); - List columns = options.globalIndexColumns(type); - String optionKey = options.globalIndexColumnsOptionKey(type); - validateNoDuplicatePrimaryKeyIndexColumns(columns, optionKey); - for (String column : columns) { - checkArgument(!column.isEmpty(), "%s must contain non-empty columns.", optionKey); - checkArgument( - schema.nameToFieldMap().containsKey(column), - "%s entry '%s' must reference an existing column.", - optionKey, - column); - if (CoreOptions.isVectorIndexType(type)) { - VectorIndexOptions.resolve(options.toConfiguration(), type, column); - } else { - options.globalIndexOptions(type, column); - } - } - } - if (options.dataEvolutionEnabled()) { - for (String type : - Arrays.asList("vector", "btree", "bitmap", "multivalue", "full-text", "fm")) { - checkArgument( - !schema.options().containsKey("pk-" + type + ".index.columns"), - "pk-%s.index.columns requires a primary-key table; use global-index..index-columns for Data Evolution tables.", - type); - } - } - } - - private static void validatePrimaryKeyVectorIndex(TableSchema schema, CoreOptions options) { - if (!options.primaryKeyVectorIndexEnabled()) { - return; - } - - List indexColumns = options.primaryKeyVectorIndexColumns(); - checkArgument( - new HashSet<>(indexColumns).size() == indexColumns.size(), - "%s must not contain duplicate columns, but is %s.", - options.globalIndexColumnsOptionKey("vector"), - indexColumns); - checkArgument( - indexColumns.size() == 1, - "%s must contain exactly one column in the first release, but is %s.", - options.globalIndexColumnsOptionKey("vector"), - indexColumns); - String indexColumn = indexColumns.get(0); - String indexType = options.primaryKeyVectorIndexType(indexColumn); - checkArgument( - !StringUtils.isNullOrWhitespaceOnly(indexColumn), - "%s must contain a non-empty column.", - options.globalIndexColumnsOptionKey("vector")); - checkArgument( - !StringUtils.isNullOrWhitespaceOnly(indexType), - "fields.%s.pk-vector.index.type must be configured when a primary-key vector index is defined.", - indexColumn); - checkArgument( - !schema.primaryKeys().isEmpty(), - "Primary-key vector index requires a primary-key table."); - checkArgument( - options.mergeEngine() == MergeEngine.FIRST_ROW || options.deletionVectorsEnabled(), - "Primary-key vector index requires deletion-vectors.enabled = true."); - checkArgument( - !options.deletionVectorsEnabled() || !options.deletionVectorsMergeOnRead(), - "Primary-key vector index with merge-engine = %s requires deletion-vectors.merge-on-read = false.", - options.mergeEngine()); - checkArgument( - options.bucket() > 0 || options.bucket() == BucketMode.POSTPONE_BUCKET, - "Primary-key vector index requires fixed or postpone bucket mode " - + "(bucket > 0 or bucket = -2), but bucket is %s.", - options.bucket()); - checkArgument( - !options.pkClusteringOverride(), - "Primary-key vector index does not support pk-clustering-override."); - VectorIndexOptions.resolve(options.toConfiguration(), indexType, indexColumn); - - DataField vectorField = - schema.fields().stream() - .filter(field -> field.name().equals(indexColumn)) - .findFirst() - .orElse(null); - checkArgument( - vectorField != null && vectorField.type().getTypeRoot() == VECTOR, - "%s entry '%s' must reference a VECTOR column.", - options.globalIndexColumnsOptionKey("vector"), - indexColumn); - checkArgument( - ((VectorType) vectorField.type()).getElementType().getTypeRoot() - == DataTypeRoot.FLOAT, - "%s entry '%s' must use FLOAT elements.", - options.globalIndexColumnsOptionKey("vector"), - indexColumn); - checkArgument( - Arrays.asList("l2", "cosine", "inner_product") - .contains( - VectorIndexOptions.metric( - options.toConfiguration(), indexType, indexColumn)), - "fields.%s.pk-vector.distance.metric must be one of l2, cosine, inner_product, but is %s.", - indexColumn, - VectorIndexOptions.metric(options.toConfiguration(), indexType, indexColumn)); - } - - private static void validatePrimaryKeyFullTextIndex(TableSchema schema, CoreOptions options) { - if (!options.primaryKeyFullTextIndexEnabled()) { - return; - } - - List indexColumns = options.primaryKeyFullTextIndexColumns(); - checkArgument( - indexColumns.size() == 1, - "%s must contain exactly one column in the first release, but is %s.", - options.globalIndexColumnsOptionKey("full-text"), - indexColumns); - String indexColumn = indexColumns.get(0); - checkArgument( - !StringUtils.isNullOrWhitespaceOnly(indexColumn), - "%s must contain a non-empty column.", - options.globalIndexColumnsOptionKey("full-text")); - checkArgument( - !schema.primaryKeys().isEmpty(), - "Primary-key full-text index requires a primary-key table."); - checkArgument( - options.mergeEngine() == MergeEngine.FIRST_ROW || options.deletionVectorsEnabled(), - "Primary-key full-text index requires deletion-vectors.enabled = true."); - checkArgument( - !options.deletionVectorsEnabled() || !options.deletionVectorsMergeOnRead(), - "Primary-key full-text index requires deletion-vectors.merge-on-read = false."); - checkArgument( - options.bucket() > 0 || options.bucket() == BucketMode.POSTPONE_BUCKET, - "Primary-key full-text index requires fixed or postpone bucket mode " - + "(bucket > 0 or bucket = -2), but bucket is %s.", - options.bucket()); - checkArgument( - !options.pkClusteringOverride(), - "Primary-key full-text index does not support pk-clustering-override."); - checkArgument( - schema.nameToFieldMap().containsKey(indexColumn), - "%s entry '%s' must reference an existing column.", - options.globalIndexColumnsOptionKey("full-text"), - indexColumn); - DataTypeRoot typeRoot = schema.nameToFieldMap().get(indexColumn).type().getTypeRoot(); - checkArgument( - typeRoot == DataTypeRoot.CHAR || typeRoot == DataTypeRoot.VARCHAR, - "%s entry '%s' must reference a CHAR/VARCHAR/STRING column.", - options.globalIndexColumnsOptionKey("full-text"), - indexColumn); - options.primaryKeyFullTextIndexOptions(indexColumn); - } - - private static void validatePrimaryKeyIndexColumns(CoreOptions options) { - List vectorColumns = options.primaryKeyVectorIndexColumns(); - List btreeColumns = options.primaryKeyBTreeIndexColumns(); - List bitmapColumns = options.primaryKeyBitmapIndexColumns(); - List multiValueColumns = options.primaryKeyMultiValueIndexColumns(); - List fullTextColumns = options.primaryKeyFullTextIndexColumns(); - List fmColumns = options.primaryKeyFMIndexColumns(); - validateNoDuplicatePrimaryKeyIndexColumns( - vectorColumns, options.globalIndexColumnsOptionKey("vector")); - validateNoDuplicatePrimaryKeyIndexColumns( - btreeColumns, options.globalIndexColumnsOptionKey("btree")); - validateNoDuplicatePrimaryKeyIndexColumns( - bitmapColumns, options.globalIndexColumnsOptionKey("bitmap")); - validateNoDuplicatePrimaryKeyIndexColumns( - multiValueColumns, options.globalIndexColumnsOptionKey("multivalue")); - validateNoDuplicatePrimaryKeyIndexColumns( - fullTextColumns, options.globalIndexColumnsOptionKey("full-text")); - validateNoDuplicatePrimaryKeyIndexColumns( - fmColumns, options.globalIndexColumnsOptionKey("fm")); - - Set indexedColumns = new HashSet<>(); - validateUniquePrimaryKeyIndexColumns(indexedColumns, vectorColumns); - validateUniquePrimaryKeyIndexColumns(indexedColumns, btreeColumns); - validateUniquePrimaryKeyIndexColumns(indexedColumns, bitmapColumns); - validateUniquePrimaryKeyIndexColumns(indexedColumns, multiValueColumns); - validateUniquePrimaryKeyIndexColumns(indexedColumns, fullTextColumns); - validateUniquePrimaryKeyIndexColumns(indexedColumns, fmColumns); - } - - private static void validateNoDuplicatePrimaryKeyIndexColumns( - List columns, String optionKey) { - checkArgument( - new HashSet<>(columns).size() == columns.size(), - "%s must not contain duplicate columns, but is %s.", - optionKey, - columns); - } - - private static void validateUniquePrimaryKeyIndexColumns( - Set indexedColumns, List columns) { - for (String column : columns) { - checkArgument( - indexedColumns.add(column), - "Column '%s' can own at most one primary-key index.", - column); - } - } - - private static void validatePrimaryKeySortedIndexes(TableSchema schema, CoreOptions options) { - if (options.primaryKeyBTreeIndexColumns().isEmpty() - && options.primaryKeyBitmapIndexColumns().isEmpty() - && options.primaryKeyMultiValueIndexColumns().isEmpty() - && options.primaryKeyFMIndexColumns().isEmpty()) { - return; - } - - checkArgument( - options.deletionVectorsEnabled(), - "Primary-key BTree, Bitmap, Multivalue, and FM indexes require deletion-vectors.enabled = true."); - checkArgument( - !schema.primaryKeys().isEmpty(), - "Primary-key BTree, Bitmap, Multivalue, and FM indexes require a primary-key table."); - checkArgument( - options.bucket() > 0 || options.bucket() == BucketMode.POSTPONE_BUCKET, - "Primary-key BTree, Bitmap, Multivalue, and FM indexes require fixed or postpone bucket mode " - + "(bucket > 0 or bucket = -2), but bucket is %s.", - options.bucket()); - checkArgument( - !options.deletionVectorsMergeOnRead(), - "Primary-key BTree, Bitmap, Multivalue, and FM indexes require deletion-vectors.merge-on-read = false."); - checkArgument( - !options.pkClusteringOverride(), - "Primary-key BTree, Bitmap, Multivalue, and FM indexes do not support pk-clustering-override."); - - validatePrimaryKeySortedIndexColumns( - schema, - options.primaryKeyBTreeIndexColumns(), - options.globalIndexColumnsOptionKey("btree")); - validatePrimaryKeySortedIndexColumns( - schema, - options.primaryKeyBitmapIndexColumns(), - options.globalIndexColumnsOptionKey("bitmap")); - validatePrimaryKeySortedIndexColumns( - schema, - options.primaryKeyMultiValueIndexColumns(), - options.globalIndexColumnsOptionKey("multivalue")); - validatePrimaryKeySortedIndexColumns( - schema, - options.primaryKeyFMIndexColumns(), - options.globalIndexColumnsOptionKey("fm")); - - Map fields = schema.nameToFieldMap(); - for (String column : options.primaryKeyBTreeIndexColumns()) { - GlobalIndexer.create( - BTreeGlobalIndexerFactory.IDENTIFIER, - Collections.singletonList(fields.get(column)), - options.primaryKeyBTreeIndexOptions(column)); - } - for (String column : options.primaryKeyBitmapIndexColumns()) { - GlobalIndexer.create( - BitmapGlobalIndexerFactory.IDENTIFIER, - Collections.singletonList(fields.get(column)), - options.primaryKeyBitmapIndexOptions(column)); - } - for (String column : options.primaryKeyMultiValueIndexColumns()) { - GlobalIndexer.create( - MultiValueGlobalIndexerFactory.IDENTIFIER, - Collections.singletonList(fields.get(column)), - options.primaryKeyMultiValueIndexOptions(column)); - } - for (String column : options.primaryKeyFMIndexColumns()) { - GlobalIndexer.create( - FMGlobalIndexerFactory.IDENTIFIER, - Collections.singletonList(fields.get(column)), - options.primaryKeyFMIndexOptions(column)); - } - } - - private static void validatePrimaryKeySortedIndexColumns( - TableSchema schema, List columns, String optionKey) { - for (String column : columns) { - checkArgument( - schema.fieldNames().contains(column), - "%s entry '%s' must reference an existing column.", - optionKey, - column); - } - } - private static void validateSequenceField(TableSchema schema, CoreOptions options) { List sequenceField = options.sequenceField(); if (!sequenceField.isEmpty()) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java index baa6dab5f980..642cb7ae3f37 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java @@ -147,7 +147,7 @@ private DataField textColumn() { } private Optional primaryKeyFullTextDefinition(DataField textColumn) { - if (table.coreOptions().dataEvolutionEnabled()) { + if (table.schema().primaryKeys().isEmpty() || table.coreOptions().dataEvolutionEnabled()) { return Optional.empty(); } if (table.coreOptions().primaryKeyFullTextIndexColumns().isEmpty()) { diff --git a/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java index d1bec9fc9e6b..c775e0812c9c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java @@ -255,29 +255,42 @@ void testPrimaryKeyAndDataEvolutionValidationUseSameDeclarations() { .doesNotThrowAnyException(); } - @Test - void testPrimaryKeyRequirementsStillApply() { + @ParameterizedTest + @CsvSource({ + "btree,btree", + "bitmap,bitmap", + "multivalue,multivalue", + "fm,fm", + "full-text,full-text", + "ivf-pq,vector" + }) + void testIndexOptionsAreNotValidatedWithSchema(String type, String legacyFamily) { Map values = new HashMap<>(); - values.put("global-index.btree.index-columns", "amount"); + values.put("global-index." + type + ".index-columns", "unknown,,amount,amount"); + values.put("pk-" + legacyFamily + ".index.columns", "unknown,,amount,amount"); + values.put("fields.amount.index-options", "invalid JSON"); + values.put("fields.amount.pk-" + legacyFamily + ".index.options", "invalid JSON"); values.put("bucket", "1"); - assertThatThrownBy(() -> SchemaValidation.validateTableSchema(schema(values, true))) - .hasMessageContaining("require deletion-vectors.enabled = true"); - values.put("deletion-vectors.enabled", "true"); - values.put("global-index.bitmap.index-columns", "amount"); - assertThatThrownBy(() -> SchemaValidation.validateTableSchema(schema(values, true))) - .hasMessageContaining("at most one primary-key index"); + assertThatCode(() -> SchemaValidation.validateTableSchema(schema(values, true))) + .doesNotThrowAnyException(); + + values.put("bucket", "-1"); + values.put("row-tracking.enabled", "true"); + values.put("data-evolution.enabled", "true"); + assertThatCode(() -> SchemaValidation.validateTableSchema(schema(values, false))) + .doesNotThrowAnyException(); } @ParameterizedTest @ValueSource(strings = {"amount,amount", "unknown", "amount,", ""}) - void testInvalidColumnDeclarationsAreRejected(String columns) { + void testSchemaDoesNotValidateIndexColumnDeclarations(String columns) { Map values = new HashMap<>(); values.put("global-index.btree.index-columns", columns); values.put("bucket", "-1"); values.put("row-tracking.enabled", "true"); values.put("data-evolution.enabled", "true"); - assertThatThrownBy(() -> SchemaValidation.validateTableSchema(schema(values, false))) - .hasMessageContaining("global-index.btree.index-columns"); + assertThatCode(() -> SchemaValidation.validateTableSchema(schema(values, false))) + .doesNotThrowAnyException(); } private static TableSchema schema(Map options, boolean primaryKey) { diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyFMIndexValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyFMIndexValidationTest.java deleted file mode 100644 index a279ce771fd8..000000000000 --- a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyFMIndexValidationTest.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.schema; - -import org.apache.paimon.CoreOptions; -import org.apache.paimon.types.DataField; -import org.apache.paimon.types.DataTypes; - -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import static org.apache.paimon.schema.SchemaValidation.validateTableSchema; -import static org.assertj.core.api.Assertions.assertThatCode; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -/** Tests for primary-key FM index option validation. */ -class PrimaryKeyFMIndexValidationTest { - - @Test - void testValidConfiguration() { - Map options = enabledOptions(); - options.put( - "fields.content.pk-fm.index.options", - "{\"partition-row-count\":\"2\",\"sa-sample-rate\":\"16\"}"); - - assertThatCode(() -> validateTableSchema(schema(options))).doesNotThrowAnyException(); - } - - @Test - void testRequiresCharacterColumn() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_FM_INDEX_COLUMNS.key(), "id"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("FM index requires a character string column"); - } - - @Test - void testRequiresDeletionVectors() { - Map options = enabledOptions(); - options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "false"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("FM indexes require deletion-vectors.enabled = true"); - } - - @Test - void testRejectsUnknownColumn() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_FM_INDEX_COLUMNS.key(), "unknown"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining(CoreOptions.PK_FM_INDEX_COLUMNS.key()) - .hasMessageContaining("entry 'unknown'"); - } - - @Test - void testRejectsColumnConfiguredForAnotherFamily() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "content"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("content") - .hasMessageContaining("at most one primary-key index"); - } - - @Test - void testRejectsMalformedFieldOptions() { - Map options = enabledOptions(); - options.put("fields.content.pk-fm.index.options", "{not-json"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("fields.content.pk-fm.index.options must be a JSON object"); - } - - @Test - void testRejectsInvalidFMOptions() { - Map options = enabledOptions(); - options.put("fields.content.pk-fm.index.options", "{\"partition-row-count\":\"0\"}"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("FM index partition row count must be positive"); - } - - @Test - void testRequiresPrimaryKeyTable() { - Map options = enabledOptions(); - options.put(CoreOptions.BUCKET_KEY.key(), "id"); - TableSchema appendTable = - new TableSchema( - 0, - fields(), - 0, - Collections.emptyList(), - Collections.emptyList(), - options, - ""); - - assertThatThrownBy(() -> validateTableSchema(appendTable)) - .hasMessageContaining("FM indexes require a primary-key table"); - } - - private static Map enabledOptions() { - Map options = new HashMap<>(); - options.put(CoreOptions.BUCKET.key(), "1"); - options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_FM_INDEX_COLUMNS.key(), "content"); - return options; - } - - private static java.util.List fields() { - return Arrays.asList( - new DataField(0, "id", DataTypes.INT().notNull()), - new DataField(1, "content", DataTypes.STRING())); - } - - private static TableSchema schema(Map options) { - return new TableSchema( - 0, - fields(), - 0, - Collections.emptyList(), - Collections.singletonList("id"), - options, - ""); - } -} diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyFullTextIndexValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyFullTextIndexValidationTest.java deleted file mode 100644 index e5a927f47049..000000000000 --- a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyFullTextIndexValidationTest.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.schema; - -import org.apache.paimon.CoreOptions; -import org.apache.paimon.types.DataField; -import org.apache.paimon.types.DataTypes; - -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import static org.apache.paimon.schema.SchemaValidation.validateTableSchema; -import static org.assertj.core.api.Assertions.assertThatCode; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -/** Tests for primary-key full-text index option validation. */ -class PrimaryKeyFullTextIndexValidationTest { - - @Test - void testValidConfiguration() { - assertThatCode(() -> validateTableSchema(schema(enabledOptions()))) - .doesNotThrowAnyException(); - } - - @Test - void testRejectsMultipleColumnsForFirstRelease() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key(), "content,other_content"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining(CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key()) - .hasMessageContaining("exactly one column"); - } - - @Test - void testRejectsDuplicateColumns() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key(), "content,content"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining(CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key()) - .hasMessageContaining("duplicate"); - } - - @Test - void testRejectsEmptyColumn() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key(), " "); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining(CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key()) - .hasMessageContaining("non-empty column"); - } - - @Test - void testRejectsUnknownColumn() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key(), "unknown"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("entry 'unknown'") - .hasMessageContaining("existing column"); - } - - @Test - void testRejectsNonCharacterColumn() { - TableSchema intContentSchema = - new TableSchema( - 0, - Arrays.asList( - new DataField(0, "id", DataTypes.INT().notNull()), - new DataField(1, "content", DataTypes.INT())), - 0, - Collections.emptyList(), - Collections.singletonList("id"), - enabledOptions(), - ""); - - assertThatThrownBy(() -> validateTableSchema(intContentSchema)) - .hasMessageContaining("entry 'content'") - .hasMessageContaining("CHAR/VARCHAR/STRING"); - } - - @Test - void testRequiresDeletionVectors() { - Map options = enabledOptions(); - options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "false"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("requires deletion-vectors.enabled = true"); - } - - @Test - void testIgnoresMergeOnReadForFirstRowWithoutDeletionVectors() { - Map options = enabledOptions(); - options.put(CoreOptions.MERGE_ENGINE.key(), "first-row"); - options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "false"); - options.put(CoreOptions.DELETION_VECTORS_MERGE_ON_READ.key(), "true"); - - assertThatCode(() -> validateTableSchema(schema(options))).doesNotThrowAnyException(); - } - - @Test - void testRejectsDeletionVectorMergeOnRead() { - Map options = enabledOptions(); - options.put(CoreOptions.DELETION_VECTORS_MERGE_ON_READ.key(), "true"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("requires deletion-vectors.merge-on-read = false"); - } - - @Test - void testRejectsDynamicBucket() { - Map options = enabledOptions(); - options.put(CoreOptions.BUCKET.key(), "-1"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("requires fixed or postpone bucket mode"); - } - - @Test - void testSupportsPostponeBucket() { - Map options = enabledOptions(); - options.put(CoreOptions.BUCKET.key(), "-2"); - - assertThatCode(() -> validateTableSchema(schema(options))).doesNotThrowAnyException(); - } - - @Test - void testRejectsPkClusteringOverride() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_CLUSTERING_OVERRIDE.key(), "true"); - options.put(CoreOptions.CLUSTERING_COLUMNS.key(), "content"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("does not support pk-clustering-override"); - } - - @Test - void testRejectsColumnConfiguredForAnotherPrimaryKeyIndexFamily() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "content"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("content") - .hasMessageContaining("at most one primary-key index"); - } - - @Test - void testRejectsMalformedIndexOptions() { - Map options = enabledOptions(); - options.put("fields.content.pk-full-text.index.options", "{not-json"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining( - "fields.content.pk-full-text.index.options must be a JSON object"); - } - - @Test - void testRejectsConflictingGlobalAndFieldIndexOptions() { - Map options = enabledOptions(); - options.put("full-text.tokenizer", "standard"); - options.put("fields.content.pk-full-text.index.options", "{\"tokenizer\":\"jieba\"}"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining( - "fields.content.pk-full-text.index.options defines conflicting values for full-text.tokenizer"); - } - - @Test - void testRequiresPrimaryKeyTable() { - Map options = enabledOptions(); - options.put(CoreOptions.BUCKET_KEY.key(), "id"); - - TableSchema appendTable = - new TableSchema( - 0, - fields(), - 0, - Collections.emptyList(), - Collections.emptyList(), - options, - ""); - - assertThatThrownBy(() -> validateTableSchema(appendTable)) - .hasMessageContaining("Primary-key full-text index requires a primary-key table"); - } - - private static Map enabledOptions() { - Map options = new HashMap<>(); - options.put(CoreOptions.BUCKET.key(), "1"); - options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key(), "content"); - return options; - } - - private static java.util.List fields() { - return Arrays.asList( - new DataField(0, "id", DataTypes.INT().notNull()), - new DataField(1, "content", DataTypes.STRING()), - new DataField(2, "other_content", DataTypes.STRING())); - } - - private static TableSchema schema(Map options) { - return new TableSchema( - 0, - fields(), - 0, - Collections.emptyList(), - Collections.singletonList("id"), - options, - ""); - } -} diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeySortedIndexValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeySortedIndexValidationTest.java deleted file mode 100644 index 1d1ac2f9c5f6..000000000000 --- a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeySortedIndexValidationTest.java +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.schema; - -import org.apache.paimon.CoreOptions; -import org.apache.paimon.types.DataField; -import org.apache.paimon.types.DataTypes; - -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import static org.apache.paimon.schema.SchemaValidation.validateTableSchema; -import static org.assertj.core.api.Assertions.assertThatCode; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -/** Tests for source-backed primary-key sorted index validation. */ -class PrimaryKeySortedIndexValidationTest { - - @Test - void testRejectsColumnConfiguredForMultipleIndexFamilies() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "payload"); - options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "payload"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("payload") - .hasMessageContaining("at most one primary-key index"); - } - - @Test - void testRejectsDuplicateBTreeIndexColumns() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "payload,payload"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining(CoreOptions.PK_BTREE_INDEX_COLUMNS.key()) - .hasMessageContaining("duplicate"); - } - - @Test - void testRejectsDuplicateBitmapIndexColumns() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "payload,payload"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key()) - .hasMessageContaining("duplicate"); - } - - @Test - void testSupportsMultiValueIndexOnArrayColumn() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_MULTIVALUE_INDEX_COLUMNS.key(), "tags"); - - assertThatCode(() -> validateTableSchema(arraySchema(options))).doesNotThrowAnyException(); - } - - @Test - void testRejectsDuplicateMultiValueIndexColumns() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_MULTIVALUE_INDEX_COLUMNS.key(), "tags,tags"); - - assertThatThrownBy(() -> validateTableSchema(arraySchema(options))) - .hasMessageContaining(CoreOptions.PK_MULTIVALUE_INDEX_COLUMNS.key()) - .hasMessageContaining("duplicate"); - } - - @Test - void testRejectsMultiValueIndexOnScalarColumn() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_MULTIVALUE_INDEX_COLUMNS.key(), "payload"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("Multivalue index requires an ARRAY column") - .hasMessageContaining("payload"); - } - - @Test - void testRequiresDeletionVectors() { - Map options = enabledOptions(); - options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "false"); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "payload"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining( - "Primary-key BTree, Bitmap, Multivalue, and FM indexes require deletion-vectors.enabled = true"); - } - - @Test - void testRequiresPrimaryKeyTable() { - Map options = enabledOptions(); - options.put(CoreOptions.BUCKET_KEY.key(), "id"); - options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "payload"); - - assertThatThrownBy(() -> validateTableSchema(schema(options, Collections.emptyList()))) - .hasMessageContaining( - "Primary-key BTree, Bitmap, Multivalue, and FM indexes require a primary-key table"); - } - - @Test - void testRequiresFixedBucketMode() { - Map options = enabledOptions(); - options.put(CoreOptions.BUCKET.key(), "-1"); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "payload"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining( - "Primary-key BTree, Bitmap, Multivalue, and FM indexes require fixed or postpone bucket mode"); - } - - @Test - void testSupportsPostponeBucket() { - Map options = enabledOptions(); - options.put(CoreOptions.BUCKET.key(), "-2"); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "payload"); - - assertThatCode(() -> validateTableSchema(schema(options))).doesNotThrowAnyException(); - } - - @Test - void testRejectsDeletionVectorMergeOnRead() { - Map options = enabledOptions(); - options.put(CoreOptions.DELETION_VECTORS_MERGE_ON_READ.key(), "true"); - options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "payload"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining( - "Primary-key BTree, Bitmap, Multivalue, and FM indexes require deletion-vectors.merge-on-read = false"); - } - - @Test - void testRejectsPkClusteringOverride() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_CLUSTERING_OVERRIDE.key(), "true"); - options.put(CoreOptions.CLUSTERING_COLUMNS.key(), "payload"); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "payload"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining( - "Primary-key BTree, Bitmap, Multivalue, and FM indexes do not support pk-clustering-override"); - } - - @Test - void testRejectsUnknownIndexColumn() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "unknown"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("pk-bitmap.index.columns entry 'unknown'") - .hasMessageContaining("must reference an existing column"); - } - - @Test - void testRejectsUnsupportedIndexColumnType() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "payload"); - - TableSchema schema = - new TableSchema( - 0, - Arrays.asList( - new DataField(0, "id", DataTypes.INT().notNull()), - new DataField(1, "payload", DataTypes.ARRAY(DataTypes.INT()))), - 0, - Collections.emptyList(), - Collections.singletonList("id"), - options, - ""); - - assertThatThrownBy(() -> validateTableSchema(schema)) - .hasMessageContaining("ARRAY") - .hasMessageContaining("not supported by global index"); - } - - private static Map enabledOptions() { - Map options = new HashMap<>(); - options.put(CoreOptions.BUCKET.key(), "1"); - options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - return options; - } - - private static TableSchema schema(Map options) { - return schema(options, Collections.singletonList("id")); - } - - private static TableSchema arraySchema(Map options) { - return new TableSchema( - 0, - Arrays.asList( - new DataField(0, "id", DataTypes.INT().notNull()), - new DataField(1, "tags", DataTypes.ARRAY(DataTypes.STRING()))), - 0, - Collections.emptyList(), - Collections.singletonList("id"), - options, - ""); - } - - private static TableSchema schema( - Map options, java.util.List primaryKeys) { - return new TableSchema( - 0, - Arrays.asList( - new DataField(0, "id", DataTypes.INT().notNull()), - new DataField(1, "payload", DataTypes.STRING())), - 0, - Collections.emptyList(), - primaryKeys, - options, - ""); - } -} diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java deleted file mode 100644 index 748b697a2641..000000000000 --- a/paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeyVectorIndexValidationTest.java +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.schema; - -import org.apache.paimon.CoreOptions; -import org.apache.paimon.types.DataField; -import org.apache.paimon.types.DataTypes; - -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import static org.apache.paimon.schema.SchemaValidation.validateTableSchema; -import static org.assertj.core.api.Assertions.assertThatCode; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -/** Tests for primary-key vector index option validation. */ -class PrimaryKeyVectorIndexValidationTest { - - @Test - void testValidPluralPrimaryKeyVectorIndexConfiguration() { - Map options = new HashMap<>(); - options.put(CoreOptions.BUCKET.key(), "1"); - options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), " embedding "); - options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); - - assertThatCode(() -> validateTableSchema(schema(options))).doesNotThrowAnyException(); - } - - @Test - void testRejectsMultiplePrimaryKeyVectorIndexColumnsForFirstRelease() { - Map options = new HashMap<>(); - options.put(CoreOptions.BUCKET.key(), "1"); - options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding,other_embedding"); - options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("exactly one column") - .hasMessageContaining("pk-vector.index.columns"); - } - - @Test - void testRejectsDuplicatePrimaryKeyVectorIndexColumns() { - Map options = new HashMap<>(); - options.put(CoreOptions.BUCKET.key(), "1"); - options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding,embedding"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("pk-vector.index.columns") - .hasMessageContaining("duplicate"); - } - - @Test - void testValidPrimaryKeyVectorIndex() { - assertThatCode(() -> validateTableSchema(schema(enabledOptions()))) - .doesNotThrowAnyException(); - } - - @Test - void testRequiresFieldScopedIndexType() { - Map options = enabledOptions(); - options.remove("fields.embedding.pk-vector.index.type"); - options.put("pk-vector.index.type", "ivf-pq"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("fields.embedding.pk-vector.index.type"); - } - - @Test - void testRequiresPrimaryKeyTable() { - Map options = enabledOptions(); - options.put(CoreOptions.BUCKET_KEY.key(), "id"); - assertThatThrownBy( - () -> - validateTableSchema( - new TableSchema( - 0, - fields(), - 0, - Collections.emptyList(), - Collections.emptyList(), - options, - ""))) - .hasMessageContaining("requires a primary-key table"); - } - - @Test - void testRequiresDeletionVectors() { - Map options = enabledOptions(); - options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "false"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("requires deletion-vectors.enabled = true"); - } - - @Test - void testSupportsPartialUpdateMergeEngine() { - Map options = enabledOptions(); - options.put(CoreOptions.MERGE_ENGINE.key(), "partial-update"); - - assertThatCode(() -> validateTableSchema(schema(options))).doesNotThrowAnyException(); - } - - @Test - void testSupportsAggregationMergeEngine() { - Map options = enabledOptions(); - options.put(CoreOptions.MERGE_ENGINE.key(), "aggregation"); - - assertThatCode(() -> validateTableSchema(schema(options))).doesNotThrowAnyException(); - } - - @Test - void testIgnoresMergeOnReadForFirstRowWithoutDeletionVectors() { - Map options = enabledOptions(); - options.put(CoreOptions.MERGE_ENGINE.key(), "first-row"); - options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "false"); - options.put(CoreOptions.DELETION_VECTORS_MERGE_ON_READ.key(), "true"); - - assertThatCode(() -> validateTableSchema(schema(options))).doesNotThrowAnyException(); - } - - @Test - void testRejectsFirstRowWithDeletionVectors() { - Map options = enabledOptions(); - options.put(CoreOptions.MERGE_ENGINE.key(), "first-row"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining( - "First row merge engine does not need deletion vectors because there is no deletion of old data in this merge engine"); - } - - @Test - void testPartialUpdateRejectsDeletionVectorMergeOnRead() { - Map options = enabledOptions(); - options.put(CoreOptions.MERGE_ENGINE.key(), "partial-update"); - options.put(CoreOptions.DELETION_VECTORS_MERGE_ON_READ.key(), "true"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining( - "partial-update requires deletion-vectors.merge-on-read = false"); - } - - @Test - void testDeduplicateRejectsDeletionVectorMergeOnRead() { - Map options = enabledOptions(); - options.put(CoreOptions.DELETION_VECTORS_MERGE_ON_READ.key(), "true"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("requires deletion-vectors.merge-on-read = false"); - } - - @Test - void testSupportsPostponeBucket() { - Map options = enabledOptions(); - options.put(CoreOptions.BUCKET.key(), "-2"); - - assertThatCode(() -> validateTableSchema(schema(options))).doesNotThrowAnyException(); - } - - @Test - void testRejectsDynamicBucket() { - Map options = enabledOptions(); - options.put(CoreOptions.BUCKET.key(), "-1"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("requires fixed or postpone bucket mode"); - } - - @Test - void testRejectsPkClusteringOverride() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_CLUSTERING_OVERRIDE.key(), "true"); - options.put(CoreOptions.CLUSTERING_COLUMNS.key(), "payload"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("does not support pk-clustering-override"); - } - - @Test - void testRequiresVectorColumn() { - Map options = enabledOptions(); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "payload"); - options.put("fields.payload.pk-vector.index.type", "ivf-pq"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("must reference a VECTOR column"); - } - - @Test - void testIndexNameIsNotRequired() { - Map options = enabledOptions(); - assertThatCode(() -> validateTableSchema(schema(options))).doesNotThrowAnyException(); - } - - @Test - void testRequiresFloatVectorElements() { - TableSchema schema = - new TableSchema( - 0, - Arrays.asList( - new DataField(0, "id", DataTypes.INT().notNull()), - new DataField( - 1, "embedding", DataTypes.VECTOR(8, DataTypes.DOUBLE()))), - 1, - Collections.emptyList(), - Collections.singletonList("id"), - enabledOptions(), - ""); - - assertThatThrownBy(() -> validateTableSchema(schema)) - .hasMessageContaining("must use FLOAT elements"); - } - - @Test - void testRejectsUnsupportedDistanceMetric() { - Map options = enabledOptions(); - options.put("fields.embedding.pk-vector.distance.metric", "manhattan"); - - assertThatThrownBy(() -> validateTableSchema(schema(options))) - .hasMessageContaining("pk-vector.distance.metric") - .hasMessageContaining("l2, cosine, inner_product"); - } - - private static Map enabledOptions() { - Map options = new HashMap<>(); - options.put(CoreOptions.BUCKET.key(), "1"); - options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); - options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); - return options; - } - - private static TableSchema schema(Map options) { - return new TableSchema( - 0, - fields(), - 0, - Collections.emptyList(), - Collections.singletonList("id"), - options, - ""); - } - - private static java.util.List fields() { - return Arrays.asList( - new DataField(0, "id", DataTypes.INT().notNull()), - new DataField(1, "embedding", DataTypes.VECTOR(8, DataTypes.FLOAT())), - new DataField(2, "payload", DataTypes.STRING())); - } -} diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java index 7baeed2e102f..f43b7c590a44 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java @@ -135,6 +135,31 @@ public void testFullTextSearchEndToEnd() throws Exception { assertThat(ids).containsAnyOf(0, 1, 3); } + @Test + public void testDeclaredFullTextIndexOnOrdinaryAppendTable() throws Exception { + Identifier identifier = identifier("declared_full_text_append"); + catalog.createTable( + identifier, + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column(TEXT_FIELD_NAME, DataTypes.STRING()) + .option(CoreOptions.BUCKET.key(), "-1") + .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true") + .option("global-index.full-text.index-columns", TEXT_FIELD_NAME) + .build(), + false); + FileStoreTable table = getTable(identifier); + String[] documents = {"Apache Paimon", "Paimon lake", "other document"}; + writeDocuments(table, documents); + buildAndCommitIndex(table, documents); + GlobalIndexResult result = + table.newFullTextSearchBuilder() + .withQuery(TEXT_FIELD_NAME, matchQuery("Paimon")) + .withLimit(10) + .executeLocal(); + assertThat(result.results()).containsExactlyInAnyOrder(0L, 1L); + } + @ParameterizedTest @CsvSource({"full,true", "detail,true", "full,false", "detail,false"}) public void testPartialCompositeKeepsScalarFilterCoverage(String mode, boolean scalar) diff --git a/paimon-python/pypaimon/table/source/full_text_search_builder.py b/paimon-python/pypaimon/table/source/full_text_search_builder.py index 21bfbed2be2c..5c791785a655 100644 --- a/paimon-python/pypaimon/table/source/full_text_search_builder.py +++ b/paimon-python/pypaimon/table/source/full_text_search_builder.py @@ -182,7 +182,8 @@ def _text_columns(self): def _primary_key_full_text_definition(self): text_column = self._text_columns()[0] core = CoreOptions(Options(dict(self._table.table_schema.options))) - if (core.data_evolution_enabled() + if (not getattr(self._table.table_schema, 'primary_keys', None) + or core.data_evolution_enabled() or not core.primary_key_full_text_index_columns()): return None for definition in PrimaryKeyIndexDefinitions.create( diff --git a/paimon-python/pypaimon/tests/primary_key_index_definitions_test.py b/paimon-python/pypaimon/tests/primary_key_index_definitions_test.py index 33c37b52f86e..663a39ed158c 100644 --- a/paimon-python/pypaimon/tests/primary_key_index_definitions_test.py +++ b/paimon-python/pypaimon/tests/primary_key_index_definitions_test.py @@ -182,6 +182,7 @@ def test_primary_key_full_text_only_supports_fast_mode(self): def builder(mode): schema = TableSchema( fields=fields, + primary_keys=['id'], options={"pk-full-text.index.columns": "content", "global-index.search-mode": mode}) table = SimpleNamespace( @@ -193,6 +194,7 @@ def builder(mode): builder("fast").new_full_text_read(), PrimaryKeyFullTextRead) schema = TableSchema( fields=fields, + primary_keys=['id'], options={ "pk-full-text.index.columns": "content", "global-index.search-mode": "full", @@ -212,6 +214,16 @@ def builder(mode): NotImplementedError, "only supports the FAST"): builder(mode).new_full_text_read() + def test_append_full_text_declarations_do_not_select_primary_key_reads(self): + field = DataField(0, 'content', AtomicType('STRING')) + for key in ('global-index.full-text.index-columns', 'global-index.full-text.index-column', + 'pk-full-text.index.columns'): + with self.subTest(key=key): + schema = TableSchema(fields=[field], options={key: 'content'}) + table = SimpleNamespace(fields=[field], table_schema=schema, partition_keys=[]) + builder = FullTextSearchBuilderImpl(table).with_query('content', 'paimon').with_limit(2) + self.assertIsNone(builder._primary_key_full_text_definition()) + def test_primary_key_full_text_reader_cannot_bypass_fast_mode(self): field = DataField(1, "content", AtomicType("STRING")) for mode in ("full", "detail"): From 8f5de640fba529e125a091f667bc3798d91c0d7a Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 1 Nov 2026 09:43:19 +0800 Subject: [PATCH 04/14] [core][python] Move vector type hints to shared resolver --- .../java/org/apache/paimon/CoreOptions.java | 26 +--------- .../paimon/options/VectorIndexOptions.java | 31 +++++++++++- .../AbstractDataEvolutionVectorRead.java | 3 +- .../table/source/DataEvolutionVectorScan.java | 5 +- .../apache/paimon/GlobalIndexOptionsTest.java | 16 +++++++ .../options/VectorIndexOptionsTest.java | 36 ++++++++++++++ .../table/source/VectorSearchBuilderTest.java | 47 +++++++++++++++++++ .../pypaimon/common/options/core_options.py | 16 ++----- .../common/options/vector_index_options.py | 18 ++++++- .../execution/native/index_options.py | 2 +- .../native/read/native_search_plan.py | 2 +- .../python/read/search/vector_search_read.py | 2 +- .../python/read/search/vector_search_scan.py | 3 +- .../tests/vector_index_options_test.py | 28 +++++++++++ .../tests/vector_search_filter_test.py | 25 +++++----- 15 files changed, 198 insertions(+), 62 deletions(-) diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 1dc222b57c84..361f08042a94 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -5218,10 +5218,7 @@ public String globalIndexColumnsOptionKey(String type) { public Options globalIndexOptions(@Nullable String type, String column) { Options resolved = new Options(options.toMap(), options); if (type == null) { - type = vectorIndexType(column); - if (type == null) { - return resolved; - } + return resolved; } boolean vector = isVectorIndexType(type); String prefix = vector || "full-text".equals(type) ? type + "." : type + "-index."; @@ -5355,26 +5352,7 @@ public String primaryKeyVectorIndexType(String column) { @Nullable public String vectorIndexType(String column) { - List declared = - vectorIndexTypes().stream() - .filter(type -> globalIndexColumns(type).contains(column)) - .collect(Collectors.toList()); - if (!declared.isEmpty()) { - return declared.size() == 1 ? declared.get(0) : null; - } - for (String key : - Arrays.asList( - "fields." + column + ".pk-vector.index.type", - "index_type", - "index-type", - "vector.index-type", - "fields." + column + ".index-type")) { - String type = options.get(key); - if (type != null) { - return type.toLowerCase(Locale.ROOT).trim(); - } - } - return null; + return VectorIndexOptions.indexType(options, column); } /** Compatibility delegate to the vector resolver shared by both table modes. */ diff --git a/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java b/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java index a8a69fa8a3c8..49e7ee104d73 100644 --- a/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java @@ -25,12 +25,39 @@ import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.stream.Collectors; /** Vector parameter resolution shared by primary-key and Data Evolution indexes. */ public final class VectorIndexOptions { private VectorIndexOptions() {} + @Nullable + public static String indexType(Options options, String column) { + CoreOptions core = new CoreOptions(options); + List declared = + core.globalIndexTypes().stream() + .filter(CoreOptions::isVectorIndexType) + .filter(type -> core.globalIndexColumns(type).contains(column)) + .collect(Collectors.toList()); + if (!declared.isEmpty()) { + return declared.size() == 1 ? declared.get(0) : null; + } + for (String key : + Arrays.asList( + "fields." + column + ".pk-vector.index.type", + "index_type", + "index-type", + "vector.index-type", + "fields." + column + ".index-type")) { + String type = options.get(key); + if (type != null) { + return type.toLowerCase(Locale.ROOT).trim(); + } + } + return null; + } + public static Options resolve(Options options, @Nullable String type, String column) { return resolve(options, type, column, true); } @@ -39,7 +66,7 @@ public static Options resolve(Options options, @Nullable String type, String col public static Options resolve( Options options, @Nullable String type, String column, boolean defaultMetric) { if (type == null) { - type = new CoreOptions(options).vectorIndexType(column); + type = indexType(options, column); if (type == null) { return new Options(options.toMap(), options); } @@ -87,7 +114,7 @@ public static Options resolve( public static String metric(Options options, @Nullable String type, String column) { if (type == null) { - type = new CoreOptions(options).vectorIndexType(column); + type = indexType(options, column); } return type == null ? normalizeMetric(options.get("fields." + column + ".pk-vector.distance.metric")) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java index c68ae0d0b359..761c883aa574 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java @@ -783,7 +783,8 @@ private String configuredRawSearchMetric(Map options) { String fieldPrefix = "fields." + vectorColumn.name() + "."; if (options.containsKey(fieldPrefix + "index-options")) { CoreOptions configured = new CoreOptions(new Options(table.options(), options)); - String type = configured.primaryKeyVectorIndexType(vectorColumn.name()); + String type = + VectorIndexOptions.indexType(configured.toConfiguration(), vectorColumn.name()); if (type == null) { type = "vector"; } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java index f088fcee1830..e7c38e82bbb7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java @@ -27,6 +27,7 @@ import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.options.Options; +import org.apache.paimon.options.VectorIndexOptions; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.table.FileStoreTable; @@ -293,8 +294,8 @@ private String vectorIndexType(List indexFiles) { @Nullable private String configuredVectorIndexType() { - return new CoreOptions(new Options(table.options(), options)) - .vectorIndexType(vectorColumn.name()); + return VectorIndexOptions.indexType( + new Options(table.options(), options), vectorColumn.name()); } private static boolean containsField(GlobalIndexMeta meta, int fieldId) { diff --git a/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java index c775e0812c9c..380f35872035 100644 --- a/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java @@ -47,6 +47,22 @@ /** Configuration contracts shared by primary-key and Data Evolution indexes. */ class GlobalIndexOptionsTest { + @ParameterizedTest + @ValueSource( + strings = { + "index_type", + "index-type", + "vector.index-type", + "fields.embedding.index-type" + }) + void testGenericMergeDoesNotInferHistoricalVectorTypes(String key) { + Map values = new HashMap<>(); + values.put(key, "ivf-pq"); + values.put("fields.embedding.index-options", "{\"nlist\":64}"); + assertThat(new CoreOptions(values).globalIndexOptions(null, "embedding").toMap()) + .isEqualTo(values); + } + @Test void testGenericMergeDoesNotInterpretVectorParameters() { Map values = new HashMap<>(); diff --git a/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java index 7ad519f641b5..81499faf0b8b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java @@ -23,6 +23,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import java.util.Collections; import java.util.HashMap; @@ -33,6 +34,41 @@ /** Tests for vector parameter resolution shared by both table modes. */ class VectorIndexOptionsTest { + @ParameterizedTest + @ValueSource( + strings = { + "fields.embedding.pk-vector.index.type", + "index_type", + "index-type", + "vector.index-type", + "fields.embedding.index-type" + }) + void testLegacyTypeAliasesResolveFieldParameters(String key) { + Map values = new HashMap<>(); + values.put(key, " IVF-PQ "); + values.put("ivf-pq.dimension", "768"); + values.put("fields.embedding.index-options", "{\"nlist\":64,\"metric\":\"COSINE\"}"); + Options input = new Options(values); + assertThat(VectorIndexOptions.indexType(input, "embedding")).isEqualTo("ivf-pq"); + assertThat(new CoreOptions(input).vectorIndexType("embedding")).isEqualTo("ivf-pq"); + Options resolved = VectorIndexOptions.resolve(input, null, "embedding", false); + assertMetric(resolved, "cosine"); + assertThat(resolved.get("ivf-pq.dimension")).isEqualTo("768"); + assertThat(resolved.get("ivf-pq.nlist")).isEqualTo("64"); + assertThat(VectorIndexOptions.resolve(resolved, null, "embedding", false).toMap()) + .isEqualTo(resolved.toMap()); + assertThat(input.toMap()).isEqualTo(values); + } + + @Test + void testAmbiguousDeclarationsDoNotFallBackToHistoricalHint() { + Options options = new Options(); + options.setString("global-index.ivf-pq.index-columns", "embedding"); + options.setString("global-index.diskann.index-column", "embedding"); + options.setString("index_type", "ivf-flat"); + assertThat(VectorIndexOptions.indexType(options, "embedding")).isNull(); + } + @ParameterizedTest @CsvSource({ "fields.embedding.pk-vector.distance.metric,COSINE,cosine", diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java index 38a7258fc667..bff2b3bc76e9 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java @@ -19,6 +19,7 @@ package org.apache.paimon.table.source; import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.Identifier; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericArray; @@ -69,6 +70,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; import java.io.ByteArrayInputStream; @@ -528,6 +530,51 @@ public void testVectorSearchFullModeScansUnindexedData() throws Exception { assertThat(TestVectorGlobalIndexer.metricCalls()).isGreaterThan(0); } + @ParameterizedTest + @CsvSource({ + "index_type,true", "index_type,false", + "index-type,true", "index-type,false", + "vector.index-type,true", "vector.index-type,false", + "fields.vec.index-type,true", "fields.vec.index-type,false" + }) + public void testHistoricalTypeHintsReachRawSingleAndBatchQueries( + String key, boolean tableOption) throws Exception { + Identifier identifier = identifier("raw_type_hint_" + key.replace('.', '_') + tableOption); + Schema.Builder schema = + vectorSchemaBuilder(VECTOR_FIELD_NAME) + .option(CoreOptions.GLOBAL_INDEX_SEARCH_MODE.key(), "full"); + if (tableOption) { + schema.option(key, TestVectorGlobalIndexerFactory.IDENTIFIER); + } + catalog.createTable(identifier, schema.build(), false); + FileStoreTable table = getTable(identifier); + writeVectors(table, new float[][] {{1.0f, 0.0f}, {0.0f, 1.0f}}); + Map queryOptions = + tableOption + ? Collections.emptyMap() + : Collections.singletonMap(key, TestVectorGlobalIndexerFactory.IDENTIFIER); + VectorSearchBuilder single = + table.newVectorSearchBuilder() + .withVectorColumn(VECTOR_FIELD_NAME) + .withVector(new float[] {1.0f, 0.0f}) + .withLimit(1) + .withOptions(queryOptions); + RawVectorSearchSplit raw = + rawVectorSearchSplits(single.newVectorScan().scan().splits()).get(0); + assertThat(raw.indexType()).isEqualTo(TestVectorGlobalIndexerFactory.IDENTIFIER); + assertThat(single.executeLocal().results()).containsExactly(0L); + List batch = + table.newBatchVectorSearchBuilder() + .withVectorColumn(VECTOR_FIELD_NAME) + .withVectors(new float[][] {{1.0f, 0.0f}, {0.0f, 1.0f}}) + .withLimit(1) + .withOptions(queryOptions) + .executeBatchLocal(); + assertThat(batch).hasSize(2); + assertThat(batch.get(0).results()).containsExactly(0L); + assertThat(batch.get(1).results()).containsExactly(1L); + } + @Test public void testVectorSearchFastModeSkipsUnindexedDataByDefault() throws Exception { catalog.createTable( diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py index a1dc1e4fbf32..fc1ade9afd57 100644 --- a/paimon-python/pypaimon/common/options/core_options.py +++ b/paimon-python/pypaimon/common/options/core_options.py @@ -1910,9 +1910,7 @@ def global_index_options(self, index_type: Optional[str], column: str) -> Option """Resolve the same implementation and field options for either table mode.""" values = dict(self.options.to_map()) if index_type is None: - index_type = self.vector_index_type(column) - if index_type is None: - return Options(values) + return Options(values) vector = self._is_vector_index_type(index_type) prefix = index_type + ("." if vector or index_type == "full-text" else "-index.") field_prefix = "fields.%s." % column @@ -1950,16 +1948,8 @@ def primary_key_vector_index_type(self, column: str): return self.vector_index_type(column) def vector_index_type(self, column: str): - declared = [index_type for index_type in self._vector_index_types() - if column in self.global_index_columns(index_type)] - if declared: - return declared[0] if len(declared) == 1 else None - for key in ("fields.%s.pk-vector.index.type" % column, "index_type", "index-type", - "vector.index-type", "fields.%s.index-type" % column): - value = self.options.to_map().get(key) - if value is not None: - return str(value).lower().strip() - return None + from pypaimon.common.options.vector_index_options import VectorIndexOptions + return VectorIndexOptions.index_type(self.options, column) def primary_key_vector_index_options(self, column: str) -> Options: from pypaimon.common.options.vector_index_options import VectorIndexOptions diff --git a/paimon-python/pypaimon/common/options/vector_index_options.py b/paimon-python/pypaimon/common/options/vector_index_options.py index 6442f2c04e40..2025c9bb5886 100644 --- a/paimon-python/pypaimon/common/options/vector_index_options.py +++ b/paimon-python/pypaimon/common/options/vector_index_options.py @@ -22,11 +22,25 @@ class VectorIndexOptions: + @staticmethod + def index_type(options, column): + core = CoreOptions(options) + declared = [kind for kind in core.global_index_types() + if CoreOptions._is_vector_index_type(kind) and column in core.global_index_columns(kind)] + if declared: + return declared[0] if len(declared) == 1 else None + for key in ('fields.%s.pk-vector.index.type' % column, 'index_type', 'index-type', + 'vector.index-type', 'fields.%s.index-type' % column): + value = options.to_map().get(key) + if value is not None: + return str(value).lower().strip() + return None + @staticmethod def resolve(options, index_type, column, *, default_metric=True): values = dict(options.to_map()) if index_type is None: - index_type = CoreOptions(options).vector_index_type(column) + index_type = VectorIndexOptions.index_type(options, column) if index_type is None: return Options(values) prefix = index_type + '.' @@ -52,7 +66,7 @@ def resolve(options, index_type, column, *, default_metric=True): @staticmethod def metric(options, index_type, column): if index_type is None: - index_type = CoreOptions(options).vector_index_type(column) + index_type = VectorIndexOptions.index_type(options, column) if index_type is None: return _normalize_metric(options.to_map().get('fields.%s.pk-vector.distance.metric' % column)) return VectorIndexOptions.resolve(options, index_type, column).to_map()[index_type + '.metric'] diff --git a/paimon-python/pypaimon/execution/native/index_options.py b/paimon-python/pypaimon/execution/native/index_options.py index 49190300c18c..3d731926cd92 100644 --- a/paimon-python/pypaimon/execution/native/index_options.py +++ b/paimon-python/pypaimon/execution/native/index_options.py @@ -53,7 +53,7 @@ def _resolve(schema): definitions += [(kind, core.global_index_columns(kind)) for kind in declarations if kind not in families] if primary_key and not any(kind not in families for kind in declarations): - definitions += [(core.vector_index_type(column), [column]) + definitions += [(VectorIndexOptions.index_type(core.options, column), [column]) for column in core.primary_key_vector_index_columns()] known_columns = set() native_columns = {} diff --git a/paimon-python/pypaimon/execution/native/read/native_search_plan.py b/paimon-python/pypaimon/execution/native/read/native_search_plan.py index a1ba3fa2507d..e63f1ccbe7c2 100644 --- a/paimon-python/pypaimon/execution/native/read/native_search_plan.py +++ b/paimon-python/pypaimon/execution/native/read/native_search_plan.py @@ -48,7 +48,7 @@ def _index_search_options(table, column, index_type=None, overrides=None, scalar return values core = CoreOptions(Options(values)) if index_type is None: - index_type = core.vector_index_type(column) + index_type = VectorIndexOptions.index_type(Options(values), column) if index_type is None: # The Python reader can resolve field options once persisted index metadata is known. if any(values.get(key) is not None for key in ( diff --git a/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py b/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py index a1a099bd9f5f..b94211ee5803 100644 --- a/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py +++ b/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py @@ -994,7 +994,7 @@ def _configured_vector_metric(options, vector_column, index_type=None): field_prefix = "fields.%s." % vector_column.name if field_prefix + "index-options" in options: core = CoreOptions(Options(options)) - selected_type = index_type or core.primary_key_vector_index_type(vector_column.name) or "vector" + selected_type = index_type or VectorIndexOptions.index_type(core.options, vector_column.name) or "vector" metric = VectorIndexOptions.resolve(Options(options), selected_type, vector_column.name, default_metric=False).to_map().get(selected_type + ".metric") if metric is not None: diff --git a/paimon-python/pypaimon/execution/python/read/search/vector_search_scan.py b/paimon-python/pypaimon/execution/python/read/search/vector_search_scan.py index 2c50e5f5b9b6..21ace7fc67d2 100644 --- a/paimon-python/pypaimon/execution/python/read/search/vector_search_scan.py +++ b/paimon-python/pypaimon/execution/python/read/search/vector_search_scan.py @@ -23,6 +23,7 @@ from pypaimon.common.options.core_options import CoreOptions, GlobalIndexSearchMode from pypaimon.common.options.options import Options +from pypaimon.common.options.vector_index_options import VectorIndexOptions from pypaimon.execution.python.globalindex.data_evolution_global_index_coverage import DataEvolutionGlobalIndexCoverage from pypaimon.execution.python.globalindex.data_evolution_global_index_scanner import ( is_supported_scalar_index, @@ -244,4 +245,4 @@ def _configured_vector_index_type(table, vector_column, options): table_options = getattr(getattr(table, "options", None), "options", None) values = dict(table_options.to_map()) if table_options is not None else {} values.update(options) - return CoreOptions(Options(values)).vector_index_type(vector_column.name) + return VectorIndexOptions.index_type(Options(values), vector_column.name) diff --git a/paimon-python/pypaimon/tests/vector_index_options_test.py b/paimon-python/pypaimon/tests/vector_index_options_test.py index 89d18dbc2727..ef1626a66ffe 100644 --- a/paimon-python/pypaimon/tests/vector_index_options_test.py +++ b/paimon-python/pypaimon/tests/vector_index_options_test.py @@ -30,6 +30,34 @@ def _assert_metric(options, expected): assert options.to_map()[key] == expected +@pytest.mark.parametrize('key', ['index_type', 'index-type', 'vector.index-type', 'fields.embedding.index-type']) +def test_generic_merge_does_not_infer_historical_vector_types(key): + values = {key: 'ivf-pq', 'fields.embedding.index-options': '{"nlist":64}'} + assert CoreOptions(Options(values)).global_index_options(None, 'embedding').to_map() == values + + +@pytest.mark.parametrize('key', ['fields.embedding.pk-vector.index.type', 'index_type', 'index-type', + 'vector.index-type', 'fields.embedding.index-type']) +def test_legacy_type_aliases_resolve_field_parameters(key): + values = {key: ' IVF-PQ ', 'ivf-pq.dimension': '768', + 'fields.embedding.index-options': '{"nlist":64,"metric":"COSINE"}'} + options = Options(values) + assert VectorIndexOptions.index_type(options, 'embedding') == 'ivf-pq' + assert CoreOptions(options).vector_index_type('embedding') == 'ivf-pq' + resolved = VectorIndexOptions.resolve(options, None, 'embedding', default_metric=False) + _assert_metric(resolved, 'cosine') + assert resolved.to_map()['ivf-pq.dimension'] == '768' + assert resolved.to_map()['ivf-pq.nlist'] == '64' + assert VectorIndexOptions.resolve(resolved, None, 'embedding', default_metric=False).to_map() == resolved.to_map() + assert options.to_map() == values + + +def test_ambiguous_declarations_do_not_fall_back_to_historical_hint(): + options = Options({'global-index.ivf-pq.index-columns': 'embedding', + 'global-index.diskann.index-column': 'embedding', 'index_type': 'ivf-flat'}) + assert VectorIndexOptions.index_type(options, 'embedding') is None + + @pytest.mark.parametrize('key,value,expected', [ ('fields.embedding.pk-vector.distance.metric', 'COSINE', 'cosine'), ('fields.embedding.distance.metric', 'COSINE', 'cosine'), diff --git a/paimon-python/pypaimon/tests/vector_search_filter_test.py b/paimon-python/pypaimon/tests/vector_search_filter_test.py index 1cf307ef8a4f..073eab277503 100644 --- a/paimon-python/pypaimon/tests/vector_search_filter_test.py +++ b/paimon-python/pypaimon/tests/vector_search_filter_test.py @@ -1832,6 +1832,7 @@ def test_raw_vector_read_passes_ranges_without_bitmap(self): def test_scan_threads_builder_options_to_raw_split_index_type(self): from pypaimon.table.source.vector_search_split import RawVectorSearchSplit + from pypaimon.table.source.batch_vector_search_builder import BatchVectorSearchBuilderImpl class _Snapshots: def get_latest_snapshot(self_inner): @@ -1846,20 +1847,16 @@ def get_latest_snapshot(self_inner): _patch_snapshot(self, []) self._travel_patch.stop() - splits = ( - VectorSearchBuilderImpl(table) - .with_vector_column("embedding") - .with_query_vector([1.0, 0.0, 0.0, 0.0]) - .with_limit(3) - .with_option("index-type", "ivf-flat") - .new_vector_search_scan() - .scan() - .splits() - ) - - raw = [s for s in splits if isinstance(s, RawVectorSearchSplit)] - self.assertEqual(1, len(raw)) - self.assertEqual("ivf-flat", raw[0].index_type) + for key in ('index_type', 'index-type', 'vector.index-type', 'fields.embedding.index-type'): + for batch in (False, True): + with self.subTest(key=key, batch=batch): + builder = (BatchVectorSearchBuilderImpl(table).with_query_vectors([[1.0, 0.0, 0.0, 0.0]]) + if batch else VectorSearchBuilderImpl(table).with_query_vector([1.0, 0.0, 0.0, 0.0])) + splits = (builder.with_vector_column("embedding").with_limit(3).with_option(key, "ivf-flat") + .new_vector_search_scan().scan().splits()) + raw = [s for s in splits if isinstance(s, RawVectorSearchSplit)] + self.assertEqual(1, len(raw)) + self.assertEqual("ivf-flat", raw[0].index_type) def test_scan_excludes_composite_btree_when_filter_hits_extra_field(self): id_name_index = _entry(None, field_id=2, index_type="btree", From c735c280b59d0d44c6182e8b55ddda4d872394fa Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 1 Nov 2026 10:35:08 +0800 Subject: [PATCH 05/14] [core][python] Remove legacy vector type hints --- docs/docs/multimodal-table/global-index.mdx | 6 + docs/generated/core_configuration.html | 30 +++++ .../paimon/options/VectorIndexOptions.java | 15 +-- .../options/VectorIndexOptionsTest.java | 18 ++- .../table/source/VectorSearchBuilderTest.java | 24 ++-- paimon-python/dev/native-vector-type-api.md | 77 +++++++++++ .../common/options/vector_index_options.py | 8 +- .../execution/native/index_options.py | 28 +++- .../native/read/native_hybrid_search.py | 4 + .../native/read/native_search_plan.py | 10 +- .../native/read/native_vector_search.py | 6 +- .../pypaimon/tests/global_index_build_test.py | 12 +- .../pypaimon/tests/global_index_test.py | 6 +- .../tests/lazy_search_execution_test.py | 124 +++++++++++++++++- .../tests/native_index_options_test.py | 40 +++++- .../pypaimon/tests/native_search_plan_test.py | 8 +- .../tests/vector_index_options_test.py | 18 ++- .../tests/vector_metric_consistency_test.py | 3 +- .../tests/vector_search_filter_test.py | 7 +- 19 files changed, 379 insertions(+), 65 deletions(-) create mode 100644 paimon-python/dev/native-vector-type-api.md diff --git a/docs/docs/multimodal-table/global-index.mdx b/docs/docs/multimodal-table/global-index.mdx index 11f704be512f..e2fcfad00768 100644 --- a/docs/docs/multimodal-table/global-index.mdx +++ b/docs/docs/multimodal-table/global-index.mdx @@ -75,6 +75,12 @@ ivf-pq.dimension=768 fields.embedding.index-options={"nlist":64,"distance.metric":"cosine"} ``` +For raw vector searches without existing index files, declare the implementation +with `global-index..index-columns` (or singular `index-column`). The former +`index_type`, `index-type`, `vector.index-type`, and `fields..index-type` +table/query option aliases are no longer interpreted. Legacy primary-key index +configuration remains supported, including `fields..pk-vector.index.type`. + Use the actual implementation identifier, such as `ivf-pq`, in the declaration. Each listed column declares an independent index. The singular `index-column` spelling remains an alias; `index-columns` takes precedence. Field JSON options diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 0fc8c42fd261..7538370c51fa 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -872,6 +872,18 @@

Enum

Search mode for full-text index queries.

Possible values:
  • "fast": Only search indexed data.
  • "full": Use snapshot next row id and global index coverage to detect missing row ids, and scan raw data only when a gap exists.
  • "detail": Scan data files to find exact unindexed rows. This can handle index invalidation caused by updates or rewrites.
+ +
global-index.bitmap.index-columns
+ (none) + String + Comma-separated columns to index with Bitmap indexes. + + +
global-index.btree.index-columns
+ (none) + String + Comma-separated columns to index with BTree indexes. +
global-index.build.max-parallelism
4096 @@ -908,6 +920,24 @@ Boolean Whether a vector, hybrid or full-text search may read the filter columns of candidate rows to verify a row filter that the scalar global index can only answer with a superset, such as contains, ends-with or like on a BTree index or a conjunction with a member no index can evaluate. When false, such candidates are excluded from the search, which never returns a non-matching row but may return fewer than the requested top-k. When true, the read runs on the caller and may cover every candidate row. + +
global-index.fm.index-columns
+ (none) + String + Comma-separated character columns to index with FM indexes. + + +
global-index.full-text.index-columns
+ (none) + String + Comma-separated character columns to index with full-text indexes. + + +
global-index.multivalue.index-columns
+ (none) + String + Comma-separated ARRAY columns to index with Multivalue indexes. +
global-index.query-in-reader.enabled
false diff --git a/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java b/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java index 49e7ee104d73..5785567e4974 100644 --- a/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java @@ -43,19 +43,8 @@ public static String indexType(Options options, String column) { if (!declared.isEmpty()) { return declared.size() == 1 ? declared.get(0) : null; } - for (String key : - Arrays.asList( - "fields." + column + ".pk-vector.index.type", - "index_type", - "index-type", - "vector.index-type", - "fields." + column + ".index-type")) { - String type = options.get(key); - if (type != null) { - return type.toLowerCase(Locale.ROOT).trim(); - } - } - return null; + String type = options.get("fields." + column + ".pk-vector.index.type"); + return type == null ? null : type.toLowerCase(Locale.ROOT).trim(); } public static Options resolve(Options options, @Nullable String type, String column) { diff --git a/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java index 81499faf0b8b..51aef5b4e557 100644 --- a/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java @@ -37,15 +37,27 @@ class VectorIndexOptionsTest { @ParameterizedTest @ValueSource( strings = { - "fields.embedding.pk-vector.index.type", "index_type", "index-type", "vector.index-type", "fields.embedding.index-type" }) - void testLegacyTypeAliasesResolveFieldParameters(String key) { + void testRemovedTypeAliasesAreIgnored(String key) { + Options input = Options.fromMap(Collections.singletonMap(key, "ivf-pq")); + assertThat(VectorIndexOptions.indexType(input, "embedding")).isNull(); + assertThat(new CoreOptions(input).vectorIndexType("embedding")).isNull(); + assertThat(VectorIndexOptions.resolve(input, null, "embedding", false).toMap()) + .isEqualTo(input.toMap()); + input.setString("fields.embedding.pk-vector.index.type", "diskann"); + assertThat(VectorIndexOptions.indexType(input, "embedding")).isEqualTo("diskann"); + input.setString("global-index.ivf-flat.index-columns", "embedding"); + assertThat(VectorIndexOptions.indexType(input, "embedding")).isEqualTo("ivf-flat"); + } + + @Test + void testLegacyPrimaryKeyTypeResolvesFieldParameters() { Map values = new HashMap<>(); - values.put(key, " IVF-PQ "); + values.put("fields.embedding.pk-vector.index.type", " IVF-PQ "); values.put("ivf-pq.dimension", "768"); values.put("fields.embedding.index-options", "{\"nlist\":64,\"metric\":\"COSINE\"}"); Options input = new Options(values); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java index bff2b3bc76e9..9f4d5fe30f25 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java @@ -532,27 +532,31 @@ public void testVectorSearchFullModeScansUnindexedData() throws Exception { @ParameterizedTest @CsvSource({ - "index_type,true", "index_type,false", - "index-type,true", "index-type,false", - "vector.index-type,true", "vector.index-type,false", - "fields.vec.index-type,true", "fields.vec.index-type,false" + "global-index.test-vector-ann.index-columns,true", + "global-index.test-vector-ann.index-columns,false", + "global-index.test-vector-ann.index-column,true", + "global-index.test-vector-ann.index-column,false", + "fields.vec.pk-vector.index.type,true", + "fields.vec.pk-vector.index.type,false" }) - public void testHistoricalTypeHintsReachRawSingleAndBatchQueries( - String key, boolean tableOption) throws Exception { + public void testConfiguredTypesReachRawSingleAndBatchQueries(String key, boolean tableOption) + throws Exception { Identifier identifier = identifier("raw_type_hint_" + key.replace('.', '_') + tableOption); Schema.Builder schema = vectorSchemaBuilder(VECTOR_FIELD_NAME) .option(CoreOptions.GLOBAL_INDEX_SEARCH_MODE.key(), "full"); + String value = + key.startsWith("global-index.") + ? VECTOR_FIELD_NAME + : TestVectorGlobalIndexerFactory.IDENTIFIER; if (tableOption) { - schema.option(key, TestVectorGlobalIndexerFactory.IDENTIFIER); + schema.option(key, value); } catalog.createTable(identifier, schema.build(), false); FileStoreTable table = getTable(identifier); writeVectors(table, new float[][] {{1.0f, 0.0f}, {0.0f, 1.0f}}); Map queryOptions = - tableOption - ? Collections.emptyMap() - : Collections.singletonMap(key, TestVectorGlobalIndexerFactory.IDENTIFIER); + tableOption ? Collections.emptyMap() : Collections.singletonMap(key, value); VectorSearchBuilder single = table.newVectorSearchBuilder() .withVectorColumn(VECTOR_FIELD_NAME) diff --git a/paimon-python/dev/native-vector-type-api.md b/paimon-python/dev/native-vector-type-api.md new file mode 100644 index 000000000000..e365f268b29a --- /dev/null +++ b/paimon-python/dev/native-vector-type-api.md @@ -0,0 +1,77 @@ + + +# Native vector implementation API + +The Python adapter expects the following method on all three +`pypaimon_rust.datafusion` builders: `VectorSearchBuilder`, +`BatchVectorSearchBuilder`, and `HybridSearchBuilder`. + +```python +def with_vector_index_type(self, column: str, index_type: str) -> Self: + ... +``` + +Expose equivalent typed state in the Rust core builders, rather than encoding +it as a table or query option. The Python adapter already calls this method; +the Rust implementation and binding are a separate change. + +* `column` is the schema column name. `index_type` is a normalized implementation + identifier such as `ivf-pq`, `ivf-flat`, or `diskann`. +* The method mutates builder configuration and returns the same builder, like + `with_vector_column`. Calling it must not load a snapshot, plan or execute a + search. Repeating it for a column replaces that column's previous value. +* Single and batch queries use the entry for the selected vector column. + Hybrid queries use the entry for each vector route's column, independently of + route order. Full-text routes do not consume it. Configuration may be supplied + before routes are added. +* Persisted index metadata continues to determine the implementation for indexed + ranges. The explicit value supplies the implementation for a raw search when + no vector index metadata is available, matching the Java planner. An absent + value leaves existing metadata/default behavior in place. +* Keep this state through scan/read construction, batch reader reconstruction, + captured hybrid route builders, and distributed worker serialization. + `RawVectorSearchSplit.index_type` remains a typed transport field. +* This API sets only the implementation. Resolved vector parameters still use + `with_options`/route options; an unspecified metric must remain unspecified so + persisted index metadata can provide it. +* Native code must stop interpreting `index_type`, `index-type`, + `vector.index-type`, and `fields..index-type` as option aliases. + Legitimate API argument names and index metadata fields named `index_type` + remain supported. + +Python resolves the implementation once through `VectorIndexOptions`, using +`global-index..index-columns` (or singular `index-column`) and the retained +legacy PK key `fields..pk-vector.index.type`. It strips the four removed +aliases before transferring schema or query options to older SDKs. + +When an implementation is configured and this method is unavailable, Python +returns to its own execution path before planning, reading, executing, or +capturing hybrid routes. SDK configuration and execution errors propagate; +there is no retry after execution. Queries with no configured implementation +can continue to use older SDKs and persisted index metadata. + +## Rust verification + +Cover single, batch, hybrid, scan/read, captured routes, and serialized workers +with both canonical and legacy PK declarations. Test raw searches without index +files with and without a metric, metadata taking precedence over the explicit +value, and independent columns in a hybrid query. The Python adapter contract +and old-SDK fallback are covered in `lazy_search_execution_test.py`; native +integration coverage is in `native_search_plan_test.py` and requires this API. diff --git a/paimon-python/pypaimon/common/options/vector_index_options.py b/paimon-python/pypaimon/common/options/vector_index_options.py index 2025c9bb5886..c3aebc74cd22 100644 --- a/paimon-python/pypaimon/common/options/vector_index_options.py +++ b/paimon-python/pypaimon/common/options/vector_index_options.py @@ -29,12 +29,8 @@ def index_type(options, column): if CoreOptions._is_vector_index_type(kind) and column in core.global_index_columns(kind)] if declared: return declared[0] if len(declared) == 1 else None - for key in ('fields.%s.pk-vector.index.type' % column, 'index_type', 'index-type', - 'vector.index-type', 'fields.%s.index-type' % column): - value = options.to_map().get(key) - if value is not None: - return str(value).lower().strip() - return None + value = options.to_map().get('fields.%s.pk-vector.index.type' % column) + return None if value is None else str(value).lower().strip() @staticmethod def resolve(options, index_type, column, *, default_metric=True): diff --git a/paimon-python/pypaimon/execution/native/index_options.py b/paimon-python/pypaimon/execution/native/index_options.py index 3d731926cd92..d05e3507a478 100644 --- a/paimon-python/pypaimon/execution/native/index_options.py +++ b/paimon-python/pypaimon/execution/native/index_options.py @@ -29,6 +29,27 @@ def native_index_options(schema): return _resolve(schema)[0] +def native_search_options(options): + """Do not let an older SDK interpret removed external type hints.""" + return {key: value for key, value in options.items() + if key not in ('index_type', 'index-type', 'vector.index-type') + and not (key.startswith('fields.') and key.endswith('.index-type'))} + + +def configure_vector_index_type(native, options, column): + """Use the explicit native API, qualifying SDK support before execution.""" + if column is None: + return True + index_type = VectorIndexOptions.index_type(Options(options), column) + if index_type is None: + return True + configure = getattr(native, 'with_vector_index_type', None) + if not callable(configure): + return False + configure(column, index_type) + return True + + def native_scalar_index_read_supported(table): if not table.options.global_index_enabled(): return True @@ -38,7 +59,7 @@ def native_scalar_index_read_supported(table): def _resolve(schema): if not isinstance(getattr(schema, 'options', None), Mapping): return {}, True - values = dict(schema.options) + values = native_search_options(schema.options) core = CoreOptions(Options(values)) result = dict(values) declarations = core.global_index_types() @@ -72,13 +93,15 @@ def _resolve(schema): known_columns.add(column) resolved = (VectorIndexOptions.resolve(core.options, kind, column, default_metric=False).to_map() if vector else core.global_index_options(kind, column).to_map()) + resolved = native_search_options(resolved) profile = {key: value for key, value in resolved.items() if key.startswith(prefix) and value is not None} field_prefix = 'fields.%s.' % column if primary_key: if vector: result[field_prefix + 'pk-vector.index.type'] = kind - result[field_prefix + 'pk-vector.distance.metric'] = VectorIndexOptions.metric(core.options, kind, column) + result[field_prefix + 'pk-vector.distance.metric'] = VectorIndexOptions.metric( + core.options, kind, column) profile.update({key: value for key, value in resolved.items() if key.startswith(field_prefix) and not key.startswith(field_prefix + 'pk-') and key != field_prefix + 'index-options' and value is not None}) @@ -88,7 +111,6 @@ def _resolve(schema): result[field_prefix + 'pk-%s.index.options' % family] = json.dumps(profile) result.pop(field_prefix + 'index-options', None) elif vector: - result[field_prefix + 'index-type'] = kind result.update({key: value for key, value in resolved.items() if key.startswith(field_prefix) and key != field_prefix + 'index-options'}) elif kind != 'full-text': diff --git a/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py b/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py index 2325ec6c9cce..b1560f9ab948 100644 --- a/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py +++ b/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py @@ -26,6 +26,7 @@ from pypaimon.table.file_store_table import FileStoreTable from pypaimon.execution.native.write.native_commit import _rest_catalog_supported from pypaimon.execution.native.read.native_search_plan import _index_search_options +from pypaimon.execution.native.index_options import configure_vector_index_type def _native_builder(builder, executor=None): @@ -64,6 +65,9 @@ def _native_builder(builder, executor=None): if full_text_options is not None: table = table.copy(full_text_options) native = _native_table(table).new_hybrid_search_builder() + for route, options in zip(builder._routes, route_options): + if route.is_vector() and not configure_vector_index_type(native, options, route.field_name): + return None native.with_limit(builder._limit).with_ranker(builder._ranker) for route, options in zip(builder._routes, route_options): if route.is_vector(): diff --git a/paimon-python/pypaimon/execution/native/read/native_search_plan.py b/paimon-python/pypaimon/execution/native/read/native_search_plan.py index e63f1ccbe7c2..79b19286992a 100644 --- a/paimon-python/pypaimon/execution/native/read/native_search_plan.py +++ b/paimon-python/pypaimon/execution/native/read/native_search_plan.py @@ -23,7 +23,7 @@ from pypaimon.common.options.core_options import CoreOptions from pypaimon.common.options.options import Options from pypaimon.common.options.vector_index_options import VectorIndexOptions -from pypaimon.execution.native.index_options import native_scalar_index_read_supported +from pypaimon.execution.native.index_options import native_scalar_index_read_supported, native_search_options from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult from pypaimon.read.split_serializer import deserialize_split_v1, serialize_split_v1 from pypaimon.execution.native.table import (_partition_fields) @@ -42,6 +42,7 @@ def _index_search_options(table, column, index_type=None, overrides=None, scalar """Use the common resolver before transferring search options to Core.""" values = dict(table.options.options.to_map()) values.update(overrides or {}) + values = native_search_options(values) if scalar_filter is not None and not native_scalar_index_read_supported(table.copy(values)): return None if column is None: @@ -56,9 +57,10 @@ def _index_search_options(table, column, index_type=None, overrides=None, scalar return None return values # Keep an unspecified metric absent so readers can use persisted index metadata. - return (VectorIndexOptions.resolve(Options(values), index_type, column, default_metric=False).to_map() - if CoreOptions._is_vector_index_type(index_type) else - core.global_index_options(index_type, column).to_map()) + resolved = (VectorIndexOptions.resolve(Options(values), index_type, column, default_metric=False).to_map() + if CoreOptions._is_vector_index_type(index_type) else + core.global_index_options(index_type, column).to_map()) + return native_search_options(resolved) def _range(value): diff --git a/paimon-python/pypaimon/execution/native/read/native_vector_search.py b/paimon-python/pypaimon/execution/native/read/native_vector_search.py index 32c492c0d9f6..7f81865e4208 100644 --- a/paimon-python/pypaimon/execution/native/read/native_vector_search.py +++ b/paimon-python/pypaimon/execution/native/read/native_vector_search.py @@ -23,6 +23,7 @@ from pypaimon.execution.native.read.builder import (_predicate_to_native) from pypaimon.execution.native.runtime import (native_runtime_available) from pypaimon.execution.native.read.native_search_plan import _result, _index_search_options +from pypaimon.execution.native.index_options import configure_vector_index_type, native_search_options def _native_vector_builder(builder, batch=False): @@ -51,9 +52,12 @@ def _native_vector_builder(builder, batch=False): native_table = _native_table(table.copy(options)) native = (native_table.new_batch_vector_search_builder() if batch else native_table.new_vector_search_builder()) + if not configure_vector_index_type(native, options, + builder._vector_column.name if builder._vector_column else None): + return None if builder._vector_column is not None: native.with_vector_column(builder._vector_column.name) - native.with_limit(builder._limit).with_options(builder._options) + native.with_limit(builder._limit).with_options(native_search_options(builder._options)) try: if builder._filter is not None: native.with_filter(_predicate_to_native(builder._filter)) diff --git a/paimon-python/pypaimon/tests/global_index_build_test.py b/paimon-python/pypaimon/tests/global_index_build_test.py index cc9a4d3e649d..97b3758ce3e8 100644 --- a/paimon-python/pypaimon/tests/global_index_build_test.py +++ b/paimon-python/pypaimon/tests/global_index_build_test.py @@ -261,7 +261,8 @@ def test_native_scalar_and_full_text_builds_use_resolved_field_options(self): ('full-text', 'full-text.tokenizer', 'raw')): with self.subTest(index_type=kind): short_key = key.split('.', 1)[1] - options = dict(self.table_options, **{'write.native.enabled': 'true', + options = dict(self.table_options, **{ + 'write.native.enabled': 'true', 'global-index.%s.index-columns' % kind: 'name', 'fields.name.index-options': '{"%s":"%s"}' % (short_key, value)}) table = self._create_table(pa_schema=schema, options=options) @@ -293,7 +294,8 @@ def test_full_text_build_and_raw_fallback_share_field_options(self): schema = pa.schema([('id', pa.int32()), ('content', pa.string())]) configurations = { - 'shared': {'fields.content.index-options': '{"tokenizer":"raw","lower-case":false,"stop-words":["paimon","lake"]}'}, + 'shared': {'fields.content.index-options': + '{"tokenizer":"raw","lower-case":false,"stop-words":["paimon","lake"]}'}, 'legacy': {'fields.content.pk-full-text.index.options': '{"tokenizer":"raw"}'}, 'dlf': {'full-text.tokenizer': 'raw'}, } @@ -329,8 +331,10 @@ def test_declared_vector_index_keeps_data_evolution_single_and_batch_reads(self) 'read.native.enabled': 'false', 'scan.native-plan.enabled': 'false'}) table = self._create_table(pa_schema=schema, options=options) self._write_arrow(table, pa.table({'id': [1, 2], 'embedding': [[1.0, 0.0], [0.0, 1.0]]}, schema=schema)) - single = table.new_vector_search_builder().with_vector_column('embedding').with_limit(1).with_query_vector([1.0, 0.0]) - batch = table.new_batch_vector_search_builder().with_vector_column('embedding').with_limit(1).with_query_vectors([[0.0, 1.0]]) + single = (table.new_vector_search_builder().with_vector_column('embedding').with_limit(1) + .with_query_vector([1.0, 0.0])) + batch = (table.new_batch_vector_search_builder().with_vector_column('embedding').with_limit(1) + .with_query_vectors([[0.0, 1.0]])) self.assertFalse(single._is_primary_key_vector_search()) self.assertFalse(batch._is_primary_key_vector_search()) self.assertEqual([0], list(single.execute_local().results())) diff --git a/paimon-python/pypaimon/tests/global_index_test.py b/paimon-python/pypaimon/tests/global_index_test.py index bbd5620303f4..f9ad3186ed63 100644 --- a/paimon-python/pypaimon/tests/global_index_test.py +++ b/paimon-python/pypaimon/tests/global_index_test.py @@ -84,9 +84,9 @@ def test_shared_index_column_precedence_and_legacy_fallback(self): def test_shared_field_options_override_type_defaults(self): for index_type, prefix, key in (("btree", "btree-index.", "block-size"), - ("bitmap", "bitmap-index.", "dictionary-block-size"), - ("full-text", "full-text.", "tokenizer"), - ("ivf-pq", "ivf-pq.", "nlist")): + ("bitmap", "bitmap-index.", "dictionary-block-size"), + ("full-text", "full-text.", "tokenizer"), + ("ivf-pq", "ivf-pq.", "nlist")): with self.subTest(index_type=index_type): values = {prefix + key: "old", "fields.payload.index-options": '{"%s":"new"}' % key, diff --git a/paimon-python/pypaimon/tests/lazy_search_execution_test.py b/paimon-python/pypaimon/tests/lazy_search_execution_test.py index fc5942c49790..33633508e55e 100644 --- a/paimon-python/pypaimon/tests/lazy_search_execution_test.py +++ b/paimon-python/pypaimon/tests/lazy_search_execution_test.py @@ -53,18 +53,20 @@ def _builder(table, family): @pytest.mark.parametrize('family', ['vector', 'batch', 'text', 'hybrid']) @pytest.mark.parametrize('configure_metric', [True, False]) -def test_native_search_routes_use_shared_index_options(search_table, family, configure_metric): +@pytest.mark.parametrize('legacy', [True, False]) +def test_native_search_routes_use_shared_index_options(search_table, family, configure_metric, legacy): from pypaimon.execution.native.read import native_vector_search, native_full_text_search, native_hybrid_search from pypaimon.table.source.hybrid_search_builder import HybridSearchRoute - table = search_table.copy({ - 'global-index.ivf-pq.index-columns': 'embedding', + declaration = ({'pk-vector.index.columns': 'embedding', 'fields.embedding.pk-vector.index.type': 'ivf-pq'} + if legacy else {'global-index.ivf-pq.index-columns': 'embedding'}) + table = search_table.copy(dict(declaration, **{ 'global-index.full-text.index-column': 'body', 'ivf-pq.dimension': '2', 'fields.embedding.index-options': json.dumps( {'distance.metric': 'COSINE', 'refine_factor': 3} if configure_metric else {'refine_factor': 3}), 'fields.body.index-options': '{"tokenizer":"raw","lower-case":false}', - }) + })) core = Mock() core.with_limit.return_value = core core.add_full_text_route.side_effect = HybridSearchRoute.full_text_route @@ -113,6 +115,120 @@ def test_native_search_routes_use_shared_index_options(search_table, family, con assert 'fields.embedding.metric' not in options assert 'ivf-pq.metric' not in options assert options['fields.embedding.refine_factor'] == '3' + if family != 'text': + core.with_vector_index_type.assert_called_once_with('embedding', 'ivf-pq') + + +@pytest.mark.parametrize('family', ['vector', 'batch', 'hybrid']) +@pytest.mark.parametrize('configured', [True, False]) +@pytest.mark.parametrize('legacy', [True, False]) +def test_native_type_api_is_qualified_before_execution(search_table, family, configured, legacy): + from pypaimon.execution.native.read import native_vector_search, native_hybrid_search + + declaration = ({'pk-vector.index.columns': 'embedding', 'fields.embedding.pk-vector.index.type': 'ivf-flat'} + if legacy else {'global-index.ivf-flat.index-columns': 'embedding'}) + table = search_table.copy(declaration if configured else {}) + core = Mock(spec=['with_limit', 'with_ranker', 'with_options', 'with_vector_column', + 'add_vector_route', 'execute', 'route_builders']) + core.with_limit.return_value = core + proxy = SimpleNamespace(new_vector_search_builder=lambda: core, + new_batch_vector_search_builder=lambda: core, + new_hybrid_search_builder=lambda: core) + module = native_hybrid_search if family == 'hybrid' else native_vector_search + with ExitStack() as stack: + stack.enter_context(patch.object(module, 'native_runtime_available', return_value=True)) + stack.enter_context(patch.object(module, '_native_table', return_value=proxy)) + if family == 'hybrid': + stack.enter_context(patch.object(module, '_rest_catalog_supported', return_value=True)) + builder = table.new_hybrid_search_builder().with_limit(2).add_vector_route('embedding', [1., 0.], 2) + result = module._native_builder(builder) + else: + stack.enter_context(patch.object(module, '_catalog_metastore', return_value='rest')) + result = module._native_vector_builder(_builder(table, family), family == 'batch') + assert (result is None) == configured + core.execute.assert_not_called() + core.route_builders.assert_not_called() + if configured: + core.with_limit.assert_not_called() + + +@pytest.mark.parametrize('family', ['vector', 'batch', 'hybrid']) +@pytest.mark.parametrize('key', ['index_type', 'index-type', 'vector.index-type', 'fields.embedding.index-type']) +def test_removed_type_hints_never_reach_native_search(search_table, family, key): + from pypaimon.execution.native.read import native_vector_search, native_hybrid_search + + table = search_table.copy({key: 'diskann'}) + core = Mock() + core.with_limit.return_value = core + proxy = SimpleNamespace(new_vector_search_builder=lambda: core, + new_batch_vector_search_builder=lambda: core, + new_hybrid_search_builder=lambda: core) + module = native_hybrid_search if family == 'hybrid' else native_vector_search + with ExitStack() as stack: + stack.enter_context(patch.object(module, 'native_runtime_available', return_value=True)) + stack.enter_context(patch.object(module, '_native_table', return_value=proxy)) + if family == 'hybrid': + stack.enter_context(patch.object(module, '_rest_catalog_supported', return_value=True)) + builder = (table.new_hybrid_search_builder().with_limit(2) + .add_vector_route('embedding', [1., 0.], 2, options={key: 'ivf-pq'})) + assert module._native_builder(builder) is not None + assert key not in core.add_vector_route.call_args[0][-1] + else: + stack.enter_context(patch.object(module, '_catalog_metastore', return_value='rest')) + builder = _builder(table, family).with_option(key, 'ivf-pq') + assert module._native_vector_builder(builder, family == 'batch') is core + assert key not in core.with_options.call_args[0][0] + core.with_vector_index_type.assert_not_called() + + +@pytest.mark.parametrize('family', ['vector', 'batch', 'hybrid']) +def test_native_type_configuration_errors_propagate(search_table, family): + from pypaimon.execution.native.read import native_vector_search, native_hybrid_search + + table = search_table.copy({'global-index.ivf-flat.index-columns': 'embedding'}) + core = Mock() + core.with_vector_index_type.side_effect = ValueError('unsupported native vector implementation') + proxy = SimpleNamespace(new_vector_search_builder=lambda: core, + new_batch_vector_search_builder=lambda: core, + new_hybrid_search_builder=lambda: core) + module = native_hybrid_search if family == 'hybrid' else native_vector_search + with ExitStack() as stack: + stack.enter_context(patch.object(module, 'native_runtime_available', return_value=True)) + stack.enter_context(patch.object(module, '_native_table', return_value=proxy)) + stack.enter_context(pytest.raises(ValueError, match='unsupported native vector implementation')) + if family == 'hybrid': + stack.enter_context(patch.object(module, '_rest_catalog_supported', return_value=True)) + builder = table.new_hybrid_search_builder().with_limit(2).add_vector_route('embedding', [1., 0.], 2) + module._native_builder(builder) + else: + stack.enter_context(patch.object(module, '_catalog_metastore', return_value='rest')) + module._native_vector_builder(_builder(table, family), family == 'batch') + core.execute.assert_not_called() + core.route_builders.assert_not_called() + + +def test_native_hybrid_vector_columns_keep_independent_types(tmp_path): + from pypaimon.execution.native.read import native_hybrid_search + + catalog = CatalogFactory.create({'warehouse': str(tmp_path / 'warehouse')}) + catalog.create_database('default', True) + catalog.create_table('default.search', Schema.from_pyarrow_schema( + pa.schema([('embedding', pa.list_(pa.float32())), ('other', pa.list_(pa.float32()))]), + options={'row-tracking.enabled': 'true', 'data-evolution.enabled': 'true', + 'read.native.enabled': 'true', 'global-index.ivf-pq.index-columns': 'embedding', + 'global-index.diskann.index-column': 'other'}), False) + table = catalog.get_table('default.search') + builder = (table.new_hybrid_search_builder().with_limit(2) + .add_vector_route('other', [0., 1.], 2).add_vector_route('embedding', [1., 0.], 2)) + core = Mock() + core.with_limit.return_value = core + proxy = SimpleNamespace(new_hybrid_search_builder=lambda: core) + with patch.object(native_hybrid_search, 'native_runtime_available', return_value=True), \ + patch.object(native_hybrid_search, '_rest_catalog_supported', return_value=True), \ + patch.object(native_hybrid_search, '_native_table', return_value=proxy): + assert native_hybrid_search._native_builder(builder) is not None + assert core.with_vector_index_type.call_args_list == [(('other', 'diskann'),), (('embedding', 'ivf-pq'),)] + assert [call[0][0] for call in core.add_vector_route.call_args_list] == ['other', 'embedding'] @pytest.mark.parametrize('same_analyzer', [True, False]) diff --git a/paimon-python/pypaimon/tests/native_index_options_test.py b/paimon-python/pypaimon/tests/native_index_options_test.py index 190db364ef97..7092f7f4050f 100644 --- a/paimon-python/pypaimon/tests/native_index_options_test.py +++ b/paimon-python/pypaimon/tests/native_index_options_test.py @@ -45,9 +45,8 @@ def _table(tmp_path, primary_key, extra=None): DataField(2, 'other', AtomicType('INT')), DataField(3, 'embedding', VectorType(True, AtomicType('FLOAT'), 2)), DataField(4, 'body', AtomicType('STRING'))] - catalog.create_table('default.search', Schema(fields=fields, - primary_keys=['id'] if primary_key else [], - options=options), False) + catalog.create_table('default.search', Schema( + fields=fields, primary_keys=['id'] if primary_key else [], options=options), False) return catalog.get_table('default.search') @@ -92,7 +91,7 @@ def test_native_read_and_write_schemas_share_resolved_index_contract(tmp_path, p assert by_type['full-text'].options.to_map()['full-text.lower-case'] == 'false' else: assert not any(key.startswith('pk-') for key in options) - assert options['fields.embedding.index-type'] == 'ivf-pq' + assert 'fields.embedding.index-type' not in options assert options['fields.embedding.nlist'] == '64' assert options['fields.embedding.metric'] == 'cosine' assert options['btree-index.fallback-scan-max-size'] == '1 kb' @@ -142,11 +141,42 @@ def test_native_data_evolution_does_not_invent_a_vector_metric(tmp_path): }) options = json.loads(_resolved_schema_json(table))['options'] assert options['fields.embedding.nlist'] == '64' - assert options['fields.embedding.index-type'] == 'ivf-pq' + assert 'fields.embedding.index-type' not in options assert 'fields.embedding.metric' not in options assert 'ivf-pq.metric' not in options +@pytest.mark.parametrize('primary_key', [False, True]) +@pytest.mark.parametrize('key', ['index_type', 'index-type', 'vector.index-type', 'fields.embedding.index-type']) +def test_native_schema_drops_removed_type_aliases(tmp_path, primary_key, key): + table = _table(tmp_path, primary_key, { + 'global-index.ivf-pq.index-columns': 'embedding', key: 'diskann', + 'fields.embedding.index-options': '{"nlist":64}', + }) + original = dict(table.table_schema.options) + options = json.loads(_resolved_schema_json(table))['options'] + assert key not in options + assert 'fields.embedding.index-type' not in options + if primary_key: + assert options['fields.embedding.pk-vector.index.type'] == 'ivf-pq' + assert key not in json.loads(options['fields.embedding.pk-vector.index.options']) + assert table.table_schema.options == original + + +@pytest.mark.parametrize('primary_key', [False, True]) +@pytest.mark.parametrize('suffix', ['index-options', 'pk-vector.index.options']) +def test_field_json_cannot_restore_removed_native_type_hint(tmp_path, primary_key, suffix): + table = _table(tmp_path, primary_key, { + 'global-index.ivf-pq.index-columns': 'embedding', + 'fields.embedding.' + suffix: '{"index-type":"diskann","nlist":64}', + }) + options = json.loads(_resolved_schema_json(table))['options'] + assert 'fields.embedding.index-type' not in options + if primary_key: + assert 'fields.embedding.index-type' not in json.loads(options['fields.embedding.pk-vector.index.options']) + assert options['fields.embedding.pk-vector.index.type'] == 'ivf-pq' + + def test_native_qualifies_distinct_scalar_field_options_before_reading(tmp_path): table = _table(tmp_path, False, { 'global-index.btree.index-columns': 'amount,other', diff --git a/paimon-python/pypaimon/tests/native_search_plan_test.py b/paimon-python/pypaimon/tests/native_search_plan_test.py index e1ab9ee1050f..24d5f2814a19 100644 --- a/paimon-python/pypaimon/tests/native_search_plan_test.py +++ b/paimon-python/pypaimon/tests/native_search_plan_test.py @@ -234,10 +234,14 @@ def test_reader_table_options_apply_to_captured_sources(rest_catalog, family): @pytest.mark.parametrize('family', ['vector', 'batch']) @pytest.mark.parametrize('metric', [None, 'cosine']) -@pytest.mark.parametrize('index_type', [' IVF-FLAT ', 'diskann']) +@pytest.mark.parametrize('index_type', ['ivf-flat', 'diskann']) def test_raw_native_plans_use_java_configured_indexer_default(rest_catalog, family, metric, index_type): + from pypaimon_rust import datafusion as sdk + cls = sdk.VectorSearchBuilder if family == 'vector' else sdk.BatchVectorSearchBuilder + if not hasattr(cls, 'with_vector_index_type'): + pytest.skip('Native SDK does not yet implement the explicit vector type API') options = { - 'fields.embedding.index-type': index_type, 'write.native.enabled': 'false', + 'global-index.%s.index-columns' % index_type: 'embedding', 'write.native.enabled': 'false', 'read.native.enabled': 'false', 'scan.native-plan.enabled': 'false', 'fields.other.distance.metric': 'l2', } diff --git a/paimon-python/pypaimon/tests/vector_index_options_test.py b/paimon-python/pypaimon/tests/vector_index_options_test.py index ef1626a66ffe..c84006f04adb 100644 --- a/paimon-python/pypaimon/tests/vector_index_options_test.py +++ b/paimon-python/pypaimon/tests/vector_index_options_test.py @@ -36,10 +36,20 @@ def test_generic_merge_does_not_infer_historical_vector_types(key): assert CoreOptions(Options(values)).global_index_options(None, 'embedding').to_map() == values -@pytest.mark.parametrize('key', ['fields.embedding.pk-vector.index.type', 'index_type', 'index-type', - 'vector.index-type', 'fields.embedding.index-type']) -def test_legacy_type_aliases_resolve_field_parameters(key): - values = {key: ' IVF-PQ ', 'ivf-pq.dimension': '768', +@pytest.mark.parametrize('key', ['index_type', 'index-type', 'vector.index-type', 'fields.embedding.index-type']) +def test_removed_type_aliases_are_ignored(key): + options = Options({key: 'ivf-pq'}) + assert VectorIndexOptions.index_type(options, 'embedding') is None + assert CoreOptions(options).vector_index_type('embedding') is None + assert VectorIndexOptions.resolve(options, None, 'embedding').to_map() == options.to_map() + options.to_map()['fields.embedding.pk-vector.index.type'] = 'diskann' + assert VectorIndexOptions.index_type(options, 'embedding') == 'diskann' + options.to_map()['global-index.ivf-flat.index-columns'] = 'embedding' + assert VectorIndexOptions.index_type(options, 'embedding') == 'ivf-flat' + + +def test_legacy_primary_key_type_resolves_field_parameters(): + values = {'fields.embedding.pk-vector.index.type': ' IVF-PQ ', 'ivf-pq.dimension': '768', 'fields.embedding.index-options': '{"nlist":64,"metric":"COSINE"}'} options = Options(values) assert VectorIndexOptions.index_type(options, 'embedding') == 'ivf-pq' diff --git a/paimon-python/pypaimon/tests/vector_metric_consistency_test.py b/paimon-python/pypaimon/tests/vector_metric_consistency_test.py index 58994188e4d6..1086ff012075 100644 --- a/paimon-python/pypaimon/tests/vector_metric_consistency_test.py +++ b/paimon-python/pypaimon/tests/vector_metric_consistency_test.py @@ -130,7 +130,8 @@ def test_default_metric_is_consistent_before_and_after_build(self): if indexed: self._build(table) for batch in (False, True): - builder = self._builder(table, batch).with_option('index-type', 'ivf-flat').with_limit(1) + builder = (self._builder(table, batch) + .with_option('global-index.ivf-flat.index-columns', 'embedding').with_limit(1)) for result in self._execute(builder, batch): self.assertEqual({1: 3.0}, _scores(result)) diff --git a/paimon-python/pypaimon/tests/vector_search_filter_test.py b/paimon-python/pypaimon/tests/vector_search_filter_test.py index 073eab277503..8089eba11564 100644 --- a/paimon-python/pypaimon/tests/vector_search_filter_test.py +++ b/paimon-python/pypaimon/tests/vector_search_filter_test.py @@ -1847,12 +1847,15 @@ def get_latest_snapshot(self_inner): _patch_snapshot(self, []) self._travel_patch.stop() - for key in ('index_type', 'index-type', 'vector.index-type', 'fields.embedding.index-type'): + for key, value in ( + ('global-index.ivf-flat.index-columns', 'embedding'), + ('global-index.ivf-flat.index-column', 'embedding'), + ('fields.embedding.pk-vector.index.type', 'ivf-flat')): for batch in (False, True): with self.subTest(key=key, batch=batch): builder = (BatchVectorSearchBuilderImpl(table).with_query_vectors([[1.0, 0.0, 0.0, 0.0]]) if batch else VectorSearchBuilderImpl(table).with_query_vector([1.0, 0.0, 0.0, 0.0])) - splits = (builder.with_vector_column("embedding").with_limit(3).with_option(key, "ivf-flat") + splits = (builder.with_vector_column("embedding").with_limit(3).with_option(key, value) .new_vector_search_scan().scan().splits()) raw = [s for s in splits if isinstance(s, RawVectorSearchSplit)] self.assertEqual(1, len(raw)) From 69c555f17f5ddfa3a5bcef806569687f1f7c222b Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 1 Nov 2026 10:42:14 +0800 Subject: [PATCH 06/14] [python] Require native vector type API when configured --- paimon-python/dev/native-vector-type-api.md | 10 ++--- .../execution/native/index_options.py | 13 ++---- .../native/read/native_hybrid_search.py | 4 +- .../native/read/native_vector_search.py | 4 +- .../tests/lazy_search_execution_test.py | 42 ++++++++++++++++++- 5 files changed, 52 insertions(+), 21 deletions(-) diff --git a/paimon-python/dev/native-vector-type-api.md b/paimon-python/dev/native-vector-type-api.md index e365f268b29a..2faf8959c0c3 100644 --- a/paimon-python/dev/native-vector-type-api.md +++ b/paimon-python/dev/native-vector-type-api.md @@ -62,10 +62,10 @@ legacy PK key `fields..pk-vector.index.type`. It strips the four removed aliases before transferring schema or query options to older SDKs. When an implementation is configured and this method is unavailable, Python -returns to its own execution path before planning, reading, executing, or -capturing hybrid routes. SDK configuration and execution errors propagate; -there is no retry after execution. Queries with no configured implementation -can continue to use older SDKs and persisted index metadata. +raises `AttributeError` before planning, reading, executing, or capturing hybrid +routes. It does not fall back to Python execution. SDK configuration and execution +errors propagate; there is no retry after execution. Queries with no configured +implementation can continue to use older SDKs and persisted index metadata. ## Rust verification @@ -73,5 +73,5 @@ Cover single, batch, hybrid, scan/read, captured routes, and serialized workers with both canonical and legacy PK declarations. Test raw searches without index files with and without a metric, metadata taking precedence over the explicit value, and independent columns in a hybrid query. The Python adapter contract -and old-SDK fallback are covered in `lazy_search_execution_test.py`; native +and missing-SDK-method errors are covered in `lazy_search_execution_test.py`; native integration coverage is in `native_search_plan_test.py` and requires this API. diff --git a/paimon-python/pypaimon/execution/native/index_options.py b/paimon-python/pypaimon/execution/native/index_options.py index d05e3507a478..1e42869ddf41 100644 --- a/paimon-python/pypaimon/execution/native/index_options.py +++ b/paimon-python/pypaimon/execution/native/index_options.py @@ -37,17 +37,12 @@ def native_search_options(options): def configure_vector_index_type(native, options, column): - """Use the explicit native API, qualifying SDK support before execution.""" + """Use the required native API and propagate missing-method errors.""" if column is None: - return True + return index_type = VectorIndexOptions.index_type(Options(options), column) - if index_type is None: - return True - configure = getattr(native, 'with_vector_index_type', None) - if not callable(configure): - return False - configure(column, index_type) - return True + if index_type is not None: + native.with_vector_index_type(column, index_type) def native_scalar_index_read_supported(table): diff --git a/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py b/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py index b1560f9ab948..5052b879f3a2 100644 --- a/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py +++ b/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py @@ -66,8 +66,8 @@ def _native_builder(builder, executor=None): table = table.copy(full_text_options) native = _native_table(table).new_hybrid_search_builder() for route, options in zip(builder._routes, route_options): - if route.is_vector() and not configure_vector_index_type(native, options, route.field_name): - return None + if route.is_vector(): + configure_vector_index_type(native, options, route.field_name) native.with_limit(builder._limit).with_ranker(builder._ranker) for route, options in zip(builder._routes, route_options): if route.is_vector(): diff --git a/paimon-python/pypaimon/execution/native/read/native_vector_search.py b/paimon-python/pypaimon/execution/native/read/native_vector_search.py index 7f81865e4208..e7ecc2318a0c 100644 --- a/paimon-python/pypaimon/execution/native/read/native_vector_search.py +++ b/paimon-python/pypaimon/execution/native/read/native_vector_search.py @@ -52,9 +52,7 @@ def _native_vector_builder(builder, batch=False): native_table = _native_table(table.copy(options)) native = (native_table.new_batch_vector_search_builder() if batch else native_table.new_vector_search_builder()) - if not configure_vector_index_type(native, options, - builder._vector_column.name if builder._vector_column else None): - return None + configure_vector_index_type(native, options, builder._vector_column.name if builder._vector_column else None) if builder._vector_column is not None: native.with_vector_column(builder._vector_column.name) native.with_limit(builder._limit).with_options(native_search_options(builder._options)) diff --git a/paimon-python/pypaimon/tests/lazy_search_execution_test.py b/paimon-python/pypaimon/tests/lazy_search_execution_test.py index 33633508e55e..b5b7feccc52e 100644 --- a/paimon-python/pypaimon/tests/lazy_search_execution_test.py +++ b/paimon-python/pypaimon/tests/lazy_search_execution_test.py @@ -122,7 +122,7 @@ def test_native_search_routes_use_shared_index_options(search_table, family, con @pytest.mark.parametrize('family', ['vector', 'batch', 'hybrid']) @pytest.mark.parametrize('configured', [True, False]) @pytest.mark.parametrize('legacy', [True, False]) -def test_native_type_api_is_qualified_before_execution(search_table, family, configured, legacy): +def test_missing_native_type_api_errors_before_execution(search_table, family, configured, legacy): from pypaimon.execution.native.read import native_vector_search, native_hybrid_search declaration = ({'pk-vector.index.columns': 'embedding', 'fields.embedding.pk-vector.index.type': 'ivf-flat'} @@ -138,6 +138,8 @@ def test_native_type_api_is_qualified_before_execution(search_table, family, con with ExitStack() as stack: stack.enter_context(patch.object(module, 'native_runtime_available', return_value=True)) stack.enter_context(patch.object(module, '_native_table', return_value=proxy)) + if configured: + stack.enter_context(pytest.raises(AttributeError, match='with_vector_index_type')) if family == 'hybrid': stack.enter_context(patch.object(module, '_rest_catalog_supported', return_value=True)) builder = table.new_hybrid_search_builder().with_limit(2).add_vector_route('embedding', [1., 0.], 2) @@ -145,13 +147,49 @@ def test_native_type_api_is_qualified_before_execution(search_table, family, con else: stack.enter_context(patch.object(module, '_catalog_metastore', return_value='rest')) result = module._native_vector_builder(_builder(table, family), family == 'batch') - assert (result is None) == configured + if not configured: + assert result is not None core.execute.assert_not_called() core.route_builders.assert_not_called() if configured: core.with_limit.assert_not_called() +@pytest.mark.parametrize('family,entry', [ + ('vector', 'execute'), ('vector', 'scan'), ('vector', 'read'), + ('batch', 'execute'), ('batch', 'scan'), ('batch', 'read'), + ('hybrid', 'execute'), ('hybrid', 'routes'), +]) +def test_missing_native_type_api_never_selects_python(search_table, family, entry): + from pypaimon.execution.native.read import native_vector_search, native_hybrid_search + + table = search_table.copy({'global-index.ivf-flat.index-columns': 'embedding'}) + core = Mock(spec=['execute', 'route_builders']) + proxy = SimpleNamespace(new_vector_search_builder=lambda: core, + new_batch_vector_search_builder=lambda: core, + new_hybrid_search_builder=lambda: core) + module = native_hybrid_search if family == 'hybrid' else native_vector_search + with ExitStack() as stack: + stack.enter_context(_forbid_python_execution()) + stack.enter_context(patch.object(module, 'native_runtime_available', return_value=True)) + stack.enter_context(patch.object(module, '_native_table', return_value=proxy)) + if family == 'hybrid': + stack.enter_context(patch.object(module, '_rest_catalog_supported', return_value=True)) + builder = table.new_hybrid_search_builder().with_limit(2).add_vector_route('embedding', [1., 0.], 2) + invoke = builder.execute_local if entry == 'execute' else builder.route_builders + else: + stack.enter_context(patch.object(module, '_catalog_metastore', return_value='rest')) + builder = _builder(table, family) + if entry == 'execute': + invoke = builder.execute_batch_local if family == 'batch' else builder.execute_local + else: + invoke = _factories(builder, family)[0 if entry == 'scan' else 1] + with pytest.raises(AttributeError, match='with_vector_index_type'): + invoke() + core.execute.assert_not_called() + core.route_builders.assert_not_called() + + @pytest.mark.parametrize('family', ['vector', 'batch', 'hybrid']) @pytest.mark.parametrize('key', ['index_type', 'index-type', 'vector.index-type', 'fields.embedding.index-type']) def test_removed_type_hints_never_reach_native_search(search_table, family, key): From b86b92264bad04a365bd61d7fd616132c1f1a073 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 1 Nov 2026 11:00:11 +0800 Subject: [PATCH 07/14] [python] Preserve vector query parameter scope across readers --- paimon-python/dev/native-vector-type-api.md | 5 ++ .../native/read/native_hybrid_search.py | 5 +- .../native/read/native_search_plan.py | 10 +++ .../native/read/native_vector_search.py | 12 +-- .../python/read/search/vector_search_read.py | 21 ++--- .../tests/lazy_search_execution_test.py | 90 ++++++++++++++++--- .../tests/vector_metric_consistency_test.py | 52 +++++++++++ 7 files changed, 160 insertions(+), 35 deletions(-) diff --git a/paimon-python/dev/native-vector-type-api.md b/paimon-python/dev/native-vector-type-api.md index 2faf8959c0c3..d1ae6292a51c 100644 --- a/paimon-python/dev/native-vector-type-api.md +++ b/paimon-python/dev/native-vector-type-api.md @@ -51,6 +51,11 @@ the Rust implementation and binding are a separate change. * This API sets only the implementation. Resolved vector parameters still use `with_options`/route options; an unspecified metric must remain unspecified so persisted index metadata can provide it. +* Query and route options contain only explicitly supplied query parameters, + resolved through the shared vector parser. Table defaults remain in the schema; + a table metric must not become an explicit query metric that conflicts with + persisted index metadata. New and legacy field JSON query options follow the + same rule. * Native code must stop interpreting `index_type`, `index-type`, `vector.index-type`, and `fields..index-type` as option aliases. Legitimate API argument names and index metadata fields named `index_type` diff --git a/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py b/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py index 5052b879f3a2..b3eea3ccba2f 100644 --- a/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py +++ b/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py @@ -25,7 +25,7 @@ from pypaimon.execution.native.runtime import (native_runtime_available) from pypaimon.table.file_store_table import FileStoreTable from pypaimon.execution.native.write.native_commit import _rest_catalog_supported -from pypaimon.execution.native.read.native_search_plan import _index_search_options +from pypaimon.execution.native.read.native_search_plan import _index_search_options, _vector_query_options from pypaimon.execution.native.index_options import configure_vector_index_type @@ -71,7 +71,8 @@ def _native_builder(builder, executor=None): native.with_limit(builder._limit).with_ranker(builder._ranker) for route, options in zip(builder._routes, route_options): if route.is_vector(): - native.add_vector_route(route.field_name, route.vector, route.limit, route.weight, options) + native.add_vector_route(route.field_name, route.vector, route.limit, route.weight, + _vector_query_options(route.options, route.field_name, options)) else: native.add_full_text_route( route.field_name, route.full_text_query, route.limit, route.weight, options) diff --git a/paimon-python/pypaimon/execution/native/read/native_search_plan.py b/paimon-python/pypaimon/execution/native/read/native_search_plan.py index 79b19286992a..e92c83eb4a5b 100644 --- a/paimon-python/pypaimon/execution/native/read/native_search_plan.py +++ b/paimon-python/pypaimon/execution/native/read/native_search_plan.py @@ -63,6 +63,16 @@ def _index_search_options(table, column, index_type=None, overrides=None, scalar return native_search_options(resolved) +def _vector_query_options(options, column, resolved_options): + """Resolve explicit query parameters without treating table defaults as overrides.""" + values = native_search_options(options) + if column is None: + return values + index_type = VectorIndexOptions.index_type(Options(resolved_options), column) + return native_search_options(VectorIndexOptions.resolve( + Options(values), index_type, column, default_metric=False).to_map()) + + def _range(value): return Range(*value) diff --git a/paimon-python/pypaimon/execution/native/read/native_vector_search.py b/paimon-python/pypaimon/execution/native/read/native_vector_search.py index e7ecc2318a0c..f9cb82793c9b 100644 --- a/paimon-python/pypaimon/execution/native/read/native_vector_search.py +++ b/paimon-python/pypaimon/execution/native/read/native_vector_search.py @@ -22,8 +22,8 @@ from pypaimon.execution.native.table import (_catalog_metastore, _native_table) from pypaimon.execution.native.read.builder import (_predicate_to_native) from pypaimon.execution.native.runtime import (native_runtime_available) -from pypaimon.execution.native.read.native_search_plan import _result, _index_search_options -from pypaimon.execution.native.index_options import configure_vector_index_type, native_search_options +from pypaimon.execution.native.read.native_search_plan import _result, _index_search_options, _vector_query_options +from pypaimon.execution.native.index_options import configure_vector_index_type def _native_vector_builder(builder, batch=False): @@ -45,17 +45,17 @@ def _native_vector_builder(builder, batch=False): return None if not native_runtime_available(): return None - options = _index_search_options(table, builder._vector_column.name if builder._vector_column else None, - overrides=builder._options, scalar_filter=builder._filter) + column = builder._vector_column.name if builder._vector_column else None + options = _index_search_options(table, column, overrides=builder._options, scalar_filter=builder._filter) if options is None: return None native_table = _native_table(table.copy(options)) native = (native_table.new_batch_vector_search_builder() if batch else native_table.new_vector_search_builder()) - configure_vector_index_type(native, options, builder._vector_column.name if builder._vector_column else None) + configure_vector_index_type(native, options, column) if builder._vector_column is not None: native.with_vector_column(builder._vector_column.name) - native.with_limit(builder._limit).with_options(native_search_options(builder._options)) + native.with_limit(builder._limit).with_options(_vector_query_options(builder._options, column, options)) try: if builder._filter is not None: native.with_filter(_predicate_to_native(builder._filter)) diff --git a/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py b/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py index b94211ee5803..ea90b799bdb7 100644 --- a/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py +++ b/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py @@ -991,21 +991,12 @@ def _table_options_map(table): def _configured_vector_metric(options, vector_column, index_type=None): - field_prefix = "fields.%s." % vector_column.name - if field_prefix + "index-options" in options: - core = CoreOptions(Options(options)) - selected_type = index_type or VectorIndexOptions.index_type(core.options, vector_column.name) or "vector" - metric = VectorIndexOptions.resolve(Options(options), selected_type, vector_column.name, - default_metric=False).to_map().get(selected_type + ".metric") - if metric is not None: - return metric - index_prefix = "%s." % index_type if index_type else None - keys = [field_prefix + "pk-vector.distance.metric", - field_prefix + "distance.metric", field_prefix + "metric"] - if index_prefix is not None: - keys.extend([index_prefix + "distance.metric", index_prefix + "metric"]) - keys.extend(["test.vector.metric", "lumina.distance.metric", "distance.metric", "metric"]) - for key in keys: + selected_type = index_type or VectorIndexOptions.index_type(Options(options), vector_column.name) or "vector" + metric = VectorIndexOptions.resolve(Options(options), selected_type, vector_column.name, + default_metric=False).to_map().get(selected_type + ".metric") + if metric is not None: + return metric + for key in ("test.vector.metric", "lumina.distance.metric"): if key in options: return _normalize_metric(options[key]) return None diff --git a/paimon-python/pypaimon/tests/lazy_search_execution_test.py b/paimon-python/pypaimon/tests/lazy_search_execution_test.py index b5b7feccc52e..f8df7b88ce44 100644 --- a/paimon-python/pypaimon/tests/lazy_search_execution_test.py +++ b/paimon-python/pypaimon/tests/lazy_search_execution_test.py @@ -51,22 +51,28 @@ def _builder(table, family): return table.new_vector_search_builder().with_vector_column('embedding').with_limit(2).with_query_vector([1.0, 0.0]) -@pytest.mark.parametrize('family', ['vector', 'batch', 'text', 'hybrid']) +@pytest.mark.parametrize('family,legacy', [ + ('vector', False), ('batch', False), ('text', False), ('hybrid', False), ('vector', True), +]) @pytest.mark.parametrize('configure_metric', [True, False]) -@pytest.mark.parametrize('legacy', [True, False]) -def test_native_search_routes_use_shared_index_options(search_table, family, configure_metric, legacy): +def test_native_search_routes_use_shared_index_options(search_table, tmp_path, family, configure_metric, legacy): from pypaimon.execution.native.read import native_vector_search, native_full_text_search, native_hybrid_search from pypaimon.table.source.hybrid_search_builder import HybridSearchRoute declaration = ({'pk-vector.index.columns': 'embedding', 'fields.embedding.pk-vector.index.type': 'ivf-pq'} if legacy else {'global-index.ivf-pq.index-columns': 'embedding'}) - table = search_table.copy(dict(declaration, **{ + options = dict(declaration, **{ 'global-index.full-text.index-column': 'body', 'ivf-pq.dimension': '2', 'fields.embedding.index-options': json.dumps( {'distance.metric': 'COSINE', 'refine_factor': 3} if configure_metric else {'refine_factor': 3}), 'fields.body.index-options': '{"tokenizer":"raw","lower-case":false}', - })) + }) + if legacy: + from pypaimon.tests.native_index_options_test import _table + table = _table(tmp_path / 'pk', True, dict(options, **{'read.native.enabled': 'true'})) + else: + table = search_table.copy(options) core = Mock() core.with_limit.return_value = core core.add_full_text_route.side_effect = HybridSearchRoute.full_text_route @@ -92,15 +98,19 @@ def test_native_search_routes_use_shared_index_options(search_table, family, con .add_full_text_route('body', '{}', 2)) module._native_builder(builder) if family == 'hybrid': + from pypaimon.execution.native.index_options import native_index_options + vector = core.add_vector_route.call_args[0][-1] + assert vector == {} assert core.add_full_text_route.call_args[0][-1] == {} text = bridge.call_args[0][0].options.options.to_map() + wire = native_index_options(bridge.call_args[0][0].table_schema) if configure_metric: - assert vector['fields.embedding.metric'] == 'cosine' + assert wire['fields.embedding.metric'] == 'cosine' else: - assert 'fields.embedding.metric' not in vector - assert 'ivf-pq.metric' not in vector - assert vector['fields.embedding.refine_factor'] == '3' + assert 'fields.embedding.metric' not in wire + assert 'ivf-pq.metric' not in wire + assert wire['fields.embedding.refine_factor'] == '3' assert text['full-text.tokenizer'] == 'raw' assert text['full-text.lower-case'] == 'false' else: @@ -120,14 +130,70 @@ def test_native_search_routes_use_shared_index_options(search_table, family, con @pytest.mark.parametrize('family', ['vector', 'batch', 'hybrid']) +@pytest.mark.parametrize('query_options,metric', [ + ({}, None), + ({'fields.embedding.index-options': '{"metric":"COSINE","refine_factor":4}'}, 'cosine'), + ({'fields.embedding.pk-vector.index.options': '{"distance.metric":"COSINE","refine_factor":4}'}, 'cosine'), + ({'fields.embedding.metric': 'COSINE'}, 'cosine'), + ({'ivf-pq.distance.metric': 'COSINE'}, 'cosine'), + ({'metric': 'COSINE'}, 'cosine'), + ({'fields.embedding.index-options': '{"refine_factor":4}'}, None), +]) +def test_native_vector_query_options_preserve_explicit_metric_scope(search_table, family, query_options, metric): + from pypaimon.execution.native.read import native_vector_search, native_hybrid_search + + table = search_table.copy({'global-index.ivf-pq.index-columns': 'embedding', + 'ivf-pq.dimension': '2', 'fields.embedding.distance.metric': 'l2', + 'fields.embedding.refine_factor': '3'}) + core = Mock() + core.with_limit.return_value = core + proxy = SimpleNamespace(new_vector_search_builder=lambda: core, + new_batch_vector_search_builder=lambda: core, + new_hybrid_search_builder=lambda: core) + module = native_hybrid_search if family == 'hybrid' else native_vector_search + original = dict(query_options) + with ExitStack() as stack: + stack.enter_context(patch.object(module, 'native_runtime_available', return_value=True)) + bridge = stack.enter_context(patch.object(module, '_native_table', return_value=proxy)) + if family == 'hybrid': + stack.enter_context(patch.object(module, '_rest_catalog_supported', return_value=True)) + builder = (table.new_hybrid_search_builder().with_limit(2) + .add_vector_route('embedding', [1., 0.], 2, options=query_options)) + assert module._native_builder(builder) is not None + actual = core.add_vector_route.call_args[0][-1] + else: + stack.enter_context(patch.object(module, '_catalog_metastore', return_value='rest')) + builder = _builder(table, family).with_options(query_options) + assert module._native_vector_builder(builder, family == 'batch') is core + actual = core.with_options.call_args[0][0] + assert 'ivf-pq.dimension' not in actual + for key in ('metric', 'distance.metric'): + assert actual.get('fields.embedding.' + key) == metric + assert actual.get('ivf-pq.' + key) == metric + if any(key.endswith('index-options') or key.endswith('index.options') for key in query_options): + assert actual['ivf-pq.refine_factor'] == '4' + else: + assert 'fields.embedding.refine_factor' not in actual + if not query_options: + assert bridge.call_args[0][0].options.options.to_map()['fields.embedding.distance.metric'] == 'l2' + assert query_options == original + + +@pytest.mark.parametrize('family,legacy', [ + ('vector', False), ('batch', False), ('hybrid', False), ('vector', True), +]) @pytest.mark.parametrize('configured', [True, False]) -@pytest.mark.parametrize('legacy', [True, False]) -def test_missing_native_type_api_errors_before_execution(search_table, family, configured, legacy): +def test_missing_native_type_api_errors_before_execution(search_table, tmp_path, family, configured, legacy): from pypaimon.execution.native.read import native_vector_search, native_hybrid_search declaration = ({'pk-vector.index.columns': 'embedding', 'fields.embedding.pk-vector.index.type': 'ivf-flat'} if legacy else {'global-index.ivf-flat.index-columns': 'embedding'}) - table = search_table.copy(declaration if configured else {}) + if legacy: + from pypaimon.tests.native_index_options_test import _table + table = _table(tmp_path / 'pk', True, dict( + declaration if configured else {}, **{'read.native.enabled': 'true'})) + else: + table = search_table.copy(declaration if configured else {}) core = Mock(spec=['with_limit', 'with_ranker', 'with_options', 'with_vector_column', 'add_vector_route', 'execute', 'route_builders']) core.with_limit.return_value = core diff --git a/paimon-python/pypaimon/tests/vector_metric_consistency_test.py b/paimon-python/pypaimon/tests/vector_metric_consistency_test.py index 1086ff012075..b0fe97e7bdda 100644 --- a/paimon-python/pypaimon/tests/vector_metric_consistency_test.py +++ b/paimon-python/pypaimon/tests/vector_metric_consistency_test.py @@ -16,6 +16,8 @@ # under the License. import importlib.util +import json +from types import SimpleNamespace import unittest from unittest.mock import Mock, patch @@ -192,6 +194,56 @@ def test_shared_field_metric_is_used_by_raw_and_indexed_search(self): with self.assertRaisesRegex(ValueError, 'does not match index metric'): self._reader(options)._record_index_metric(native, 'ivf-flat') + def test_legacy_field_json_metric_reaches_raw_single_and_batch_search(self): + _install_raw_vector_read_builder(self.table, 'embedding', {0: [2.0], 1: [3.0]}) + split = RawVectorSearchSplit([Range(0, 1)], [], 'ivf-flat') + for key in ('metric', 'distance.metric', 'fields.embedding.metric'): + for scope in ('table', 'query'): + for batch in (False, True): + with self.subTest(key=key, scope=scope, batch=batch): + options = {'pk-vector.index.columns': 'embedding', + 'fields.embedding.pk-vector.index.type': 'ivf-flat', + 'fields.embedding.pk-vector.index.options': json.dumps({key: 'COSINE'})} + self.table.options.options.to_map().clear() + if scope == 'table': + self.table.options.options.to_map().update(options) + reader = self._reader(options if scope == 'query' else {}, batch) + results = reader.read_batch([split]) if batch else [reader.read([split])] + self.assertEqual([{0: 1.0}], [_scores(result) for result in results]) + + def test_legacy_field_json_query_metric_is_validated_against_index_metadata(self): + options = {'fields.embedding.pk-vector.index.options': '{"metric":"cosine"}'} + native = Mock(spec=['vector_metric']) + native.vector_metric.return_value = 'cosine' + reader = self._reader(options) + reader._record_index_metric(native, 'ivf-flat') + self.assertEqual('cosine', reader._search_metric('ivf-flat')) + native.vector_metric.return_value = 'inner_product' + with self.assertRaisesRegex(ValueError, 'does not match index metric'): + self._reader(options)._record_index_metric(native, 'ivf-flat') + + def test_primary_key_uncovered_rows_use_legacy_field_json_metric(self): + from pypaimon.execution.python.read.search.primary_key_vector_read import PrimaryKeyVectorRead + from pypaimon.table.row.generic_row import GenericRow + + self.table.options.options.to_map().update({ + 'pk-vector.index.columns': 'embedding', + 'fields.embedding.pk-vector.index.type': 'ivf-flat', + 'fields.embedding.pk-vector.index.options': '{"metric":"cosine"}', + }) + read_builder = Mock() + read_builder.with_projection.return_value = read_builder + read_builder.new_read.return_value.to_arrow.return_value = pa.table({'embedding': [[2.0], [3.0]]}) + self.table.new_read_builder = lambda: read_builder + data_file = SimpleNamespace(file_name='raw.parquet', row_count=2) + data_split = SimpleNamespace(files=[data_file], partition=GenericRow([], []), bucket=0, + data_deletion_files=None) + split = SimpleNamespace(uncovered_data_files=['raw.parquet'], data_split=data_split, row_ranges_by_file={}) + plan = SimpleNamespace(splits=lambda: [split]) + reader = PrimaryKeyVectorRead(self.table, 1, self.column, [1.0]) + candidates = list(reader._raw_candidates(plan)) + self.assertEqual([(0, 1.0), (1, 1.0)], [(row.row_position, row.score) for row in candidates]) + def test_unrelated_field_options_do_not_override_persisted_metric(self): native = Mock(spec=['vector_metric']) native.vector_metric.return_value = 'cosine' From 33454c0f1acdb87b42b0e4d3782989c5ad6a2204 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 1 Nov 2026 11:46:14 +0800 Subject: [PATCH 08/14] [core][python] Hide legacy vector configuration behind shared index APIs --- docs/generated/core_configuration.html | 6 - .../java/org/apache/paimon/CoreOptions.java | 131 ++++-------------- .../paimon/options/VectorIndexOptions.java | 56 +++++++- .../org/apache/paimon/KeyValueFileStore.java | 13 +- .../index/pk/PrimaryKeyIndexDefinitions.java | 30 ++-- .../LookupMergeTreeCompactRewriter.java | 4 +- .../paimon/schema/SchemaManagerUtils.java | 13 +- .../source/BatchVectorSearchBuilderImpl.java | 7 +- .../source/FullTextSearchBuilderImpl.java | 2 +- .../table/source/PrimaryKeyVectorRead.java | 4 +- .../table/source/PrimaryKeyVectorScan.java | 3 +- .../table/source/VectorSearchBuilderImpl.java | 7 +- .../apache/paimon/GlobalIndexOptionsTest.java | 26 ++-- .../PrimaryKeySortedIndexMaintenanceTest.java | 2 +- .../PrimaryKeySortedIndexOptionsTest.java | 20 ++- .../PrimaryKeyVectorIndexOptionsTest.java | 44 +++--- .../LookupMergeTreeCompactRewriterTest.java | 2 +- .../operation/PrimaryKeyIndexWriteTest.java | 2 +- .../options/VectorIndexOptionsTest.java | 32 ++++- .../paimon/schema/SchemaManagerTest.java | 2 +- .../source/PrimaryKeyVectorScanTest.java | 4 +- .../source/PrimaryKeyVectorSearchTest.java | 18 +-- .../procedure/VectorSearchProcedure.java | 4 +- paimon-python/dev/native-vector-type-api.md | 6 + .../pypaimon/common/options/core_options.py | 64 ++------- .../common/options/vector_index_options.py | 39 +++++- .../execution/native/index_options.py | 2 +- .../read/search/primary_key_vector_read.py | 7 +- .../read/search/primary_key_vector_scan.py | 7 +- .../index/pk/primary_key_index_definitions.py | 21 +-- .../table/source/full_text_search_builder.py | 2 +- .../table/source/vector_search_builder.py | 5 +- .../pypaimon/tests/global_index_test.py | 21 +-- .../primary_key_index_definitions_test.py | 4 +- .../tests/sorted_index_options_test.py | 2 +- .../tests/vector_index_options_test.py | 27 +++- .../paimon/spark/PaimonScanBuilder.scala | 7 +- 37 files changed, 346 insertions(+), 300 deletions(-) diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 7538370c51fa..e2aa57b0fbba 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -1452,12 +1452,6 @@ String Comma-separated ARRAY columns indexed by primary-key Multivalue indexes. - -
pk-vector.index.columns
- (none) - String - Comma-separated VECTOR columns indexed by primary-key vector indexes. Each column owns one index and must define fields.<column>.pk-vector.index.type. Index options and distance metric are also field-scoped. The first release supports exactly one column. -
postpone.batch-write-fixed-bucket
true diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 361f08042a94..f8f24adfe65f 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -3254,18 +3254,6 @@ public String toString() { .withFallbackKeys("global-index.fm.index-column", "pk-fm.index.columns") .withDescription("Comma-separated character columns to index with FM indexes."); - // Retain these option keys for applications using the primary-key index API. - public static final ConfigOption PK_VECTOR_INDEX_COLUMNS = - key("pk-vector.index.columns") - .stringType() - .noDefaultValue() - .withDescription( - "Comma-separated VECTOR columns indexed by primary-key vector indexes. " - + "Each column owns one index and must define " - + "fields..pk-vector.index.type. Index options and distance " - + "metric are also field-scoped. The first release supports exactly " - + "one column."); - public static final ConfigOption PK_BTREE_INDEX_COLUMNS = key("pk-btree.index.columns") .stringType() @@ -5091,48 +5079,38 @@ public int vectorSearchLateralJoinBatchSize() { return options.get(VECTOR_SEARCH_LATERAL_JOIN_BATCH_SIZE); } - public boolean primaryKeyVectorIndexEnabled() { - return !primaryKeyVectorIndexColumns().isEmpty(); + public boolean fullTextIndexEnabled() { + return !fullTextIndexColumns().isEmpty(); } - public boolean primaryKeyFullTextIndexEnabled() { - return !primaryKeyFullTextIndexColumns().isEmpty(); - } - - public boolean primaryKeyFMIndexEnabled() { - return !primaryKeyFMIndexColumns().isEmpty(); - } - - public List primaryKeyVectorIndexColumns() { - List types = vectorIndexTypes(); - if (!types.isEmpty()) { - return types.stream() - .flatMap(type -> globalIndexColumns(type).stream()) - .collect(Collectors.toList()); - } - return indexColumns(options.get(PK_VECTOR_INDEX_COLUMNS)); + public boolean fmIndexEnabled() { + return !fmIndexColumns().isEmpty(); } - public List primaryKeyBTreeIndexColumns() { + public List btreeIndexColumns() { return globalIndexColumns("btree"); } - public List primaryKeyBitmapIndexColumns() { + public List bitmapIndexColumns() { return globalIndexColumns("bitmap"); } - public List primaryKeyMultiValueIndexColumns() { + public List multiValueIndexColumns() { return globalIndexColumns("multivalue"); } - public List primaryKeyFullTextIndexColumns() { + public List fullTextIndexColumns() { return globalIndexColumns("full-text"); } - public List primaryKeyFMIndexColumns() { + public List fmIndexColumns() { return globalIndexColumns("fm"); } + public List vectorIndexColumns(String type) { + return VectorIndexOptions.columns(options, type); + } + /** Index implementation identifiers declared with {@code global-index..index-columns}. */ public List globalIndexTypes() { String prefix = "global-index."; @@ -5153,12 +5131,6 @@ public static boolean isVectorIndexType(String type) { return !Arrays.asList("btree", "bitmap", "multivalue", "full-text", "fm").contains(type); } - private List vectorIndexTypes() { - return globalIndexTypes().stream() - .filter(CoreOptions::isVectorIndexType) - .collect(Collectors.toList()); - } - public List globalIndexColumns(String type) { String columns = options.get("global-index." + type + ".index-columns"); if (columns == null) { @@ -5167,15 +5139,7 @@ public List globalIndexColumns(String type) { if (columns == null) { columns = options.get("pk-" + type + ".index.columns"); } - if (columns != null) { - return indexColumns(columns); - } - return indexColumns(options.get(PK_VECTOR_INDEX_COLUMNS)).stream() - .filter( - column -> - type.equals( - options.get("fields." + column + ".pk-vector.index.type"))) - .collect(Collectors.toList()); + return indexColumns(columns); } private static List indexColumns(@Nullable String columns) { @@ -5187,18 +5151,6 @@ private static List indexColumns(@Nullable String columns) { } public String globalIndexColumnsOptionKey(String type) { - if ("vector".equals(type) && !vectorIndexTypes().isEmpty()) { - return vectorIndexTypes().stream() - .map( - t -> - "global-index." - + t - + (options.get("global-index." + t + ".index-columns") - != null - ? ".index-columns" - : ".index-column")) - .collect(Collectors.joining(", ")); - } for (String suffix : Arrays.asList(".index-columns", ".index-column")) { String key = "global-index." + type + suffix; if (options.get(key) != null) { @@ -5306,70 +5258,35 @@ private static LinkedHashMap parseIndexOptions( } } - public Options primaryKeyBTreeIndexOptions(String column) { - return primaryKeySortedIndexOptions(column, "btree"); + public Options btreeIndexOptions(String column) { + return sortedIndexOptions(column, "btree"); } - public Options primaryKeyBitmapIndexOptions(String column) { - return primaryKeySortedIndexOptions(column, "bitmap"); + public Options bitmapIndexOptions(String column) { + return sortedIndexOptions(column, "bitmap"); } - public Options primaryKeyMultiValueIndexOptions(String column) { - return primaryKeySortedIndexOptions(column, "multivalue"); + public Options multiValueIndexOptions(String column) { + return sortedIndexOptions(column, "multivalue"); } - public Options primaryKeyFMIndexOptions(String column) { - return primaryKeySortedIndexOptions(column, "fm"); + public Options fmIndexOptions(String column) { + return sortedIndexOptions(column, "fm"); } - public Options primaryKeyFullTextIndexOptions(String column) { + public Options fullTextIndexOptions(String column) { Map resolved = globalIndexOptions("full-text", column).toMap(); resolved.keySet().removeIf(key -> !key.startsWith("full-text.")); return new Options(resolved); } - private Options primaryKeySortedIndexOptions(String column, String type) { + private Options sortedIndexOptions(String column, String type) { Options resolved = globalIndexOptions(type, column); resolved.remove("sorted-index.records-per-file"); resolved.remove("sorted-index.records-per-range"); return resolved; } - public String primaryKeyVectorIndexColumn() { - List columns = primaryKeyVectorIndexColumns(); - checkArgument( - columns.size() == 1, - "%s must contain exactly one column in the first release, but is %s.", - globalIndexColumnsOptionKey("vector"), - columns); - return columns.get(0); - } - - @Nullable - public String primaryKeyVectorIndexType(String column) { - return vectorIndexType(column); - } - - @Nullable - public String vectorIndexType(String column) { - return VectorIndexOptions.indexType(options, column); - } - - /** Compatibility delegate to the vector resolver shared by both table modes. */ - public Options primaryKeyVectorIndexOptions(String column) { - String type = primaryKeyVectorIndexType(column); - checkArgument( - type != null && !type.trim().isEmpty(), - "fields.%s.pk-vector.index.type must be configured before resolving index options.", - column); - return VectorIndexOptions.resolve(options, type, column); - } - - /** Compatibility delegate to the vector resolver shared by both table modes. */ - public String primaryKeyVectorDistanceMetric(String column) { - return VectorIndexOptions.metric(options, primaryKeyVectorIndexType(column), column); - } - /** Specifies the merge engine for table with primary key. */ public enum MergeEngine implements DescribedEnum { DEDUPLICATE("deduplicate", "De-duplicate and keep the last row."), diff --git a/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java b/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java index 5785567e4974..294e7ba80acb 100644 --- a/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java @@ -23,6 +23,7 @@ import javax.annotation.Nullable; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.stream.Collectors; @@ -30,19 +31,70 @@ /** Vector parameter resolution shared by primary-key and Data Evolution indexes. */ public final class VectorIndexOptions { + private static final String LEGACY_INDEX_COLUMNS = "pk-vector.index.columns"; + private VectorIndexOptions() {} + public static List columns(Options options) { + CoreOptions core = new CoreOptions(options); + List types = indexTypes(core); + if (!types.isEmpty()) { + return types.stream() + .flatMap(type -> columns(options, type).stream()) + .collect(Collectors.toList()); + } + return legacyColumns(options); + } + + public static List columns(Options options, String type) { + List declared = new CoreOptions(options).globalIndexColumns(type); + return declared.isEmpty() + ? legacyColumns(options).stream() + .filter(column -> type.equals(legacyIndexType(options, column))) + .collect(Collectors.toList()) + : declared; + } + + private static List legacyColumns(Options options) { + String columns = options.get(LEGACY_INDEX_COLUMNS); + return columns == null + ? Collections.emptyList() + : Arrays.stream(columns.split(",", -1)) + .map(String::trim) + .collect(Collectors.toList()); + } + + public static String columnsOptionKey(Options options) { + CoreOptions core = new CoreOptions(options); + List types = indexTypes(core); + return types.isEmpty() + ? LEGACY_INDEX_COLUMNS + : types.stream() + .map(core::globalIndexColumnsOptionKey) + .collect(Collectors.joining(", ")); + } + + private static List indexTypes(CoreOptions core) { + return core.globalIndexTypes().stream() + .filter(CoreOptions::isVectorIndexType) + .collect(Collectors.toList()); + } + @Nullable public static String indexType(Options options, String column) { CoreOptions core = new CoreOptions(options); List declared = - core.globalIndexTypes().stream() - .filter(CoreOptions::isVectorIndexType) + indexTypes(core).stream() .filter(type -> core.globalIndexColumns(type).contains(column)) .collect(Collectors.toList()); if (!declared.isEmpty()) { return declared.size() == 1 ? declared.get(0) : null; } + return legacyIndexType(options, column); + } + + @Nullable + private static String legacyIndexType(Options options, String column) { String type = options.get("fields." + column + ".pk-vector.index.type"); return type == null ? null : type.toLowerCase(Locale.ROOT).trim(); } diff --git a/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java b/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java index 7675fb752523..462b0ba162bb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java +++ b/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java @@ -34,6 +34,7 @@ import org.apache.paimon.operation.MergeFileSplitRead; import org.apache.paimon.operation.RawFileSplitRead; import org.apache.paimon.options.Options; +import org.apache.paimon.options.VectorIndexOptions; import org.apache.paimon.postpone.PostponeBucketFileStoreWrite; import org.apache.paimon.schema.KeyValueFieldsExtractor; import org.apache.paimon.schema.SchemaManager; @@ -188,12 +189,12 @@ private AbstractFileStoreWrite newFixedBucketWrite( dvMaintainerFactory = BucketedDvMaintainer.factory(newIndexFileHandler()); } BucketedPrimaryKeyIndexMaintainer.Factory primaryKeyIndexMaintainerFactory = null; - if (writeOptions.primaryKeyVectorIndexEnabled() - || writeOptions.primaryKeyFullTextIndexEnabled() - || writeOptions.primaryKeyFMIndexEnabled() - || !writeOptions.primaryKeyBTreeIndexColumns().isEmpty() - || !writeOptions.primaryKeyBitmapIndexColumns().isEmpty() - || !writeOptions.primaryKeyMultiValueIndexColumns().isEmpty()) { + if (!VectorIndexOptions.columns(writeOptions.toConfiguration()).isEmpty() + || writeOptions.fullTextIndexEnabled() + || writeOptions.fmIndexEnabled() + || !writeOptions.btreeIndexColumns().isEmpty() + || !writeOptions.bitmapIndexColumns().isEmpty() + || !writeOptions.multiValueIndexColumns().isEmpty()) { primaryKeyIndexMaintainerFactory = BucketedPrimaryKeyIndexMaintainer.Factory.create( newIndexFileHandler(), newReaderFactoryBuilder(), schema); diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java b/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java index 8dbaeff52cb0..cca199e9ee67 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java @@ -46,13 +46,14 @@ private PrimaryKeyIndexDefinitions(List definitions) public static PrimaryKeyIndexDefinitions create(TableSchema schema) { CoreOptions options = new CoreOptions(schema.options()); - List vectorColumns = options.primaryKeyVectorIndexColumns(); - List btreeColumns = options.primaryKeyBTreeIndexColumns(); - List bitmapColumns = options.primaryKeyBitmapIndexColumns(); - List multiValueColumns = options.primaryKeyMultiValueIndexColumns(); - List fullTextColumns = options.primaryKeyFullTextIndexColumns(); - List fmColumns = options.primaryKeyFMIndexColumns(); - validateNoDuplicates(vectorColumns, options.globalIndexColumnsOptionKey("vector")); + List vectorColumns = VectorIndexOptions.columns(options.toConfiguration()); + List btreeColumns = options.btreeIndexColumns(); + List bitmapColumns = options.bitmapIndexColumns(); + List multiValueColumns = options.multiValueIndexColumns(); + List fullTextColumns = options.fullTextIndexColumns(); + List fmColumns = options.fmIndexColumns(); + validateNoDuplicates( + vectorColumns, VectorIndexOptions.columnsOptionKey(options.toConfiguration())); validateNoDuplicates(btreeColumns, options.globalIndexColumnsOptionKey("btree")); validateNoDuplicates(bitmapColumns, options.globalIndexColumnsOptionKey("bitmap")); validateNoDuplicates(multiValueColumns, options.globalIndexColumnsOptionKey("multivalue")); @@ -75,7 +76,7 @@ public static PrimaryKeyIndexDefinitions create(TableSchema schema) { column, field.id(), BTreeGlobalIndexerFactory.IDENTIFIER, - options.primaryKeyBTreeIndexOptions(column), + options.btreeIndexOptions(column), PrimaryKeyIndexDefinition.Family.BTREE)); } else if (bitmapColumns.contains(column)) { definitions.add( @@ -83,7 +84,7 @@ public static PrimaryKeyIndexDefinitions create(TableSchema schema) { column, field.id(), BitmapGlobalIndexerFactory.IDENTIFIER, - options.primaryKeyBitmapIndexOptions(column), + options.bitmapIndexOptions(column), PrimaryKeyIndexDefinition.Family.BITMAP)); } else if (multiValueColumns.contains(column)) { definitions.add( @@ -91,17 +92,18 @@ public static PrimaryKeyIndexDefinitions create(TableSchema schema) { column, field.id(), MultiValueGlobalIndexerFactory.IDENTIFIER, - options.primaryKeyMultiValueIndexOptions(column), + options.multiValueIndexOptions(column), PrimaryKeyIndexDefinition.Family.MULTI_VALUE)); } else if (vectorColumns.contains(column)) { definitions.add( new PrimaryKeyIndexDefinition( column, field.id(), - options.primaryKeyVectorIndexType(column), + VectorIndexOptions.indexType(options.toConfiguration(), column), VectorIndexOptions.resolve( options.toConfiguration(), - options.primaryKeyVectorIndexType(column), + VectorIndexOptions.indexType( + options.toConfiguration(), column), column), PrimaryKeyIndexDefinition.Family.VECTOR)); } else if (fullTextColumns.contains(column)) { @@ -110,7 +112,7 @@ public static PrimaryKeyIndexDefinitions create(TableSchema schema) { column, field.id(), "full-text", - options.primaryKeyFullTextIndexOptions(column), + options.fullTextIndexOptions(column), PrimaryKeyIndexDefinition.Family.FULL_TEXT)); } else if (fmColumns.contains(column)) { definitions.add( @@ -118,7 +120,7 @@ public static PrimaryKeyIndexDefinitions create(TableSchema schema) { column, field.id(), FMGlobalIndexerFactory.IDENTIFIER, - options.primaryKeyFMIndexOptions(column), + options.fmIndexOptions(column), PrimaryKeyIndexDefinition.Family.FM)); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/LookupMergeTreeCompactRewriter.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/LookupMergeTreeCompactRewriter.java index 36cc4cd6a4d5..f2d98c51158a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/LookupMergeTreeCompactRewriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/LookupMergeTreeCompactRewriter.java @@ -32,6 +32,7 @@ import org.apache.paimon.mergetree.MergeSorter; import org.apache.paimon.mergetree.SortedRun; import org.apache.paimon.mergetree.lookup.RemoteLookupFileManager; +import org.apache.paimon.options.VectorIndexOptions; import org.apache.paimon.utils.FieldsComparator; import org.apache.paimon.utils.UserDefinedSeqComparator; @@ -98,7 +99,8 @@ public LookupMergeTreeCompactRewriter( this.lookupLevels = lookupLevels; this.wrapperFactory = wrapperFactory; this.noSequenceField = options.sequenceField().isEmpty(); - this.forceRewriteLevelZeroForVectorIndex = options.primaryKeyVectorIndexEnabled(); + this.forceRewriteLevelZeroForVectorIndex = + !VectorIndexOptions.columns(options.toConfiguration()).isEmpty(); String fileFormat = options.fileFormatString(); Map fileFormatPerLevel = options.fileFormatPerLevel(); this.level2FileFormat = level -> fileFormatPerLevel.getOrDefault(level, fileFormat); diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManagerUtils.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManagerUtils.java index c7642c44865b..1717a8e61b50 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManagerUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManagerUtils.java @@ -24,6 +24,7 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.options.VectorIndexOptions; import org.apache.paimon.schema.ColumnDirectiveUtils.ConvertedColumn; import org.apache.paimon.schema.SchemaChange.AddColumn; import org.apache.paimon.schema.SchemaChange.DropColumn; @@ -809,12 +810,12 @@ static void assertNotUpdatingPrimaryKeyIndexColumn( } String fieldName = fieldNames[0]; CoreOptions options = new CoreOptions(schema.options()); - if (options.primaryKeyVectorIndexColumns().contains(fieldName) - || options.primaryKeyBTreeIndexColumns().contains(fieldName) - || options.primaryKeyBitmapIndexColumns().contains(fieldName) - || options.primaryKeyMultiValueIndexColumns().contains(fieldName) - || options.primaryKeyFMIndexColumns().contains(fieldName) - || options.primaryKeyFullTextIndexColumns().contains(fieldName)) { + if (VectorIndexOptions.columns(options.toConfiguration()).contains(fieldName) + || options.btreeIndexColumns().contains(fieldName) + || options.bitmapIndexColumns().contains(fieldName) + || options.multiValueIndexColumns().contains(fieldName) + || options.fmIndexColumns().contains(fieldName) + || options.fullTextIndexColumns().contains(fieldName)) { throw new UnsupportedOperationException( String.format( "Cannot %s primary-key index column: [%s]", operation, fieldName)); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java index 8c3b738985e1..6b2349370b6f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java @@ -19,6 +19,7 @@ package org.apache.paimon.table.source; import org.apache.paimon.catalog.TableQueryAuthResult; +import org.apache.paimon.options.VectorIndexOptions; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; @@ -131,7 +132,8 @@ public VectorScan newVectorScan() { return new PrimaryKeyVectorScan( table, vectorColumn.id(), - table.coreOptions().primaryKeyVectorIndexType(vectorColumn.name()), + VectorIndexOptions.indexType( + table.coreOptions().toConfiguration(), vectorColumn.name()), partitionFilter, filter); } @@ -159,6 +161,7 @@ public BatchVectorRead newBatchVectorRead() { protected boolean isPrimaryKeyVectorSearch() { return !table.schema().primaryKeys().isEmpty() && vectorColumn != null - && table.coreOptions().primaryKeyVectorIndexColumns().contains(vectorColumn.name()); + && VectorIndexOptions.columns(table.coreOptions().toConfiguration()) + .contains(vectorColumn.name()); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java index 642cb7ae3f37..66666f5c0e46 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextSearchBuilderImpl.java @@ -150,7 +150,7 @@ private Optional primaryKeyFullTextDefinition(DataFie if (table.schema().primaryKeys().isEmpty() || table.coreOptions().dataEvolutionEnabled()) { return Optional.empty(); } - if (table.coreOptions().primaryKeyFullTextIndexColumns().isEmpty()) { + if (table.coreOptions().fullTextIndexColumns().isEmpty()) { return Optional.empty(); } for (PrimaryKeyIndexDefinition definition : diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java index e709cd734d20..39bfeb7b1aaa 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java @@ -136,7 +136,9 @@ public PrimaryKeyVectorRead( checkArgument(limit > 0, "Vector search limit must be positive: %s.", limit); this.table = table; this.vectorField = vectorField; - this.indexType = table.coreOptions().primaryKeyVectorIndexType(vectorField.name()); + this.indexType = + VectorIndexOptions.indexType( + table.coreOptions().toConfiguration(), vectorField.name()); this.indexOptions = VectorIndexOptions.resolve( table.coreOptions().toConfiguration(), indexType, vectorField.name()); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorScan.java index f02af9228dc1..dba04bde38a1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorScan.java @@ -27,6 +27,7 @@ import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.IndexManifestEntry; +import org.apache.paimon.options.VectorIndexOptions; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.table.FileStoreTable; @@ -83,7 +84,7 @@ public PrimaryKeyVectorScan( @Override public Plan scan() { checkArgument( - table.coreOptions().primaryKeyVectorIndexEnabled(), + !VectorIndexOptions.columns(table.coreOptions().toConfiguration()).isEmpty(), "Primary-key vector search requires a configured primary-key vector index."); checkArgument( filter == null diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java index 897cea418531..297492aed077 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java @@ -20,6 +20,7 @@ import org.apache.paimon.Snapshot; import org.apache.paimon.catalog.TableQueryAuthResult; +import org.apache.paimon.options.VectorIndexOptions; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; @@ -134,7 +135,8 @@ public VectorScan newVectorScan() { return new PrimaryKeyVectorScan( table, vectorColumn.id(), - table.coreOptions().primaryKeyVectorIndexType(vectorColumn.name()), + VectorIndexOptions.indexType( + table.coreOptions().toConfiguration(), vectorColumn.name()), partitionFilter, filter, pinnedSnapshot); @@ -157,7 +159,8 @@ public VectorRead newVectorRead() { protected boolean isPrimaryKeyVectorSearch() { return !table.schema().primaryKeys().isEmpty() && vectorColumn != null - && table.coreOptions().primaryKeyVectorIndexColumns().contains(vectorColumn.name()); + && VectorIndexOptions.columns(table.coreOptions().toConfiguration()) + .contains(vectorColumn.name()); } public VectorSearchBuilderImpl withSnapshot(Snapshot snapshot) { diff --git a/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java index 380f35872035..3d1a9294b5f8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java @@ -91,7 +91,11 @@ void testColumnDeclarationsAndLegacyFallback(String type, String legacyFamily) { values.put("pk-" + legacyFamily + ".index.columns", " old_column "); values.put("fields.old_column.pk-vector.index.type", "ivf-pq"); CoreOptions options = new CoreOptions(values); - assertThat(options.globalIndexColumns(type)).containsExactly("old_column"); + assertThat( + CoreOptions.isVectorIndexType(type) + ? VectorIndexOptions.columns(options.toConfiguration()) + : options.globalIndexColumns(type)) + .containsExactly("old_column"); values.put("global-index." + type + ".index-column", "singular_column"); options = new CoreOptions(values); @@ -102,8 +106,10 @@ void testColumnDeclarationsAndLegacyFallback(String type, String legacyFamily) { options = new CoreOptions(values); assertThat(options.globalIndexColumns(type)).containsExactly("first", "second"); if ("ivf-pq".equals(type)) { - assertThat(options.primaryKeyVectorIndexColumns()).containsExactly("first", "second"); - assertThat(options.primaryKeyVectorIndexType("first")).isEqualTo("ivf-pq"); + assertThat(VectorIndexOptions.columns(options.toConfiguration())) + .containsExactly("first", "second"); + assertThat(VectorIndexOptions.indexType(options.toConfiguration(), "first")) + .isEqualTo("ivf-pq"); } } @@ -113,7 +119,7 @@ void testSingularColumnOptionIsRecognized() { new CoreOptions( Collections.singletonMap("global-index.btree.index-column", "amount")); assertThat(options.globalIndexTypes()).containsExactly("btree"); - assertThat(options.primaryKeyBTreeIndexColumns()).containsExactly("amount"); + assertThat(options.btreeIndexColumns()).containsExactly("amount"); } @ParameterizedTest @@ -224,16 +230,20 @@ void testDocumentedVectorOptionsAndFieldOverrides() { values.put("fields.embedding.nlist", "48"); values.put("fields.embedding.distance.metric", "cosine"); CoreOptions options = new CoreOptions(values); - assertThat(options.primaryKeyVectorDistanceMetric("embedding")).isEqualTo("cosine"); - assertThat(options.primaryKeyVectorIndexOptions("embedding").get("ivf-pq.dimension")) + assertThat(VectorIndexOptions.metric(options.toConfiguration(), null, "embedding")) + .isEqualTo("cosine"); + assertThat( + VectorIndexOptions.resolve(options.toConfiguration(), null, "embedding") + .get("ivf-pq.dimension")) .isEqualTo("768"); values.put( "fields.embedding.index-options", "{\"nlist\":64,\"distance.metric\":\"inner_product\"}"); options = new CoreOptions(values); - assertThat(options.primaryKeyVectorDistanceMetric("embedding")).isEqualTo("inner_product"); - Options resolved = options.primaryKeyVectorIndexOptions("embedding"); + assertThat(VectorIndexOptions.metric(options.toConfiguration(), null, "embedding")) + .isEqualTo("inner_product"); + Options resolved = VectorIndexOptions.resolve(options.toConfiguration(), null, "embedding"); assertThat(resolved.get("ivf-pq.nlist")).isEqualTo("64"); assertThat(resolved.get("fields.embedding.nlist")).isEqualTo("64"); assertThat(resolved.get("ivf-pq.metric")).isEqualTo("inner_product"); diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexMaintenanceTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexMaintenanceTest.java index 0acb484d3c4e..f790a4127817 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexMaintenanceTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexMaintenanceTest.java @@ -370,7 +370,7 @@ private TestFileStore createMixedStore() throws Exception { options.put(CoreOptions.COMPACTION_FORCE_REWRITE_ALL_FILES.key(), "true"); options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "itemId"); options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "comment"); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("pk-vector.index.columns", "embedding"); options.put("fields.embedding.pk-vector.index.type", "test-vector-ann"); options.put("fields.embedding.pk-vector.distance.metric", "l2"); options.put("fields.embedding.pk-vector.index.options", "{\"metric\":\"L2\"}"); diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexOptionsTest.java index 2035fb7805ee..3be51d01b93e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexOptionsTest.java @@ -37,7 +37,7 @@ void testResolvesBTreeIndexColumns() { CoreOptions options = new CoreOptions(Collections.singletonMap("pk-btree.index.columns", " name, age ")); - assertThat(options.primaryKeyBTreeIndexColumns()).containsExactly("name", "age"); + assertThat(options.btreeIndexColumns()).containsExactly("name", "age"); } @Test @@ -46,7 +46,7 @@ void testResolvesBitmapIndexColumns() { new CoreOptions( Collections.singletonMap("pk-bitmap.index.columns", " status, region ")); - assertThat(options.primaryKeyBitmapIndexColumns()).containsExactly("status", "region"); + assertThat(options.bitmapIndexColumns()).containsExactly("status", "region"); } @Test @@ -56,8 +56,7 @@ void testResolvesMultiValueIndexColumns() { Collections.singletonMap( "pk-multivalue.index.columns", " tags, categories ")); - assertThat(options.primaryKeyMultiValueIndexColumns()) - .containsExactly("tags", "categories"); + assertThat(options.multiValueIndexColumns()).containsExactly("tags", "categories"); } @Test @@ -69,11 +68,10 @@ void testResolvesFMIndexColumnsAndOptions() { "{\"partition-row-count\":\"2000\",\"fm-index.sa-sample-rate\":\"16\"}"); CoreOptions coreOptions = new CoreOptions(values); - Options options = coreOptions.primaryKeyFMIndexOptions("content"); + Options options = coreOptions.fmIndexOptions("content"); - assertThat(coreOptions.primaryKeyFMIndexEnabled()).isTrue(); - assertThat(coreOptions.primaryKeyFMIndexColumns()) - .containsExactly("content", "description"); + assertThat(coreOptions.fmIndexEnabled()).isTrue(); + assertThat(coreOptions.fmIndexColumns()).containsExactly("content", "description"); assertThat(options.get("fm-index.partition-row-count")).isEqualTo("2000"); assertThat(options.get("fm-index.sa-sample-rate")).isEqualTo("16"); } @@ -90,7 +88,7 @@ void testResolvesBTreeIndexAndSortOptions() { values.put("write-buffer-spill.max-disk-size", "1 gb"); values.put("fields.name.pk-btree.index.options", "{\"block-size\":\"4 kb\"}"); - Options options = new CoreOptions(values).primaryKeyBTreeIndexOptions("name"); + Options options = new CoreOptions(values).btreeIndexOptions("name"); assertThat(options.get("btree-index.block-size")).isEqualTo("4 kb"); assertThat(options.get("write-buffer-size")).isEqualTo("8 mb"); @@ -107,7 +105,7 @@ void testResolvesBitmapIndexOptions() { Map values = new HashMap<>(); values.put("fields.status.pk-bitmap.index.options", "{\"dictionary-block-size\":\"8 kb\"}"); - Options options = new CoreOptions(values).primaryKeyBitmapIndexOptions("status"); + Options options = new CoreOptions(values).bitmapIndexOptions("status"); assertThat(options.get("bitmap-index.dictionary-block-size")).isEqualTo("8 kb"); } @@ -118,7 +116,7 @@ void testResolvesMultiValueIndexOptions() { values.put( "fields.tags.pk-multivalue.index.options", "{\"dictionary-block-size\":\"8 kb\"}"); - Options options = new CoreOptions(values).primaryKeyMultiValueIndexOptions("tags"); + Options options = new CoreOptions(values).multiValueIndexOptions("tags"); assertThat(options.get("multivalue-index.dictionary-block-size")).isEqualTo("8 kb"); } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java index 449d640aa3f6..bbda51eb075e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java @@ -20,6 +20,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.options.Options; +import org.apache.paimon.options.VectorIndexOptions; import org.junit.jupiter.api.Test; @@ -29,23 +30,23 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -/** Tests for primary-key vector index options in {@link CoreOptions}. */ +/** Tests for legacy primary-key vector configuration handled by {@link VectorIndexOptions}. */ class PrimaryKeyVectorIndexOptionsTest { @Test void testPluralFieldRegistryEnablesIndex() { Map options = new HashMap<>(); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("pk-vector.index.columns", "embedding"); - assertThat(new CoreOptions(options).primaryKeyVectorIndexEnabled()).isTrue(); + assertThat(VectorIndexOptions.columns(new Options(options))).containsExactly("embedding"); } @Test void testResolvesSingleIndexColumn() { Map options = new HashMap<>(); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("pk-vector.index.columns", "embedding"); - assertThat(new CoreOptions(options).primaryKeyVectorIndexColumn()).isEqualTo("embedding"); + assertThat(VectorIndexOptions.columns(new Options(options))).containsExactly("embedding"); } @Test @@ -54,28 +55,27 @@ void testFieldRegistryIsTheOnlyEnableSwitch() { options.put("pk-vector.index.column", "embedding"); options.put("pk-vector.index.type", "ivf-pq"); - assertThat(new CoreOptions(options).primaryKeyVectorIndexEnabled()).isFalse(); + assertThat(VectorIndexOptions.columns(new Options(options))).isEmpty(); } @Test void testIndexTypeMustBeFieldScoped() { Map options = new HashMap<>(); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("pk-vector.index.columns", "embedding"); options.put("pk-vector.index.type", "ivf-pq"); - assertThat(new CoreOptions(options).primaryKeyVectorIndexType("embedding")).isNull(); + assertThat(VectorIndexOptions.indexType(new Options(options), "embedding")).isNull(); } @Test void testIndexOptionsMustBeFieldScoped() { Map options = new HashMap<>(); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("pk-vector.index.columns", "embedding"); options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); options.put("pk-vector.index.options", "{\"nlist\":64}"); assertThat( - new CoreOptions(options) - .primaryKeyVectorIndexOptions("embedding") + VectorIndexOptions.resolve(new Options(options), null, "embedding") .get("ivf-pq.nlist")) .isNull(); } @@ -83,33 +83,33 @@ void testIndexOptionsMustBeFieldScoped() { @Test void testDistanceMetricMustBeFieldScoped() { Map options = new HashMap<>(); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("pk-vector.index.columns", "embedding"); options.put("pk-vector.distance.metric", "l2"); - assertThat(new CoreOptions(options).primaryKeyVectorDistanceMetric("embedding")) + assertThat(VectorIndexOptions.metric(new Options(options), null, "embedding")) .isEqualTo("inner_product"); } @Test void testFieldScopedDistanceMetricOverridesTableDefault() { Map options = new HashMap<>(); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("pk-vector.index.columns", "embedding"); options.put("pk-vector.distance.metric", "l2"); options.put("fields.embedding.pk-vector.distance.metric", "cosine"); - assertThat(new CoreOptions(options).primaryKeyVectorDistanceMetric("embedding")) + assertThat(VectorIndexOptions.metric(new Options(options), null, "embedding")) .isEqualTo("cosine"); } @Test void testFieldScopedJsonOptionsOverrideTableDefault() { Map options = new HashMap<>(); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("pk-vector.index.columns", "embedding"); options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); options.put("pk-vector.index.options", "{\"nlist\":64}"); options.put("fields.embedding.pk-vector.index.options", "{\"nlist\":128}"); - Options resolved = new CoreOptions(options).primaryKeyVectorIndexOptions("embedding"); + Options resolved = VectorIndexOptions.resolve(new Options(options), null, "embedding"); assertThat(resolved.get("ivf-pq.nlist")).isEqualTo("128"); } @@ -120,7 +120,8 @@ void testResolvesShortAndQualifiedAlgorithmOptions() { coreOptions( "{\"nlist\":64,\"ivf-pq.pq.m\":\"8\"," + "\"fields.embedding.hnsw.m\":16}"); - Options resolved = coreOptions.primaryKeyVectorIndexOptions("embedding"); + Options resolved = + VectorIndexOptions.resolve(coreOptions.toConfiguration(), null, "embedding"); assertThat(resolved.get("ivf-pq.nlist")).isEqualTo("64"); assertThat(resolved.get("ivf-pq.pq.m")).isEqualTo("8"); @@ -130,7 +131,10 @@ void testResolvesShortAndQualifiedAlgorithmOptions() { @Test void testRejectsNonObjectOptions() { - assertThatThrownBy(() -> coreOptions("[1,2]").primaryKeyVectorIndexOptions("embedding")) + assertThatThrownBy( + () -> + VectorIndexOptions.resolve( + coreOptions("[1,2]").toConfiguration(), null, "embedding")) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("pk-vector.index.options") .hasMessageContaining("JSON object"); @@ -144,7 +148,7 @@ private static CoreOptions coreOptions( String indexOptions, String additionalKey, String additionalValue) { Map options = new HashMap<>(); options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("pk-vector.index.columns", "embedding"); options.put("fields.embedding.pk-vector.distance.metric", "l2"); if (indexOptions != null) { options.put("fields.embedding.pk-vector.index.options", indexOptions); diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/LookupMergeTreeCompactRewriterTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/LookupMergeTreeCompactRewriterTest.java index 2ef97f54f0b1..7fba59a619e0 100644 --- a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/LookupMergeTreeCompactRewriterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/LookupMergeTreeCompactRewriterTest.java @@ -48,7 +48,7 @@ class LookupMergeTreeCompactRewriterTest { void testVectorIndexForcesLevelZeroRewrite(CoreOptions.MergeEngine mergeEngine) { Options rawOptions = new Options(); rawOptions.set(CoreOptions.MERGE_ENGINE, mergeEngine); - rawOptions.set(CoreOptions.PK_VECTOR_INDEX_COLUMNS, "embedding"); + rawOptions.setString("pk-vector.index.columns", "embedding"); CoreOptions options = new CoreOptions(rawOptions); LookupMergeTreeCompactRewriter rewriter = new LookupMergeTreeCompactRewriter<>( diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java index 1c5e4587ffc4..d0cacf1883ba 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java @@ -121,7 +121,7 @@ void testCreatesCoordinatorForMultipleScalarDefinitionsAndVector() throws Except options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "itemId,score"); options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "comment,state"); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("pk-vector.index.columns", "embedding"); options.put("fields.embedding.pk-vector.index.type", "hnsw"); List fields = diff --git a/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java index 51aef5b4e557..012959c785af 100644 --- a/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java @@ -34,6 +34,34 @@ /** Tests for vector parameter resolution shared by both table modes. */ class VectorIndexOptionsTest { + @ParameterizedTest + @ValueSource(strings = {"index-column", "index-columns"}) + void testLegacyRegistrationIsOnlyACompatibilityFallback(String suffix) { + Options input = new Options(); + input.setString("pk-vector.index.columns", " old, other_old "); + input.setString("fields.old.pk-vector.index.type", "ivf-pq"); + input.setString("fields.other_old.pk-vector.index.type", "diskann"); + assertThat(VectorIndexOptions.columns(input)).containsExactly("old", "other_old"); + assertThat(VectorIndexOptions.indexType(input, "old")).isEqualTo("ivf-pq"); + assertThat(new CoreOptions(input).vectorIndexColumns("ivf-pq")).containsExactly("old"); + assertThat(new CoreOptions(input).vectorIndexColumns("diskann")) + .containsExactly("other_old"); + assertThat(new CoreOptions(input).vectorIndexColumns("ivf-flat")).isEmpty(); + assertThat(VectorIndexOptions.columnsOptionKey(input)).isEqualTo("pk-vector.index.columns"); + + input.setString("global-index.ivf-pq." + suffix, " embedding, other "); + input.setString("global-index.diskann." + suffix, "video"); + assertThat(VectorIndexOptions.columns(input)) + .containsExactly("video", "embedding", "other"); + assertThat(VectorIndexOptions.indexType(input, "embedding")).isEqualTo("ivf-pq"); + assertThat(VectorIndexOptions.indexType(input, "video")).isEqualTo("diskann"); + assertThat(new CoreOptions(input).vectorIndexColumns("ivf-pq")) + .containsExactly("embedding", "other"); + assertThat(new CoreOptions(input).vectorIndexColumns("diskann")).containsExactly("video"); + assertThat(VectorIndexOptions.columnsOptionKey(input)) + .isEqualTo("global-index.diskann." + suffix + ", global-index.ivf-pq." + suffix); + } + @ParameterizedTest @ValueSource( strings = { @@ -45,7 +73,6 @@ class VectorIndexOptionsTest { void testRemovedTypeAliasesAreIgnored(String key) { Options input = Options.fromMap(Collections.singletonMap(key, "ivf-pq")); assertThat(VectorIndexOptions.indexType(input, "embedding")).isNull(); - assertThat(new CoreOptions(input).vectorIndexType("embedding")).isNull(); assertThat(VectorIndexOptions.resolve(input, null, "embedding", false).toMap()) .isEqualTo(input.toMap()); input.setString("fields.embedding.pk-vector.index.type", "diskann"); @@ -62,7 +89,6 @@ void testLegacyPrimaryKeyTypeResolvesFieldParameters() { values.put("fields.embedding.index-options", "{\"nlist\":64,\"metric\":\"COSINE\"}"); Options input = new Options(values); assertThat(VectorIndexOptions.indexType(input, "embedding")).isEqualTo("ivf-pq"); - assertThat(new CoreOptions(input).vectorIndexType("embedding")).isEqualTo("ivf-pq"); Options resolved = VectorIndexOptions.resolve(input, null, "embedding", false); assertMetric(resolved, "cosine"); assertThat(resolved.get("ivf-pq.dimension")).isEqualTo("768"); @@ -123,7 +149,7 @@ void testFieldJsonAliasesRemainIdempotent(String suffix, String key) { assertThat(resolved.get("ivf-pq.nlist")).isEqualTo("64"); assertThat(VectorIndexOptions.resolve(resolved, "ivf-pq", "embedding").toMap()) .isEqualTo(resolved.toMap()); - assertThat(new CoreOptions(values).primaryKeyVectorIndexOptions("embedding").toMap()) + assertThat(VectorIndexOptions.resolve(new Options(values), null, "embedding").toMap()) .isEqualTo(resolved.toMap()); } diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java index f6fceda8db1c..06d18cffed42 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java @@ -640,7 +640,7 @@ public void testRejectRenamePrimaryKeyVectorIndexColumn() throws Exception { Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "1"); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding"); + options.put("pk-vector.index.columns", "embedding"); options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); Schema schema = new Schema( diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorScanTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorScanTest.java index 7569dab5138c..9d95be9b0f41 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorScanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorScanTest.java @@ -73,7 +73,7 @@ class PrimaryKeyVectorScanTest { void testPostponeTableOnlyScansRealBuckets() { Options options = new Options(); options.set(CoreOptions.BUCKET, BucketMode.POSTPONE_BUCKET); - options.set(CoreOptions.PK_VECTOR_INDEX_COLUMNS, "embedding"); + options.setString("pk-vector.index.columns", "embedding"); options.setString("fields.embedding.pk-vector.index.type", "ivf-pq"); FileStoreTable table = mock(FileStoreTable.class); @@ -252,7 +252,7 @@ private static IndexFileMeta payloadFile(String indexType, int fieldId, String f private static CoreOptions coreOptions() { Options options = new Options(); - options.set(CoreOptions.PK_VECTOR_INDEX_COLUMNS, "embedding"); + options.setString("pk-vector.index.columns", "embedding"); options.setString("fields.embedding.pk-vector.index.type", "ivf-pq"); return new CoreOptions(options); } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorSearchTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorSearchTest.java index 06d376be2626..cae582368807 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorSearchTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorSearchTest.java @@ -67,11 +67,12 @@ private Schema vectorSchema( .option( CoreOptions.DELETION_VECTORS_ENABLED.key(), Boolean.toString(deletionVectorsEnabled)) - .option(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding") .option( - "fields.embedding.pk-vector.index.type", - TestVectorGlobalIndexerFactory.IDENTIFIER) - .option("fields.embedding.pk-vector.distance.metric", "l2") + "global-index." + + TestVectorGlobalIndexerFactory.IDENTIFIER + + ".index-columns", + "embedding") + .option("fields.embedding.index-options", "{\"metric\":\"l2\"}") .option("test.vector.dimension", "2") .option("test.vector.metric", "l2"); if (reverseScore) { @@ -218,11 +219,12 @@ void testVectorSearchUsesSortedIndexPreFilter() throws Exception { .option(CoreOptions.BUCKET.key(), "1") .option(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true") .option(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "score") - .option(CoreOptions.PK_VECTOR_INDEX_COLUMNS.key(), "embedding") .option( - "fields.embedding.pk-vector.index.type", - TestVectorGlobalIndexerFactory.IDENTIFIER) - .option("fields.embedding.pk-vector.distance.metric", "l2") + "global-index." + + TestVectorGlobalIndexerFactory.IDENTIFIER + + ".index-columns", + "embedding") + .option("fields.embedding.index-options", "{\"metric\":\"l2\"}") .option("test.vector.dimension", "2") .option("test.vector.metric", "l2") .build(); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/VectorSearchProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/VectorSearchProcedure.java index d9370eebf294..5368835437c5 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/VectorSearchProcedure.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/VectorSearchProcedure.java @@ -29,6 +29,7 @@ import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.globalindex.GlobalIndexResult; import org.apache.paimon.options.Options; +import org.apache.paimon.options.VectorIndexOptions; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; @@ -261,7 +262,8 @@ private static PartitionPredicate parsePartitions( private static void validatePrimaryKeyFilter( FileStoreTable table, String vectorColumn, FilterParts filterParts) { if (table.schema().primaryKeys().isEmpty() - || !table.coreOptions().primaryKeyVectorIndexColumns().contains(vectorColumn) + || !VectorIndexOptions.columns(table.coreOptions().toConfiguration()) + .contains(vectorColumn) || !filterParts.hasDataFilter()) { return; } diff --git a/paimon-python/dev/native-vector-type-api.md b/paimon-python/dev/native-vector-type-api.md index d1ae6292a51c..4a014a69cee4 100644 --- a/paimon-python/dev/native-vector-type-api.md +++ b/paimon-python/dev/native-vector-type-api.md @@ -66,6 +66,12 @@ Python resolves the implementation once through `VectorIndexOptions`, using legacy PK key `fields..pk-vector.index.type`. It strips the four removed aliases before transferring schema or query options to older SDKs. +Callers use the shared vector columns/type/parameter resolver. The legacy PK +column registry and field type pairing are private compatibility inputs; +`CoreOptions.vectorIndexColumns(type)` (Python: `vector_index_columns`) delegates +to the type-scoped resolver. Other index accessors also use names without a +`primaryKey` prefix. No public method compatibility wrappers are retained. + When an implementation is configured and this method is unavailable, Python raises `AttributeError` before planning, reading, executing, or capturing hybrid routes. It does not fall back to Python execution. SDK configuration and execution diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py index fc1ade9afd57..5cde028ccd42 100644 --- a/paimon-python/pypaimon/common/options/core_options.py +++ b/paimon-python/pypaimon/common/options/core_options.py @@ -1038,13 +1038,6 @@ class CoreOptions: ) ) - PK_VECTOR_INDEX_COLUMNS: ConfigOption[str] = ( - ConfigOptions.key("pk-vector.index.columns") - .string_type() - .no_default_value() - .with_description("Comma-separated VECTOR columns indexed by primary-key vector indexes.") - ) - PK_BTREE_INDEX_COLUMNS: ConfigOption[str] = ( ConfigOptions.key("pk-btree.index.columns") .string_type() @@ -1859,22 +1852,19 @@ def global_index_row_count_per_shard(self) -> int: def global_index_build_parallelism(self) -> int: return self.options.get(CoreOptions.GLOBAL_INDEX_BUILD_PARALLELISM) - def primary_key_btree_index_columns(self) -> List[str]: + def btree_index_columns(self) -> List[str]: return self.global_index_columns("btree") - def primary_key_bitmap_index_columns(self) -> List[str]: + def bitmap_index_columns(self) -> List[str]: return self.global_index_columns("bitmap") - def primary_key_vector_index_columns(self) -> List[str]: - types = self._vector_index_types() - if types: - return [column for index_type in types - for column in self.global_index_columns(index_type)] - return self._index_columns(self.options.get(CoreOptions.PK_VECTOR_INDEX_COLUMNS)) - - def primary_key_full_text_index_columns(self) -> List[str]: + def full_text_index_columns(self) -> List[str]: return self.global_index_columns("full-text") + def vector_index_columns(self, index_type: str) -> List[str]: + from pypaimon.common.options.vector_index_options import VectorIndexOptions + return VectorIndexOptions.columns(self.options, index_type) + def global_index_types(self) -> List[str]: prefix = "global-index." return sorted({key[len(prefix):key.rfind('.')] @@ -1886,10 +1876,6 @@ def global_index_types(self) -> List[str]: def _is_vector_index_type(index_type: str) -> bool: return index_type not in ("btree", "bitmap", "multivalue", "fm", "full-text") - def _vector_index_types(self) -> List[str]: - return [index_type for index_type in self.global_index_types() - if self._is_vector_index_type(index_type)] - def global_index_columns(self, index_type: str) -> List[str]: values = self.options.to_map() columns = values.get("global-index.%s.index-columns" % index_type) @@ -1897,10 +1883,7 @@ def global_index_columns(self, index_type: str) -> List[str]: columns = values.get("global-index.%s.index-column" % index_type) if columns is None: columns = values.get("pk-%s.index.columns" % index_type) - if columns is not None: - return self._index_columns(columns) - return [column for column in self._index_columns(values.get("pk-vector.index.columns")) - if values.get("fields.%s.pk-vector.index.type" % column) == index_type] + return self._index_columns(columns) @staticmethod def _index_columns(columns) -> List[str]: @@ -1944,36 +1927,17 @@ def global_index_options(self, index_type: Optional[str], column: str) -> Option values[field_prefix + short_key] = str(value) return Options(values) - def primary_key_vector_index_type(self, column: str): - return self.vector_index_type(column) - - def vector_index_type(self, column: str): - from pypaimon.common.options.vector_index_options import VectorIndexOptions - return VectorIndexOptions.index_type(self.options, column) - - def primary_key_vector_index_options(self, column: str) -> Options: - from pypaimon.common.options.vector_index_options import VectorIndexOptions - index_type = self.primary_key_vector_index_type(column) - if index_type is None or not str(index_type).strip(): - raise ValueError("fields.%s.pk-vector.index.type must be configured before " - "resolving index options." % column) - return VectorIndexOptions.resolve(self.options, str(index_type), column) - - def primary_key_vector_distance_metric(self, column: str) -> str: - from pypaimon.common.options.vector_index_options import VectorIndexOptions - return VectorIndexOptions.metric(self.options, self.primary_key_vector_index_type(column), column) - - def primary_key_full_text_index_options(self, column: str) -> Options: + def full_text_index_options(self, column: str) -> Options: return Options({key: value for key, value in self.global_index_options( "full-text", column).to_map().items() if key.startswith("full-text.")}) - def primary_key_btree_index_options(self, column: str) -> Options: - return self._primary_key_sorted_index_options(column, "btree") + def btree_index_options(self, column: str) -> Options: + return self._sorted_index_options(column, "btree") - def primary_key_bitmap_index_options(self, column: str) -> Options: - return self._primary_key_sorted_index_options(column, "bitmap") + def bitmap_index_options(self, column: str) -> Options: + return self._sorted_index_options(column, "bitmap") - def _primary_key_sorted_index_options(self, column: str, index_type: str) -> Options: + def _sorted_index_options(self, column: str, index_type: str) -> Options: resolved = self.global_index_options(index_type, column).to_map() resolved.pop("sorted-index.records-per-file", None) resolved.pop("sorted-index.records-per-range", None) diff --git a/paimon-python/pypaimon/common/options/vector_index_options.py b/paimon-python/pypaimon/common/options/vector_index_options.py index c3aebc74cd22..154ab2c307a1 100644 --- a/paimon-python/pypaimon/common/options/vector_index_options.py +++ b/paimon-python/pypaimon/common/options/vector_index_options.py @@ -22,13 +22,48 @@ class VectorIndexOptions: + @staticmethod + def columns(options, index_type=None): + core = CoreOptions(options) + if index_type is not None: + declared = core.global_index_columns(index_type) + return declared if declared else [column for column in VectorIndexOptions._legacy_columns(options) + if VectorIndexOptions._legacy_index_type(options, column) == index_type] + types = VectorIndexOptions._index_types(core) + if types: + return [column for kind in types for column in VectorIndexOptions.columns(options, kind)] + return VectorIndexOptions._legacy_columns(options) + + @staticmethod + def _legacy_columns(options): + columns = options.to_map().get('pk-vector.index.columns') + return [] if columns is None else [column.strip() for column in str(columns).split(',')] + + @staticmethod + def columns_option_key(options): + types = VectorIndexOptions._index_types(CoreOptions(options)) + if not types: + return 'pk-vector.index.columns' + values = options.to_map() + return ', '.join('global-index.%s.%s' % ( + kind, 'index-columns' if 'global-index.%s.index-columns' % kind in values else 'index-column') + for kind in types) + + @staticmethod + def _index_types(core): + return [kind for kind in core.global_index_types() if CoreOptions._is_vector_index_type(kind)] + @staticmethod def index_type(options, column): core = CoreOptions(options) - declared = [kind for kind in core.global_index_types() - if CoreOptions._is_vector_index_type(kind) and column in core.global_index_columns(kind)] + declared = [kind for kind in VectorIndexOptions._index_types(core) + if column in core.global_index_columns(kind)] if declared: return declared[0] if len(declared) == 1 else None + return VectorIndexOptions._legacy_index_type(options, column) + + @staticmethod + def _legacy_index_type(options, column): value = options.to_map().get('fields.%s.pk-vector.index.type' % column) return None if value is None else str(value).lower().strip() diff --git a/paimon-python/pypaimon/execution/native/index_options.py b/paimon-python/pypaimon/execution/native/index_options.py index 1e42869ddf41..576624496ff0 100644 --- a/paimon-python/pypaimon/execution/native/index_options.py +++ b/paimon-python/pypaimon/execution/native/index_options.py @@ -70,7 +70,7 @@ def _resolve(schema): if kind not in families] if primary_key and not any(kind not in families for kind in declarations): definitions += [(VectorIndexOptions.index_type(core.options, column), [column]) - for column in core.primary_key_vector_index_columns()] + for column in VectorIndexOptions.columns(core.options)] known_columns = set() native_columns = {} profiles = {} diff --git a/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_read.py b/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_read.py index 227af32445c0..c00dcd470871 100644 --- a/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_read.py +++ b/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_read.py @@ -18,6 +18,7 @@ from concurrent.futures import wait from heapq import nsmallest +from pypaimon.common.options.vector_index_options import VectorIndexOptions from pypaimon.common.options.core_options import GlobalIndexSearchMode from pypaimon.index.pk.primary_key_index_source_meta import PrimaryKeyIndexSourceMeta from pypaimon.table.source.primary_key_scored_result import ( @@ -41,8 +42,7 @@ def read_plan(self, plan, executor=None): if not isinstance(plan, PrimaryKeyVectorScanPlan): raise ValueError("Primary-key vector read requires a PrimaryKeyVectorScanPlan.") self._index_metric = None - index_type = self._table.options.primary_key_vector_index_type( - self._vector_column.name) + index_type = VectorIndexOptions.index_type(self._table.options.options, self._vector_column.name) indexed_limit = self._indexed_search_limit(index_type) if indexed_limit > 2147483647: raise ValueError( @@ -170,8 +170,7 @@ def reranked_iter(): def _raw_candidates(self, plan): metric = self._search_metric( - self._table.options.primary_key_vector_index_type( - self._vector_column.name)) + VectorIndexOptions.index_type(self._table.options.options, self._vector_column.name)) read_table = self._table.copy_without_time_travel({ 'read.native.enabled': 'false', 'scan.native-plan.enabled': 'false'}) read_builder = read_table.new_read_builder().with_projection( diff --git a/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_scan.py b/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_scan.py index b7c5beba1d00..627248b4adb8 100644 --- a/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_scan.py +++ b/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_scan.py @@ -18,9 +18,9 @@ from copy import copy +from pypaimon.common.options.vector_index_options import VectorIndexOptions from pypaimon.table.source.search_types import (PrimaryKeyVectorSearchSplit, PrimaryKeyVectorScanPlan) -from pypaimon.common.options.core_options import CoreOptions from pypaimon.common.options.options import Options from pypaimon.index.index_file_handler import IndexFileHandler from pypaimon.index.pk.primary_key_index_source_meta import PrimaryKeyIndexSourceMeta @@ -83,9 +83,8 @@ def scan(self): index_type = self._index_type if index_type is None: - index_type = CoreOptions( - Options(dict(scan_table.table_schema.options)) - ).primary_key_vector_index_type(self._vector_column.name) + index_type = VectorIndexOptions.index_type( + Options(dict(scan_table.table_schema.options)), self._vector_column.name) entries = IndexFileHandler(table=scan_table).scan( snapshot, lambda entry: _matches(entry, self._vector_column.id, index_type, diff --git a/paimon-python/pypaimon/index/pk/primary_key_index_definitions.py b/paimon-python/pypaimon/index/pk/primary_key_index_definitions.py index 22a1cbac0caa..507c1c357b30 100644 --- a/paimon-python/pypaimon/index/pk/primary_key_index_definitions.py +++ b/paimon-python/pypaimon/index/pk/primary_key_index_definitions.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from pypaimon.common.options.vector_index_options import VectorIndexOptions from pypaimon.common.options.core_options import CoreOptions from pypaimon.common.options.options import Options from pypaimon.index.pk.primary_key_index_definition import ( @@ -29,13 +30,13 @@ def __init__(self, definitions): @staticmethod def create(schema): core = CoreOptions(Options(dict(schema.options))) - btree = core.primary_key_btree_index_columns() - bitmap = core.primary_key_bitmap_index_columns() - vector = core.primary_key_vector_index_columns() - full_text = core.primary_key_full_text_index_columns() + btree = core.btree_index_columns() + bitmap = core.bitmap_index_columns() + vector = VectorIndexOptions.columns(core.options) + full_text = core.full_text_index_columns() _validate_columns(btree, CoreOptions.PK_BTREE_INDEX_COLUMNS.key()) _validate_columns(bitmap, CoreOptions.PK_BITMAP_INDEX_COLUMNS.key()) - _validate_columns(vector, CoreOptions.PK_VECTOR_INDEX_COLUMNS.key()) + _validate_columns(vector, VectorIndexOptions.columns_option_key(core.options)) _validate_columns(full_text, CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key()) families = (btree, bitmap, vector, full_text) owned = set() @@ -50,23 +51,23 @@ def create(schema): if field.name in btree: definitions.append(PrimaryKeyIndexDefinition( field.name, field.id, "btree", - core.primary_key_btree_index_options(field.name), + core.btree_index_options(field.name), PrimaryKeyIndexFamily.BTREE)) elif field.name in bitmap: definitions.append(PrimaryKeyIndexDefinition( field.name, field.id, "bitmap", - core.primary_key_bitmap_index_options(field.name), + core.bitmap_index_options(field.name), PrimaryKeyIndexFamily.BITMAP)) elif field.name in vector: definitions.append(PrimaryKeyIndexDefinition( field.name, field.id, - core.primary_key_vector_index_type(field.name), - core.primary_key_vector_index_options(field.name), + VectorIndexOptions.index_type(core.options, field.name), + VectorIndexOptions.resolve(core.options, None, field.name), PrimaryKeyIndexFamily.VECTOR)) elif field.name in full_text: definitions.append(PrimaryKeyIndexDefinition( field.name, field.id, "full-text", - core.primary_key_full_text_index_options(field.name), + core.full_text_index_options(field.name), PrimaryKeyIndexFamily.FULL_TEXT)) return PrimaryKeyIndexDefinitions(definitions) diff --git a/paimon-python/pypaimon/table/source/full_text_search_builder.py b/paimon-python/pypaimon/table/source/full_text_search_builder.py index 5c791785a655..3cc8d4a220f1 100644 --- a/paimon-python/pypaimon/table/source/full_text_search_builder.py +++ b/paimon-python/pypaimon/table/source/full_text_search_builder.py @@ -184,7 +184,7 @@ def _primary_key_full_text_definition(self): core = CoreOptions(Options(dict(self._table.table_schema.options))) if (not getattr(self._table.table_schema, 'primary_keys', None) or core.data_evolution_enabled() - or not core.primary_key_full_text_index_columns()): + or not core.full_text_index_columns()): return None for definition in PrimaryKeyIndexDefinitions.create( self._table.table_schema).definitions: diff --git a/paimon-python/pypaimon/table/source/vector_search_builder.py b/paimon-python/pypaimon/table/source/vector_search_builder.py index 0bd98a2a8c0b..9e8c501655d4 100644 --- a/paimon-python/pypaimon/table/source/vector_search_builder.py +++ b/paimon-python/pypaimon/table/source/vector_search_builder.py @@ -20,9 +20,9 @@ from abc import ABC, abstractmethod from copy import copy +from pypaimon.common.options.vector_index_options import VectorIndexOptions from pypaimon.catalog.table_query_auth import reject_search_under_query_auth from pypaimon.common.predicate_builder import PredicateBuilder -from pypaimon.common.options.core_options import CoreOptions from pypaimon.common.options.options import Options @@ -228,8 +228,7 @@ def new_vector_search_scan(self): def _is_primary_key_vector_search(self): if self._vector_column is None or not getattr(self._table.table_schema, 'primary_keys', None): return False - core = CoreOptions(Options(dict(self._table.table_schema.options))) - return self._vector_column.name in core.primary_key_vector_index_columns() + return self._vector_column.name in VectorIndexOptions.columns(Options(dict(self._table.table_schema.options))) class VectorSearchBuilderImpl(AbstractVectorSearchBuilderImpl, VectorSearchBuilder): diff --git a/paimon-python/pypaimon/tests/global_index_test.py b/paimon-python/pypaimon/tests/global_index_test.py index f9ad3186ed63..0270f9a0c527 100644 --- a/paimon-python/pypaimon/tests/global_index_test.py +++ b/paimon-python/pypaimon/tests/global_index_test.py @@ -61,7 +61,7 @@ def test_legacy_primary_key_metric_resolution_is_idempotent(self): 'fields.embedding.pk-vector.index.type': 'ivf-pq', 'fields.embedding.pk-vector.distance.metric': metric.lower().replace('-', '_'), 'fields.embedding.pk-vector.index.options': '{"metric":"%s"}' % metric} - first = CoreOptions(Options(values)).primary_key_vector_index_options('embedding') + first = VectorIndexOptions.resolve(Options(values), None, 'embedding') second = VectorIndexOptions.resolve(first, 'ivf-pq', 'embedding') self.assertEqual(first.to_map(), second.to_map()) self.assertEqual('{"metric":"%s"}' % metric, @@ -74,7 +74,10 @@ def test_shared_index_column_precedence_and_legacy_fallback(self): with self.subTest(index_type=index_type): values = {"pk-%s.index.columns" % legacy: "old", "fields.old.pk-vector.index.type": "ivf-pq"} - self.assertEqual(["old"], CoreOptions(Options(values)).global_index_columns(index_type)) + core = CoreOptions(Options(values)) + columns = (VectorIndexOptions.columns(core.options) if CoreOptions._is_vector_index_type(index_type) + else core.global_index_columns(index_type)) + self.assertEqual(["old"], columns) values["global-index.%s.index-column" % index_type] = "single" self.assertEqual(["single"], CoreOptions(Options(values)).global_index_columns(index_type)) values["global-index.%s.index-columns" % index_type] = " first, second " @@ -106,13 +109,13 @@ def test_shared_vector_type_and_documented_metric(self): "fields.embedding.nlist": "48", "fields.embedding.distance.metric": "cosine"} core = CoreOptions(Options(values)) - self.assertEqual(["embedding"], core.primary_key_vector_index_columns()) - self.assertEqual("ivf-pq", core.primary_key_vector_index_type("embedding")) - self.assertEqual("cosine", core.primary_key_vector_distance_metric("embedding")) + self.assertEqual(["embedding"], VectorIndexOptions.columns(core.options)) + self.assertEqual("ivf-pq", VectorIndexOptions.index_type(core.options, "embedding")) + self.assertEqual("cosine", VectorIndexOptions.metric(core.options, None, "embedding")) values["fields.embedding.index-options"] = '{"nlist":64,"distance.metric":"inner_product"}' core = CoreOptions(Options(values)) - self.assertEqual("inner_product", core.primary_key_vector_distance_metric("embedding")) - resolved = core.primary_key_vector_index_options("embedding").to_map() + self.assertEqual("inner_product", VectorIndexOptions.metric(core.options, None, "embedding")) + resolved = VectorIndexOptions.resolve(core.options, None, "embedding").to_map() self.assertEqual("768", resolved["ivf-pq.dimension"]) self.assertEqual("64", resolved["fields.embedding.nlist"]) self.assertEqual("inner_product", resolved["ivf-pq.distance.metric"]) @@ -130,8 +133,8 @@ def test_primary_key_sorted_index_columns_preserve_java_split_semantics(self): "pk-bitmap.index.columns": "category", })) - self.assertEqual(["name", "id"], options.primary_key_btree_index_columns()) - self.assertEqual(["category"], options.primary_key_bitmap_index_columns()) + self.assertEqual(["name", "id"], options.btree_index_columns()) + self.assertEqual(["category"], options.bitmap_index_columns()) def test_global_index_meta_preserves_primary_key_source_meta(self): source_meta = b"\x00\x00\x00\x01source-file" diff --git a/paimon-python/pypaimon/tests/primary_key_index_definitions_test.py b/paimon-python/pypaimon/tests/primary_key_index_definitions_test.py index 663a39ed158c..b5cb5297c111 100644 --- a/paimon-python/pypaimon/tests/primary_key_index_definitions_test.py +++ b/paimon-python/pypaimon/tests/primary_key_index_definitions_test.py @@ -302,7 +302,7 @@ def test_empty_pk_vector_ranges_mean_no_candidates(self): def test_pk_vector_refine_factor_overflow_is_rejected_before_search(self): table = SimpleNamespace(options=SimpleNamespace( - primary_key_vector_index_type=lambda column: "ivf-pq", + options=Options({"global-index.ivf-pq.index-columns": "embedding"}), vector_index_search_mode=lambda: GlobalIndexSearchMode.FAST)) reader = PrimaryKeyVectorRead( table, 2147483647, DataField(1, "embedding", AtomicType("VECTOR(1)")), @@ -323,7 +323,7 @@ def test_pk_vector_raw_fallback_uses_vector_search_mode(self): (GlobalIndexSearchMode.FULL, 1), (GlobalIndexSearchMode.DETAIL, 1)): table = SimpleNamespace(options=SimpleNamespace( - primary_key_vector_index_type=lambda column: "ivf-pq", + options=Options({"global-index.ivf-pq.index-columns": "embedding"}), vector_index_search_mode=lambda mode=mode: mode)) reader = PrimaryKeyVectorRead(table, 1, field, [0.0]) with self.subTest(mode=mode), mock.patch.object( diff --git a/paimon-python/pypaimon/tests/sorted_index_options_test.py b/paimon-python/pypaimon/tests/sorted_index_options_test.py index 611a92eaa61c..b1ffcaa96a86 100644 --- a/paimon-python/pypaimon/tests/sorted_index_options_test.py +++ b/paimon-python/pypaimon/tests/sorted_index_options_test.py @@ -43,7 +43,7 @@ def test_primary_key_options_exclude_global_index_file_size(index_type): "write-buffer-size": "8 mb", }) - resolve_options = getattr(core_options, "primary_key_%s_index_options" % index_type) + resolve_options = getattr(core_options, "%s_index_options" % index_type) options = resolve_options("name").to_map() assert "sorted-index.records-per-file" not in options diff --git a/paimon-python/pypaimon/tests/vector_index_options_test.py b/paimon-python/pypaimon/tests/vector_index_options_test.py index c84006f04adb..72af7980b565 100644 --- a/paimon-python/pypaimon/tests/vector_index_options_test.py +++ b/paimon-python/pypaimon/tests/vector_index_options_test.py @@ -30,6 +30,29 @@ def _assert_metric(options, expected): assert options.to_map()[key] == expected +@pytest.mark.parametrize('suffix', ['index-column', 'index-columns']) +def test_legacy_registration_is_only_a_compatibility_fallback(suffix): + values = {'pk-vector.index.columns': ' old, other_old ', + 'fields.old.pk-vector.index.type': 'ivf-pq', + 'fields.other_old.pk-vector.index.type': 'diskann'} + options = Options(values) + assert VectorIndexOptions.columns(options) == ['old', 'other_old'] + assert VectorIndexOptions.index_type(options, 'old') == 'ivf-pq' + assert CoreOptions(options).vector_index_columns('ivf-pq') == ['old'] + assert CoreOptions(options).vector_index_columns('diskann') == ['other_old'] + assert CoreOptions(options).vector_index_columns('ivf-flat') == [] + assert VectorIndexOptions.columns_option_key(options) == 'pk-vector.index.columns' + values.update({'global-index.ivf-pq.' + suffix: ' embedding, other ', + 'global-index.diskann.' + suffix: 'video'}) + assert VectorIndexOptions.columns(options) == ['video', 'embedding', 'other'] + assert VectorIndexOptions.index_type(options, 'embedding') == 'ivf-pq' + assert VectorIndexOptions.index_type(options, 'video') == 'diskann' + assert CoreOptions(options).vector_index_columns('ivf-pq') == ['embedding', 'other'] + assert CoreOptions(options).vector_index_columns('diskann') == ['video'] + assert VectorIndexOptions.columns_option_key(options) == ( + 'global-index.diskann.%s, global-index.ivf-pq.%s' % (suffix, suffix)) + + @pytest.mark.parametrize('key', ['index_type', 'index-type', 'vector.index-type', 'fields.embedding.index-type']) def test_generic_merge_does_not_infer_historical_vector_types(key): values = {key: 'ivf-pq', 'fields.embedding.index-options': '{"nlist":64}'} @@ -40,7 +63,6 @@ def test_generic_merge_does_not_infer_historical_vector_types(key): def test_removed_type_aliases_are_ignored(key): options = Options({key: 'ivf-pq'}) assert VectorIndexOptions.index_type(options, 'embedding') is None - assert CoreOptions(options).vector_index_type('embedding') is None assert VectorIndexOptions.resolve(options, None, 'embedding').to_map() == options.to_map() options.to_map()['fields.embedding.pk-vector.index.type'] = 'diskann' assert VectorIndexOptions.index_type(options, 'embedding') == 'diskann' @@ -53,7 +75,6 @@ def test_legacy_primary_key_type_resolves_field_parameters(): 'fields.embedding.index-options': '{"nlist":64,"metric":"COSINE"}'} options = Options(values) assert VectorIndexOptions.index_type(options, 'embedding') == 'ivf-pq' - assert CoreOptions(options).vector_index_type('embedding') == 'ivf-pq' resolved = VectorIndexOptions.resolve(options, None, 'embedding', default_metric=False) _assert_metric(resolved, 'cosine') assert resolved.to_map()['ivf-pq.dimension'] == '768' @@ -101,7 +122,7 @@ def test_field_json_aliases_remain_idempotent(suffix, key): _assert_metric(resolved, 'cosine') assert resolved.to_map()['ivf-pq.nlist'] == '64' assert VectorIndexOptions.resolve(resolved, 'ivf-pq', 'embedding').to_map() == resolved.to_map() - assert CoreOptions(Options(values)).primary_key_vector_index_options('embedding').to_map() == resolved.to_map() + assert VectorIndexOptions.resolve(Options(values), None, 'embedding').to_map() == resolved.to_map() def test_reader_keeps_unspecified_metric_and_writer_uses_default(): diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScanBuilder.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScanBuilder.scala index 8f6d56d80f1d..565f4b7e82c1 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScanBuilder.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScanBuilder.scala @@ -18,8 +18,8 @@ package org.apache.paimon.spark -import org.apache.paimon.CoreOptions import org.apache.paimon.options.Options +import org.apache.paimon.options.VectorIndexOptions import org.apache.paimon.partition.PartitionPredicate import org.apache.paimon.predicate._ import org.apache.paimon.predicate.SortValue.{NullOrdering, SortDirection} @@ -148,9 +148,8 @@ class PaimonScanBuilder(val table: InnerTable) if ( vectorSearch.isDefined && - !CoreOptions - .fromMap(actualTable.options) - .primaryKeyVectorIndexColumns() + !VectorIndexOptions + .columns(Options.fromMap(actualTable.options)) .contains(vectorSearch.get.fieldName()) && VectorSearchResultUtils.isVectorSearchMetaOnly(requiredSchema.fieldNames.toSeq) ) { From 98582ca2de04d6fc57ddc561221b6e85368a029e Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 1 Nov 2026 11:53:40 +0800 Subject: [PATCH 09/14] [core][python] Remove redundant legacy PK index option definitions --- docs/generated/core_configuration.html | 30 ---------------- .../java/org/apache/paimon/CoreOptions.java | 36 ------------------- ...BucketedPrimaryKeyIndexMaintainerTest.java | 2 +- .../pk/PrimaryKeyIndexDefinitionsTest.java | 8 ++--- .../PrimaryKeySortedIndexMaintenanceTest.java | 12 +++---- .../operation/PrimaryKeyIndexWriteTest.java | 16 ++++----- .../paimon/schema/SchemaManagerTest.java | 12 +++---- .../source/PrimaryKeyFullTextScanTest.java | 2 +- .../source/PrimaryKeyFullTextSearchTest.java | 2 +- .../PrimaryKeySortedIndexBatchScanTest.java | 2 +- .../source/PrimaryKeySortedIndexReadTest.java | 6 ++-- .../source/PrimaryKeyVectorSearchTest.java | 2 +- .../table/system/AuditLogTableTest.java | 2 +- .../pypaimon/common/options/core_options.py | 30 +++++----------- .../common/options/vector_index_options.py | 8 ++--- .../index/pk/primary_key_index_definitions.py | 6 ++-- .../primary_key_index_definitions_test.py | 13 +++++++ 17 files changed, 59 insertions(+), 130 deletions(-) diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index e2aa57b0fbba..79764b318f0c 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -1416,42 +1416,12 @@ String You can specify a pattern to get a timestamp from partitions. The formatter pattern is defined by 'partition.timestamp-formatter'.
  • By default, read from the first field.
  • If the timestamp in the partition is a single field called 'dt', you can use '$dt'.
  • If it is spread across multiple fields for year, month, day, and hour, you can use '$year-$month-$day $hour:00:00'.
  • If the timestamp is in fields dt and hour, you can use '$dt $hour:00:00'.
- -
pk-bitmap.index.columns
- (none) - String - Comma-separated columns indexed by primary-key Bitmap indexes. - - -
pk-btree.index.columns
- (none) - String - Comma-separated columns indexed by primary-key BTree indexes. -
pk-clustering-override
false Boolean Enables clustering by non-primary key fields. When set to true, the physical sort order of data files is determined by the configured 'clustering.columns' instead of the primary key, optimizing query performance for non-PK columns. - -
pk-fm.index.columns
- (none) - String - Comma-separated character columns indexed by primary-key FM indexes. - - -
pk-full-text.index.columns
- (none) - String - Comma-separated character columns indexed by primary-key full-text indexes. The first release supports exactly one column. - - -
pk-multivalue.index.columns
- (none) - String - Comma-separated ARRAY columns indexed by primary-key Multivalue indexes. -
postpone.batch-write-fixed-bucket
true diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index f8f24adfe65f..8206a5080bdb 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -3254,42 +3254,6 @@ public String toString() { .withFallbackKeys("global-index.fm.index-column", "pk-fm.index.columns") .withDescription("Comma-separated character columns to index with FM indexes."); - public static final ConfigOption PK_BTREE_INDEX_COLUMNS = - key("pk-btree.index.columns") - .stringType() - .noDefaultValue() - .withDescription( - "Comma-separated columns indexed by primary-key BTree indexes."); - - public static final ConfigOption PK_BITMAP_INDEX_COLUMNS = - key("pk-bitmap.index.columns") - .stringType() - .noDefaultValue() - .withDescription( - "Comma-separated columns indexed by primary-key Bitmap indexes."); - - public static final ConfigOption PK_MULTIVALUE_INDEX_COLUMNS = - key("pk-multivalue.index.columns") - .stringType() - .noDefaultValue() - .withDescription( - "Comma-separated ARRAY columns indexed by primary-key Multivalue indexes."); - - public static final ConfigOption PK_FULL_TEXT_INDEX_COLUMNS = - key("pk-full-text.index.columns") - .stringType() - .noDefaultValue() - .withDescription( - "Comma-separated character columns indexed by primary-key full-text indexes. " - + "The first release supports exactly one column."); - - public static final ConfigOption PK_FM_INDEX_COLUMNS = - key("pk-fm.index.columns") - .stringType() - .noDefaultValue() - .withDescription( - "Comma-separated character columns indexed by primary-key FM indexes."); - @Immutable public static final ConfigOption PK_CLUSTERING_OVERRIDE = key("pk-clustering-override") diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainerTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainerTest.java index 531f440930c8..91db135a0d60 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainerTest.java @@ -128,7 +128,7 @@ void testDelegatesFullTextLifecycleAndMergesCommit() throws Exception { @Test void testFactoryCreatesConfiguredFullTextMaintainer() { Map options = new HashMap<>(); - options.put(CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key(), "content"); + options.put(CoreOptions.GLOBAL_INDEX_FULL_TEXT_INDEX_COLUMNS.key(), "content"); TableSchema schema = new TableSchema( 0, diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitionsTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitionsTest.java index 6a951f333f11..d28f8b62d694 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitionsTest.java @@ -128,10 +128,10 @@ void testCreatesFMDefinitionAndResolvesOptions(boolean sharedOptions) { @Test void testRejectsDuplicateColumnWithinFamily() { Map options = new HashMap<>(); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "name,name"); + options.put(CoreOptions.GLOBAL_INDEX_BTREE_INDEX_COLUMNS.key(), "name,name"); assertThatThrownBy(() -> PrimaryKeyIndexDefinitions.create(schema(options))) - .hasMessageContaining("pk-btree.index.columns") + .hasMessageContaining("global-index.btree.index-columns") .hasMessageContaining("duplicate") .hasMessageContaining("name"); } @@ -139,8 +139,8 @@ void testRejectsDuplicateColumnWithinFamily() { @Test void testRejectsColumnConfiguredByMultipleFamilies() { Map options = new HashMap<>(); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "name"); - options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "name"); + options.put(CoreOptions.GLOBAL_INDEX_BTREE_INDEX_COLUMNS.key(), "name"); + options.put(CoreOptions.GLOBAL_INDEX_BITMAP_INDEX_COLUMNS.key(), "name"); assertThatThrownBy(() -> PrimaryKeyIndexDefinitions.create(schema(options))) .hasMessageContaining("name") diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexMaintenanceTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexMaintenanceTest.java index f790a4127817..1d337e4030c4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexMaintenanceTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexMaintenanceTest.java @@ -326,14 +326,10 @@ private TestFileStore createStore(boolean sharedOptions) throws Exception { options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); options.put(CoreOptions.COMPACTION_FORCE_REWRITE_ALL_FILES.key(), "true"); options.put( - sharedOptions - ? "global-index.btree.index-columns" - : CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), + sharedOptions ? "global-index.btree.index-columns" : "pk-btree.index.columns", "itemId"); options.put( - sharedOptions - ? "global-index.bitmap.index-columns" - : CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), + sharedOptions ? "global-index.bitmap.index-columns" : "pk-bitmap.index.columns", "comment"); SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); @@ -368,8 +364,8 @@ private TestFileStore createMixedStore() throws Exception { options.put(CoreOptions.BUCKET.key(), "1"); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); options.put(CoreOptions.COMPACTION_FORCE_REWRITE_ALL_FILES.key(), "true"); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "itemId"); - options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "comment"); + options.put("pk-btree.index.columns", "itemId"); + options.put("pk-bitmap.index.columns", "comment"); options.put("pk-vector.index.columns", "embedding"); options.put("fields.embedding.pk-vector.index.type", "test-vector-ann"); options.put("fields.embedding.pk-vector.distance.metric", "l2"); diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java index d0cacf1883ba..0ac8c5dd4606 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java @@ -74,8 +74,8 @@ void testCreatesCoordinatorForBTreeAndBitmapDefinitions() throws Exception { Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "10"); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "itemId"); - options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "comment"); + options.put(CoreOptions.GLOBAL_INDEX_BTREE_INDEX_COLUMNS.key(), "itemId"); + options.put(CoreOptions.GLOBAL_INDEX_BITMAP_INDEX_COLUMNS.key(), "comment"); TestFileStore store = createStore(options); KeyValueFileStoreWrite write = (KeyValueFileStoreWrite) store.newWrite(); write.withIOManager(ioManager); @@ -95,7 +95,7 @@ void testCreatesCoordinatorForMultiValueDefinition() throws Exception { Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "10"); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_MULTIVALUE_INDEX_COLUMNS.key(), "tags"); + options.put(CoreOptions.GLOBAL_INDEX_MULTIVALUE_INDEX_COLUMNS.key(), "tags"); List fields = new ArrayList<>(TestKeyValueGenerator.DEFAULT_ROW_TYPE.getFields()); fields.add(new DataField(7, "tags", DataTypes.ARRAY(DataTypes.STRING()))); @@ -119,8 +119,8 @@ void testCreatesCoordinatorForMultipleScalarDefinitionsAndVector() throws Except Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "10"); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "itemId,score"); - options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "comment,state"); + options.put(CoreOptions.GLOBAL_INDEX_BTREE_INDEX_COLUMNS.key(), "itemId,score"); + options.put(CoreOptions.GLOBAL_INDEX_BITMAP_INDEX_COLUMNS.key(), "comment,state"); options.put("pk-vector.index.columns", "embedding"); options.put("fields.embedding.pk-vector.index.type", "hnsw"); @@ -158,7 +158,7 @@ void testCreatesCoordinatorForFullTextDefinition() throws Exception { Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "10"); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key(), "comment"); + options.put(CoreOptions.GLOBAL_INDEX_FULL_TEXT_INDEX_COLUMNS.key(), "comment"); TestFileStore store = createStore(options); KeyValueFileStoreWrite write = (KeyValueFileStoreWrite) store.newWrite(); write.withIOManager(ioManager); @@ -181,8 +181,8 @@ void testPostponeBucketDefersCoordinatorUntilFixedBucketCompaction() throws Exce Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), String.valueOf(BucketMode.POSTPONE_BUCKET)); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "itemId"); - options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "comment"); + options.put(CoreOptions.GLOBAL_INDEX_BTREE_INDEX_COLUMNS.key(), "itemId"); + options.put(CoreOptions.GLOBAL_INDEX_BITMAP_INDEX_COLUMNS.key(), "comment"); TableSchema schema = createSchema(options, TestKeyValueGenerator.DEFAULT_ROW_TYPE); TestKeyValueGenerator generator = new TestKeyValueGenerator(); KeyValue record = generator.next(); diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java index 06d18cffed42..2d24b971c712 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java @@ -669,7 +669,7 @@ public void testRejectRenamePrimaryKeyBTreeIndexColumn() throws Exception { Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "1"); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "name"); + options.put(CoreOptions.GLOBAL_INDEX_BTREE_INDEX_COLUMNS.key(), "name"); Schema schema = new Schema( Arrays.asList( @@ -696,7 +696,7 @@ public void testRejectDestructivePrimaryKeyFullTextIndexColumnChanges() throws E Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "1"); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key(), "content"); + options.put(CoreOptions.GLOBAL_INDEX_FULL_TEXT_INDEX_COLUMNS.key(), "content"); Schema schema = new Schema( Arrays.asList( @@ -732,7 +732,7 @@ public void testRejectDestructivePrimaryKeyFMIndexColumnChanges() throws Excepti Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "1"); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_FM_INDEX_COLUMNS.key(), "content"); + options.put(CoreOptions.GLOBAL_INDEX_FM_INDEX_COLUMNS.key(), "content"); Schema schema = new Schema( Arrays.asList( @@ -768,7 +768,7 @@ public void testRejectDropPrimaryKeyBitmapIndexColumn() throws Exception { Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "1"); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "status"); + options.put(CoreOptions.GLOBAL_INDEX_BITMAP_INDEX_COLUMNS.key(), "status"); Schema schema = new Schema( Arrays.asList( @@ -791,7 +791,7 @@ public void testRejectTypeChangeOfPrimaryKeyBitmapIndexColumn() throws Exception Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "1"); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "status"); + options.put(CoreOptions.GLOBAL_INDEX_BITMAP_INDEX_COLUMNS.key(), "status"); Schema schema = new Schema( Arrays.asList( @@ -818,7 +818,7 @@ public void testRejectChangeOfPrimaryKeyMultiValueIndexColumn() throws Exception Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "1"); options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); - options.put(CoreOptions.PK_MULTIVALUE_INDEX_COLUMNS.key(), "tags"); + options.put(CoreOptions.GLOBAL_INDEX_MULTIVALUE_INDEX_COLUMNS.key(), "tags"); Schema schema = new Schema( Arrays.asList( diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextScanTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextScanTest.java index 29d843d3ff1f..8abf2e5ba61c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextScanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextScanTest.java @@ -147,7 +147,7 @@ void testCapturesOneSnapshotAndPrunesPartitions() { Snapshot snapshot = mock(Snapshot.class); when(snapshot.id()).thenReturn(11L); Options tableOptions = new Options(); - tableOptions.set(CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS, "content"); + tableOptions.set(CoreOptions.GLOBAL_INDEX_FULL_TEXT_INDEX_COLUMNS, "content"); when(table.coreOptions()).thenReturn(new CoreOptions(tableOptions)); when(table.copy(Collections.singletonMap(CoreOptions.SCAN_SNAPSHOT_ID.key(), "11"))) .thenReturn(table); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextSearchTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextSearchTest.java index 400a609b3895..671ab49532f9 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextSearchTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextSearchTest.java @@ -127,7 +127,7 @@ void testHybridRouteUsesPrimaryKeyFullText() { private static FileStoreTable table(boolean dataEvolution) { Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "2"); - options.put(CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key(), "content"); + options.put(CoreOptions.GLOBAL_INDEX_FULL_TEXT_INDEX_COLUMNS.key(), "content"); options.put(CoreOptions.DATA_EVOLUTION_ENABLED.key(), Boolean.toString(dataEvolution)); TableSchema schema = new TableSchema( diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexBatchScanTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexBatchScanTest.java index 426374562a72..6804becd9a9d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexBatchScanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexBatchScanTest.java @@ -162,7 +162,7 @@ private static TableSchema tableSchema(boolean globalIndexEnabled, int bucket) { options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"); options.put(CoreOptions.DELETION_VECTORS_MERGE_ON_READ.key(), "false"); options.put(CoreOptions.GLOBAL_INDEX_ENABLED.key(), Boolean.toString(globalIndexEnabled)); - options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "f7"); + options.put(CoreOptions.GLOBAL_INDEX_BTREE_INDEX_COLUMNS.key(), "f7"); return new TableSchema( 5, Arrays.asList( diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexReadTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexReadTest.java index c76ba9ddb7b6..1bc51aa6bb7e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexReadTest.java @@ -69,7 +69,7 @@ protected Schema schemaDefault() { .primaryKey("id") .option(CoreOptions.BUCKET.key(), "1") .option(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true") - .option(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "score") + .option(CoreOptions.GLOBAL_INDEX_BTREE_INDEX_COLUMNS.key(), "score") .build(); } @@ -169,7 +169,7 @@ void testReadAfterIndexCompaction() throws Exception { .option(CoreOptions.BUCKET.key(), "1") .option(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true") .option(CoreOptions.TARGET_FILE_SIZE.key(), "1 b") - .option(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "score") + .option(CoreOptions.GLOBAL_INDEX_BTREE_INDEX_COLUMNS.key(), "score") .build(); catalog.createTable(identifier(), schema, false); FileStoreTable table = getTableDefault(); @@ -319,7 +319,7 @@ void testReadWithMultiValueIndex() throws Exception { .primaryKey("id") .option(CoreOptions.BUCKET.key(), "1") .option(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true") - .option(CoreOptions.PK_MULTIVALUE_INDEX_COLUMNS.key(), "tags") + .option(CoreOptions.GLOBAL_INDEX_MULTIVALUE_INDEX_COLUMNS.key(), "tags") .build(); catalog.createTable(identifier(), schema, false); FileStoreTable table = getTableDefault(); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorSearchTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorSearchTest.java index cae582368807..967349940ef4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorSearchTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorSearchTest.java @@ -218,7 +218,7 @@ void testVectorSearchUsesSortedIndexPreFilter() throws Exception { .primaryKey("id") .option(CoreOptions.BUCKET.key(), "1") .option(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true") - .option(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "score") + .option(CoreOptions.GLOBAL_INDEX_BTREE_INDEX_COLUMNS.key(), "score") .option( "global-index." + TestVectorGlobalIndexerFactory.IDENTIFIER diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/AuditLogTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/AuditLogTableTest.java index a6c4b46c3c57..7f8f3d3ea5c2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/AuditLogTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/AuditLogTableTest.java @@ -115,7 +115,7 @@ public void testReadAuditLogWithPrimaryKeySortedIndex() throws Exception { .option(CoreOptions.CHANGELOG_PRODUCER.key(), "input") .option(CoreOptions.BUCKET.key(), "1") .option(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true") - .option(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "score") + .option(CoreOptions.GLOBAL_INDEX_BTREE_INDEX_COLUMNS.key(), "score") .build()); FileStoreTable dataTable = FileStoreTableFactory.create(fileIO, tablePath, tableSchema); write(dataTable, ioManager, GenericRow.of(1, 10)); diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py index 5cde028ccd42..2cadf56d1aa5 100644 --- a/paimon-python/pypaimon/common/options/core_options.py +++ b/paimon-python/pypaimon/common/options/core_options.py @@ -1038,27 +1038,6 @@ class CoreOptions: ) ) - PK_BTREE_INDEX_COLUMNS: ConfigOption[str] = ( - ConfigOptions.key("pk-btree.index.columns") - .string_type() - .no_default_value() - .with_description("Comma-separated columns indexed by primary-key BTree indexes.") - ) - - PK_BITMAP_INDEX_COLUMNS: ConfigOption[str] = ( - ConfigOptions.key("pk-bitmap.index.columns") - .string_type() - .no_default_value() - .with_description("Comma-separated columns indexed by primary-key Bitmap indexes.") - ) - - PK_FULL_TEXT_INDEX_COLUMNS: ConfigOption[str] = ( - ConfigOptions.key("pk-full-text.index.columns") - .string_type() - .no_default_value() - .with_description("Comma-separated character columns indexed by primary-key full-text indexes.") - ) - GLOBAL_INDEX_COLUMN_UPDATE_ACTION: ConfigOption[GlobalIndexColumnUpdateAction] = ( ConfigOptions.key("global-index.column-update-action") .enum_type(GlobalIndexColumnUpdateAction) @@ -1885,6 +1864,15 @@ def global_index_columns(self, index_type: str) -> List[str]: columns = values.get("pk-%s.index.columns" % index_type) return self._index_columns(columns) + def global_index_columns_option_key(self, index_type: str) -> str: + values = self.options.to_map() + for suffix in ("index-columns", "index-column"): + key = "global-index.%s.%s" % (index_type, suffix) + if values.get(key) is not None: + return key + legacy = "pk-%s.index.columns" % index_type + return legacy if values.get(legacy) is not None else "global-index.%s.index-columns" % index_type + @staticmethod def _index_columns(columns) -> List[str]: return [] if columns is None else [column.strip() for column in str(columns).split(',')] diff --git a/paimon-python/pypaimon/common/options/vector_index_options.py b/paimon-python/pypaimon/common/options/vector_index_options.py index 154ab2c307a1..c47df4a2141e 100644 --- a/paimon-python/pypaimon/common/options/vector_index_options.py +++ b/paimon-python/pypaimon/common/options/vector_index_options.py @@ -41,13 +41,11 @@ def _legacy_columns(options): @staticmethod def columns_option_key(options): - types = VectorIndexOptions._index_types(CoreOptions(options)) + core = CoreOptions(options) + types = VectorIndexOptions._index_types(core) if not types: return 'pk-vector.index.columns' - values = options.to_map() - return ', '.join('global-index.%s.%s' % ( - kind, 'index-columns' if 'global-index.%s.index-columns' % kind in values else 'index-column') - for kind in types) + return ', '.join(core.global_index_columns_option_key(kind) for kind in types) @staticmethod def _index_types(core): diff --git a/paimon-python/pypaimon/index/pk/primary_key_index_definitions.py b/paimon-python/pypaimon/index/pk/primary_key_index_definitions.py index 507c1c357b30..6b7366a40d29 100644 --- a/paimon-python/pypaimon/index/pk/primary_key_index_definitions.py +++ b/paimon-python/pypaimon/index/pk/primary_key_index_definitions.py @@ -34,10 +34,10 @@ def create(schema): bitmap = core.bitmap_index_columns() vector = VectorIndexOptions.columns(core.options) full_text = core.full_text_index_columns() - _validate_columns(btree, CoreOptions.PK_BTREE_INDEX_COLUMNS.key()) - _validate_columns(bitmap, CoreOptions.PK_BITMAP_INDEX_COLUMNS.key()) + _validate_columns(btree, core.global_index_columns_option_key("btree")) + _validate_columns(bitmap, core.global_index_columns_option_key("bitmap")) _validate_columns(vector, VectorIndexOptions.columns_option_key(core.options)) - _validate_columns(full_text, CoreOptions.PK_FULL_TEXT_INDEX_COLUMNS.key()) + _validate_columns(full_text, core.global_index_columns_option_key("full-text")) families = (btree, bitmap, vector, full_text) owned = set() for columns in families: diff --git a/paimon-python/pypaimon/tests/primary_key_index_definitions_test.py b/paimon-python/pypaimon/tests/primary_key_index_definitions_test.py index b5cb5297c111..31a279b9e3dc 100644 --- a/paimon-python/pypaimon/tests/primary_key_index_definitions_test.py +++ b/paimon-python/pypaimon/tests/primary_key_index_definitions_test.py @@ -53,6 +53,19 @@ class PrimaryKeyIndexDefinitionsTest(unittest.TestCase): + + def test_duplicate_columns_report_the_configured_declaration_key(self): + for kind in ('btree', 'bitmap', 'full-text'): + for key in ('global-index.%s.index-columns' % kind, + 'global-index.%s.index-column' % kind, 'pk-%s.index.columns' % kind): + with self.subTest(key=key): + schema = TableSchema(fields=[DataField(0, 'name', AtomicType('STRING'))], + options={key: 'name,name'}) + with self.assertRaises(ValueError) as error: + PrimaryKeyIndexDefinitions.create(schema) + self.assertIn(key, str(error.exception)) + self.assertIn('duplicate column', str(error.exception)) + def test_shared_vector_and_full_text_definitions(self): schema = TableSchema( fields=[DataField(0, "embedding", AtomicType("VECTOR(3)")), From 37452d661a69126ce524bd1c552736cf44451e3b Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 1 Nov 2026 12:06:39 +0800 Subject: [PATCH 10/14] Resolve PK vector read parameters from snapshot index entries --- .../source/BatchVectorSearchBuilderImpl.java | 8 +- .../source/PrimaryKeyBatchVectorRead.java | 9 +- .../table/source/PrimaryKeyVectorRead.java | 152 ++++++++++++------ .../table/source/PrimaryKeyVectorScan.java | 8 +- .../table/source/VectorSearchBuilderImpl.java | 8 +- .../source/PrimaryKeyVectorReadTest.java | 96 ++++++++++- .../source/PrimaryKeyVectorScanTest.java | 9 +- .../source/PrimaryKeyVectorSearchTest.java | 50 ++++++ .../FlinkPrimaryKeyVectorRead.java | 17 +- .../FlinkPrimaryKeyVectorReadTest.java | 17 +- .../spark/read/SparkPrimaryKeyVectorRead.java | 7 +- .../sql/PrimaryKeyVectorSearchTest.scala | 8 +- 12 files changed, 303 insertions(+), 86 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java index 6b2349370b6f..bbcfc4b7b522 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java @@ -129,13 +129,7 @@ public BatchVectorSearchBuilder withOption(String key, String value) { public VectorScan newVectorScan() { TableQueryAuthResult.rejectSearchUnderQueryAuth(table); if (isPrimaryKeyVectorSearch()) { - return new PrimaryKeyVectorScan( - table, - vectorColumn.id(), - VectorIndexOptions.indexType( - table.coreOptions().toConfiguration(), vectorColumn.name()), - partitionFilter, - filter); + return new PrimaryKeyVectorScan(table, vectorColumn.id(), partitionFilter, filter); } return new DataEvolutionVectorScan(table, partitionFilter, filter, vectorColumn, options); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyBatchVectorRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyBatchVectorRead.java index bc6361412636..5a6053bd3167 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyBatchVectorRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyBatchVectorRead.java @@ -65,15 +65,20 @@ public PrimaryKeyBatchVectorRead( public List readBatch(VectorScan.Plan plan) { PrimaryKeyVectorRead firstRead = queryReads.get(0); PrimaryKeyVectorScan.Plan primaryKeyPlan = firstRead.primaryKeyPlan(plan); + List splits = firstRead.bucketSplits(primaryKeyPlan); + PrimaryKeyVectorRead.SearchParameters parameters = firstRead.searchParameters(splits); List searchResults = - firstRead.searchBuckets(firstRead.bucketSplits(primaryKeyPlan), queries); + firstRead.searchBuckets(splits, queries, parameters); checkArgument( searchResults.size() == queryReads.size(), "Primary-key vector batch result count does not match query count."); List results = new ArrayList<>(queryReads.size()); for (int i = 0; i < queryReads.size(); i++) { - results.add(queryReads.get(i).createResult(primaryKeyPlan, searchResults.get(i))); + results.add( + queryReads + .get(i) + .createResult(primaryKeyPlan, searchResults.get(i), parameters)); } return Collections.unmodifiableList(results); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java index 39bfeb7b1aaa..577f5c66138e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java @@ -28,6 +28,7 @@ import org.apache.paimon.globalindex.IndexedSplit; import org.apache.paimon.globalindex.VectorSearchMetric; import org.apache.paimon.index.IndexFileHandler; +import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.index.pk.PrimaryKeyIndexSourcePolicy; import org.apache.paimon.index.pkvector.PkVectorAnnSegmentSearcher; import org.apache.paimon.index.pkvector.PkVectorBucketIndexState; @@ -59,6 +60,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -100,14 +102,9 @@ public class PrimaryKeyVectorRead implements VectorRead, Serializable { protected final FileStoreTable table; protected final DataField vectorField; - private final String indexType; - private final Options indexOptions; private final Map searchOptions; private final float[] query; protected final int limit; - private final String metric; - private final int refineFactor; - private final int indexedLimit; @Nullable private final Predicate filter; public PrimaryKeyVectorRead( @@ -136,24 +133,9 @@ public PrimaryKeyVectorRead( checkArgument(limit > 0, "Vector search limit must be positive: %s.", limit); this.table = table; this.vectorField = vectorField; - this.indexType = - VectorIndexOptions.indexType( - table.coreOptions().toConfiguration(), vectorField.name()); - this.indexOptions = - VectorIndexOptions.resolve( - table.coreOptions().toConfiguration(), indexType, vectorField.name()); this.searchOptions = Collections.unmodifiableMap(new HashMap<>(searchOptions)); this.query = query.clone(); this.limit = limit; - this.metric = - normalize(VectorIndexOptions.metric(indexOptions, indexType, vectorField.name())); - this.refineFactor = - VectorSearchRefineOptions.resolve( - this.searchOptions, - this.indexOptions.toMap(), - vectorField.name(), - indexType); - this.indexedLimit = VectorSearchRefineOptions.searchLimit(limit, refineFactor); this.filter = filter; } @@ -171,7 +153,9 @@ private static KeyValueFileStore keyValueStore(FileStoreTable table) { @Override public GlobalIndexResult read(VectorScan.Plan plan) { PrimaryKeyVectorScan.Plan primaryKeyPlan = primaryKeyPlan(plan); - return createResult(primaryKeyPlan, searchBuckets(bucketSplits(primaryKeyPlan))); + List splits = bucketSplits(primaryKeyPlan); + SearchParameters parameters = searchParameters(splits); + return createResult(primaryKeyPlan, searchBuckets(splits, parameters), parameters); } protected PrimaryKeyVectorScan.Plan primaryKeyPlan(VectorScan.Plan plan) { @@ -196,9 +180,54 @@ protected List bucketSplits(PrimaryKeyVectorScan.Plan p return splits; } - protected SearchResult searchBuckets(List splits) { + protected SearchParameters searchParameters(List splits) { + String indexType = null; + for (BucketVectorSearchSplit split : splits) { + Set payloadTypes = new HashSet<>(); + for (IndexFileMeta payload : split.payloadFiles()) { + payloadTypes.add(payload.indexType()); + } + for (String payloadType : payloadTypes) { + PkVectorBucketIndexState state = + PkVectorBucketIndexState.fromActiveDataFiles( + vectorField.id(), + payloadType, + split.dataSplit().dataFiles(), + split.payloadFiles()); + if (state.annSegments().isEmpty()) { + continue; + } + checkArgument( + indexType == null || indexType.equals(payloadType), + "Primary-key vector search cannot mix index types %s and %s.", + indexType, + payloadType); + indexType = payloadType; + } + } + Options tableOptions = table.coreOptions().toConfiguration(); + if (indexType == null) { + indexType = VectorIndexOptions.indexType(tableOptions, vectorField.name()); + } + Options indexOptions = + VectorIndexOptions.resolve(tableOptions, indexType, vectorField.name()); + String metric = + normalize(VectorIndexOptions.metric(indexOptions, indexType, vectorField.name())); + int refineFactor = + VectorSearchRefineOptions.resolve( + searchOptions, indexOptions.toMap(), vectorField.name(), indexType); + return new SearchParameters( + indexType, + indexOptions, + metric, + refineFactor, + VectorSearchRefineOptions.searchLimit(limit, refineFactor)); + } + + protected SearchResult searchBuckets( + List splits, SearchParameters parameters) { try { - SearchContext context = createSearchContext(); + SearchContext context = createSearchContext(parameters); List> futures = new ArrayList<>(splits.size()); // Start from the caller because each bucket submits leaf searches to the same executor. // Wrapping the whole bucket in that executor and waiting inside it can starve leaf @@ -211,7 +240,7 @@ protected SearchResult searchBuckets(List splits) { for (CompletableFuture future : futures) { results.add(future.join()); } - return mergeSearchResults(results); + return mergeSearchResults(results, parameters); } catch (IOException e) { throw new RuntimeException("Failed to search primary-key vector index.", e); } catch (CompletionException e) { @@ -220,9 +249,9 @@ protected SearchResult searchBuckets(List splits) { } protected List searchBuckets( - List splits, float[][] queries) { + List splits, float[][] queries, SearchParameters parameters) { try { - SearchContext context = createSearchContext(); + SearchContext context = createSearchContext(parameters); List>> futures = new ArrayList<>(splits.size()); for (BucketVectorSearchSplit split : splits) { futures.add(searchBatchAsync(split, context, queries)); @@ -243,7 +272,7 @@ protected List searchBuckets( } List results = new ArrayList<>(queries.length); for (List queryResults : resultsByQuery) { - results.add(mergeSearchResults(queryResults)); + results.add(mergeSearchResults(queryResults, parameters)); } return Collections.unmodifiableList(results); } catch (IOException e) { @@ -253,24 +282,28 @@ protected List searchBuckets( } } - SearchContext createSearchContext() { - return new SearchContext(table); + SearchContext createSearchContext(SearchParameters parameters) { + return new SearchContext(table, parameters); } protected GlobalIndexResult createResult( - PrimaryKeyVectorScan.Plan plan, SearchResult searchResult) { - List indexedCandidates = topK(searchResult.indexedCandidates(), indexedLimit); - if (refineFactor > 0 && !indexedCandidates.isEmpty()) { - indexedCandidates = rerank(plan, indexedCandidates); + PrimaryKeyVectorScan.Plan plan, + SearchResult searchResult, + SearchParameters parameters) { + List indexedCandidates = + topK(searchResult.indexedCandidates(), parameters.indexedLimit); + if (parameters.refineFactor > 0 && !indexedCandidates.isEmpty()) { + indexedCandidates = rerank(plan, indexedCandidates, parameters.metric); } else { indexedCandidates = topK(indexedCandidates, limit); } List candidates = new ArrayList<>(indexedCandidates); candidates.addAll(topK(searchResult.exactCandidates(), limit)); - return new PrimaryKeyVectorResult(plan, topK(candidates, limit), metric); + return new PrimaryKeyVectorResult(plan, topK(candidates, limit), parameters.metric); } - protected SearchResult mergeSearchResults(List results) { + protected SearchResult mergeSearchResults( + List results, SearchParameters parameters) { List indexedCandidates = new ArrayList<>(); List exactCandidates = new ArrayList<>(); for (SearchResult result : results) { @@ -278,7 +311,7 @@ protected SearchResult mergeSearchResults(List results) { exactCandidates.addAll(result.exactCandidates()); } return new SearchResult( - topK(indexedCandidates, indexedLimit), topK(exactCandidates, limit)); + topK(indexedCandidates, parameters.indexedLimit), topK(exactCandidates, limit)); } CompletableFuture searchAsync( @@ -291,13 +324,14 @@ CompletableFuture> searchBatchAsync( BucketVectorSearchSplit split, SearchContext context, float[][] queries) throws IOException { DataSplit dataSplit = split.dataSplit(); + SearchParameters parameters = context.parameters; List activeFiles = dataSplit.dataFiles().stream() .filter(PrimaryKeyIndexSourcePolicy::shouldRead) .collect(Collectors.toList()); PkVectorBucketIndexState state = PkVectorBucketIndexState.fromActiveDataFiles( - vectorField.id(), indexType, activeFiles, split.payloadFiles()); + vectorField.id(), parameters.indexType, activeFiles, split.payloadFiles()); Map deletionVectors = deletionVectors(dataSplit, context.fileIO); PkVectorDataFileReader.Factory readerFactory = new PkVectorDataFileReader.Factory( @@ -312,15 +346,15 @@ CompletableFuture> searchBatchAsync( context.indexFileHandler.pkVectorAnnSegment( dataSplit.partition(), dataSplit.bucket()), vectorField, - indexOptions, - metric, + parameters.indexOptions, + parameters.metric, context.executor); PrimaryKeyVectorBucketSearch bucketSearch = new PrimaryKeyVectorBucketSearch( readerFactory, annSearcher, searchOptions, - metric, + parameters.metric, table.coreOptions().vectorIndexSearchMode()); return bucketSearch .searchBatchAsync( @@ -329,7 +363,7 @@ CompletableFuture> searchBatchAsync( deletionVectors, rowRangesByFile(split), queries, - indexedLimit, + parameters.indexedLimit, limit, context.executor) .thenApply( @@ -445,7 +479,7 @@ private static List candidates( } private List rerank( - PrimaryKeyVectorScan.Plan plan, List indexedCandidates) { + PrimaryKeyVectorScan.Plan plan, List indexedCandidates, String metric) { Map candidatesByPosition = new HashMap<>(); for (Candidate candidate : indexedCandidates) { PhysicalPosition position = new PhysicalPosition(candidate); @@ -461,7 +495,7 @@ private List rerank( List reranked = new ArrayList<>(indexedCandidates.size()); try { for (IndexedSplit split : splits) { - rerankSplit(read, split, candidatesByPosition, reranked); + rerankSplit(read, split, candidatesByPosition, reranked, metric); } } catch (IOException e) { throw new RuntimeException("Failed to rerank primary-key vector candidates.", e); @@ -478,7 +512,8 @@ private void rerankSplit( TableRead read, IndexedSplit split, Map candidatesByPosition, - List reranked) + List reranked, + String metric) throws IOException { DataSplit dataSplit = split.dataSplit(); checkArgument( @@ -691,14 +726,41 @@ public String toString() { } } + /** Query parameters resolved once from the captured plan and shared by all bucket tasks. */ + protected static class SearchParameters implements Serializable { + + private static final long serialVersionUID = 1L; + + @Nullable private final String indexType; + private final Options indexOptions; + private final String metric; + private final int refineFactor; + private final int indexedLimit; + + private SearchParameters( + @Nullable String indexType, + Options indexOptions, + String metric, + int refineFactor, + int indexedLimit) { + this.indexType = indexType; + this.indexOptions = indexOptions; + this.metric = metric; + this.refineFactor = refineFactor; + this.indexedLimit = indexedLimit; + } + } + static class SearchContext { private final FileIO fileIO; private final IndexFileHandler indexFileHandler; private final KeyValueFileReaderFactory.Builder readerFactoryBuilder; private final ExecutorService executor; + private final SearchParameters parameters; - private SearchContext(FileStoreTable table) { + private SearchContext(FileStoreTable table, SearchParameters parameters) { + this.parameters = parameters; this.fileIO = table.fileIO(); this.indexFileHandler = table.store().newIndexFileHandler(); this.readerFactoryBuilder = keyValueStore(table).newReaderFactoryBuilder(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorScan.java index dba04bde38a1..8838742d5f21 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorScan.java @@ -52,7 +52,6 @@ public class PrimaryKeyVectorScan implements VectorScan { private final FileStoreTable table; private final int vectorFieldId; - private final String indexType; @Nullable private final PartitionPredicate partitionFilter; @Nullable private final Predicate filter; @Nullable private final Snapshot pinnedSnapshot; @@ -60,22 +59,19 @@ public class PrimaryKeyVectorScan implements VectorScan { public PrimaryKeyVectorScan( FileStoreTable table, int vectorFieldId, - String indexType, @Nullable PartitionPredicate partitionFilter, @Nullable Predicate filter) { - this(table, vectorFieldId, indexType, partitionFilter, filter, null); + this(table, vectorFieldId, partitionFilter, filter, null); } PrimaryKeyVectorScan( FileStoreTable table, int vectorFieldId, - String indexType, @Nullable PartitionPredicate partitionFilter, @Nullable Predicate filter, @Nullable Snapshot pinnedSnapshot) { this.table = table; this.vectorFieldId = vectorFieldId; - this.indexType = indexType; this.partitionFilter = partitionFilter; this.filter = filter; this.pinnedSnapshot = pinnedSnapshot; @@ -132,7 +128,7 @@ public Plan scan() { indexFileHandler.scan( snapshot, entry -> - entry.indexFile().indexType().equals(indexType) + CoreOptions.isVectorIndexType(entry.indexFile().indexType()) && entry.indexFile().globalIndexMeta() != null && entry.indexFile().globalIndexMeta().sourceMeta() != null && entry.indexFile().globalIndexMeta().indexFieldId() diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java index 297492aed077..200a8d593fe2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java @@ -133,13 +133,7 @@ public VectorScan newVectorScan() { rejectUnderQueryAuth(); if (isPrimaryKeyVectorSearch()) { return new PrimaryKeyVectorScan( - table, - vectorColumn.id(), - VectorIndexOptions.indexType( - table.coreOptions().toConfiguration(), vectorColumn.name()), - partitionFilter, - filter, - pinnedSnapshot); + table, vectorColumn.id(), partitionFilter, filter, pinnedSnapshot); } return new DataEvolutionVectorScan( table, partitionFilter, filter, vectorColumn, options, pinnedSnapshot); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorReadTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorReadTest.java index 0164c38cbd05..8d748c130458 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorReadTest.java @@ -20,10 +20,17 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.index.GlobalIndexMeta; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.index.pk.PrimaryKeyIndexSourceFile; +import org.apache.paimon.index.pk.PrimaryKeyIndexSourceMeta; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.manifest.FileSource; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.types.DataField; import org.apache.paimon.types.FloatType; import org.apache.paimon.types.VectorType; +import org.apache.paimon.utils.InstantiationUtil; import org.junit.jupiter.api.Test; @@ -32,6 +39,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -40,12 +48,76 @@ import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; /** Tests global candidate merging for primary-key vector search. */ class PrimaryKeyVectorReadTest { + @Test + void testSnapshotParametersSurviveSerializationAndBoundGlobalCandidates() throws Exception { + Map options = new HashMap<>(); + options.put("global-index.future-vector.index-columns", "vector"); + options.put("future-vector.refine_factor", "invalid"); + options.put("test-vector-ann.refine_factor", "3"); + FileStoreTable table = mock(FileStoreTable.class); + when(table.coreOptions()).thenReturn(new CoreOptions(options)); + PrimaryKeyVectorRead read = + new PrimaryKeyVectorRead( + table, + new DataField(1, "vector", new VectorType(2, new FloatType())), + new float[] {0, 0}, + 1, + Collections.emptyMap()); + BucketVectorSearchSplit indexed = mock(BucketVectorSearchSplit.class); + DataFileMeta dataFile = mock(DataFileMeta.class); + when(dataFile.fileName()).thenReturn("data"); + when(dataFile.rowCount()).thenReturn(2L); + when(dataFile.level()).thenReturn(1); + when(dataFile.fileSource()).thenReturn(Optional.of(FileSource.COMPACT)); + DataSplit dataSplit = mock(DataSplit.class); + when(dataSplit.dataFiles()).thenReturn(Collections.singletonList(dataFile)); + when(indexed.dataSplit()).thenReturn(dataSplit); + IndexFileMeta payload = payload("test-vector-ann", "data"); + IndexFileMeta stale = payload("obsolete-vector", "removed"); + when(indexed.payloadFiles()).thenReturn(Arrays.asList(payload, stale)); + BucketVectorSearchSplit raw = mock(BucketVectorSearchSplit.class); + + PrimaryKeyVectorRead.SearchParameters parameters = + InstantiationUtil.deserializeObject( + InstantiationUtil.serializeObject( + read.searchParameters(Arrays.asList(indexed, raw))), + getClass().getClassLoader()); + PrimaryKeyVectorRead.SearchResult result = + read.mergeSearchResults( + Collections.singletonList( + new PrimaryKeyVectorRead.SearchResult( + Arrays.asList( + candidate(0, "data", 0, 1F), + candidate(1, "data", 0, 2F), + candidate(2, "data", 0, 3F), + candidate(3, "data", 0, 4F)), + Collections.emptyList())), + parameters); + assertThat(result.indexedCandidates()) + .extracting(PrimaryKeyVectorRead.Candidate::bucket) + .containsExactly(0, 1, 2); + // Configuration is used only when the entire plan has no index payloads. + assertThatThrownBy(() -> read.searchParameters(Collections.singletonList(raw))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid vector refine factor"); + when(indexed.payloadFiles()).thenReturn(Collections.singletonList(stale)); + assertThatThrownBy(() -> read.searchParameters(Collections.singletonList(indexed))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid vector refine factor"); + when(indexed.payloadFiles()) + .thenReturn(Arrays.asList(payload, payload("another-vector", "data"))); + assertThatThrownBy(() -> read.searchParameters(Collections.singletonList(indexed))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot mix index types"); + } + @Test void testStartsBucketsBeforeWaitingForResults() throws Exception { BucketVectorSearchSplit split1 = mock(BucketVectorSearchSplit.class); @@ -59,7 +131,11 @@ void testStartsBucketsBeforeWaitingForResults() throws Exception { try { Future resultFuture = - executor.submit(() -> read.searchBuckets(Arrays.asList(split1, split2))); + executor.submit( + () -> + read.searchBuckets( + Arrays.asList(split1, split2), + read.searchParameters(Arrays.asList(split1, split2)))); assertThat(started.await(5, TimeUnit.SECONDS)).isTrue(); future1.complete( @@ -107,6 +183,22 @@ private static PrimaryKeyVectorRead.Candidate candidate( BinaryRow.EMPTY_ROW, bucket, fileName, position, distance); } + private static IndexFileMeta payload(String type, String sourceFile) { + byte[] source = + new PrimaryKeyIndexSourceMeta( + 1, + Collections.singletonList( + new PrimaryKeyIndexSourceFile(sourceFile, 2))) + .serialize(); + return new IndexFileMeta( + type, + type + ".index", + 100, + 2, + new GlobalIndexMeta(0, 1, 1, null, null, source), + null); + } + private static FileStoreTable table() { Map options = new HashMap<>(); options.put("fields.vector.pk-vector.index.type", "test-vector-ann"); @@ -145,7 +237,7 @@ private TestingPrimaryKeyVectorRead( } @Override - protected SearchContext createSearchContext() { + protected SearchContext createSearchContext(SearchParameters parameters) { return null; } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorScanTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorScanTest.java index 9d95be9b0f41..30de45f4e209 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorScanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorScanTest.java @@ -95,7 +95,7 @@ void testPostponeTableOnlyScansRealBuckets() { when(reader.indexFileHandler()).thenReturn(indexFileHandler); configureBatchScan(table, reader, snapshot); - new PrimaryKeyVectorScan(table, 7, "ivf-pq", null, null).scan(); + new PrimaryKeyVectorScan(table, 7, null, null).scan(); verify(reader).onlyReadRealBuckets(); } @@ -124,7 +124,8 @@ void testScansOneSnapshotAndFiltersVectorIdentity() { Arrays.asList( payloadEntry("ivf-pq", 7, "ann-match"), payloadEntry("ivf-pq", 8, "ann-other-field"), - payloadEntry("hnsw", 7, "ann-other-type")); + payloadEntry("hnsw", 7, "ann-other-type"), + payloadEntry("btree", 7, "scalar-index")); IndexFileHandler indexFileHandler = mock(IndexFileHandler.class); when(indexFileHandler.scan(eq(snapshot), any(Filter.class))) .thenAnswer( @@ -142,14 +143,14 @@ void testScansOneSnapshotAndFiltersVectorIdentity() { configureBatchScan(table, snapshotReader, snapshot); PrimaryKeyVectorScan.Plan plan = - new PrimaryKeyVectorScan(table, 7, "ivf-pq", null, null, snapshot).scan(); + new PrimaryKeyVectorScan(table, 7, null, null, snapshot).scan(); assertThat(plan.snapshotId()).isEqualTo(11); assertThat(plan.splits()).hasSize(1); BucketVectorSearchSplit bucketSplit = (BucketVectorSearchSplit) plan.splits().get(0); assertThat(bucketSplit.payloadFiles()) .extracting(IndexFileMeta::fileName) - .containsExactly("ann-match"); + .containsExactly("ann-match", "ann-other-type"); verify(table).copy(Collections.singletonMap(CoreOptions.SCAN_SNAPSHOT_ID.key(), "11")); } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorSearchTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorSearchTest.java index 967349940ef4..3311da65e660 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorSearchTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorSearchTest.java @@ -37,9 +37,13 @@ import org.apache.paimon.types.DataTypes; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -121,6 +125,52 @@ void testRefineFactorReranksAnnCandidates() throws Exception { .containsExactly(0.1F); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testReadsIndexTypeFromSnapshotEntries(boolean refine) throws Exception { + catalog.createTable(identifier(), vectorSchema("deduplicate", true, true), false); + FileStoreTable table = getTableDefault(); + write( + table, + ioManager, + GenericRow.of(1, BinaryVector.fromPrimitiveArray(new float[] {1, 0})), + GenericRow.of(2, BinaryVector.fromPrimitiveArray(new float[] {2, 0})), + GenericRow.of(3, BinaryVector.fromPrimitiveArray(new float[] {3, 0}))); + + Map options = new HashMap<>(); + options.put( + "global-index." + TestVectorGlobalIndexerFactory.IDENTIFIER + ".index-columns", ""); + options.put("global-index.future-vector.index-columns", "embedding"); + // Query parameters must follow the persisted algorithm, including on the batch path. + Map searchOptions = new HashMap<>(); + searchOptions.put("future-vector.refine_factor", "invalid"); + if (refine) { + searchOptions.put(TestVectorGlobalIndexerFactory.IDENTIFIER + ".refine_factor", "3"); + } + FileStoreTable readTable = table.copy(options); + + GlobalIndexResult result = + readTable + .newVectorSearchBuilder() + .withVectorColumn("embedding") + .withVector(new float[] {0, 0}) + .withLimit(1) + .withOptions(searchOptions) + .executeLocal(); + assertThat(readIds(table, result)).containsExactly(refine ? 1 : 3); + + List batch = + readTable + .newBatchVectorSearchBuilder() + .withVectorColumn("embedding") + .withVectors(new float[][] {{0, 0}}) + .withLimit(1) + .withOptions(searchOptions) + .executeBatchLocal(); + assertThat(batch).hasSize(1); + assertThat(readIds(table, batch.get(0))).containsExactly(refine ? 1 : 3); + } + @Test void testVectorSearchMaterializesPhysicalRows() throws Exception { createTableDefault(); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/vectorsearch/FlinkPrimaryKeyVectorRead.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/vectorsearch/FlinkPrimaryKeyVectorRead.java index 613c59465bcf..99e203efd944 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/vectorsearch/FlinkPrimaryKeyVectorRead.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/vectorsearch/FlinkPrimaryKeyVectorRead.java @@ -72,6 +72,7 @@ public GlobalIndexResult read(VectorScan.Plan plan) { if (splits.size() < parallelism * 2L) { return super.read(plan); } + SearchParameters parameters = searchParameters(splits); List serializedSplits = new ArrayList<>(splits.size()); for (BucketVectorSearchSplit split : splits) { @@ -83,12 +84,13 @@ public GlobalIndexResult read(VectorScan.Plan plan) { } List> groups = splitGroups(serializedSplits, parallelism); - List groupResults = executeBucketSearchGroups(groups, parallelism); + List groupResults = executeBucketSearchGroups(groups, parallelism, parameters); List searchResults = new ArrayList<>(groupResults.size()); for (byte[] groupResult : groupResults) { searchResults.add(deserializeSearchResult(groupResult)); } - return createResult(primaryKeyPlan, mergeSearchResults(searchResults)); + return createResult( + primaryKeyPlan, mergeSearchResults(searchResults, parameters), parameters); } protected int flinkParallelism() { @@ -100,7 +102,8 @@ protected int flinkParallelism() { : maxParallelism; } - protected List executeBucketSearchGroups(List> groups, int parallelism) { + protected List executeBucketSearchGroups( + List> groups, int parallelism, SearchParameters parameters) { String operatorName = "Primary-Key Vector Search"; List serializedGroups = new ArrayList<>(groups.size()); try { @@ -121,7 +124,9 @@ protected List executeBucketSearchGroups(List> groups, int .rebalance() .map( (MapFunction) - bytes -> executeBucketSearchGroup(deserializeGroup(bytes))) + bytes -> + executeBucketSearchGroup( + deserializeGroup(bytes), parameters)) .returns(PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO) .name(operatorName) .setParallelism(Math.min(parallelism, groups.size())); @@ -140,13 +145,13 @@ protected String flinkJobName(String operatorName) { return "Vector Search - " + operatorName + " : " + table.fullName(); } - protected byte[] executeBucketSearchGroup(List group) { + protected byte[] executeBucketSearchGroup(List group, SearchParameters parameters) { List taskSplits = new ArrayList<>(group.size()); for (byte[] bytes : group) { taskSplits.add(deserializeSplit(bytes)); } try { - return InstantiationUtil.serializeObject(searchBuckets(taskSplits)); + return InstantiationUtil.serializeObject(searchBuckets(taskSplits, parameters)); } catch (IOException e) { throw new RuntimeException("Failed to serialize primary-key vector candidates.", e); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/vectorsearch/FlinkPrimaryKeyVectorReadTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/vectorsearch/FlinkPrimaryKeyVectorReadTest.java index e09dfdaec306..2c5ee0721612 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/vectorsearch/FlinkPrimaryKeyVectorReadTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/vectorsearch/FlinkPrimaryKeyVectorReadTest.java @@ -103,6 +103,7 @@ private static class TestingFlinkPrimaryKeyVectorRead extends FlinkPrimaryKeyVec private final List dispatchedBuckets = new ArrayList<>(); private int operatorParallelism; private boolean createResultCalled; + private SearchParameters planParameters; private TestingFlinkPrimaryKeyVectorRead(List splits) { super( @@ -135,14 +136,19 @@ protected int flinkParallelism() { @Override protected List executeBucketSearchGroups( - List> groups, int parallelism) { + List> groups, int parallelism, SearchParameters parameters) { + planParameters = parameters; operatorParallelism = parallelism; taskGroupSizes.addAll(groups.stream().map(List::size).collect(Collectors.toList())); - return groups.stream().map(this::executeBucketSearchGroup).collect(Collectors.toList()); + return groups.stream() + .map(group -> executeBucketSearchGroup(group, parameters)) + .collect(Collectors.toList()); } @Override - protected SearchResult searchBuckets(List taskSplits) { + protected SearchResult searchBuckets( + List taskSplits, SearchParameters parameters) { + assertThat(parameters).isSameAs(planParameters); dispatchedBuckets.addAll( taskSplits.stream() .map(split -> split.dataSplit().bucket()) @@ -152,7 +158,10 @@ protected SearchResult searchBuckets(List taskSplits) { @Override protected GlobalIndexResult createResult( - PrimaryKeyVectorScan.Plan plan, PrimaryKeyVectorRead.SearchResult searchResult) { + PrimaryKeyVectorScan.Plan plan, + PrimaryKeyVectorRead.SearchResult searchResult, + SearchParameters parameters) { + assertThat(parameters).isSameAs(planParameters); assertThat(plan).isSameAs(primaryKeyPlan); assertThat(searchResult.indexedCandidates()).isEmpty(); assertThat(searchResult.exactCandidates()).isEmpty(); diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkPrimaryKeyVectorRead.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkPrimaryKeyVectorRead.java index 3e4a35f142d6..bbe7bf3dc5d5 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkPrimaryKeyVectorRead.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkPrimaryKeyVectorRead.java @@ -70,6 +70,7 @@ public GlobalIndexResult read(VectorScan.Plan plan) { if (splits.size() < parallelism * 2) { return super.read(plan); } + SearchParameters parameters = searchParameters(splits); List serializedSplits = new ArrayList<>(splits.size()); for (BucketVectorSearchSplit split : splits) { @@ -87,7 +88,8 @@ public GlobalIndexResult read(VectorScan.Plan plan) { taskSplits.add(deserializeSplit(bytes)); } try { - return InstantiationUtil.serializeObject(searchBuckets(taskSplits)); + return InstantiationUtil.serializeObject( + searchBuckets(taskSplits, parameters)); } catch (IOException e) { throw new RuntimeException( "Failed to serialize primary-key vector candidates.", e); @@ -98,7 +100,8 @@ public GlobalIndexResult read(VectorScan.Plan plan) { for (byte[] groupResult : groupResults) { searchResults.add(deserializeSearchResult(groupResult)); } - return createResult(primaryKeyPlan, mergeSearchResults(searchResults)); + return createResult( + primaryKeyPlan, mergeSearchResults(searchResults, parameters), parameters); } protected int sparkParallelism() { diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PrimaryKeyVectorSearchTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PrimaryKeyVectorSearchTest.scala index 80806858289c..c020412b8876 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PrimaryKeyVectorSearchTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PrimaryKeyVectorSearchTest.scala @@ -53,11 +53,17 @@ class PrimaryKeyVectorSearchTest extends PaimonSparkTestBase { | (4, array(4.0f, 0.0f)) |""".stripMargin) - val builder = new SparkVectorSearchBuilderImpl(loadTable("T")) + val table = loadTable("T").copy( + Map( + s"global-index.${TestVectorGlobalIndexerFactory.IDENTIFIER}.index-columns" -> "", + "global-index.future-vector.index-columns" -> "embedding").asJava) + val builder = new SparkVectorSearchBuilderImpl(table) builder .withVectorColumn("embedding") .withVector(Array(0.0f, 0.0f)) .withLimit(2) + .withOption("future-vector.refine_factor", "invalid") + .withOption(s"${TestVectorGlobalIndexerFactory.IDENTIFIER}.refine_factor", "2") val jobGroup = s"primary-key-vector-${System.nanoTime()}" spark.sparkContext.setJobGroup(jobGroup, jobGroup) From 913093b2ef7a5fa57f8a310488741ded9cee8119 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 1 Nov 2026 12:22:07 +0800 Subject: [PATCH 11/14] Return vector index type and options from one resolver --- .../paimon/options/VectorIndexOptions.java | 75 ++++++++++------- .../paimon/globalindex/GlobalIndexer.java | 2 +- .../pk/BucketedPrimaryKeyIndexMaintainer.java | 9 +- .../index/pk/PrimaryKeyIndexDefinitions.java | 10 +-- .../AbstractDataEvolutionVectorRead.java | 20 ++--- .../table/source/DataEvolutionVectorScan.java | 5 +- .../table/source/PrimaryKeyVectorRead.java | 49 +++++------ .../apache/paimon/GlobalIndexOptionsTest.java | 19 ++++- .../pk/PrimaryKeyIndexDefinitionsTest.java | 5 ++ .../PrimaryKeyVectorIndexOptionsTest.java | 21 +++-- .../options/VectorIndexOptionsTest.java | 82 ++++++++++++++----- .../source/PrimaryKeyVectorSearchTest.java | 69 ++++++++++++++++ paimon-python/dev/native-vector-type-api.md | 8 ++ .../common/options/vector_index_options.py | 37 ++++----- .../execution/native/index_options.py | 21 +++-- .../native/read/native_search_plan.py | 26 +++--- .../read/search/primary_key_vector_read.py | 6 +- .../read/search/primary_key_vector_scan.py | 5 +- .../python/read/search/vector_search_read.py | 9 +- .../python/read/search/vector_search_scan.py | 2 +- .../globalindex/create_global_index.py | 2 +- .../index/pk/primary_key_index_definitions.py | 4 +- .../pypaimon/tests/global_index_test.py | 13 +-- .../tests/vector_index_options_test.py | 60 +++++++++----- 24 files changed, 370 insertions(+), 189 deletions(-) diff --git a/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java b/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java index 294e7ba80acb..80f1c4e7c258 100644 --- a/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/options/VectorIndexOptions.java @@ -22,6 +22,7 @@ import javax.annotation.Nullable; +import java.io.Serializable; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -29,11 +30,34 @@ import java.util.stream.Collectors; /** Vector parameter resolution shared by primary-key and Data Evolution indexes. */ -public final class VectorIndexOptions { +public final class VectorIndexOptions implements Serializable { + + private static final long serialVersionUID = 1L; private static final String LEGACY_INDEX_COLUMNS = "pk-vector.index.columns"; - private VectorIndexOptions() {} + @Nullable private final String type; + private final Options options; + private final String metric; + + private VectorIndexOptions(@Nullable String type, Options options, String metric) { + this.type = type; + this.options = options; + this.metric = metric; + } + + @Nullable + public String type() { + return type; + } + + public Options options() { + return options; + } + + public String metric() { + return metric; + } public static List columns(Options options) { CoreOptions core = new CoreOptions(options); @@ -80,36 +104,38 @@ private static List indexTypes(CoreOptions core) { .collect(Collectors.toList()); } - @Nullable - public static String indexType(Options options, String column) { - CoreOptions core = new CoreOptions(options); - List declared = - indexTypes(core).stream() - .filter(type -> core.globalIndexColumns(type).contains(column)) - .collect(Collectors.toList()); - if (!declared.isEmpty()) { - return declared.size() == 1 ? declared.get(0) : null; - } - return legacyIndexType(options, column); - } - @Nullable private static String legacyIndexType(Options options, String column) { String type = options.get("fields." + column + ".pk-vector.index.type"); return type == null ? null : type.toLowerCase(Locale.ROOT).trim(); } - public static Options resolve(Options options, @Nullable String type, String column) { + public static VectorIndexOptions resolve( + Options options, @Nullable String type, String column) { return resolve(options, type, column, true); } /** Retains an unspecified metric when readers must obtain it from persisted index metadata. */ - public static Options resolve( + public static VectorIndexOptions resolve( Options options, @Nullable String type, String column, boolean defaultMetric) { if (type == null) { - type = indexType(options, column); + CoreOptions core = new CoreOptions(options); + List declared = + indexTypes(core).stream() + .filter( + candidate -> + core.globalIndexColumns(candidate).contains(column)) + .collect(Collectors.toList()); + type = + declared.isEmpty() + ? legacyIndexType(options, column) + : (declared.size() == 1 ? declared.get(0) : null); if (type == null) { - return new Options(options.toMap(), options); + return new VectorIndexOptions( + null, + new Options(options.toMap(), options), + normalizeMetric( + options.get("fields." + column + ".pk-vector.distance.metric"))); } } String prefix = type + "."; @@ -150,16 +176,7 @@ public static Options resolve( String normalized = normalizeMetric(metric); aliases.forEach(key -> resolved.setString(key, normalized)); } - return resolved; - } - - public static String metric(Options options, @Nullable String type, String column) { - if (type == null) { - type = indexType(options, column); - } - return type == null - ? normalizeMetric(options.get("fields." + column + ".pk-vector.distance.metric")) - : resolve(options, type, column).get(type + ".metric"); + return new VectorIndexOptions(type, resolved, normalizeMetric(metric)); } private static String normalizeMetric(@Nullable String metric) { diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexer.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexer.java index b9d84693fd83..2b74c433bbe2 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexer.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexer.java @@ -58,7 +58,7 @@ static GlobalIndexer create(String type, List indexFields, Options op String column = indexFields.get(0).name(); resolved = CoreOptions.isVectorIndexType(type) - ? VectorIndexOptions.resolve(options, type, column) + ? VectorIndexOptions.resolve(options, type, column).options() : new CoreOptions(options).globalIndexOptions(type, column); } return GlobalIndexerFactoryUtils.load(type).create(indexFields, resolved); diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainer.java b/paimon-core/src/main/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainer.java index 92a9513a91cd..0c2167e8b03b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainer.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainer.java @@ -371,10 +371,11 @@ public static Factory create( handler, readerFactoryBuilder, field, - VectorIndexOptions.metric( - definition.options(), - definition.indexType(), - definition.column()), + VectorIndexOptions.resolve( + definition.options(), + definition.indexType(), + definition.column()) + .metric(), definition.indexType()) .withIndexOptions(definition.options()); break; diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java b/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java index cca199e9ee67..e0c16eec8a0b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java @@ -95,16 +95,14 @@ public static PrimaryKeyIndexDefinitions create(TableSchema schema) { options.multiValueIndexOptions(column), PrimaryKeyIndexDefinition.Family.MULTI_VALUE)); } else if (vectorColumns.contains(column)) { + VectorIndexOptions resolved = + VectorIndexOptions.resolve(options.toConfiguration(), null, column); definitions.add( new PrimaryKeyIndexDefinition( column, field.id(), - VectorIndexOptions.indexType(options.toConfiguration(), column), - VectorIndexOptions.resolve( - options.toConfiguration(), - VectorIndexOptions.indexType( - options.toConfiguration(), column), - column), + resolved.type(), + resolved.options(), PrimaryKeyIndexDefinition.Family.VECTOR)); } else if (fullTextColumns.contains(column)) { definitions.add( diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java index 761c883aa574..e3206db3dec1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java @@ -783,15 +783,15 @@ private String configuredRawSearchMetric(Map options) { String fieldPrefix = "fields." + vectorColumn.name() + "."; if (options.containsKey(fieldPrefix + "index-options")) { CoreOptions configured = new CoreOptions(new Options(table.options(), options)); - String type = - VectorIndexOptions.indexType(configured.toConfiguration(), vectorColumn.name()); - if (type == null) { - type = "vector"; - } - String configuredMetric = + VectorIndexOptions resolved = VectorIndexOptions.resolve( - configured.toConfiguration(), type, vectorColumn.name(), false) - .get(type + ".metric"); + configured.toConfiguration(), null, vectorColumn.name(), false); + if (resolved.type() == null) { + resolved = + VectorIndexOptions.resolve( + configured.toConfiguration(), "vector", vectorColumn.name(), false); + } + String configuredMetric = resolved.options().get(resolved.type() + ".metric"); if (configuredMetric != null) { return configuredMetric; } @@ -842,14 +842,14 @@ private static boolean isRawSearchMetric(String metric) { } protected int configuredRefineFactor(String indexType) { - Options resolved = + VectorIndexOptions resolved = VectorIndexOptions.resolve( new Options( table == null ? Collections.emptyMap() : table.options(), options), indexType, vectorColumn.name()); return VectorSearchRefineOptions.resolve( - options, resolved.toMap(), vectorColumn.name(), indexType); + options, resolved.options().toMap(), vectorColumn.name(), resolved.type()); } private static IndexFileMeta firstVectorIndexFile(List splits) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java index e7c38e82bbb7..d1e7ba58a0b2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java @@ -294,8 +294,9 @@ private String vectorIndexType(List indexFiles) { @Nullable private String configuredVectorIndexType() { - return VectorIndexOptions.indexType( - new Options(table.options(), options), vectorColumn.name()); + return VectorIndexOptions.resolve( + new Options(table.options(), options), null, vectorColumn.name(), false) + .type(); } private static boolean containsField(GlobalIndexMeta meta, int fieldId) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java index 577f5c66138e..08faf9e3140f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java @@ -74,7 +74,6 @@ import java.util.stream.Collectors; import static org.apache.paimon.CoreOptions.GLOBAL_INDEX_THREAD_NUM; -import static org.apache.paimon.globalindex.VectorSearchMetric.normalize; import static org.apache.paimon.utils.Preconditions.checkArgument; /** Executes bucket-local primary-key vector search and merges physical candidates globally. */ @@ -206,22 +205,16 @@ protected SearchParameters searchParameters(List splits } } Options tableOptions = table.coreOptions().toConfiguration(); - if (indexType == null) { - indexType = VectorIndexOptions.indexType(tableOptions, vectorField.name()); - } - Options indexOptions = + VectorIndexOptions resolved = VectorIndexOptions.resolve(tableOptions, indexType, vectorField.name()); - String metric = - normalize(VectorIndexOptions.metric(indexOptions, indexType, vectorField.name())); int refineFactor = VectorSearchRefineOptions.resolve( - searchOptions, indexOptions.toMap(), vectorField.name(), indexType); + searchOptions, + resolved.options().toMap(), + vectorField.name(), + resolved.type()); return new SearchParameters( - indexType, - indexOptions, - metric, - refineFactor, - VectorSearchRefineOptions.searchLimit(limit, refineFactor)); + resolved, refineFactor, VectorSearchRefineOptions.searchLimit(limit, refineFactor)); } protected SearchResult searchBuckets( @@ -293,13 +286,14 @@ protected GlobalIndexResult createResult( List indexedCandidates = topK(searchResult.indexedCandidates(), parameters.indexedLimit); if (parameters.refineFactor > 0 && !indexedCandidates.isEmpty()) { - indexedCandidates = rerank(plan, indexedCandidates, parameters.metric); + indexedCandidates = rerank(plan, indexedCandidates, parameters.vectorOptions.metric()); } else { indexedCandidates = topK(indexedCandidates, limit); } List candidates = new ArrayList<>(indexedCandidates); candidates.addAll(topK(searchResult.exactCandidates(), limit)); - return new PrimaryKeyVectorResult(plan, topK(candidates, limit), parameters.metric); + return new PrimaryKeyVectorResult( + plan, topK(candidates, limit), parameters.vectorOptions.metric()); } protected SearchResult mergeSearchResults( @@ -331,7 +325,10 @@ CompletableFuture> searchBatchAsync( .collect(Collectors.toList()); PkVectorBucketIndexState state = PkVectorBucketIndexState.fromActiveDataFiles( - vectorField.id(), parameters.indexType, activeFiles, split.payloadFiles()); + vectorField.id(), + parameters.vectorOptions.type(), + activeFiles, + split.payloadFiles()); Map deletionVectors = deletionVectors(dataSplit, context.fileIO); PkVectorDataFileReader.Factory readerFactory = new PkVectorDataFileReader.Factory( @@ -346,15 +343,15 @@ CompletableFuture> searchBatchAsync( context.indexFileHandler.pkVectorAnnSegment( dataSplit.partition(), dataSplit.bucket()), vectorField, - parameters.indexOptions, - parameters.metric, + parameters.vectorOptions.options(), + parameters.vectorOptions.metric(), context.executor); PrimaryKeyVectorBucketSearch bucketSearch = new PrimaryKeyVectorBucketSearch( readerFactory, annSearcher, searchOptions, - parameters.metric, + parameters.vectorOptions.metric(), table.coreOptions().vectorIndexSearchMode()); return bucketSearch .searchBatchAsync( @@ -731,21 +728,13 @@ protected static class SearchParameters implements Serializable { private static final long serialVersionUID = 1L; - @Nullable private final String indexType; - private final Options indexOptions; - private final String metric; + private final VectorIndexOptions vectorOptions; private final int refineFactor; private final int indexedLimit; private SearchParameters( - @Nullable String indexType, - Options indexOptions, - String metric, - int refineFactor, - int indexedLimit) { - this.indexType = indexType; - this.indexOptions = indexOptions; - this.metric = metric; + VectorIndexOptions vectorOptions, int refineFactor, int indexedLimit) { + this.vectorOptions = vectorOptions; this.refineFactor = refineFactor; this.indexedLimit = indexedLimit; } diff --git a/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java index 3d1a9294b5f8..b909074e37cb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/GlobalIndexOptionsTest.java @@ -108,7 +108,10 @@ void testColumnDeclarationsAndLegacyFallback(String type, String legacyFamily) { if ("ivf-pq".equals(type)) { assertThat(VectorIndexOptions.columns(options.toConfiguration())) .containsExactly("first", "second"); - assertThat(VectorIndexOptions.indexType(options.toConfiguration(), "first")) + assertThat( + VectorIndexOptions.resolve( + options.toConfiguration(), null, "first", false) + .type()) .isEqualTo("ivf-pq"); } } @@ -165,6 +168,7 @@ void testLegacyPrimaryKeyMetricSurvivesFactoryResolution(String metric) { assertThat( VectorIndexOptions.resolve( definition.options(), definition.indexType(), "embedding") + .options() .toMap()) .isEqualTo(definition.options().toMap()); assertThat( @@ -216,6 +220,7 @@ void testUnrelatedFieldOptionsDoNotRequestDefaultMetric() { assertThat( VectorIndexOptions.resolve( options.toConfiguration(), "ivf-pq", "embedding", false) + .options() .get("ivf-pq.metric")) .isNull(); } @@ -230,10 +235,13 @@ void testDocumentedVectorOptionsAndFieldOverrides() { values.put("fields.embedding.nlist", "48"); values.put("fields.embedding.distance.metric", "cosine"); CoreOptions options = new CoreOptions(values); - assertThat(VectorIndexOptions.metric(options.toConfiguration(), null, "embedding")) + assertThat( + VectorIndexOptions.resolve(options.toConfiguration(), null, "embedding") + .metric()) .isEqualTo("cosine"); assertThat( VectorIndexOptions.resolve(options.toConfiguration(), null, "embedding") + .options() .get("ivf-pq.dimension")) .isEqualTo("768"); @@ -241,9 +249,12 @@ void testDocumentedVectorOptionsAndFieldOverrides() { "fields.embedding.index-options", "{\"nlist\":64,\"distance.metric\":\"inner_product\"}"); options = new CoreOptions(values); - assertThat(VectorIndexOptions.metric(options.toConfiguration(), null, "embedding")) + assertThat( + VectorIndexOptions.resolve(options.toConfiguration(), null, "embedding") + .metric()) .isEqualTo("inner_product"); - Options resolved = VectorIndexOptions.resolve(options.toConfiguration(), null, "embedding"); + Options resolved = + VectorIndexOptions.resolve(options.toConfiguration(), null, "embedding").options(); assertThat(resolved.get("ivf-pq.nlist")).isEqualTo("64"); assertThat(resolved.get("fields.embedding.nlist")).isEqualTo("64"); assertThat(resolved.get("ivf-pq.metric")).isEqualTo("inner_product"); diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitionsTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitionsTest.java index d28f8b62d694..f3c98176801e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitionsTest.java @@ -47,6 +47,9 @@ void testCreatesMixedDefinitionsInSchemaFieldOrder(boolean sharedOptions) { if (!sharedOptions) { options.put("fields.embedding.pk-vector.index.type", "ivf-pq"); } + options.put( + "fields.embedding." + (sharedOptions ? "index-options" : "pk-vector.index.options"), + "{\"nlist\":64,\"metric\":\"cosine\"}"); options.put(columnsKey("btree", sharedOptions), "name"); options.put(columnsKey("bitmap", sharedOptions), "status"); options.put(columnsKey("multivalue", sharedOptions), "tags"); @@ -70,6 +73,8 @@ void testCreatesMixedDefinitionsInSchemaFieldOrder(boolean sharedOptions) { assertThat(definitions) .extracting(PrimaryKeyIndexDefinition::indexType) .containsExactly("btree", "bitmap", "ivf-pq", "multivalue"); + assertThat(definitions.get(2).options().get("ivf-pq.nlist")).isEqualTo("64"); + assertThat(definitions.get(2).options().get("ivf-pq.metric")).isEqualTo("cosine"); } @ParameterizedTest diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java index bbda51eb075e..f1cd6d47bc7c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PrimaryKeyVectorIndexOptionsTest.java @@ -64,7 +64,10 @@ void testIndexTypeMustBeFieldScoped() { options.put("pk-vector.index.columns", "embedding"); options.put("pk-vector.index.type", "ivf-pq"); - assertThat(VectorIndexOptions.indexType(new Options(options), "embedding")).isNull(); + assertThat( + VectorIndexOptions.resolve(new Options(options), null, "embedding", false) + .type()) + .isNull(); } @Test @@ -76,6 +79,7 @@ void testIndexOptionsMustBeFieldScoped() { assertThat( VectorIndexOptions.resolve(new Options(options), null, "embedding") + .options() .get("ivf-pq.nlist")) .isNull(); } @@ -86,7 +90,7 @@ void testDistanceMetricMustBeFieldScoped() { options.put("pk-vector.index.columns", "embedding"); options.put("pk-vector.distance.metric", "l2"); - assertThat(VectorIndexOptions.metric(new Options(options), null, "embedding")) + assertThat(VectorIndexOptions.resolve(new Options(options), null, "embedding").metric()) .isEqualTo("inner_product"); } @@ -97,7 +101,7 @@ void testFieldScopedDistanceMetricOverridesTableDefault() { options.put("pk-vector.distance.metric", "l2"); options.put("fields.embedding.pk-vector.distance.metric", "cosine"); - assertThat(VectorIndexOptions.metric(new Options(options), null, "embedding")) + assertThat(VectorIndexOptions.resolve(new Options(options), null, "embedding").metric()) .isEqualTo("cosine"); } @@ -109,7 +113,8 @@ void testFieldScopedJsonOptionsOverrideTableDefault() { options.put("pk-vector.index.options", "{\"nlist\":64}"); options.put("fields.embedding.pk-vector.index.options", "{\"nlist\":128}"); - Options resolved = VectorIndexOptions.resolve(new Options(options), null, "embedding"); + Options resolved = + VectorIndexOptions.resolve(new Options(options), null, "embedding").options(); assertThat(resolved.get("ivf-pq.nlist")).isEqualTo("128"); } @@ -121,7 +126,8 @@ void testResolvesShortAndQualifiedAlgorithmOptions() { "{\"nlist\":64,\"ivf-pq.pq.m\":\"8\"," + "\"fields.embedding.hnsw.m\":16}"); Options resolved = - VectorIndexOptions.resolve(coreOptions.toConfiguration(), null, "embedding"); + VectorIndexOptions.resolve(coreOptions.toConfiguration(), null, "embedding") + .options(); assertThat(resolved.get("ivf-pq.nlist")).isEqualTo("64"); assertThat(resolved.get("ivf-pq.pq.m")).isEqualTo("8"); @@ -134,7 +140,10 @@ void testRejectsNonObjectOptions() { assertThatThrownBy( () -> VectorIndexOptions.resolve( - coreOptions("[1,2]").toConfiguration(), null, "embedding")) + coreOptions("[1,2]").toConfiguration(), + null, + "embedding") + .options()) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("pk-vector.index.options") .hasMessageContaining("JSON object"); diff --git a/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java index 012959c785af..6dbd5bf27886 100644 --- a/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/options/VectorIndexOptionsTest.java @@ -34,6 +34,37 @@ /** Tests for vector parameter resolution shared by both table modes. */ class VectorIndexOptionsTest { + @ParameterizedTest + @ValueSource( + strings = { + "pk-vector.index.columns", + "global-index.ivf-pq.index-column", + "global-index.ivf-pq.index-columns" + }) + void testResolutionReturnsMatchingTypeAndOptions(String declaration) { + Map values = new HashMap<>(); + values.put(declaration, "embedding"); + values.put("fields.embedding.pk-vector.index.type", " IVF-PQ "); + values.put("ivf-pq.dimension", "768"); + String suffix = declaration.startsWith("pk-") ? "pk-vector.index.options" : "index-options"; + values.put("fields.embedding." + suffix, "{\"nlist\":64,\"metric\":\"COSINE\"}"); + Map original = new HashMap<>(values); + + VectorIndexOptions resolved = + VectorIndexOptions.resolve(new Options(values), null, "embedding"); + assertThat(resolved.type()).isEqualTo("ivf-pq"); + assertThat(resolved.options().get("ivf-pq.nlist")).isEqualTo("64"); + assertThat(resolved.options().get("ivf-pq.dimension")).isEqualTo("768"); + assertThat(resolved.metric()).isEqualTo("cosine"); + assertThat(values).isEqualTo(original); + + VectorIndexOptions persisted = + VectorIndexOptions.resolve(new Options(values), "ivf-flat", "embedding", false); + assertThat(persisted.type()).isEqualTo("ivf-flat"); + assertThat(persisted.options().get("ivf-flat.nlist")).isEqualTo("64"); + assertThat(persisted.metric()).isEqualTo("cosine"); + } + @ParameterizedTest @ValueSource(strings = {"index-column", "index-columns"}) void testLegacyRegistrationIsOnlyACompatibilityFallback(String suffix) { @@ -42,7 +73,8 @@ void testLegacyRegistrationIsOnlyACompatibilityFallback(String suffix) { input.setString("fields.old.pk-vector.index.type", "ivf-pq"); input.setString("fields.other_old.pk-vector.index.type", "diskann"); assertThat(VectorIndexOptions.columns(input)).containsExactly("old", "other_old"); - assertThat(VectorIndexOptions.indexType(input, "old")).isEqualTo("ivf-pq"); + assertThat(VectorIndexOptions.resolve(input, null, "old", false).type()) + .isEqualTo("ivf-pq"); assertThat(new CoreOptions(input).vectorIndexColumns("ivf-pq")).containsExactly("old"); assertThat(new CoreOptions(input).vectorIndexColumns("diskann")) .containsExactly("other_old"); @@ -53,8 +85,10 @@ void testLegacyRegistrationIsOnlyACompatibilityFallback(String suffix) { input.setString("global-index.diskann." + suffix, "video"); assertThat(VectorIndexOptions.columns(input)) .containsExactly("video", "embedding", "other"); - assertThat(VectorIndexOptions.indexType(input, "embedding")).isEqualTo("ivf-pq"); - assertThat(VectorIndexOptions.indexType(input, "video")).isEqualTo("diskann"); + assertThat(VectorIndexOptions.resolve(input, null, "embedding", false).type()) + .isEqualTo("ivf-pq"); + assertThat(VectorIndexOptions.resolve(input, null, "video", false).type()) + .isEqualTo("diskann"); assertThat(new CoreOptions(input).vectorIndexColumns("ivf-pq")) .containsExactly("embedding", "other"); assertThat(new CoreOptions(input).vectorIndexColumns("diskann")).containsExactly("video"); @@ -72,13 +106,15 @@ void testLegacyRegistrationIsOnlyACompatibilityFallback(String suffix) { }) void testRemovedTypeAliasesAreIgnored(String key) { Options input = Options.fromMap(Collections.singletonMap(key, "ivf-pq")); - assertThat(VectorIndexOptions.indexType(input, "embedding")).isNull(); - assertThat(VectorIndexOptions.resolve(input, null, "embedding", false).toMap()) + assertThat(VectorIndexOptions.resolve(input, null, "embedding", false).type()).isNull(); + assertThat(VectorIndexOptions.resolve(input, null, "embedding", false).options().toMap()) .isEqualTo(input.toMap()); input.setString("fields.embedding.pk-vector.index.type", "diskann"); - assertThat(VectorIndexOptions.indexType(input, "embedding")).isEqualTo("diskann"); + assertThat(VectorIndexOptions.resolve(input, null, "embedding", false).type()) + .isEqualTo("diskann"); input.setString("global-index.ivf-flat.index-columns", "embedding"); - assertThat(VectorIndexOptions.indexType(input, "embedding")).isEqualTo("ivf-flat"); + assertThat(VectorIndexOptions.resolve(input, null, "embedding", false).type()) + .isEqualTo("ivf-flat"); } @Test @@ -88,12 +124,13 @@ void testLegacyPrimaryKeyTypeResolvesFieldParameters() { values.put("ivf-pq.dimension", "768"); values.put("fields.embedding.index-options", "{\"nlist\":64,\"metric\":\"COSINE\"}"); Options input = new Options(values); - assertThat(VectorIndexOptions.indexType(input, "embedding")).isEqualTo("ivf-pq"); - Options resolved = VectorIndexOptions.resolve(input, null, "embedding", false); + assertThat(VectorIndexOptions.resolve(input, null, "embedding", false).type()) + .isEqualTo("ivf-pq"); + Options resolved = VectorIndexOptions.resolve(input, null, "embedding", false).options(); assertMetric(resolved, "cosine"); assertThat(resolved.get("ivf-pq.dimension")).isEqualTo("768"); assertThat(resolved.get("ivf-pq.nlist")).isEqualTo("64"); - assertThat(VectorIndexOptions.resolve(resolved, null, "embedding", false).toMap()) + assertThat(VectorIndexOptions.resolve(resolved, null, "embedding", false).options().toMap()) .isEqualTo(resolved.toMap()); assertThat(input.toMap()).isEqualTo(values); } @@ -104,7 +141,7 @@ void testAmbiguousDeclarationsDoNotFallBackToHistoricalHint() { options.setString("global-index.ivf-pq.index-columns", "embedding"); options.setString("global-index.diskann.index-column", "embedding"); options.setString("index_type", "ivf-flat"); - assertThat(VectorIndexOptions.indexType(options, "embedding")).isNull(); + assertThat(VectorIndexOptions.resolve(options, null, "embedding", false).type()).isNull(); } @ParameterizedTest @@ -123,11 +160,12 @@ void testLegacyAndDlfMetricAliases(String key, String value, String expected) { values.put("ivf-pq.dimension", "768"); values.put("fields.embedding.nlist", "48"); values.put(key, value); - Options resolved = VectorIndexOptions.resolve(new Options(values), null, "embedding"); + Options resolved = + VectorIndexOptions.resolve(new Options(values), null, "embedding").options(); assertMetric(resolved, expected); assertThat(resolved.get("ivf-pq.dimension")).isEqualTo("768"); assertThat(resolved.get("fields.embedding.nlist")).isEqualTo("48"); - assertThat(VectorIndexOptions.resolve(resolved, "ivf-pq", "embedding").toMap()) + assertThat(VectorIndexOptions.resolve(resolved, "ivf-pq", "embedding").options().toMap()) .isEqualTo(resolved.toMap()); assertThat(values.get(key)).isEqualTo(value); } @@ -144,12 +182,16 @@ void testFieldJsonAliasesRemainIdempotent(String suffix, String key) { values.put("fields.embedding.pk-vector.index.type", "ivf-pq"); values.put("fields.embedding.pk-vector.distance.metric", "cosine"); values.put("fields.embedding." + suffix, "{\"" + key + "\":\"COSINE\",\"nlist\":64}"); - Options resolved = VectorIndexOptions.resolve(new Options(values), null, "embedding"); + Options resolved = + VectorIndexOptions.resolve(new Options(values), null, "embedding").options(); assertMetric(resolved, "cosine"); assertThat(resolved.get("ivf-pq.nlist")).isEqualTo("64"); - assertThat(VectorIndexOptions.resolve(resolved, "ivf-pq", "embedding").toMap()) + assertThat(VectorIndexOptions.resolve(resolved, "ivf-pq", "embedding").options().toMap()) .isEqualTo(resolved.toMap()); - assertThat(VectorIndexOptions.resolve(new Options(values), null, "embedding").toMap()) + assertThat( + VectorIndexOptions.resolve(new Options(values), null, "embedding") + .options() + .toMap()) .isEqualTo(resolved.toMap()); } @@ -159,12 +201,14 @@ void testReaderKeepsUnspecifiedMetricAndWriterUsesDefault() { new Options( Collections.singletonMap( "fields.embedding.index-options", "{\"nlist\":64}")); - Options read = VectorIndexOptions.resolve(input, "ivf-pq", "embedding", false); + Options read = VectorIndexOptions.resolve(input, "ivf-pq", "embedding", false).options(); assertThat(read.get("ivf-pq.nlist")).isEqualTo("64"); assertThat(read.get("ivf-pq.metric")).isNull(); assertThat(read.get("fields.embedding.distance.metric")).isNull(); - assertMetric(VectorIndexOptions.resolve(input, "ivf-pq", "embedding"), "inner_product"); - assertThat(VectorIndexOptions.resolve(input, null, "embedding").toMap()) + assertMetric( + VectorIndexOptions.resolve(input, "ivf-pq", "embedding").options(), + "inner_product"); + assertThat(VectorIndexOptions.resolve(input, null, "embedding").options().toMap()) .isEqualTo(input.toMap()); } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorSearchTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorSearchTest.java index 3311da65e660..f23ea1f0a19a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorSearchTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorSearchTest.java @@ -41,6 +41,7 @@ import org.junit.jupiter.params.provider.ValueSource; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -55,6 +56,74 @@ protected Schema schemaDefault() { return vectorSchema("deduplicate", true); } + @ParameterizedTest + @ValueSource(strings = {"legacy", "index-column", "index-columns"}) + void testTypedConfigurationReachesWritesAndRawFallback(String declaration) throws Exception { + Schema base = schemaDefault(); + Map options = new HashMap<>(base.options()); + options.put("vector-index.search-mode", "full"); + String type = TestVectorGlobalIndexerFactory.IDENTIFIER; + options.remove("global-index." + type + ".index-columns"); + options.remove("fields.embedding.index-options"); + if (declaration.equals("legacy")) { + options.put("pk-vector.index.columns", "embedding"); + options.put("fields.embedding.pk-vector.index.type", type); + options.put("fields.embedding.pk-vector.index.options", "{\"metric\":\"l2\"}"); + } else { + options.put("global-index." + type + "." + declaration, "embedding"); + options.put("fields.embedding.index-options", "{\"metric\":\"l2\"}"); + } + catalog.createTable( + identifier(), + new Schema( + base.fields(), + base.partitionKeys(), + base.primaryKeys(), + options, + base.comment()), + false); + FileStoreTable table = getTableDefault(); + write( + table, + ioManager, + GenericRow.of(1, BinaryVector.fromPrimitiveArray(new float[] {3, 0})), + GenericRow.of(2, BinaryVector.fromPrimitiveArray(new float[] {1, 0})), + GenericRow.of(3, BinaryVector.fromPrimitiveArray(new float[] {2, 0}))); + + VectorSearchBuilder builder = + table.newVectorSearchBuilder() + .withVectorColumn("embedding") + .withVector(new float[] {0, 0}) + .withLimit(1); + PrimaryKeyVectorScan.Plan indexed = + (PrimaryKeyVectorScan.Plan) builder.newVectorScan().scan(); + List dataSplits = new ArrayList<>(); + for (VectorSearchSplit split : indexed.splits()) { + BucketVectorSearchSplit bucket = (BucketVectorSearchSplit) split; + assertThat(bucket.payloadFiles()).isNotEmpty(); + assertThat(bucket.payloadFiles()).allMatch(payload -> payload.indexType().equals(type)); + dataSplits.add(bucket.dataSplit()); + } + assertThat(readIds(table, builder.newVectorRead().read(indexed))).containsExactly(2); + + PrimaryKeyVectorScan.Plan raw = + PrimaryKeyVectorScan.plan( + indexed.snapshotId(), dataSplits, Collections.emptyList()); + GlobalIndexResult rawResult = builder.newVectorRead().read(raw); + assertThat(readIds(table, rawResult)).containsExactly(2); + assertThat(((PrimaryKeyVectorResult) rawResult).splits().get(0).scores()) + .containsExactly(0.5F); + List batch = + table.newBatchVectorSearchBuilder() + .withVectorColumn("embedding") + .withVectors(new float[][] {{0, 0}, {4, 0}}) + .withLimit(1) + .newBatchVectorRead() + .readBatch(raw); + assertThat(readIds(table, batch.get(0))).containsExactly(2); + assertThat(readIds(table, batch.get(1))).containsExactly(1); + } + private Schema vectorSchema(String mergeEngine, boolean deletionVectorsEnabled) { return vectorSchema(mergeEngine, deletionVectorsEnabled, false); } diff --git a/paimon-python/dev/native-vector-type-api.md b/paimon-python/dev/native-vector-type-api.md index 4a014a69cee4..eb5a75b8ad1e 100644 --- a/paimon-python/dev/native-vector-type-api.md +++ b/paimon-python/dev/native-vector-type-api.md @@ -72,6 +72,14 @@ column registry and field type pairing are private compatibility inputs; to the type-scoped resolver. Other index accessors also use names without a `primaryKey` prefix. No public method compatibility wrappers are retained. +`VectorIndexOptions.resolve(...)` returns the selected implementation together +with its merged options (Java: `type()` and `options()`; Python: `index_type` +and `options`). There is no separate `indexType`/`index_type` lookup method. +Writers and raw-query fallback use this result; readers pass the implementation +from persisted index metadata when available. `metric()` (Python: `metric`) +provides the normalized metric default without inserting it into query options +when `defaultMetric`/`default_metric` is false. + When an implementation is configured and this method is unavailable, Python raises `AttributeError` before planning, reading, executing, or capturing hybrid routes. It does not fall back to Python execution. SDK configuration and execution diff --git a/paimon-python/pypaimon/common/options/vector_index_options.py b/paimon-python/pypaimon/common/options/vector_index_options.py index c47df4a2141e..e56f76f9581d 100644 --- a/paimon-python/pypaimon/common/options/vector_index_options.py +++ b/paimon-python/pypaimon/common/options/vector_index_options.py @@ -17,11 +17,19 @@ """Vector parameter resolution shared by primary-key and Data Evolution indexes.""" +from dataclasses import dataclass +from typing import Optional + from pypaimon.common.options.core_options import CoreOptions from pypaimon.common.options.options import Options +@dataclass(frozen=True) class VectorIndexOptions: + index_type: Optional[str] + options: Options + metric: str + @staticmethod def columns(options, index_type=None): core = CoreOptions(options) @@ -51,15 +59,6 @@ def columns_option_key(options): def _index_types(core): return [kind for kind in core.global_index_types() if CoreOptions._is_vector_index_type(kind)] - @staticmethod - def index_type(options, column): - core = CoreOptions(options) - declared = [kind for kind in VectorIndexOptions._index_types(core) - if column in core.global_index_columns(kind)] - if declared: - return declared[0] if len(declared) == 1 else None - return VectorIndexOptions._legacy_index_type(options, column) - @staticmethod def _legacy_index_type(options, column): value = options.to_map().get('fields.%s.pk-vector.index.type' % column) @@ -69,9 +68,15 @@ def _legacy_index_type(options, column): def resolve(options, index_type, column, *, default_metric=True): values = dict(options.to_map()) if index_type is None: - index_type = VectorIndexOptions.index_type(options, column) + core = CoreOptions(options) + declared = [kind for kind in VectorIndexOptions._index_types(core) + if column in core.global_index_columns(kind)] + index_type = ((declared[0] if len(declared) == 1 else None) if declared + else VectorIndexOptions._legacy_index_type(options, column)) if index_type is None: - return Options(values) + return VectorIndexOptions( + None, Options(values), + _normalize_metric(values.get('fields.%s.pk-vector.distance.metric' % column))) prefix = index_type + '.' field_prefix = 'fields.%s.' % column aliases = (field_prefix + 'distance.metric', field_prefix + 'metric', @@ -90,15 +95,7 @@ def resolve(options, index_type, column, *, default_metric=True): normalized = _normalize_metric(metric) for key in aliases: resolved[key] = normalized - return Options(resolved) - - @staticmethod - def metric(options, index_type, column): - if index_type is None: - index_type = VectorIndexOptions.index_type(options, column) - if index_type is None: - return _normalize_metric(options.to_map().get('fields.%s.pk-vector.distance.metric' % column)) - return VectorIndexOptions.resolve(options, index_type, column).to_map()[index_type + '.metric'] + return VectorIndexOptions(index_type, Options(resolved), _normalize_metric(metric)) def _normalize_metric(metric): diff --git a/paimon-python/pypaimon/execution/native/index_options.py b/paimon-python/pypaimon/execution/native/index_options.py index 576624496ff0..09006b011aa6 100644 --- a/paimon-python/pypaimon/execution/native/index_options.py +++ b/paimon-python/pypaimon/execution/native/index_options.py @@ -40,7 +40,7 @@ def configure_vector_index_type(native, options, column): """Use the required native API and propagate missing-method errors.""" if column is None: return - index_type = VectorIndexOptions.index_type(Options(options), column) + index_type = VectorIndexOptions.resolve(Options(options), None, column, default_metric=False).index_type if index_type is not None: native.with_vector_index_type(column, index_type) @@ -68,9 +68,12 @@ def _resolve(schema): definitions = [(family, core.global_index_columns(family)) for family in families] definitions += [(kind, core.global_index_columns(kind)) for kind in declarations if kind not in families] + vector_options = {} if primary_key and not any(kind not in families for kind in declarations): - definitions += [(VectorIndexOptions.index_type(core.options, column), [column]) - for column in VectorIndexOptions.columns(core.options)] + for column in VectorIndexOptions.columns(core.options): + resolved = VectorIndexOptions.resolve(core.options, None, column, default_metric=False) + vector_options[(resolved.index_type, column)] = resolved + definitions.append((resolved.index_type, [column])) known_columns = set() native_columns = {} profiles = {} @@ -86,8 +89,13 @@ def _resolve(schema): result['pk-%s.index.columns' % family] = ','.join(native_columns[family]) for column in columns: known_columns.add(column) - resolved = (VectorIndexOptions.resolve(core.options, kind, column, default_metric=False).to_map() - if vector else core.global_index_options(kind, column).to_map()) + if vector: + parsed = vector_options.get((kind, column)) + if parsed is None: + parsed = VectorIndexOptions.resolve(core.options, kind, column, default_metric=False) + resolved = parsed.options.to_map() + else: + resolved = core.global_index_options(kind, column).to_map() resolved = native_search_options(resolved) profile = {key: value for key, value in resolved.items() if key.startswith(prefix) and value is not None} @@ -95,8 +103,7 @@ def _resolve(schema): if primary_key: if vector: result[field_prefix + 'pk-vector.index.type'] = kind - result[field_prefix + 'pk-vector.distance.metric'] = VectorIndexOptions.metric( - core.options, kind, column) + result[field_prefix + 'pk-vector.distance.metric'] = parsed.metric profile.update({key: value for key, value in resolved.items() if key.startswith(field_prefix) and not key.startswith(field_prefix + 'pk-') and key != field_prefix + 'index-options' and value is not None}) diff --git a/paimon-python/pypaimon/execution/native/read/native_search_plan.py b/paimon-python/pypaimon/execution/native/read/native_search_plan.py index e92c83eb4a5b..a713834a3763 100644 --- a/paimon-python/pypaimon/execution/native/read/native_search_plan.py +++ b/paimon-python/pypaimon/execution/native/read/native_search_plan.py @@ -48,19 +48,17 @@ def _index_search_options(table, column, index_type=None, overrides=None, scalar if column is None: return values core = CoreOptions(Options(values)) - if index_type is None: - index_type = VectorIndexOptions.index_type(Options(values), column) - if index_type is None: - # The Python reader can resolve field options once persisted index metadata is known. - if any(values.get(key) is not None for key in ( - 'fields.%s.index-options' % column, 'fields.%s.pk-vector.index.options' % column)): - return None - return values + if index_type is not None and not CoreOptions._is_vector_index_type(index_type): + return native_search_options(core.global_index_options(index_type, column).to_map()) # Keep an unspecified metric absent so readers can use persisted index metadata. - resolved = (VectorIndexOptions.resolve(Options(values), index_type, column, default_metric=False).to_map() - if CoreOptions._is_vector_index_type(index_type) else - core.global_index_options(index_type, column).to_map()) - return native_search_options(resolved) + resolved = VectorIndexOptions.resolve(Options(values), index_type, column, default_metric=False) + if resolved.index_type is None: + # The Python reader can resolve field options once persisted index metadata is known. + if any(values.get(key) is not None for key in ( + 'fields.%s.index-options' % column, 'fields.%s.pk-vector.index.options' % column)): + return None + return values + return native_search_options(resolved.options.to_map()) def _vector_query_options(options, column, resolved_options): @@ -68,9 +66,9 @@ def _vector_query_options(options, column, resolved_options): values = native_search_options(options) if column is None: return values - index_type = VectorIndexOptions.index_type(Options(resolved_options), column) + index_type = VectorIndexOptions.resolve(Options(resolved_options), None, column, default_metric=False).index_type return native_search_options(VectorIndexOptions.resolve( - Options(values), index_type, column, default_metric=False).to_map()) + Options(values), index_type, column, default_metric=False).options.to_map()) def _range(value): diff --git a/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_read.py b/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_read.py index c00dcd470871..ed8e24a12991 100644 --- a/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_read.py +++ b/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_read.py @@ -42,7 +42,8 @@ def read_plan(self, plan, executor=None): if not isinstance(plan, PrimaryKeyVectorScanPlan): raise ValueError("Primary-key vector read requires a PrimaryKeyVectorScanPlan.") self._index_metric = None - index_type = VectorIndexOptions.index_type(self._table.options.options, self._vector_column.name) + index_type = VectorIndexOptions.resolve( + self._table.options.options, None, self._vector_column.name, default_metric=False).index_type indexed_limit = self._indexed_search_limit(index_type) if indexed_limit > 2147483647: raise ValueError( @@ -170,7 +171,8 @@ def reranked_iter(): def _raw_candidates(self, plan): metric = self._search_metric( - VectorIndexOptions.index_type(self._table.options.options, self._vector_column.name)) + VectorIndexOptions.resolve( + self._table.options.options, None, self._vector_column.name, default_metric=False).index_type) read_table = self._table.copy_without_time_travel({ 'read.native.enabled': 'false', 'scan.native-plan.enabled': 'false'}) read_builder = read_table.new_read_builder().with_projection( diff --git a/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_scan.py b/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_scan.py index 627248b4adb8..c2ca6d03a2e1 100644 --- a/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_scan.py +++ b/paimon-python/pypaimon/execution/python/read/search/primary_key_vector_scan.py @@ -83,8 +83,9 @@ def scan(self): index_type = self._index_type if index_type is None: - index_type = VectorIndexOptions.index_type( - Options(dict(scan_table.table_schema.options)), self._vector_column.name) + index_type = VectorIndexOptions.resolve( + Options(dict(scan_table.table_schema.options)), None, + self._vector_column.name, default_metric=False).index_type entries = IndexFileHandler(table=scan_table).scan( snapshot, lambda entry: _matches(entry, self._vector_column.id, index_type, diff --git a/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py b/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py index ea90b799bdb7..553cb8aad187 100644 --- a/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py +++ b/paimon-python/pypaimon/execution/python/read/search/vector_search_read.py @@ -659,7 +659,7 @@ def _score_refine_splits(self, table_read, splits, queries_by_row, query_vectors def _resolved_index_options(self, index_type): values = dict(_table_options_map(self._table)) values.update(self._options) - return VectorIndexOptions.resolve(Options(values), index_type, self._vector_column.name).to_map() + return VectorIndexOptions.resolve(Options(values), index_type, self._vector_column.name).options.to_map() def _configured_refine_factor(self, index_type): value = _configured_refine_factor( @@ -991,9 +991,10 @@ def _table_options_map(table): def _configured_vector_metric(options, vector_column, index_type=None): - selected_type = index_type or VectorIndexOptions.index_type(Options(options), vector_column.name) or "vector" - metric = VectorIndexOptions.resolve(Options(options), selected_type, vector_column.name, - default_metric=False).to_map().get(selected_type + ".metric") + resolved = VectorIndexOptions.resolve(Options(options), index_type, vector_column.name, default_metric=False) + if resolved.index_type is None: + resolved = VectorIndexOptions.resolve(Options(options), "vector", vector_column.name, default_metric=False) + metric = resolved.options.to_map().get(resolved.index_type + ".metric") if metric is not None: return metric for key in ("test.vector.metric", "lumina.distance.metric"): diff --git a/paimon-python/pypaimon/execution/python/read/search/vector_search_scan.py b/paimon-python/pypaimon/execution/python/read/search/vector_search_scan.py index 21ace7fc67d2..ad3c842425c2 100644 --- a/paimon-python/pypaimon/execution/python/read/search/vector_search_scan.py +++ b/paimon-python/pypaimon/execution/python/read/search/vector_search_scan.py @@ -245,4 +245,4 @@ def _configured_vector_index_type(table, vector_column, options): table_options = getattr(getattr(table, "options", None), "options", None) values = dict(table_options.to_map()) if table_options is not None else {} values.update(options) - return VectorIndexOptions.index_type(Options(values), vector_column.name) + return VectorIndexOptions.resolve(Options(values), None, vector_column.name, default_metric=False).index_type diff --git a/paimon-python/pypaimon/globalindex/create_global_index.py b/paimon-python/pypaimon/globalindex/create_global_index.py index 516cd631206c..253e1ce13aaf 100644 --- a/paimon-python/pypaimon/globalindex/create_global_index.py +++ b/paimon-python/pypaimon/globalindex/create_global_index.py @@ -111,7 +111,7 @@ def __init__( "Python global index build currently supports one column, got %s." % self._index_columns ) - self._options = (VectorIndexOptions.resolve(self._options, self._index_type, self._index_columns[0]) + self._options = (VectorIndexOptions.resolve(self._options, self._index_type, self._index_columns[0]).options if CoreOptions._is_vector_index_type(self._index_type) else self._core_options.global_index_options(self._index_type, self._index_columns[0])) self._core_options = CoreOptions(self._options) diff --git a/paimon-python/pypaimon/index/pk/primary_key_index_definitions.py b/paimon-python/pypaimon/index/pk/primary_key_index_definitions.py index 6b7366a40d29..8de1827d1fed 100644 --- a/paimon-python/pypaimon/index/pk/primary_key_index_definitions.py +++ b/paimon-python/pypaimon/index/pk/primary_key_index_definitions.py @@ -59,10 +59,10 @@ def create(schema): core.bitmap_index_options(field.name), PrimaryKeyIndexFamily.BITMAP)) elif field.name in vector: + resolved = VectorIndexOptions.resolve(core.options, None, field.name) definitions.append(PrimaryKeyIndexDefinition( field.name, field.id, - VectorIndexOptions.index_type(core.options, field.name), - VectorIndexOptions.resolve(core.options, None, field.name), + resolved.index_type, resolved.options, PrimaryKeyIndexFamily.VECTOR)) elif field.name in full_text: definitions.append(PrimaryKeyIndexDefinition( diff --git a/paimon-python/pypaimon/tests/global_index_test.py b/paimon-python/pypaimon/tests/global_index_test.py index 0270f9a0c527..6dc655d574b7 100644 --- a/paimon-python/pypaimon/tests/global_index_test.py +++ b/paimon-python/pypaimon/tests/global_index_test.py @@ -61,8 +61,8 @@ def test_legacy_primary_key_metric_resolution_is_idempotent(self): 'fields.embedding.pk-vector.index.type': 'ivf-pq', 'fields.embedding.pk-vector.distance.metric': metric.lower().replace('-', '_'), 'fields.embedding.pk-vector.index.options': '{"metric":"%s"}' % metric} - first = VectorIndexOptions.resolve(Options(values), None, 'embedding') - second = VectorIndexOptions.resolve(first, 'ivf-pq', 'embedding') + first = VectorIndexOptions.resolve(Options(values), None, 'embedding').options + second = VectorIndexOptions.resolve(first, 'ivf-pq', 'embedding').options self.assertEqual(first.to_map(), second.to_map()) self.assertEqual('{"metric":"%s"}' % metric, values['fields.embedding.pk-vector.index.options']) @@ -110,12 +110,13 @@ def test_shared_vector_type_and_documented_metric(self): "fields.embedding.distance.metric": "cosine"} core = CoreOptions(Options(values)) self.assertEqual(["embedding"], VectorIndexOptions.columns(core.options)) - self.assertEqual("ivf-pq", VectorIndexOptions.index_type(core.options, "embedding")) - self.assertEqual("cosine", VectorIndexOptions.metric(core.options, None, "embedding")) + self.assertEqual("ivf-pq", VectorIndexOptions.resolve( + core.options, None, "embedding", default_metric=False).index_type) + self.assertEqual("cosine", VectorIndexOptions.resolve(core.options, None, "embedding").metric) values["fields.embedding.index-options"] = '{"nlist":64,"distance.metric":"inner_product"}' core = CoreOptions(Options(values)) - self.assertEqual("inner_product", VectorIndexOptions.metric(core.options, None, "embedding")) - resolved = VectorIndexOptions.resolve(core.options, None, "embedding").to_map() + self.assertEqual("inner_product", VectorIndexOptions.resolve(core.options, None, "embedding").metric) + resolved = VectorIndexOptions.resolve(core.options, None, "embedding").options.to_map() self.assertEqual("768", resolved["ivf-pq.dimension"]) self.assertEqual("64", resolved["fields.embedding.nlist"]) self.assertEqual("inner_product", resolved["ivf-pq.distance.metric"]) diff --git a/paimon-python/pypaimon/tests/vector_index_options_test.py b/paimon-python/pypaimon/tests/vector_index_options_test.py index 72af7980b565..dbec30915d4e 100644 --- a/paimon-python/pypaimon/tests/vector_index_options_test.py +++ b/paimon-python/pypaimon/tests/vector_index_options_test.py @@ -30,6 +30,27 @@ def _assert_metric(options, expected): assert options.to_map()[key] == expected +@pytest.mark.parametrize('declaration', [ + 'pk-vector.index.columns', 'global-index.ivf-pq.index-column', 'global-index.ivf-pq.index-columns']) +def test_resolution_returns_matching_type_and_options(declaration): + values = {declaration: 'embedding', 'fields.embedding.pk-vector.index.type': ' IVF-PQ ', + 'ivf-pq.dimension': '768'} + suffix = 'pk-vector.index.options' if declaration.startswith('pk-') else 'index-options' + values['fields.embedding.' + suffix] = '{"nlist":64,"metric":"COSINE"}' + original = dict(values) + resolved = VectorIndexOptions.resolve(Options(values), None, 'embedding') + assert resolved.index_type == 'ivf-pq' + assert resolved.options.to_map()['ivf-pq.nlist'] == '64' + assert resolved.options.to_map()['ivf-pq.dimension'] == '768' + assert resolved.metric == 'cosine' + assert values == original + # An algorithm supplied by persisted metadata wins over the table declaration. + persisted = VectorIndexOptions.resolve(Options(values), 'ivf-flat', 'embedding', default_metric=False) + assert persisted.index_type == 'ivf-flat' + assert persisted.options.to_map()['ivf-flat.nlist'] == '64' + assert persisted.metric == 'cosine' + + @pytest.mark.parametrize('suffix', ['index-column', 'index-columns']) def test_legacy_registration_is_only_a_compatibility_fallback(suffix): values = {'pk-vector.index.columns': ' old, other_old ', @@ -37,7 +58,7 @@ def test_legacy_registration_is_only_a_compatibility_fallback(suffix): 'fields.other_old.pk-vector.index.type': 'diskann'} options = Options(values) assert VectorIndexOptions.columns(options) == ['old', 'other_old'] - assert VectorIndexOptions.index_type(options, 'old') == 'ivf-pq' + assert VectorIndexOptions.resolve(options, None, 'old', default_metric=False).index_type == 'ivf-pq' assert CoreOptions(options).vector_index_columns('ivf-pq') == ['old'] assert CoreOptions(options).vector_index_columns('diskann') == ['other_old'] assert CoreOptions(options).vector_index_columns('ivf-flat') == [] @@ -45,8 +66,8 @@ def test_legacy_registration_is_only_a_compatibility_fallback(suffix): values.update({'global-index.ivf-pq.' + suffix: ' embedding, other ', 'global-index.diskann.' + suffix: 'video'}) assert VectorIndexOptions.columns(options) == ['video', 'embedding', 'other'] - assert VectorIndexOptions.index_type(options, 'embedding') == 'ivf-pq' - assert VectorIndexOptions.index_type(options, 'video') == 'diskann' + assert VectorIndexOptions.resolve(options, None, 'embedding', default_metric=False).index_type == 'ivf-pq' + assert VectorIndexOptions.resolve(options, None, 'video', default_metric=False).index_type == 'diskann' assert CoreOptions(options).vector_index_columns('ivf-pq') == ['embedding', 'other'] assert CoreOptions(options).vector_index_columns('diskann') == ['video'] assert VectorIndexOptions.columns_option_key(options) == ( @@ -62,31 +83,32 @@ def test_generic_merge_does_not_infer_historical_vector_types(key): @pytest.mark.parametrize('key', ['index_type', 'index-type', 'vector.index-type', 'fields.embedding.index-type']) def test_removed_type_aliases_are_ignored(key): options = Options({key: 'ivf-pq'}) - assert VectorIndexOptions.index_type(options, 'embedding') is None - assert VectorIndexOptions.resolve(options, None, 'embedding').to_map() == options.to_map() + assert VectorIndexOptions.resolve(options, None, 'embedding', default_metric=False).index_type is None + assert VectorIndexOptions.resolve(options, None, 'embedding').options.to_map() == options.to_map() options.to_map()['fields.embedding.pk-vector.index.type'] = 'diskann' - assert VectorIndexOptions.index_type(options, 'embedding') == 'diskann' + assert VectorIndexOptions.resolve(options, None, 'embedding', default_metric=False).index_type == 'diskann' options.to_map()['global-index.ivf-flat.index-columns'] = 'embedding' - assert VectorIndexOptions.index_type(options, 'embedding') == 'ivf-flat' + assert VectorIndexOptions.resolve(options, None, 'embedding', default_metric=False).index_type == 'ivf-flat' def test_legacy_primary_key_type_resolves_field_parameters(): values = {'fields.embedding.pk-vector.index.type': ' IVF-PQ ', 'ivf-pq.dimension': '768', 'fields.embedding.index-options': '{"nlist":64,"metric":"COSINE"}'} options = Options(values) - assert VectorIndexOptions.index_type(options, 'embedding') == 'ivf-pq' - resolved = VectorIndexOptions.resolve(options, None, 'embedding', default_metric=False) + assert VectorIndexOptions.resolve(options, None, 'embedding', default_metric=False).index_type == 'ivf-pq' + resolved = VectorIndexOptions.resolve(options, None, 'embedding', default_metric=False).options _assert_metric(resolved, 'cosine') assert resolved.to_map()['ivf-pq.dimension'] == '768' assert resolved.to_map()['ivf-pq.nlist'] == '64' - assert VectorIndexOptions.resolve(resolved, None, 'embedding', default_metric=False).to_map() == resolved.to_map() + assert (VectorIndexOptions.resolve(resolved, None, 'embedding', default_metric=False).options.to_map() + == resolved.to_map()) assert options.to_map() == values def test_ambiguous_declarations_do_not_fall_back_to_historical_hint(): options = Options({'global-index.ivf-pq.index-columns': 'embedding', 'global-index.diskann.index-column': 'embedding', 'index_type': 'ivf-flat'}) - assert VectorIndexOptions.index_type(options, 'embedding') is None + assert VectorIndexOptions.resolve(options, None, 'embedding', default_metric=False).index_type is None @pytest.mark.parametrize('key,value,expected', [ @@ -101,11 +123,11 @@ def test_ambiguous_declarations_do_not_fall_back_to_historical_hint(): def test_legacy_and_dlf_metric_aliases(key, value, expected): values = {'global-index.ivf-pq.index-column': 'embedding', 'ivf-pq.dimension': '768', 'fields.embedding.nlist': '48', key: value} - resolved = VectorIndexOptions.resolve(Options(values), None, 'embedding') + resolved = VectorIndexOptions.resolve(Options(values), None, 'embedding').options _assert_metric(resolved, expected) assert resolved.to_map()['ivf-pq.dimension'] == '768' assert resolved.to_map()['fields.embedding.nlist'] == '48' - assert VectorIndexOptions.resolve(resolved, 'ivf-pq', 'embedding').to_map() == resolved.to_map() + assert VectorIndexOptions.resolve(resolved, 'ivf-pq', 'embedding').options.to_map() == resolved.to_map() assert values[key] == value @@ -118,18 +140,18 @@ def test_field_json_aliases_remain_idempotent(suffix, key): 'fields.embedding.pk-vector.index.type': 'ivf-pq', 'fields.embedding.pk-vector.distance.metric': 'cosine', 'fields.embedding.' + suffix: json.dumps({key: 'COSINE', 'nlist': 64})} - resolved = VectorIndexOptions.resolve(Options(values), None, 'embedding') + resolved = VectorIndexOptions.resolve(Options(values), None, 'embedding').options _assert_metric(resolved, 'cosine') assert resolved.to_map()['ivf-pq.nlist'] == '64' - assert VectorIndexOptions.resolve(resolved, 'ivf-pq', 'embedding').to_map() == resolved.to_map() - assert VectorIndexOptions.resolve(Options(values), None, 'embedding').to_map() == resolved.to_map() + assert VectorIndexOptions.resolve(resolved, 'ivf-pq', 'embedding').options.to_map() == resolved.to_map() + assert VectorIndexOptions.resolve(Options(values), None, 'embedding').options.to_map() == resolved.to_map() def test_reader_keeps_unspecified_metric_and_writer_uses_default(): values = Options({'fields.embedding.index-options': '{"nlist":64}'}) - resolved = VectorIndexOptions.resolve(values, 'ivf-pq', 'embedding', default_metric=False).to_map() + resolved = VectorIndexOptions.resolve(values, 'ivf-pq', 'embedding', default_metric=False).options.to_map() assert resolved['ivf-pq.nlist'] == '64' assert 'ivf-pq.metric' not in resolved assert 'fields.embedding.distance.metric' not in resolved - _assert_metric(VectorIndexOptions.resolve(values, 'ivf-pq', 'embedding'), 'inner_product') - assert VectorIndexOptions.resolve(values, None, 'embedding').to_map() == values.to_map() + _assert_metric(VectorIndexOptions.resolve(values, 'ivf-pq', 'embedding').options, 'inner_product') + assert VectorIndexOptions.resolve(values, None, 'embedding').options.to_map() == values.to_map() From b544ff6a6c3be6e6031813ed050bf052c1fc4d6c Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 1 Nov 2026 13:08:46 +0800 Subject: [PATCH 12/14] [python] Align native context tests with resolved table options --- .../tests/native_vector_context_test.py | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/paimon-python/pypaimon/tests/native_vector_context_test.py b/paimon-python/pypaimon/tests/native_vector_context_test.py index 9c4608a36fde..8b51a413e465 100644 --- a/paimon-python/pypaimon/tests/native_vector_context_test.py +++ b/paimon-python/pypaimon/tests/native_vector_context_test.py @@ -24,20 +24,32 @@ from pypaimon.catalog.catalog_context import CatalogContext from pypaimon.catalog.rest.rest_catalog_loader import RESTCatalogLoader +from pypaimon.common.options.core_options import CoreOptions +from pypaimon.common.options.options import Options from pypaimon.common.predicate import Predicate from pypaimon.execution.native.read import native_vector_search @pytest.mark.parametrize('option', ['hadoop_conf', 'prefer_io_loader', 'fallback_io_loader']) @pytest.mark.parametrize('batch', [False, True], ids=['single', 'batch']) -def test_unused_context_attributes_preserve_native_vector_execution(option, batch): +@pytest.mark.parametrize('declaration', [None, 'legacy', 'dlf']) +def test_unused_context_attributes_preserve_native_vector_execution(option, batch, declaration): context = CatalogContext.create_from_options({'metastore': 'rest', 'uri': 'http://unused'}) unused = Mock(side_effect=AssertionError('An unused IO loader was invoked')) setattr(context, option, {'unused': 'value'} if option == 'hadoop_conf' else unused) loader = RESTCatalogLoader(context) + values = {'read.native.enabled': 'true', 'global-index.enabled': 'false'} + if declaration == 'legacy': + values.update({'pk-vector.index.columns': 'embedding', + 'fields.embedding.pk-vector.index.type': 'ivf-flat', + 'fields.embedding.pk-vector.index.options': '{"nlist":64}'}) + elif declaration == 'dlf': + values.update({'global-index.ivf-flat.index-columns': 'embedding', + 'fields.embedding.index-options': '{"nlist":64}'}) + environment = SimpleNamespace(catalog_loader=loader) table = SimpleNamespace( - options=SimpleNamespace(native_read_enabled=lambda: True, query_auth_enabled=False), - catalog_environment=SimpleNamespace(catalog_loader=loader)) + options=CoreOptions(Options(values)), catalog_environment=environment, + copy=lambda options: SimpleNamespace(options=CoreOptions(Options(options)), catalog_environment=environment)) builder = SimpleNamespace( _table=table, _vector_column=SimpleNamespace(name='embedding'), _limit=2, _options={'ivf-flat.metric': 'l2'}, _filter=Predicate('equal', 0, 'id', [7]), @@ -57,10 +69,20 @@ def test_unused_context_attributes_preserve_native_vector_execution(option, batc patch.object(native_vector_search, '_native_table', return_value=native_table) as resolve: result = native_vector_search.try_native_vector_search(builder, queries, batch=batch) assert result is not None - resolve.assert_called_once_with(table) + resolve.assert_called_once() + resolved_table = resolve.call_args[0][0] + assert resolved_table.catalog_environment is environment core.with_vector_column.assert_called_once_with('embedding') core.with_limit.assert_called_once_with(2) - core.with_options.assert_called_once_with({'ivf-flat.metric': 'l2'}) + expected_options = {'ivf-flat.metric': 'l2'} + if declaration is not None: + core.with_vector_index_type.assert_called_once_with('embedding', 'ivf-flat') + assert resolved_table.options.options.to_map()['ivf-flat.nlist'] == '64' + expected_options.update({'ivf-flat.distance.metric': 'l2', + 'fields.embedding.metric': 'l2', 'fields.embedding.distance.metric': 'l2'}) + else: + core.with_vector_index_type.assert_not_called() + core.with_options.assert_called_once_with(expected_options) core.with_filter.assert_called_once_with({'method': 'equal', 'field': 'id', 'literals': [7]}) core.with_partition_filter.assert_called_once_with({'method': 'equal', 'field': 'pt', 'literals': [1]}) unused.assert_not_called() From 38f5b0c39d29d096f6dd81cd4847cbeaf2d99b01 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 1 Nov 2026 13:08:48 +0800 Subject: [PATCH 13/14] [flink] Stabilize MySQL CDC test time zones across DST transitions --- .../cdc/mysql/MySqlActionITCaseBase.java | 9 ++- .../action/cdc/mysql/MySqlTimeZoneTest.java | 72 +++++++++++++++++++ .../src/test/resources/mysql/my.cnf | 4 +- 3 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTimeZoneTest.java diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionITCaseBase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionITCaseBase.java index 010041268c8a..0518363ee5ac 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionITCaseBase.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionITCaseBase.java @@ -30,7 +30,6 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; -import java.time.ZoneId; import java.util.HashMap; import java.util.Map; import java.util.stream.Stream; @@ -40,6 +39,10 @@ public class MySqlActionITCaseBase extends CdcActionITCaseBase { private static final Logger LOG = LoggerFactory.getLogger(MySqlActionITCaseBase.class); + // Keep the three-hour difference without DST-dependent offsets in MySqlValidator. + protected static final String MYSQL_SYSTEM_TIME_ZONE = "Etc/GMT+8"; + protected static final String MYSQL_SERVER_TIME_ZONE = "Etc/GMT+5"; + protected static final MySqlContainer MYSQL_CONTAINER = createMySqlContainer(MySqlVersion.V5_7); private static final String USER = "paimonuser"; private static final String PASSWORD = "paimonpw"; @@ -57,7 +60,7 @@ private static MySqlContainer createMySqlContainer(MySqlVersion version) { .withConfigurationOverride("mysql/my.cnf") .withUsername(USER) .withPassword(PASSWORD) - .withEnv("TZ", "America/Los_Angeles") + .withEnv("TZ", MYSQL_SYSTEM_TIME_ZONE) .withLogConsumer(new Slf4jLogConsumer(LOG)); } @@ -83,7 +86,7 @@ protected Map getBasicMySqlConfig() { config.put("username", USER); config.put("password", PASSWORD); // see mysql/my.cnf in test resources - config.put("server-time-zone", ZoneId.of("America/New_York").toString()); + config.put("server-time-zone", MYSQL_SERVER_TIME_ZONE); // TODO When setting the parameter scan.newly-added-table.enabled=true in version 2.4.2, the // Insert data inserted after the newly created table cannot be captured. When set to false, diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTimeZoneTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTimeZoneTest.java new file mode 100644 index 000000000000..1efa27d7cbfa --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTimeZoneTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action.cdc.mysql; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.io.InputStream; +import java.time.Clock; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Properties; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Checks the MySQL test time zone across daylight-saving transitions and JVM time zones. */ +class MySqlTimeZoneTest { + + @ParameterizedTest + @CsvSource({ + "2026-03-08T06:30:00Z,GMT+07:00", + "2026-03-08T07:30:00Z,GMT+07:00", + "2026-11-01T04:30:00Z,GMT+07:00", + "2026-11-01T06:30:00Z,America/New_York", + "2026-11-01T07:30:00Z,America/Los_Angeles", + "2026-07-01T12:00:00Z,UTC", + "2026-01-01T12:00:00Z,Asia/Shanghai" + }) + void testValidatorOffsetMatchesServer(String timestamp, String jvmZone) throws Exception { + Properties config = new Properties(); + try (InputStream input = getClass().getResourceAsStream("/mysql/my.cnf")) { + config.load(input); + } + assertThat(config.getProperty("default-time-zone")) + .isEqualTo(MySqlActionITCaseBase.MYSQL_SERVER_TIME_ZONE); + ZoneId serverZone = ZoneId.of(config.getProperty("default-time-zone")); + Instant instant = Instant.parse(timestamp); + LocalDateTime localTime = LocalDateTime.now(Clock.fixed(instant, ZoneId.of(jvmZone))); + // MySqlValidator uses the JVM's local date-time to look up the server zone offset. + assertThat(serverZone.getRules().getOffset(localTime)) + .isEqualTo(serverZone.getRules().getOffset(instant)); + } + + @Test + void testTimestampFixtureKeepsThreeHourDifference() { + LocalDateTime inserted = LocalDateTime.parse("2023-03-23T15:00:10.123456"); + LocalDateTime read = + inserted.atZone(ZoneId.of(MySqlActionITCaseBase.MYSQL_SYSTEM_TIME_ZONE)) + .withZoneSameInstant( + ZoneId.of(MySqlActionITCaseBase.MYSQL_SERVER_TIME_ZONE)) + .toLocalDateTime(); + assertThat(read).isEqualTo(LocalDateTime.parse("2023-03-23T18:00:10.123456")); + } +} diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/mysql/my.cnf b/paimon-flink/paimon-flink-cdc/src/test/resources/mysql/my.cnf index 38beb8630be3..28a7a71478d7 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/resources/mysql/my.cnf +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/mysql/my.cnf @@ -64,5 +64,5 @@ binlog_format = row # which is bad for our tests explicit_defaults_for_timestamp = ON -# We need a default time zone different from UTC+0 for test -default-time-zone = America/New_York +# Keep a non-UTC server zone without DST-dependent offsets in MySqlValidator. +default-time-zone = Etc/GMT+5 From eb56a8cf4d370fef9929e70fa590c8b10b2c1541 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 1 Nov 2026 14:10:40 +0800 Subject: [PATCH 14/14] [python] Forward native index options without compatibility rewriting --- paimon-python/dev/native-index-options.md | 76 +++++++ paimon-python/dev/native-vector-type-api.md | 96 --------- .../execution/native/index_options.py | 135 ------------ .../native/read/native_full_text_search.py | 12 +- .../native/read/native_hybrid_search.py | 33 +-- .../native/read/native_search_plan.py | 37 ---- .../native/read/native_vector_search.py | 14 +- .../pypaimon/execution/native/read/plan.py | 5 - .../execution/native/read/vindex_reader.py | 5 +- .../pypaimon/execution/native/table.py | 12 +- .../execution/native/write/native_commit.py | 6 +- .../native/write/native_index_build.py | 39 +--- .../pypaimon/tests/global_index_build_test.py | 61 +++++- .../global_index_execution_boundary_test.py | 32 ++- .../tests/lazy_search_execution_test.py | 138 +++++------- .../tests/native_index_options_test.py | 199 ++++++------------ .../pypaimon/tests/native_search_plan_test.py | 4 - .../tests/native_vector_context_test.py | 16 +- 18 files changed, 314 insertions(+), 606 deletions(-) create mode 100644 paimon-python/dev/native-index-options.md delete mode 100644 paimon-python/dev/native-vector-type-api.md delete mode 100644 paimon-python/pypaimon/execution/native/index_options.py diff --git a/paimon-python/dev/native-index-options.md b/paimon-python/dev/native-index-options.md new file mode 100644 index 000000000000..d9d4808196db --- /dev/null +++ b/paimon-python/dev/native-index-options.md @@ -0,0 +1,76 @@ + + +# Native index configuration + +Rust owns index configuration parsing for native reads, writes, builds, and +searches. Python forwards the effective table options in the schema and the +explicit query or route options in the corresponding builder call. It preserves +option keys and field JSON, including canonical declarations, legacy PK +properties, and opaque implementation parameters. + +The common FFI encoding converts scalar values to strings, booleans to lowercase +strings, and collections to JSON. A None table option represents an unset value. +Python preserves effective authorization and write-policy values at the generic +table boundary. + +## Resolution contract + +* The shared resolver returns the selected implementation together with its + merged options. Java exposes `type()` and `options()`; Python exposes + `index_type` and `options`. Rust uses the same configuration semantics. +* Support `global-index..index-columns`, singular `index-column`, + type-level parameters such as `ivf-pq.dimension`, and short keys in + `fields..index-options`. Retain legacy PK declarations, field types, + field JSON, metrics, and DLF flat field parameters. +* Configuration is interpreted in Rust, including normalization, defaults, + option precedence, per-column scalar settings, and full-text analyzers. + Python does not flatten field JSON or convert canonical declarations into + synthetic legacy PK declarations for native execution. +* PK declarations control source-backed index maintenance. Data Evolution + declarations supply implementation and parameter defaults while retaining + explicit global-index builds. Rust owns this table-mode distinction. +* Read valid existing index entries before choosing the implementation. Persisted + metadata determines the algorithm for indexed data; use configuration for + writes or raw fallback when no valid index metadata is available. +* Query and route options remain separate from table defaults. An unspecified + metric stays unspecified in query options so persisted metadata can provide + it. Each hybrid route retains its own options and column-specific analyzer. +* Preserve configuration through scan/read reconstruction, captured routes, + build shards, and distributed worker serialization. Typed metadata fields + such as `RawVectorSearchSplit.index_type` retain their existing meaning. +* The external type-hint aliases `index_type`, `index-type`, `vector.index-type`, + and `fields..index-type` are unsupported. Python forwards options + without interpreting or removing these keys; Rust must ignore them as type + selectors. Legitimate API argument and metadata names remain supported. +* SDK configuration and execution errors propagate. A failed native operation + must not trigger a second execution against a potentially newer snapshot. + +## Verification + +`native_index_options_test.py` checks lossless schema transport for reads and +writes, legacy PK/DLF declarations, per-field options, and generic encoding. +`lazy_search_execution_test.py` checks single, batch, hybrid, captured, and +serialized requests using standard builders and unmodified query options. +`global_index_build_test.py` checks that builds retain table and user options. + +Real native integration in `native_search_plan_test.py` requires the corresponding +Rust shared configuration resolver. Cover raw queries without index files, +metadata precedence, independent hybrid columns/analyzers, and PK/DE maintenance +semantics. The Rust implementation is maintained separately. diff --git a/paimon-python/dev/native-vector-type-api.md b/paimon-python/dev/native-vector-type-api.md deleted file mode 100644 index eb5a75b8ad1e..000000000000 --- a/paimon-python/dev/native-vector-type-api.md +++ /dev/null @@ -1,96 +0,0 @@ - - -# Native vector implementation API - -The Python adapter expects the following method on all three -`pypaimon_rust.datafusion` builders: `VectorSearchBuilder`, -`BatchVectorSearchBuilder`, and `HybridSearchBuilder`. - -```python -def with_vector_index_type(self, column: str, index_type: str) -> Self: - ... -``` - -Expose equivalent typed state in the Rust core builders, rather than encoding -it as a table or query option. The Python adapter already calls this method; -the Rust implementation and binding are a separate change. - -* `column` is the schema column name. `index_type` is a normalized implementation - identifier such as `ivf-pq`, `ivf-flat`, or `diskann`. -* The method mutates builder configuration and returns the same builder, like - `with_vector_column`. Calling it must not load a snapshot, plan or execute a - search. Repeating it for a column replaces that column's previous value. -* Single and batch queries use the entry for the selected vector column. - Hybrid queries use the entry for each vector route's column, independently of - route order. Full-text routes do not consume it. Configuration may be supplied - before routes are added. -* Persisted index metadata continues to determine the implementation for indexed - ranges. The explicit value supplies the implementation for a raw search when - no vector index metadata is available, matching the Java planner. An absent - value leaves existing metadata/default behavior in place. -* Keep this state through scan/read construction, batch reader reconstruction, - captured hybrid route builders, and distributed worker serialization. - `RawVectorSearchSplit.index_type` remains a typed transport field. -* This API sets only the implementation. Resolved vector parameters still use - `with_options`/route options; an unspecified metric must remain unspecified so - persisted index metadata can provide it. -* Query and route options contain only explicitly supplied query parameters, - resolved through the shared vector parser. Table defaults remain in the schema; - a table metric must not become an explicit query metric that conflicts with - persisted index metadata. New and legacy field JSON query options follow the - same rule. -* Native code must stop interpreting `index_type`, `index-type`, - `vector.index-type`, and `fields..index-type` as option aliases. - Legitimate API argument names and index metadata fields named `index_type` - remain supported. - -Python resolves the implementation once through `VectorIndexOptions`, using -`global-index..index-columns` (or singular `index-column`) and the retained -legacy PK key `fields..pk-vector.index.type`. It strips the four removed -aliases before transferring schema or query options to older SDKs. - -Callers use the shared vector columns/type/parameter resolver. The legacy PK -column registry and field type pairing are private compatibility inputs; -`CoreOptions.vectorIndexColumns(type)` (Python: `vector_index_columns`) delegates -to the type-scoped resolver. Other index accessors also use names without a -`primaryKey` prefix. No public method compatibility wrappers are retained. - -`VectorIndexOptions.resolve(...)` returns the selected implementation together -with its merged options (Java: `type()` and `options()`; Python: `index_type` -and `options`). There is no separate `indexType`/`index_type` lookup method. -Writers and raw-query fallback use this result; readers pass the implementation -from persisted index metadata when available. `metric()` (Python: `metric`) -provides the normalized metric default without inserting it into query options -when `defaultMetric`/`default_metric` is false. - -When an implementation is configured and this method is unavailable, Python -raises `AttributeError` before planning, reading, executing, or capturing hybrid -routes. It does not fall back to Python execution. SDK configuration and execution -errors propagate; there is no retry after execution. Queries with no configured -implementation can continue to use older SDKs and persisted index metadata. - -## Rust verification - -Cover single, batch, hybrid, scan/read, captured routes, and serialized workers -with both canonical and legacy PK declarations. Test raw searches without index -files with and without a metric, metadata taking precedence over the explicit -value, and independent columns in a hybrid query. The Python adapter contract -and missing-SDK-method errors are covered in `lazy_search_execution_test.py`; native -integration coverage is in `native_search_plan_test.py` and requires this API. diff --git a/paimon-python/pypaimon/execution/native/index_options.py b/paimon-python/pypaimon/execution/native/index_options.py deleted file mode 100644 index 09006b011aa6..000000000000 --- a/paimon-python/pypaimon/execution/native/index_options.py +++ /dev/null @@ -1,135 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Transport resolved index configuration through the existing native schema contract.""" - -import json -from collections.abc import Mapping - -from pypaimon.common.options.core_options import CoreOptions -from pypaimon.common.options.options import Options -from pypaimon.common.options.vector_index_options import VectorIndexOptions - - -def native_index_options(schema): - return _resolve(schema)[0] - - -def native_search_options(options): - """Do not let an older SDK interpret removed external type hints.""" - return {key: value for key, value in options.items() - if key not in ('index_type', 'index-type', 'vector.index-type') - and not (key.startswith('fields.') and key.endswith('.index-type'))} - - -def configure_vector_index_type(native, options, column): - """Use the required native API and propagate missing-method errors.""" - if column is None: - return - index_type = VectorIndexOptions.resolve(Options(options), None, column, default_metric=False).index_type - if index_type is not None: - native.with_vector_index_type(column, index_type) - - -def native_scalar_index_read_supported(table): - if not table.options.global_index_enabled(): - return True - return _resolve(table.table_schema)[1] - - -def _resolve(schema): - if not isinstance(getattr(schema, 'options', None), Mapping): - return {}, True - values = native_search_options(schema.options) - core = CoreOptions(Options(values)) - result = dict(values) - declarations = core.global_index_types() - # Native's older shared declaration keys enable automatic maintenance. Python's - # declarations select implementations and keep the explicit DE build lifecycle. - for key in list(result): - if key.startswith('global-index.') and key.endswith(('.index-columns', '.index-column')): - del result[key] - primary_key = bool(schema.primary_keys) - families = ['btree', 'bitmap', 'multivalue', 'fm', 'full-text'] - definitions = [(family, core.global_index_columns(family)) for family in families] - definitions += [(kind, core.global_index_columns(kind)) for kind in declarations - if kind not in families] - vector_options = {} - if primary_key and not any(kind not in families for kind in declarations): - for column in VectorIndexOptions.columns(core.options): - resolved = VectorIndexOptions.resolve(core.options, None, column, default_metric=False) - vector_options[(resolved.index_type, column)] = resolved - definitions.append((resolved.index_type, [column])) - known_columns = set() - native_columns = {} - profiles = {} - compatible = True - for kind, columns in definitions: - if kind is None: - continue - vector = kind not in families - family = 'vector' if vector else kind - prefix = kind + ('.' if vector or kind == 'full-text' else '-index.') - if primary_key and columns: - native_columns.setdefault(family, []).extend(columns) - result['pk-%s.index.columns' % family] = ','.join(native_columns[family]) - for column in columns: - known_columns.add(column) - if vector: - parsed = vector_options.get((kind, column)) - if parsed is None: - parsed = VectorIndexOptions.resolve(core.options, kind, column, default_metric=False) - resolved = parsed.options.to_map() - else: - resolved = core.global_index_options(kind, column).to_map() - resolved = native_search_options(resolved) - profile = {key: value for key, value in resolved.items() - if key.startswith(prefix) and value is not None} - field_prefix = 'fields.%s.' % column - if primary_key: - if vector: - result[field_prefix + 'pk-vector.index.type'] = kind - result[field_prefix + 'pk-vector.distance.metric'] = parsed.metric - profile.update({key: value for key, value in resolved.items() - if key.startswith(field_prefix) and not key.startswith(field_prefix + 'pk-') - and key != field_prefix + 'index-options' and value is not None}) - if vector or kind == 'full-text': - # Native ranked readers consume flat options, unlike scalar definitions. - result.update(profile) - result[field_prefix + 'pk-%s.index.options' % family] = json.dumps(profile) - result.pop(field_prefix + 'index-options', None) - elif vector: - result.update({key: value for key, value in resolved.items() - if key.startswith(field_prefix) and key != field_prefix + 'index-options'}) - elif kind != 'full-text': - # Native scalar scanners accept one set of read options per family. - if kind in profiles and profiles[kind] != profile: - compatible = False - profiles[kind] = profile - if primary_key and columns and not vector and kind != 'full-text': - for key in list(result): - if key.startswith(prefix): - del result[key] - if not primary_key: - for profile in profiles.values(): - result.update(profile) - for key in values: - if key.startswith('fields.') and key.endswith('.index-options'): - column = key[len('fields.'):-len('.index-options')] - if column not in known_columns: - compatible = False - return result, compatible diff --git a/paimon-python/pypaimon/execution/native/read/native_full_text_search.py b/paimon-python/pypaimon/execution/native/read/native_full_text_search.py index 37c8fdc51068..2ad37539a169 100644 --- a/paimon-python/pypaimon/execution/native/read/native_full_text_search.py +++ b/paimon-python/pypaimon/execution/native/read/native_full_text_search.py @@ -19,12 +19,11 @@ from pypaimon.catalog.table_query_auth import reject_search_under_query_auth from pypaimon.common.options.core_options import GlobalIndexSearchMode -from pypaimon.globalindex.full_text_options import NativeFullTextIndexOptions from pypaimon.execution.native.table import (_native_table) from pypaimon.execution.native.read.builder import (_predicate_to_native) from pypaimon.execution.native.runtime import (native_runtime_available) from pypaimon.table.file_store_table import FileStoreTable -from pypaimon.execution.native.read.native_search_plan import _result, _index_search_options +from pypaimon.execution.native.read.native_search_plan import _result from pypaimon.execution.native.write.native_commit import _rest_catalog_supported @@ -46,14 +45,7 @@ def _native_full_text_builder(builder, require_limit=True): raise NotImplementedError('Primary-key full-text search only supports the FAST full-text index search mode') if require_limit and (builder._limit is None or builder._limit <= 0): raise ValueError('Limit must be positive, set via with_limit()') - # The full-text adapter accepts typed lists and booleans. Reuse its mapping - # before the generic table bridge stringifies options for Rust. - resolved = _index_search_options(table, builder._field_name, 'full-text', scalar_filter=builder._filter) - if resolved is None: - return None - options = {'full-text.' + key: value for key, value in NativeFullTextIndexOptions.from_options( - resolved).to_native_options().items()} - native = _native_table(table.copy(options)).new_full_text_search_builder() + native = _native_table(table).new_full_text_search_builder() if builder._field_name is not None and builder._query is not None: native.with_query(builder._field_name, builder._query) if builder._limit is not None: diff --git a/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py b/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py index b3eea3ccba2f..998c9adc21fa 100644 --- a/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py +++ b/paimon-python/pypaimon/execution/native/read/native_hybrid_search.py @@ -18,15 +18,12 @@ """Transport hybrid queries, captured routes and weighted results to Rust Core.""" from pypaimon.catalog.table_query_auth import reject_search_under_query_auth -from pypaimon.globalindex.full_text_options import NativeFullTextIndexOptions from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult -from pypaimon.execution.native.table import (_native_table) +from pypaimon.execution.native.table import (_native_table, _options_to_native) from pypaimon.execution.native.read.builder import (_predicate_to_native) from pypaimon.execution.native.runtime import (native_runtime_available) from pypaimon.table.file_store_table import FileStoreTable from pypaimon.execution.native.write.native_commit import _rest_catalog_supported -from pypaimon.execution.native.read.native_search_plan import _index_search_options, _vector_query_options -from pypaimon.execution.native.index_options import configure_vector_index_type def _native_builder(builder, executor=None): @@ -45,34 +42,12 @@ def _native_builder(builder, executor=None): builder._validate_search() if executor is not None and not table.options.data_evolution_enabled(): raise ValueError('Distributed search supports only data-evolution tables') - route_options = [] - full_text_options = None - for route in builder._routes: - options = _index_search_options(table, route.field_name, - 'full-text' if route.is_full_text() else None, route.options, - scalar_filter=builder._filter) - if options is None: - return None - if route.is_full_text(): - resolved = {'full-text.' + key: value for key, value in - NativeFullTextIndexOptions.from_options(options).to_native_options().items()} - # Core accepts analyzer options only on the table, not on individual routes. - if full_text_options is not None and full_text_options != resolved: - return None - full_text_options = resolved - options = {} - route_options.append(options) - if full_text_options is not None: - table = table.copy(full_text_options) native = _native_table(table).new_hybrid_search_builder() - for route, options in zip(builder._routes, route_options): - if route.is_vector(): - configure_vector_index_type(native, options, route.field_name) native.with_limit(builder._limit).with_ranker(builder._ranker) - for route, options in zip(builder._routes, route_options): + for route in builder._routes: + options = _options_to_native(route.options) if route.is_vector(): - native.add_vector_route(route.field_name, route.vector, route.limit, route.weight, - _vector_query_options(route.options, route.field_name, options)) + native.add_vector_route(route.field_name, route.vector, route.limit, route.weight, options) else: native.add_full_text_route( route.field_name, route.full_text_query, route.limit, route.weight, options) diff --git a/paimon-python/pypaimon/execution/native/read/native_search_plan.py b/paimon-python/pypaimon/execution/native/read/native_search_plan.py index a713834a3763..56d29ce63cbe 100644 --- a/paimon-python/pypaimon/execution/native/read/native_search_plan.py +++ b/paimon-python/pypaimon/execution/native/read/native_search_plan.py @@ -20,10 +20,6 @@ from copy import copy, deepcopy from pypaimon.common.json_util import JSON -from pypaimon.common.options.core_options import CoreOptions -from pypaimon.common.options.options import Options -from pypaimon.common.options.vector_index_options import VectorIndexOptions -from pypaimon.execution.native.index_options import native_scalar_index_read_supported, native_search_options from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult from pypaimon.read.split_serializer import deserialize_split_v1, serialize_split_v1 from pypaimon.execution.native.table import (_partition_fields) @@ -38,39 +34,6 @@ from pypaimon.write.commit_message_serializer import _index_from_row, _index_row -def _index_search_options(table, column, index_type=None, overrides=None, scalar_filter=None): - """Use the common resolver before transferring search options to Core.""" - values = dict(table.options.options.to_map()) - values.update(overrides or {}) - values = native_search_options(values) - if scalar_filter is not None and not native_scalar_index_read_supported(table.copy(values)): - return None - if column is None: - return values - core = CoreOptions(Options(values)) - if index_type is not None and not CoreOptions._is_vector_index_type(index_type): - return native_search_options(core.global_index_options(index_type, column).to_map()) - # Keep an unspecified metric absent so readers can use persisted index metadata. - resolved = VectorIndexOptions.resolve(Options(values), index_type, column, default_metric=False) - if resolved.index_type is None: - # The Python reader can resolve field options once persisted index metadata is known. - if any(values.get(key) is not None for key in ( - 'fields.%s.index-options' % column, 'fields.%s.pk-vector.index.options' % column)): - return None - return values - return native_search_options(resolved.options.to_map()) - - -def _vector_query_options(options, column, resolved_options): - """Resolve explicit query parameters without treating table defaults as overrides.""" - values = native_search_options(options) - if column is None: - return values - index_type = VectorIndexOptions.resolve(Options(resolved_options), None, column, default_metric=False).index_type - return native_search_options(VectorIndexOptions.resolve( - Options(values), index_type, column, default_metric=False).options.to_map()) - - def _range(value): return Range(*value) diff --git a/paimon-python/pypaimon/execution/native/read/native_vector_search.py b/paimon-python/pypaimon/execution/native/read/native_vector_search.py index f9cb82793c9b..634edf9cb59f 100644 --- a/paimon-python/pypaimon/execution/native/read/native_vector_search.py +++ b/paimon-python/pypaimon/execution/native/read/native_vector_search.py @@ -19,11 +19,10 @@ from pypaimon.catalog.table_query_auth import reject_search_under_query_auth from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult -from pypaimon.execution.native.table import (_catalog_metastore, _native_table) +from pypaimon.execution.native.table import (_catalog_metastore, _native_table, _options_to_native) from pypaimon.execution.native.read.builder import (_predicate_to_native) from pypaimon.execution.native.runtime import (native_runtime_available) -from pypaimon.execution.native.read.native_search_plan import _result, _index_search_options, _vector_query_options -from pypaimon.execution.native.index_options import configure_vector_index_type +from pypaimon.execution.native.read.native_search_plan import _result def _native_vector_builder(builder, batch=False): @@ -45,17 +44,12 @@ def _native_vector_builder(builder, batch=False): return None if not native_runtime_available(): return None - column = builder._vector_column.name if builder._vector_column else None - options = _index_search_options(table, column, overrides=builder._options, scalar_filter=builder._filter) - if options is None: - return None - native_table = _native_table(table.copy(options)) + native_table = _native_table(table) native = (native_table.new_batch_vector_search_builder() if batch else native_table.new_vector_search_builder()) - configure_vector_index_type(native, options, column) if builder._vector_column is not None: native.with_vector_column(builder._vector_column.name) - native.with_limit(builder._limit).with_options(_vector_query_options(builder._options, column, options)) + native.with_limit(builder._limit).with_options(_options_to_native(builder._options)) try: if builder._filter is not None: native.with_filter(_predicate_to_native(builder._filter)) diff --git a/paimon-python/pypaimon/execution/native/read/plan.py b/paimon-python/pypaimon/execution/native/read/plan.py index a3ac9ebb3c1f..357de9d4f28d 100644 --- a/paimon-python/pypaimon/execution/native/read/plan.py +++ b/paimon-python/pypaimon/execution/native/read/plan.py @@ -26,7 +26,6 @@ from pypaimon.common.predicate import Predicate from pypaimon.execution.native import runtime from pypaimon.execution.native import table as native_table -from pypaimon.execution.native.index_options import native_scalar_index_read_supported from pypaimon.execution.native.read import builder as native_builder from pypaimon.read.plan import Plan from pypaimon.read.query_auth_split import QueryAuthSplit @@ -70,8 +69,6 @@ def native_plan( if not runtime.native_runtime_available(): raise RuntimeError( "scan.native-plan.enabled needs the pypaimon-rust extension") - if predicate is not None and not native_scalar_index_read_supported(table): - raise NotImplementedError('Native scalar reads cannot represent these field index options') builder = native_builder._configure_native_read_builder( native_builder._native_read_builder(table), predicate, limit, projection) if partition_predicate is not None: @@ -131,8 +128,6 @@ def native_plan( def native_stream_scan(table, predicate=None, read_type=None, bucket_filter=None, consumer_id=None): """Create Rust's stateful StreamTableScan with the canonical reader type.""" - if predicate is not None and not native_scalar_index_read_supported(table): - raise NotImplementedError('Native scalar reads cannot represent these field index options') builder = native_builder._configure_native_read_builder( native_builder._native_read_builder(table), predicate, None, None, read_type=read_type) scan = builder.new_stream_scan() diff --git a/paimon-python/pypaimon/execution/native/read/vindex_reader.py b/paimon-python/pypaimon/execution/native/read/vindex_reader.py index ded48f63bc09..3a89f545ce88 100644 --- a/paimon-python/pypaimon/execution/native/read/vindex_reader.py +++ b/paimon-python/pypaimon/execution/native/read/vindex_reader.py @@ -20,6 +20,7 @@ from pypaimon.globalindex.global_index_reader import GlobalIndexReader, _completed_future from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult from pypaimon.globalindex.vindex_input import VindexReaderResources, _filter_bytes +from pypaimon.execution.native.table import _options_to_native def _native_reader_type(): @@ -47,7 +48,7 @@ def __init__(self, file_io, index_path, io_metas, options, reader_type): def _create_reader(self, index_input, file_path): return self._reader_type( index_input, file_path, self._io_meta.file_size, - {str(key): str(value) for key, value in self._options.items()}) + _options_to_native(self._options)) def visit_vector_search(self, vector_search): results = self._run_search( @@ -71,7 +72,7 @@ def _run_search(self, vectors, limit, include_row_ids, query_options): self._ensure_loaded() results = self._reader.search( vectors, limit, filter_bytes=_filter_bytes(include_row_ids), - options={str(key): str(value) for key, value in (query_options or {}).items()}) + options=_options_to_native(query_options or {})) return [DictBasedScoredIndexResult(scores) if scores else None for scores in results] def vector_metric(self): diff --git a/paimon-python/pypaimon/execution/native/table.py b/paimon-python/pypaimon/execution/native/table.py index b185ea2f8c14..ba3831bf7d23 100644 --- a/paimon-python/pypaimon/execution/native/table.py +++ b/paimon-python/pypaimon/execution/native/table.py @@ -66,9 +66,17 @@ def _option_value_to_string(value) -> str: """ if isinstance(value, bool): return 'true' if value else 'false' + if isinstance(value, (list, tuple, dict)): + return json.dumps(value) return OptionsUtils.convert_to_string(value) +def _options_to_native(options) -> dict: + """Encode all effective options for Rust; None represents an unset option.""" + return {str(key): _option_value_to_string(value) + for key, value in options.items() if value is not None} + + def _catalog_options(table) -> dict: """Catalog options that built this table, to reconstruct the Rust catalog.""" loader = getattr(getattr(table, 'catalog_environment', None), 'catalog_loader', None) @@ -137,9 +145,7 @@ def _resolved_schema_file_io_options(table) -> Optional[dict]: def _resolved_schema_json(table) -> str: """Preserve all effective table options as strings for Rust.""" from pypaimon.common.json_util import JSON - from pypaimon.execution.native.index_options import native_index_options - options = {str(key): _option_value_to_string(value) - for key, value in native_index_options(table.table_schema).items() if value is not None} + options = _options_to_native(table.table_schema.options) if 'query-auth.enabled' in options: # Preserve Python's effective policy. Forwarding an enabled spelling # such as "on" unchanged would disable authorization in Rust. diff --git a/paimon-python/pypaimon/execution/native/write/native_commit.py b/paimon-python/pypaimon/execution/native/write/native_commit.py index 289e65c60c1d..6bf4910d87a3 100644 --- a/paimon-python/pypaimon/execution/native/write/native_commit.py +++ b/paimon-python/pypaimon/execution/native/write/native_commit.py @@ -20,10 +20,9 @@ import json from pypaimon.common.json_util import JSON -from pypaimon.execution.native.index_options import native_index_options from pypaimon.execution.native.runtime import native_runtime_available from pypaimon.execution.native.table import ( - _catalog_context_options, _catalog_metastore, _option_value_to_string, + _catalog_context_options, _catalog_metastore, _option_value_to_string, _options_to_native, _resolved_schema_file_io_options) from pypaimon.utils.file_store_path_factory import canonical_data_file_path from pypaimon.write.commit_message_serializer import ( @@ -154,8 +153,7 @@ def create_native_write_table(table): from pypaimon_rust.datafusion import Table as NativeTable # Preserve the resolved schema and all effective table options, including # copy() overrides. Do not inject scan options or reload catalog schemas. - options = {str(key): _option_value_to_string(value) - for key, value in native_index_options(table.table_schema).items() if value is not None} + options = _options_to_native(table.table_schema.options) # Python accepts boolean spellings such as "off"; pass the parsed value. options['dynamic-partition-overwrite'] = _option_value_to_string( table.options.dynamic_partition_overwrite()) diff --git a/paimon-python/pypaimon/execution/native/write/native_index_build.py b/paimon-python/pypaimon/execution/native/write/native_index_build.py index 265af7892180..d1738b6d03ba 100644 --- a/paimon-python/pypaimon/execution/native/write/native_index_build.py +++ b/paimon-python/pypaimon/execution/native/write/native_index_build.py @@ -17,7 +17,7 @@ """Global-index builds delegated to Rust, retaining Python's commit API.""" -from pypaimon.execution.native.table import (_native_table, _option_value_to_string) +from pypaimon.execution.native.table import (_native_table, _options_to_native) from pypaimon.execution.native.read.builder import (_predicate_to_native) from pypaimon.execution.native.runtime import (native_runtime_available) from pypaimon.execution.native.write.native_commit import _rest_catalog_supported, from_native_commit_messages @@ -43,39 +43,12 @@ def build_native_global_index(builder, partition_filter, *, execution="local", c from pypaimon_rust.datafusion import GlobalIndexBuildBuilder if not GlobalIndexBuildBuilder.supports_index_type(builder._index_type): return None - native = _native_table(table).new_global_index_build_builder() + options = dict(table.options.options.to_map()) + options.update(builder._user_options) + native_table = table.copy_without_time_travel(builder._user_options) + native = _native_table(native_table).new_global_index_build_builder() native.with_index_column(builder._index_columns[0]).with_index_type(builder._index_type) - resolved_options = builder._options.to_map() - options = {str(key): _option_value_to_string(value) - for key, value in resolved_options.items() if value is not None} - if builder._index_type == 'full-text': - from pypaimon.globalindex.full_text_options import NativeFullTextIndexOptions - options.update({'full-text.' + key: value for key, value in - NativeFullTextIndexOptions.from_options(builder._options.to_map()).to_native_options().items()}) - elif builder._index_type in VINDEX_IDENTIFIERS: - from pypaimon.globalindex.vindex_options import native_options, train_sample_ratio - column = builder._index_columns[0] - # Reuse the Java-compatible option mapping; Rust's low-level builder also - # accepts native aliases which Python's public API deliberately ignores. - mapped = native_options(table.field_dict[column].type, resolved_options, - builder._index_type, column, builder._user_options) - options = {key: value for key, value in options.items() - if key in ('global-index.row-count-per-shard', 'global-index.build.parallelism', - 'vindex.build.granule.enabled') - and (key != 'vindex.build.granule.enabled' or builder._index_type != 'diskann')} - options.update({'fields.' + column + '.' + key: value for key, value in mapped.items() - if key != 'index.type'}) - options['fields.' + column + '.train.sample-ratio'] = str(train_sample_ratio( - resolved_options, builder._index_type, column, builder._user_options)) - if builder._index_type == 'full-text' or builder._index_type in VINDEX_IDENTIFIERS: - # Resolve typed values and explicit default overrides through the same - # ConfigOption conversion as the local builder before crossing the FFI. - try: - parallelism = int(builder._core_options.global_index_build_parallelism()) - except (TypeError, ValueError, OverflowError) as error: - raise ValueError('Invalid global-index.build.parallelism: %s' % error) from error - options['global-index.build.parallelism'] = str(parallelism) - native.with_options(options) + native.with_options(_options_to_native(options)) if partition_filter is not None: native.with_partition_filter(_predicate_to_native(partition_filter)) if execution == "ray": diff --git a/paimon-python/pypaimon/tests/global_index_build_test.py b/paimon-python/pypaimon/tests/global_index_build_test.py index 97b3758ce3e8..ea910c402488 100644 --- a/paimon-python/pypaimon/tests/global_index_build_test.py +++ b/paimon-python/pypaimon/tests/global_index_build_test.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import json import unittest from datetime import date, datetime from decimal import Decimal @@ -226,7 +227,7 @@ class GlobalIndexBuildTest( 'file.format': 'parquet', } - def test_native_vector_build_uses_the_same_resolved_field_options(self): + def test_native_vector_build_preserves_field_option_formats(self): schema = pa.schema([('id', pa.int32()), ('embedding', pa.list_(pa.float32()))]) configurations = { 'shared': {'fields.embedding.index-options': @@ -249,12 +250,10 @@ def test_native_vector_build_uses_the_same_resolved_field_options(self): table = self._create_table(pa_schema=schema, options=options) request = GlobalIndexBuilder(table, 'embedding', 'ivf-pq') actual = self._native_build_options(request) - self.assertEqual('768', actual['fields.embedding.dimension']) - self.assertEqual('64', actual['fields.embedding.nlist']) - self.assertEqual('cosine', actual['fields.embedding.metric']) - self.assertEqual('0.5', actual['fields.embedding.train.sample-ratio']) + for key, value in options.items(): + self.assertEqual(value, actual[key]) - def test_native_scalar_and_full_text_builds_use_resolved_field_options(self): + def test_native_scalar_and_full_text_builds_preserve_field_options(self): schema = pa.schema([('id', pa.int32()), ('name', pa.string())]) for kind, key, value in (('btree', 'btree-index.block-size', '4 kb'), ('bitmap', 'bitmap-index.dictionary-block-size', '8 kb'), @@ -267,7 +266,55 @@ def test_native_scalar_and_full_text_builds_use_resolved_field_options(self): 'fields.name.index-options': '{"%s":"%s"}' % (short_key, value)}) table = self._create_table(pa_schema=schema, options=options) actual = self._native_build_options(GlobalIndexBuilder(table, 'name', kind)) - self.assertEqual(value, actual[key]) + for name, configured in options.items(): + self.assertEqual(configured, actual[name]) + + def test_native_build_keeps_table_and_user_options(self): + schema = pa.schema([('id', pa.int32()), ('embedding', pa.list_(pa.float32()))]) + for kind in ('ivf-pq', 'diskann'): + with self.subTest(index_type=kind): + values = dict(self.table_options, **{ + 'write.native.enabled': 'true', 'global-index.' + kind + '.index-columns': 'embedding', + 'fields.embedding.index-options': '{"dimension":2,"metric":"COSINE"}', + 'opaque.table.option': 'table'}) + table = self._create_table(pa_schema=schema, options=values) + user = {'opaque.user.option': 'user', 'vindex.build.granule.enabled': 'true'} + actual = self._native_build_options(GlobalIndexBuilder(table, 'embedding', kind, options=user)) + for key, value in dict(values, **user).items(): + self.assertEqual(value, actual[key]) + + def test_native_build_preserves_captured_view_and_applies_overrides(self): + from pypaimon.execution.native.write import native_index_build + from pypaimon.execution.native.table import _resolved_schema_json + from unittest.mock import Mock + + schema = pa.schema([('id', pa.int32()), ('embedding', pa.list_(pa.float32()))]) + table = self._create_table(pa_schema=schema, options=dict(self.table_options, **{ + 'write.native.enabled': 'true', 'ivf-pq.metric': 'l2'}))._copy_with_snapshot(None) + sdk = types.ModuleType('pypaimon_rust.datafusion') + sdk.GlobalIndexBuildBuilder = types.SimpleNamespace(supports_index_type=lambda kind: True) + for metric in ('cosine', None): + with self.subTest(metric=metric): + request = GlobalIndexBuilder(table, 'embedding', 'ivf-pq', options={'ivf-pq.metric': metric}) + native = Mock() + native.build.return_value = [] + proxy = types.SimpleNamespace(new_global_index_build_builder=lambda: native) + with patch.dict(sys.modules, {'pypaimon_rust.datafusion': sdk}), \ + patch.object(native_index_build, 'native_runtime_available', return_value=True), \ + patch.object(native_index_build, '_rest_catalog_supported', return_value=True), \ + patch.object(native_index_build, '_native_table', return_value=proxy) as bridge, \ + patch.object(native_index_build, 'from_native_commit_messages', return_value=[]): + self.assertEqual([], request.build()) + captured = bridge.call_args[0][0] + self.assertTrue(hasattr(captured, '_read_snapshot')) + self.assertIsNone(captured._read_snapshot) + schema = json.loads(_resolved_schema_json(captured)) + for actual in (schema['options'], native.with_options.call_args[0][0]): + if metric is None: + self.assertNotIn('ivf-pq.metric', actual) + else: + self.assertEqual(metric, actual['ivf-pq.metric']) + self.assertEqual('l2', table.table_schema.options['ivf-pq.metric']) def _native_build_options(self, request): from unittest.mock import Mock diff --git a/paimon-python/pypaimon/tests/global_index_execution_boundary_test.py b/paimon-python/pypaimon/tests/global_index_execution_boundary_test.py index 06b3ba44d2ab..b687dafca435 100644 --- a/paimon-python/pypaimon/tests/global_index_execution_boundary_test.py +++ b/paimon-python/pypaimon/tests/global_index_execution_boundary_test.py @@ -74,6 +74,11 @@ def find_spec(self, fullname, path=None, target=None): options=options, identifier='db.table', is_primary_key_table=False, field_dict={'embedding': DataField(0, 'embedding', ArrayType('ARRAY', AtomicType('FLOAT')))}, ) +def copy_table(values): + copied = SimpleNamespace(**vars(table)) + copied.options = CoreOptions(Options(dict(table.options.options.to_map(), **values))) + return copied +table.copy_without_time_travel = copy_table # An empty Native result is a successful build/drop, never a Python fallback. with patch.object(native_index_build, 'native_runtime_available', return_value=True), \ patch.object(native_index_build, '_rest_catalog_supported', return_value=True), \ @@ -93,15 +98,28 @@ def find_spec(self, fullname, path=None, target=None): import io from pypaimon.globalindex.global_index_meta import GlobalIndexIOMeta from pypaimon.globalindex.vector_search import VectorSearch +from pypaimon.globalindex.batch_vector_search import BatchVectorSearch from pypaimon.globalindex.vindex_reader import VindexVectorGlobalIndexReader from pypaimon.execution.native.read import vindex_reader +reader_options = { + 'vindex.read.parallelism': 1, + 'global-index.ivf-pq.index-column': 'embedding', + 'pk-vector.index.columns': 'embedding', + 'fields.embedding.pk-vector.index.type': 'ivf-pq', + 'fields.embedding.index-options': '{"nlist":64}', + 'opaque.enabled': True, +} +wire_options = dict(reader_options, **{'vindex.read.parallelism': '1', 'opaque.enabled': 'true'}) +query_options = {'ivf.nprobe': 3, 'fields.embedding.index-options': '{"refine_factor":4}', + 'index_type': 'ignored', 'opaque.query.option': 'preserved'} + class RustReader: failure_phase = None def __init__(self, input_, file_path, file_size, options): assert input_.pread_many([(0, 1)]) == [b'x'] - assert (file_path, file_size, options) == ('/indexes/file', 1, {'vindex.read.parallelism': '1'}) + assert (file_path, file_size, options) == ('/indexes/file', 1, wire_options) if self.failure_phase == 'open': raise failure @@ -111,7 +129,8 @@ def metric(self): return 'inner_product' def search(self, vectors, limit, filter_bytes=None, options=None): - assert limit == 2 and filter_bytes is None and options == {'ivf.nprobe': '3'} + assert limit == 2 and filter_bytes is None + assert options == dict(query_options, **{'ivf.nprobe': '3'}) if self.failure_phase == 'search': raise failure return [{7: .5} for _ in vectors] @@ -126,18 +145,21 @@ def close(self): stream = io.BytesIO(b'x') reader = VindexVectorGlobalIndexReader( SimpleNamespace(new_input_stream=lambda path: stream), - '/indexes', [GlobalIndexIOMeta('file', 1)], {'vindex.read.parallelism': 1}) + '/indexes', [GlobalIndexIOMeta('file', 1)], reader_options) try: if phase is None: assert reader.vector_metric() == 'inner_product' result = reader.visit_vector_search(VectorSearch([1., 2.], 2, 'embedding', - options={'ivf.nprobe': 3})).result() + options=query_options)).result() assert list(result.results()) == [7] and result.score_getter()(7) == .5 + results = reader.visit_batch_vector_search(BatchVectorSearch( + [[1., 2.], [2., 1.]], 2, 'embedding', options=query_options)).result() + assert [list(result.results()) for result in results] == [[7], [7]] else: try: if phase == 'search': reader.visit_vector_search(VectorSearch([1., 2.], 2, 'embedding', - options={'ivf.nprobe': 3})) + options=query_options)) else: reader.vector_metric() except OSError as error: diff --git a/paimon-python/pypaimon/tests/lazy_search_execution_test.py b/paimon-python/pypaimon/tests/lazy_search_execution_test.py index f8df7b88ce44..5f1d77d3ba1a 100644 --- a/paimon-python/pypaimon/tests/lazy_search_execution_test.py +++ b/paimon-python/pypaimon/tests/lazy_search_execution_test.py @@ -97,49 +97,29 @@ def test_native_search_routes_use_shared_index_options(search_table, tmp_path, f .add_vector_route('embedding', [1.0, 0.0], 2) .add_full_text_route('body', '{}', 2)) module._native_builder(builder) + assert bridge.call_args[0][0] is table + from pypaimon.execution.native.table import _resolved_schema_json + wire = json.loads(_resolved_schema_json(table))['options'] + for key, value in options.items(): + assert wire[key] == value if family == 'hybrid': - from pypaimon.execution.native.index_options import native_index_options - - vector = core.add_vector_route.call_args[0][-1] - assert vector == {} + assert core.add_vector_route.call_args[0][-1] == {} assert core.add_full_text_route.call_args[0][-1] == {} - text = bridge.call_args[0][0].options.options.to_map() - wire = native_index_options(bridge.call_args[0][0].table_schema) - if configure_metric: - assert wire['fields.embedding.metric'] == 'cosine' - else: - assert 'fields.embedding.metric' not in wire - assert 'ivf-pq.metric' not in wire - assert wire['fields.embedding.refine_factor'] == '3' - assert text['full-text.tokenizer'] == 'raw' - assert text['full-text.lower-case'] == 'false' - else: - options = bridge.call_args[0][0].options.options.to_map() - if family == 'text': - assert options['full-text.tokenizer'] == 'raw' - assert options['full-text.lower-case'] == 'false' - else: - if configure_metric: - assert options['fields.embedding.metric'] == 'cosine' - else: - assert 'fields.embedding.metric' not in options - assert 'ivf-pq.metric' not in options - assert options['fields.embedding.refine_factor'] == '3' - if family != 'text': - core.with_vector_index_type.assert_called_once_with('embedding', 'ivf-pq') + elif family in ('vector', 'batch'): + core.with_options.assert_called_once_with({}) @pytest.mark.parametrize('family', ['vector', 'batch', 'hybrid']) -@pytest.mark.parametrize('query_options,metric', [ - ({}, None), - ({'fields.embedding.index-options': '{"metric":"COSINE","refine_factor":4}'}, 'cosine'), - ({'fields.embedding.pk-vector.index.options': '{"distance.metric":"COSINE","refine_factor":4}'}, 'cosine'), - ({'fields.embedding.metric': 'COSINE'}, 'cosine'), - ({'ivf-pq.distance.metric': 'COSINE'}, 'cosine'), - ({'metric': 'COSINE'}, 'cosine'), - ({'fields.embedding.index-options': '{"refine_factor":4}'}, None), +@pytest.mark.parametrize('query_options', [ + {}, + {'fields.embedding.index-options': '{"metric":"COSINE","refine_factor":4}'}, + {'fields.embedding.pk-vector.index.options': '{"distance.metric":"COSINE","refine_factor":4}'}, + {'fields.embedding.metric': 'COSINE'}, + {'ivf-pq.distance.metric': 'COSINE'}, + {'metric': 'COSINE'}, + {'fields.embedding.index-options': '{"refine_factor":4}'}, ]) -def test_native_vector_query_options_preserve_explicit_metric_scope(search_table, family, query_options, metric): +def test_native_vector_query_options_preserve_explicit_metric_scope(search_table, family, query_options): from pypaimon.execution.native.read import native_vector_search, native_hybrid_search table = search_table.copy({'global-index.ivf-pq.index-columns': 'embedding', @@ -166,16 +146,9 @@ def test_native_vector_query_options_preserve_explicit_metric_scope(search_table builder = _builder(table, family).with_options(query_options) assert module._native_vector_builder(builder, family == 'batch') is core actual = core.with_options.call_args[0][0] - assert 'ivf-pq.dimension' not in actual - for key in ('metric', 'distance.metric'): - assert actual.get('fields.embedding.' + key) == metric - assert actual.get('ivf-pq.' + key) == metric - if any(key.endswith('index-options') or key.endswith('index.options') for key in query_options): - assert actual['ivf-pq.refine_factor'] == '4' - else: - assert 'fields.embedding.refine_factor' not in actual - if not query_options: - assert bridge.call_args[0][0].options.options.to_map()['fields.embedding.distance.metric'] == 'l2' + assert actual == original + assert bridge.call_args[0][0] is table + assert table.options.options.to_map()['fields.embedding.distance.metric'] == 'l2' assert query_options == original @@ -183,7 +156,7 @@ def test_native_vector_query_options_preserve_explicit_metric_scope(search_table ('vector', False), ('batch', False), ('hybrid', False), ('vector', True), ]) @pytest.mark.parametrize('configured', [True, False]) -def test_missing_native_type_api_errors_before_execution(search_table, tmp_path, family, configured, legacy): +def test_native_configuration_needs_only_standard_builder_api(search_table, tmp_path, family, configured, legacy): from pypaimon.execution.native.read import native_vector_search, native_hybrid_search declaration = ({'pk-vector.index.columns': 'embedding', 'fields.embedding.pk-vector.index.type': 'ivf-flat'} @@ -204,8 +177,6 @@ def test_missing_native_type_api_errors_before_execution(search_table, tmp_path, with ExitStack() as stack: stack.enter_context(patch.object(module, 'native_runtime_available', return_value=True)) stack.enter_context(patch.object(module, '_native_table', return_value=proxy)) - if configured: - stack.enter_context(pytest.raises(AttributeError, match='with_vector_index_type')) if family == 'hybrid': stack.enter_context(patch.object(module, '_rest_catalog_supported', return_value=True)) builder = table.new_hybrid_search_builder().with_limit(2).add_vector_route('embedding', [1., 0.], 2) @@ -213,12 +184,10 @@ def test_missing_native_type_api_errors_before_execution(search_table, tmp_path, else: stack.enter_context(patch.object(module, '_catalog_metastore', return_value='rest')) result = module._native_vector_builder(_builder(table, family), family == 'batch') - if not configured: - assert result is not None + assert result is not None core.execute.assert_not_called() core.route_builders.assert_not_called() - if configured: - core.with_limit.assert_not_called() + core.with_limit.assert_called_once_with(2) @pytest.mark.parametrize('family,entry', [ @@ -226,11 +195,14 @@ def test_missing_native_type_api_errors_before_execution(search_table, tmp_path, ('batch', 'execute'), ('batch', 'scan'), ('batch', 'read'), ('hybrid', 'execute'), ('hybrid', 'routes'), ]) -def test_missing_native_type_api_never_selects_python(search_table, family, entry): +def test_native_option_errors_never_select_python(search_table, family, entry): from pypaimon.execution.native.read import native_vector_search, native_hybrid_search table = search_table.copy({'global-index.ivf-flat.index-columns': 'embedding'}) - core = Mock(spec=['execute', 'route_builders']) + core = Mock() + core.with_limit.return_value = core + core.with_options.side_effect = ValueError('invalid native options') + core.add_vector_route.side_effect = ValueError('invalid native options') proxy = SimpleNamespace(new_vector_search_builder=lambda: core, new_batch_vector_search_builder=lambda: core, new_hybrid_search_builder=lambda: core) @@ -250,7 +222,7 @@ def test_missing_native_type_api_never_selects_python(search_table, family, entr invoke = builder.execute_batch_local if family == 'batch' else builder.execute_local else: invoke = _factories(builder, family)[0 if entry == 'scan' else 1] - with pytest.raises(AttributeError, match='with_vector_index_type'): + with pytest.raises(ValueError, match='invalid native options'): invoke() core.execute.assert_not_called() core.route_builders.assert_not_called() @@ -258,7 +230,7 @@ def test_missing_native_type_api_never_selects_python(search_table, family, entr @pytest.mark.parametrize('family', ['vector', 'batch', 'hybrid']) @pytest.mark.parametrize('key', ['index_type', 'index-type', 'vector.index-type', 'fields.embedding.index-type']) -def test_removed_type_hints_never_reach_native_search(search_table, family, key): +def test_native_search_keeps_uninterpreted_options(search_table, family, key): from pypaimon.execution.native.read import native_vector_search, native_hybrid_search table = search_table.copy({key: 'diskann'}) @@ -276,30 +248,26 @@ def test_removed_type_hints_never_reach_native_search(search_table, family, key) builder = (table.new_hybrid_search_builder().with_limit(2) .add_vector_route('embedding', [1., 0.], 2, options={key: 'ivf-pq'})) assert module._native_builder(builder) is not None - assert key not in core.add_vector_route.call_args[0][-1] + assert core.add_vector_route.call_args[0][-1][key] == 'ivf-pq' else: stack.enter_context(patch.object(module, '_catalog_metastore', return_value='rest')) builder = _builder(table, family).with_option(key, 'ivf-pq') assert module._native_vector_builder(builder, family == 'batch') is core - assert key not in core.with_options.call_args[0][0] - core.with_vector_index_type.assert_not_called() + assert core.with_options.call_args[0][0][key] == 'ivf-pq' @pytest.mark.parametrize('family', ['vector', 'batch', 'hybrid']) -def test_native_type_configuration_errors_propagate(search_table, family): +def test_native_table_configuration_errors_propagate(search_table, family): from pypaimon.execution.native.read import native_vector_search, native_hybrid_search table = search_table.copy({'global-index.ivf-flat.index-columns': 'embedding'}) core = Mock() - core.with_vector_index_type.side_effect = ValueError('unsupported native vector implementation') - proxy = SimpleNamespace(new_vector_search_builder=lambda: core, - new_batch_vector_search_builder=lambda: core, - new_hybrid_search_builder=lambda: core) module = native_hybrid_search if family == 'hybrid' else native_vector_search with ExitStack() as stack: stack.enter_context(patch.object(module, 'native_runtime_available', return_value=True)) - stack.enter_context(patch.object(module, '_native_table', return_value=proxy)) - stack.enter_context(pytest.raises(ValueError, match='unsupported native vector implementation')) + stack.enter_context(patch.object( + module, '_native_table', side_effect=ValueError('unsupported native index configuration'))) + stack.enter_context(pytest.raises(ValueError, match='unsupported native index configuration')) if family == 'hybrid': stack.enter_context(patch.object(module, '_rest_catalog_supported', return_value=True)) builder = table.new_hybrid_search_builder().with_limit(2).add_vector_route('embedding', [1., 0.], 2) @@ -329,9 +297,11 @@ def test_native_hybrid_vector_columns_keep_independent_types(tmp_path): proxy = SimpleNamespace(new_hybrid_search_builder=lambda: core) with patch.object(native_hybrid_search, 'native_runtime_available', return_value=True), \ patch.object(native_hybrid_search, '_rest_catalog_supported', return_value=True), \ - patch.object(native_hybrid_search, '_native_table', return_value=proxy): + patch.object(native_hybrid_search, '_native_table', return_value=proxy) as bridge: assert native_hybrid_search._native_builder(builder) is not None - assert core.with_vector_index_type.call_args_list == [(('other', 'diskann'),), (('embedding', 'ivf-pq'),)] + assert bridge.call_args[0][0] is table + assert table.table_schema.options['global-index.ivf-pq.index-columns'] == 'embedding' + assert table.table_schema.options['global-index.diskann.index-column'] == 'other' assert [call[0][0] for call in core.add_vector_route.call_args_list] == ['other', 'embedding'] @@ -360,26 +330,28 @@ def test_native_hybrid_full_text_routes_preserve_each_field_analyzer(tmp_path, s patch.object(native_hybrid_search, '_rest_catalog_supported', return_value=True), \ patch.object(native_hybrid_search, '_native_table', return_value=proxy) as bridge: configured = native_hybrid_search._native_builder(builder) - if same_analyzer: - assert configured is not None - assert bridge.call_args[0][0].options.options.to_map()['full-text.tokenizer'] == 'raw' - assert core.add_full_text_route.call_count == 2 - assert all(call.args[-1] == {} for call in core.add_full_text_route.call_args_list) - else: - assert configured is None - bridge.assert_not_called() - core.add_full_text_route.assert_not_called() + assert configured is not None + assert bridge.call_args[0][0] is table + assert table.table_schema.options['fields.body.index-options'] == '{"tokenizer":"raw"}' + assert table.table_schema.options['fields.title.index-options'] == ( + '{"tokenizer":"%s"}' % ('raw' if same_analyzer else 'jieba')) + assert core.add_full_text_route.call_count == 2 + assert all(call.args[-1] == {} for call in core.add_full_text_route.call_args_list) -def test_native_vector_defers_field_options_until_index_type_is_known(search_table): +def test_native_vector_passes_field_options_without_knowing_type(search_table): from pypaimon.execution.native.read import native_vector_search table = search_table.copy({'fields.embedding.index-options': '{"nlist":64}'}) + core = Mock(spec=['with_limit', 'with_options', 'with_vector_column']) + core.with_limit.return_value = core + proxy = SimpleNamespace(new_vector_search_builder=lambda: core) with patch.object(native_vector_search, 'native_runtime_available', return_value=True), \ patch.object(native_vector_search, '_catalog_metastore', return_value='rest'), \ - patch.object(native_vector_search, '_native_table') as bridge: - assert native_vector_search._native_vector_builder(_builder(table, 'vector')) is None - bridge.assert_not_called() + patch.object(native_vector_search, '_native_table', return_value=proxy) as bridge: + assert native_vector_search._native_vector_builder(_builder(table, 'vector')) is core + bridge.assert_called_once_with(table) + assert table.table_schema.options['fields.embedding.index-options'] == '{"nlist":64}' def _factories(builder, family): diff --git a/paimon-python/pypaimon/tests/native_index_options_test.py b/paimon-python/pypaimon/tests/native_index_options_test.py index 7092f7f4050f..93175a52b262 100644 --- a/paimon-python/pypaimon/tests/native_index_options_test.py +++ b/paimon-python/pypaimon/tests/native_index_options_test.py @@ -22,13 +22,10 @@ import pytest from pypaimon import CatalogFactory, Schema -from pypaimon.execution.native.index_options import native_scalar_index_read_supported from pypaimon.execution.native.read.builder import _native_read_builder -from pypaimon.execution.native.read.plan import native_plan -from pypaimon.execution.native.read.native_search_plan import _index_search_options -from pypaimon.execution.native.table import _resolved_schema_json +from pypaimon.execution.native.read.plan import native_plan, native_stream_scan +from pypaimon.execution.native.table import _resolved_schema_json, _options_to_native from pypaimon.execution.native.write.native_commit import create_native_write_table -from pypaimon.index.pk.primary_key_index_definitions import PrimaryKeyIndexDefinitions from pypaimon.schema.data_types import AtomicType, DataField, VectorType @@ -51,146 +48,78 @@ def _table(tmp_path, primary_key, extra=None): @pytest.mark.parametrize('primary_key', [False, True]) -@pytest.mark.parametrize('suffix', ['index-column', 'index-columns']) -def test_native_read_and_write_schemas_share_resolved_index_contract(tmp_path, primary_key, suffix): - table = _table(tmp_path, primary_key, { - 'global-index.btree.' + suffix: 'amount', - 'fields.amount.index-options': '{"fallback-scan-max-size":"1 kb"}', - 'global-index.ivf-pq.' + suffix: 'embedding', - 'ivf-pq.dimension': '2', - 'ivf-pq.refine_factor': '3', - 'fields.embedding.index-options': '{"nlist":64,"metric":"COSINE"}', - 'global-index.full-text.' + suffix: 'body', - 'fields.body.index-options': '{"tokenizer":"raw","lower-case":false}', - }) +@pytest.mark.parametrize('declaration', ['legacy', 'index-column', 'index-columns']) +def test_native_schemas_preserve_index_configuration_verbatim(tmp_path, primary_key, declaration): + values = {'ivf-pq.dimension': '2', 'opaque.native.option': 'preserved', + 'btree-index.block-size': '4 kb', + 'fields.amount.index-options': '{"block-size":"8 kb"}', + 'fields.other.index-options': '{"block-size":"16 kb"}', + 'fields.embedding.index-options': '{"nlist":64,"metric":"COSINE"}', + 'fields.embedding.pk-vector.index.options': '{"nlist":32}', + 'fields.body.index-options': '{"stop-words":["paimon","lake"],"lower-case":false}'} + if declaration == 'legacy': + values.update({'pk-vector.index.columns': 'embedding', + 'fields.embedding.pk-vector.index.type': 'IVF-PQ', + 'pk-btree.index.columns': 'amount,other', 'pk-full-text.index.columns': 'body'}) + else: + values.update({'global-index.ivf-pq.' + declaration: 'embedding', + 'global-index.btree.' + declaration: 'amount,other', + 'global-index.full-text.' + declaration: 'body'}) + table = _table(tmp_path, primary_key, values) original = dict(table.table_schema.options) - read_options = json.loads(_resolved_schema_json(table))['options'] + read = json.loads(_resolved_schema_json(table))['options'] module = ModuleType('pypaimon_rust.datafusion') module.Table = Mock() with patch.dict('sys.modules', {'pypaimon_rust.datafusion': module}): assert create_native_write_table(table) is module.Table.from_resolved_schema.return_value - write_options = json.loads(module.Table.from_resolved_schema.call_args[0][1])['options'] - for options in (read_options, write_options): - assert not any(key.startswith('global-index.') and key.endswith(('.index-column', '.index-columns')) - for key in options) - if primary_key: - assert options['pk-btree.index.columns'] == 'amount' - assert options['pk-vector.index.columns'] == 'embedding' - assert options['fields.embedding.pk-vector.index.type'] == 'ivf-pq' - assert options['fields.embedding.pk-vector.distance.metric'] == 'cosine' - assert options['ivf-pq.dimension'] == '2' - assert options['ivf-pq.refine_factor'] == '3' - assert options['full-text.tokenizer'] == 'raw' - assert options['full-text.lower-case'] == 'false' - definitions = PrimaryKeyIndexDefinitions.create(table.table_schema.copy(new_options=options)).definitions - by_type = {definition.index_type: definition for definition in definitions} - assert by_type['btree'].options.to_map()['btree-index.fallback-scan-max-size'] == '1 kb' - assert by_type['ivf-pq'].options.to_map()['ivf-pq.nlist'] == '64' - assert by_type['ivf-pq'].options.to_map()['ivf-pq.dimension'] == '2' - assert by_type['ivf-pq'].options.to_map()['ivf-pq.metric'] == 'cosine' - assert by_type['full-text'].options.to_map()['full-text.lower-case'] == 'false' - else: - assert not any(key.startswith('pk-') for key in options) - assert 'fields.embedding.index-type' not in options - assert options['fields.embedding.nlist'] == '64' - assert options['fields.embedding.metric'] == 'cosine' - assert options['btree-index.fallback-scan-max-size'] == '1 kb' + write = json.loads(module.Table.from_resolved_schema.call_args[0][1])['options'] + for actual in (read, write): + for key, value in values.items(): + assert actual[key] == value assert table.table_schema.options == original -def test_native_primary_key_fields_keep_independent_overrides(tmp_path): - table = _table(tmp_path, True, { - 'global-index.btree.index-columns': 'amount,other', - 'btree-index.block-size': '4 kb', - 'fields.amount.index-options': '{"block-size":"8 kb"}', - 'fields.other.index-options': '{"block-size":"16 kb"}', - }) - options = json.loads(_resolved_schema_json(table))['options'] - assert 'btree-index.block-size' not in options - assert json.loads(options['fields.amount.pk-btree.index.options'])['btree-index.block-size'] == '8 kb' - assert json.loads(options['fields.other.pk-btree.index.options'])['btree-index.block-size'] == '16 kb' - - -@pytest.mark.parametrize('field_options', [ - {}, - {'fields.embedding.pk-vector.index.options': '{"nlist":64}'}, - {'fields.embedding.index-options': '{"nlist":64}'}, -]) -def test_native_legacy_primary_key_keeps_flat_vector_read_parameters(tmp_path, field_options): - options = {'pk-vector.index.columns': 'embedding', - 'fields.embedding.pk-vector.index.type': 'ivf-pq', - 'fields.embedding.pk-vector.distance.metric': 'COSINE', - 'ivf-pq.dimension': '2', 'ivf-pq.refine_factor': '3'} - options.update(field_options) - table = _table(tmp_path, True, options) - wire = json.loads(_resolved_schema_json(table))['options'] - assert wire['pk-vector.index.columns'] == 'embedding' - assert wire['fields.embedding.pk-vector.index.type'] == 'ivf-pq' - assert wire['fields.embedding.pk-vector.distance.metric'] == 'cosine' - assert wire['ivf-pq.dimension'] == '2' - assert wire['ivf-pq.refine_factor'] == '3' - assert wire['ivf-pq.metric'] == 'cosine' - if field_options: - assert wire['ivf-pq.nlist'] == '64' - - -def test_native_data_evolution_does_not_invent_a_vector_metric(tmp_path): - table = _table(tmp_path, False, { - 'global-index.ivf-pq.index-columns': 'embedding', - 'fields.embedding.index-options': '{"nlist":64}', - }) - options = json.loads(_resolved_schema_json(table))['options'] - assert options['fields.embedding.nlist'] == '64' - assert 'fields.embedding.index-type' not in options - assert 'fields.embedding.metric' not in options - assert 'ivf-pq.metric' not in options - - @pytest.mark.parametrize('primary_key', [False, True]) @pytest.mark.parametrize('key', ['index_type', 'index-type', 'vector.index-type', 'fields.embedding.index-type']) -def test_native_schema_drops_removed_type_aliases(tmp_path, primary_key, key): - table = _table(tmp_path, primary_key, { - 'global-index.ivf-pq.index-columns': 'embedding', key: 'diskann', - 'fields.embedding.index-options': '{"nlist":64}', - }) - original = dict(table.table_schema.options) - options = json.loads(_resolved_schema_json(table))['options'] - assert key not in options - assert 'fields.embedding.index-type' not in options - if primary_key: - assert options['fields.embedding.pk-vector.index.type'] == 'ivf-pq' - assert key not in json.loads(options['fields.embedding.pk-vector.index.options']) - assert table.table_schema.options == original - - -@pytest.mark.parametrize('primary_key', [False, True]) -@pytest.mark.parametrize('suffix', ['index-options', 'pk-vector.index.options']) -def test_field_json_cannot_restore_removed_native_type_hint(tmp_path, primary_key, suffix): - table = _table(tmp_path, primary_key, { - 'global-index.ivf-pq.index-columns': 'embedding', - 'fields.embedding.' + suffix: '{"index-type":"diskann","nlist":64}', - }) - options = json.loads(_resolved_schema_json(table))['options'] - assert 'fields.embedding.index-type' not in options - if primary_key: - assert 'fields.embedding.index-type' not in json.loads(options['fields.embedding.pk-vector.index.options']) - assert options['fields.embedding.pk-vector.index.type'] == 'ivf-pq' - - -def test_native_qualifies_distinct_scalar_field_options_before_reading(tmp_path): +def test_native_schema_keeps_uninterpreted_options(tmp_path, primary_key, key): + values = {'global-index.ivf-pq.index-columns': 'embedding', key: 'diskann', + 'fields.embedding.index-options': '{"index-type":"other","nlist":64}'} + table = _table(tmp_path, primary_key, values) + actual = json.loads(_resolved_schema_json(table))['options'] + for name, value in values.items(): + assert actual[name] == value + assert 'ivf-pq.nlist' not in actual + assert 'fields.embedding.metric' not in actual + + +@pytest.mark.parametrize('field_options', ['{"nlist":64}', '[1,2]']) +def test_native_defers_field_option_parsing_to_rust(tmp_path, field_options): + table = _table(tmp_path, False, {'fields.embedding.index-options': field_options}) + actual = json.loads(_resolved_schema_json(table))['options'] + assert actual['fields.embedding.index-options'] == field_options + + +@pytest.mark.parametrize('streaming', [False, True]) +def test_distinct_scalar_field_options_reach_native_scan(tmp_path, streaming): table = _table(tmp_path, False, { 'global-index.btree.index-columns': 'amount,other', 'fields.amount.index-options': '{"fallback-scan-max-size":"1 kb"}', 'fields.other.index-options': '{"fallback-scan-max-size":"2 kb"}', }) - assert not native_scalar_index_read_supported(table) predicate = table.new_read_builder().new_predicate_builder().equal('amount', 1) - with patch('pypaimon.execution.native.table._native_table') as bridge, \ - patch('pypaimon.execution.native.read.plan.runtime.native_runtime_available', return_value=True): - with pytest.raises(NotImplementedError, match='field index options'): - native_plan(table, predicate=predicate) - assert _index_search_options(table, 'body', 'full-text', scalar_filter=predicate) is None - bridge.assert_not_called() + from pypaimon.execution.native.read import plan + with patch.object(plan.runtime, 'native_runtime_available', return_value=True), \ + patch.object(plan.native_builder, '_native_read_builder') as bridge, \ + patch.object(plan.native_builder, '_configure_native_read_builder', + side_effect=lambda builder, *args, **kwargs: builder), \ + patch.object(plan, '_from_native_plan') as convert: + if streaming: + assert native_stream_scan(table, predicate=predicate) is bridge.return_value.new_stream_scan.return_value + else: + assert native_plan(table, predicate=predicate) is convert.return_value + bridge.assert_called_once_with(table) + options = json.loads(_resolved_schema_json(table))['options'] + assert options['fields.amount.index-options'] != options['fields.other.index-options'] @pytest.mark.parametrize('global_index', [True, False]) @@ -199,8 +128,16 @@ def test_index_field_options_do_not_disable_native_physical_reads(tmp_path, glob 'global-index.enabled': str(global_index).lower(), 'fields.embedding.index-options': '{"nlist":64}', }) - if not global_index: - assert native_scalar_index_read_supported(table) with patch('pypaimon.execution.native.table._native_table') as bridge: assert _native_read_builder(table) is bridge.return_value.new_read_builder.return_value bridge.assert_called_once_with(table) + + +def test_native_options_only_encode_values_and_leave_input_untouched(): + original = {'boolean': True, 'number': 4, 'list': ['paimon', 'lake'], + 'object': {'value': False}, 'text': ' COSINE ', 'unset': None} + actual = _options_to_native(original) + assert actual == {'boolean': 'true', 'number': '4', 'list': '["paimon", "lake"]', + 'object': '{"value": false}', 'text': ' COSINE '} + assert original['unset'] is None + assert original['list'] == ['paimon', 'lake'] diff --git a/paimon-python/pypaimon/tests/native_search_plan_test.py b/paimon-python/pypaimon/tests/native_search_plan_test.py index 24d5f2814a19..0b8471d26401 100644 --- a/paimon-python/pypaimon/tests/native_search_plan_test.py +++ b/paimon-python/pypaimon/tests/native_search_plan_test.py @@ -236,10 +236,6 @@ def test_reader_table_options_apply_to_captured_sources(rest_catalog, family): @pytest.mark.parametrize('metric', [None, 'cosine']) @pytest.mark.parametrize('index_type', ['ivf-flat', 'diskann']) def test_raw_native_plans_use_java_configured_indexer_default(rest_catalog, family, metric, index_type): - from pypaimon_rust import datafusion as sdk - cls = sdk.VectorSearchBuilder if family == 'vector' else sdk.BatchVectorSearchBuilder - if not hasattr(cls, 'with_vector_index_type'): - pytest.skip('Native SDK does not yet implement the explicit vector type API') options = { 'global-index.%s.index-columns' % index_type: 'embedding', 'write.native.enabled': 'false', 'read.native.enabled': 'false', 'scan.native-plan.enabled': 'false', diff --git a/paimon-python/pypaimon/tests/native_vector_context_test.py b/paimon-python/pypaimon/tests/native_vector_context_test.py index 8b51a413e465..1519098b5002 100644 --- a/paimon-python/pypaimon/tests/native_vector_context_test.py +++ b/paimon-python/pypaimon/tests/native_vector_context_test.py @@ -47,9 +47,7 @@ def test_unused_context_attributes_preserve_native_vector_execution(option, batc values.update({'global-index.ivf-flat.index-columns': 'embedding', 'fields.embedding.index-options': '{"nlist":64}'}) environment = SimpleNamespace(catalog_loader=loader) - table = SimpleNamespace( - options=CoreOptions(Options(values)), catalog_environment=environment, - copy=lambda options: SimpleNamespace(options=CoreOptions(Options(options)), catalog_environment=environment)) + table = SimpleNamespace(options=CoreOptions(Options(values)), catalog_environment=environment) builder = SimpleNamespace( _table=table, _vector_column=SimpleNamespace(name='embedding'), _limit=2, _options={'ivf-flat.metric': 'l2'}, _filter=Predicate('equal', 0, 'id', [7]), @@ -74,15 +72,9 @@ def test_unused_context_attributes_preserve_native_vector_execution(option, batc assert resolved_table.catalog_environment is environment core.with_vector_column.assert_called_once_with('embedding') core.with_limit.assert_called_once_with(2) - expected_options = {'ivf-flat.metric': 'l2'} - if declaration is not None: - core.with_vector_index_type.assert_called_once_with('embedding', 'ivf-flat') - assert resolved_table.options.options.to_map()['ivf-flat.nlist'] == '64' - expected_options.update({'ivf-flat.distance.metric': 'l2', - 'fields.embedding.metric': 'l2', 'fields.embedding.distance.metric': 'l2'}) - else: - core.with_vector_index_type.assert_not_called() - core.with_options.assert_called_once_with(expected_options) + assert resolved_table is table + assert resolved_table.options.options.to_map() == values + core.with_options.assert_called_once_with({'ivf-flat.metric': 'l2'}) core.with_filter.assert_called_once_with({'method': 'equal', 'field': 'id', 'literals': [7]}) core.with_partition_filter.assert_called_once_with({'method': 'equal', 'field': 'pt', 'literals': [1]}) unused.assert_not_called()