From ec23d6fb29ab5e2d39edc0f877dfc9e14fbf32d2 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 31 Oct 2026 10:00:44 +0800 Subject: [PATCH 1/4] feat(table): maintain declared global indexes after writes --- crates/paimon/src/spec/core_options.rs | 3 +- crates/paimon/src/table/de_vector_read.rs | 88 +- .../table/full_text_index_build_builder.rs | 15 +- .../src/table/full_text_search_builder.rs | 16 +- .../src/table/global_index_build_builder.rs | 53 +- .../src/table/global_index_build_common.rs | 37 +- .../paimon/src/table/global_index_options.rs | 207 +++ crates/paimon/src/table/mod.rs | 1 + crates/paimon/src/table/search_scan_split.rs | 19 +- .../sorted_global_index_build_builder.rs | 12 +- crates/paimon/src/table/table_commit.rs | 38 +- .../src/table/table_commit/auto_compact.rs | 73 +- .../table/table_commit/auto_index_build.rs | 185 +++ .../table_commit/auto_index_build/planning.rs | 199 +++ .../table_commit/auto_index_build_tests.rs | 1163 +++++++++++++++++ .../auto_index_build_tests/search.rs | 493 +++++++ .../paimon/src/table/vector_search_common.rs | 89 +- .../src/table/vindex_index_build_builder.rs | 40 +- crates/paimon/src/vindex/mod.rs | 123 +- .../tests/rest_catalog/auto_compact_review.rs | 59 + docs/src/compaction.md | 93 ++ docs/src/sql.md | 31 +- 22 files changed, 2871 insertions(+), 166 deletions(-) create mode 100644 crates/paimon/src/table/global_index_options.rs create mode 100644 crates/paimon/src/table/table_commit/auto_index_build.rs create mode 100644 crates/paimon/src/table/table_commit/auto_index_build/planning.rs create mode 100644 crates/paimon/src/table/table_commit/auto_index_build_tests.rs create mode 100644 crates/paimon/src/table/table_commit/auto_index_build_tests/search.rs diff --git a/crates/paimon/src/spec/core_options.rs b/crates/paimon/src/spec/core_options.rs index 49794dcf4..9a0135e8e 100644 --- a/crates/paimon/src/spec/core_options.rs +++ b/crates/paimon/src/spec/core_options.rs @@ -1639,7 +1639,8 @@ impl<'a> CoreOptions<'a> { .unwrap_or(DEFAULT_TARGET_FILE_SIZE) } - /// Skip automatic compaction after a data commit. Defaults to false. + /// Skip automatic compaction and declared global-index maintenance after + /// a data commit. Defaults to false. /// `write.compaction-skip` is the legacy alias; `write-only` takes precedence. pub fn write_only(&self) -> crate::Result { match self diff --git a/crates/paimon/src/table/de_vector_read.rs b/crates/paimon/src/table/de_vector_read.rs index cba378d8f..cdfa9a33b 100644 --- a/crates/paimon/src/table/de_vector_read.rs +++ b/crates/paimon/src/table/de_vector_read.rs @@ -17,7 +17,7 @@ //! Reads global vector indexes, scores raw vectors, and materializes global-row-ID results. -use crate::io::{FileIO, FileRead}; +use crate::io::FileIO; use crate::spec::{ row_id_data_field, CoreOptions, DataField, DataType, FileKind, GlobalIndexSearchMode, IndexFileMeta, IndexManifestEntry, Predicate, ROW_ID_FIELD_NAME, @@ -38,8 +38,8 @@ use crate::table::search_task::{ use crate::table::vector_read::Read; use crate::table::vector_search_common::{ configured_refine_factor, configured_vector_index_type, indexed_search_limit, - log_vindex_range_io_stats, normalize_metric, resolve_materialize_read_type, - validate_global_vector_index_types, vindex_concurrency_limits, + log_vindex_range_io_stats, normalize_metric, read_index_vector_metric, + resolve_materialize_read_type, validate_global_vector_index_types, vindex_concurrency_limits, }; use crate::table::{ find_field_id_by_name, merge_row_ranges, ArrowRecordBatchStream, RowRange, Table, @@ -56,9 +56,7 @@ use arrow_array::{Array, FixedSizeListArray, Float32Array, Int64Array, ListArray use arrow_select::interleave::interleave_record_batch; use futures::{stream, TryStreamExt}; use paimon_vindex_core::blas::sgemm_a_bt; -use paimon_vindex_core::diskann_io::DISKANN_HEADER_SIZE; use paimon_vindex_core::distance::MetricType; -use paimon_vindex_core::index::VectorIndexReader as VIndexReader; use paimon_vindex_core::io::SeekRead; use roaring::RoaringTreemap; use serde::{Deserialize, Serialize}; @@ -900,7 +898,7 @@ async fn evaluate_batch_vector_search_with_executor( .collect::>() }; let configured_index_type = - configured_vector_index_type(evaluation.table_options, &search_options, field_name); + configured_vector_index_type(evaluation.table_options, &search_options, field_name)?; let index_type = vector_entries .first() .map(|entry| entry.index_file.index_type.as_str()) @@ -1125,7 +1123,7 @@ async fn evaluate_batch_vector_search_with_executor( })?; let metric_start = timing_enabled.then(Instant::now); let mut metric_options = vector_searches[0].options.clone(); - if configured_vector_index_type(evaluation.table_options, &metric_options, field_name) + if configured_vector_index_type(evaluation.table_options, &metric_options, field_name)? .is_none() { if let Some(index_type) = @@ -1673,7 +1671,7 @@ async fn resolve_raw_vector_metric( field_id: i32, field_name: &str, ) -> crate::Result { - let configured_type = configured_vector_index_type(table_options, search_options, field_name); + let configured_type = configured_vector_index_type(table_options, search_options, field_name)?; let index_type = index_entries .iter() .filter(|entry| entry.kind == FileKind::Add) @@ -1703,6 +1701,7 @@ async fn resolve_raw_vector_metric( continue; } if let Some(metric) = read_index_vector_metric(file_io, table_path, entry).await? { + let metric = RawVectorMetric::from_vindex(metric); check.check(metric)?; persisted = Some(metric); } @@ -1713,6 +1712,24 @@ async fn resolve_raw_vector_metric( if let Some(metric) = check.requested { return Ok(metric); } + if let Some(kind) = super::global_index_options::vector_index_type(table_options, field_name)? { + if index_type == Some(kind) { + let options = super::global_index_options::field_index_options( + table_options, + field_name, + "vector", + kind, + )?; + let native = crate::vindex::VindexVectorIndexOptions::resolve_native_options( + table_options, + &options, + &HashMap::new(), + kind, + field_name, + )?; + return RawVectorMetric::parse(&native["metric"]); + } + } let table_metric = vector_metric_override(table_options, field_name, index_type)?; if let Some(metric) = table_metric { return Ok(metric); @@ -1726,61 +1743,6 @@ async fn resolve_raw_vector_metric( }) } -/// Rerouted raw searches still validate each shard's persisted metric. -async fn read_index_vector_metric( - file_io: &FileIO, - table_path: &str, - entry: &IndexManifestEntry, -) -> crate::Result> { - let Some(global_meta) = entry.index_file.global_index_meta.as_ref() else { - return Ok(None); - }; - if !is_vindex_index_type(&entry.index_file.index_type) { - return Ok(None); - } - if let Some(index_meta) = global_meta.index_meta.as_ref() { - if let Ok(options) = serde_json::from_slice::>(index_meta) { - if let Some(metric) = options.get("metric") { - if let Some(metric) = RawVectorMetric::parse_normalized(&normalize_metric(metric)) { - return Ok(Some(metric)); - } - } - } - } - let path = IndexFileLocation::Global { table_path }.resolve( - &entry.index_file.file_name, - entry.index_file.external_path.as_deref(), - ); - let input = file_io.new_input(&path)?; - let read_error = |e| crate::Error::DataInvalid { - message: format!( - "Failed to read vindex index file '{}' for raw search metric: {}", - entry.index_file.file_name, e - ), - source: Some(Box::new(e)), - }; - let header_size = if entry.index_file.file_size > 0 { - (entry.index_file.file_size as u64).min(DISKANN_HEADER_SIZE as u64) - } else { - input - .metadata() - .await - .map_err(&read_error)? - .size - .min(DISKANN_HEADER_SIZE as u64) - }; - let file_reader = input.reader().await.map_err(&read_error)?; - let bytes = file_reader.read(0..header_size).await.map_err(read_error)?; - let reader = VIndexReader::open(Cursor::new(bytes)).map_err(|e| crate::Error::DataInvalid { - message: format!( - "Failed to open paimon-vindex-core reader for raw search metric: {}", - e - ), - source: Some(Box::new(e)), - })?; - Ok(Some(RawVectorMetric::from_vindex(reader.metadata().metric))) -} - fn vector_metric_override( options: &HashMap, field_name: &str, diff --git a/crates/paimon/src/table/full_text_index_build_builder.rs b/crates/paimon/src/table/full_text_index_build_builder.rs index 39fb547fc..c8598867b 100644 --- a/crates/paimon/src/table/full_text_index_build_builder.rs +++ b/crates/paimon/src/table/full_text_index_build_builder.rs @@ -53,6 +53,9 @@ pub(super) struct FullTextIndexBuildBuilder<'a> { index_column: Option, options: HashMap, pub(super) partition_filter: Option, + pub(super) snapshot: Option>, + pub(super) replaced_index_files: + std::collections::HashSet, } impl<'a> FullTextIndexBuildBuilder<'a> { @@ -62,6 +65,8 @@ impl<'a> FullTextIndexBuildBuilder<'a> { index_column: None, options: HashMap::new(), partition_filter: None, + snapshot: None, + replaced_index_files: Default::default(), } } @@ -110,10 +115,10 @@ impl<'a> FullTextIndexBuildBuilder<'a> { full_text_config(&native_options)?; let index_meta = serialize_index_meta(&native_options)?; - // The table's manager resolves a REST-managed table's latest snapshot - // through the catalog, like reads and commit validation do. - let snapshot_manager = self.table.snapshot_manager(); - let Some(snapshot) = snapshot_manager.get_latest_snapshot().await? else { + let Some(snapshot) = + super::global_index_build_common::build_snapshot(self.table, self.snapshot.as_ref()) + .await? + else { return Ok((None, vec![])); }; @@ -132,6 +137,7 @@ impl<'a> FullTextIndexBuildBuilder<'a> { FULL_TEXT_GLOBAL_INDEX_TYPE, index_field.id(), None, + &self.replaced_index_files, ) .await?; let shards = plan_vector_index_shards( @@ -159,6 +165,7 @@ impl<'a> FullTextIndexBuildBuilder<'a> { .iter() .map(|shard| RowRange::new(shard.row_range_start, shard.row_range_end)) .collect::>(), + &self.replaced_index_files, ) .await?; diff --git a/crates/paimon/src/table/full_text_search_builder.rs b/crates/paimon/src/table/full_text_search_builder.rs index 0f573240b..c02cc4122 100644 --- a/crates/paimon/src/table/full_text_search_builder.rs +++ b/crates/paimon/src/table/full_text_search_builder.rs @@ -697,7 +697,7 @@ async fn read_raw_full_text_splits( let read = read_builder.new_read()?; let mut stream = read.to_arrow(splits)?; - let config = raw_full_text_config(table_options)?; + let config = raw_full_text_config(table_options, &search.field_name)?; let mut writer = tokio::task::spawn_blocking(move || { FullTextIndexWriter::new(config).map_err(full_text_error) }) @@ -908,8 +908,16 @@ fn raw_full_text_document( fn raw_full_text_config( table_options: &HashMap, + column: &str, ) -> crate::Result { - let native_options: HashMap = table_options + let mut options = table_options.clone(); + options.extend(super::global_index_options::field_index_options( + table_options, + column, + "full-text", + "full-text", + )?); + let native_options: HashMap = options .iter() .filter_map(|(key, value)| { key.strip_prefix("full-text.") @@ -1321,7 +1329,7 @@ mod tests { "native_text,secondary".to_string(), )]); - let config = raw_full_text_config(&options).unwrap(); + let config = raw_full_text_config(&options, "body").unwrap(); assert_eq!(config.text_fields, vec!["native_text", "secondary"]); assert_eq!(config.default_text_field(), "native_text"); @@ -1333,7 +1341,7 @@ mod tests { "full-text.text-field".to_string(), "native_text".to_string(), )]); - let config = raw_full_text_config(&options).unwrap(); + let config = raw_full_text_config(&options, "body").unwrap(); let search = FullTextSearch::new( r#"{"match":{"column":"native_text","query":"rare common"}}"#.to_string(), 10, diff --git a/crates/paimon/src/table/global_index_build_builder.rs b/crates/paimon/src/table/global_index_build_builder.rs index 485d6b0b3..d19c9c0a7 100644 --- a/crates/paimon/src/table/global_index_build_builder.rs +++ b/crates/paimon/src/table/global_index_build_builder.rs @@ -105,6 +105,20 @@ impl<'a> GlobalIndexBuildBuilder<'a> { self } + fn configured_table(&self, columns: &[&str], index_type: &str) -> Result { + if columns.len() != 1 || crate::vindex::is_vindex_index_type(index_type) { + return Ok(self.table.clone()); + } + // Native vector builders keep declarations in their own option layer. + let defaults = super::global_index_options::declared_options( + self.table.schema().options(), + columns[0], + index_type, + index_type, + )?; + Ok(self.table.copy_with_options(defaults)) + } + /// Restrict the build to partitions. Repeated calls combine filters with AND. pub fn with_partition_filter(&mut self, filter: Predicate) -> Result<&mut Self> { super::partition_filter::validate_partition_filter(self.table, &filter)?; @@ -197,6 +211,32 @@ impl<'a> GlobalIndexBuildBuilder<'a> { } async fn prepare(&self) -> Result { + self.prepare_at_snapshot(None, Default::default()).await + } + + /// Check the complete writer configuration without scanning or creating files. + pub(crate) async fn validate(&self) -> Result<()> { + self.prepare_at_snapshot(Some(None), Default::default()) + .await + .map(|_| ()) + } + + pub(crate) async fn build_for_maintenance( + &self, + snapshot: &crate::spec::Snapshot, + replaced: std::collections::HashSet, + ) -> Result> { + Ok(self + .prepare_at_snapshot(Some(Some(snapshot.clone())), replaced) + .await? + .messages) + } + + async fn prepare_at_snapshot( + &self, + snapshot: Option>, + replaced: std::collections::HashSet, + ) -> Result { CoreOptions::new(self.table.schema().options()).ensure_read_authorized()?; self.table.ensure_not_branch_reference_for_write()?; let index_type = normalize_global_index_type(self.index_type.trim()).ok_or_else(|| { @@ -210,8 +250,11 @@ impl<'a> GlobalIndexBuildBuilder<'a> { })?; let fields = resolve_index_fields(self.table, columns, index_type)?; let columns = fields.iter().map(|field| field.name()).collect::>(); + let table = self.configured_table(&columns, index_type)?; if normalize_queryable_global_index_type(index_type).is_some() { - let mut builder = SortedGlobalIndexBuildBuilder::new(self.table); + let mut builder = SortedGlobalIndexBuildBuilder::new(&table); + builder.snapshot = snapshot; + builder.replaced_index_files = replaced; builder .with_index_columns(&columns) .with_index_type(index_type) @@ -222,14 +265,18 @@ impl<'a> GlobalIndexBuildBuilder<'a> { return Ok(builder.prepare().await?.into()); } if index_type == FULL_TEXT_GLOBAL_INDEX_TYPE { - let mut builder = FullTextIndexBuildBuilder::new(self.table); + let mut builder = FullTextIndexBuildBuilder::new(&table); + builder.snapshot = snapshot; + builder.replaced_index_files = replaced; builder .with_index_column(columns[0]) .with_options(self.options.clone()); builder.partition_filter = self.partition_filter.clone(); return Ok(builder.prepare().await?.into()); } - let mut builder = VindexIndexBuildBuilder::new(self.table, index_type); + let mut builder = VindexIndexBuildBuilder::new(&table, index_type); + builder.snapshot = snapshot; + builder.replaced_index_files = replaced; builder .with_index_column(columns[0]) .with_options(self.options.clone()); diff --git a/crates/paimon/src/table/global_index_build_common.rs b/crates/paimon/src/table/global_index_build_common.rs index eab63268f..ca8e8c20f 100644 --- a/crates/paimon/src/table/global_index_build_common.rs +++ b/crates/paimon/src/table/global_index_build_common.rs @@ -26,10 +26,11 @@ //! each builder. use crate::io::OutputFile; -use crate::spec::{DataField, FileKind, IndexManifest}; +use crate::spec::{DataField, FileKind, IndexManifest, IndexManifestEntry, Snapshot}; use crate::table::{merge_row_ranges, RowRange, Table}; use crate::{Error, Result}; use bytes::Bytes; +use std::collections::HashSet; use std::path::Path; use tokio::io::AsyncReadExt; @@ -38,6 +39,28 @@ pub(crate) mod vector; const COPY_BUFFER_SIZE: usize = 1024 * 1024; +pub(crate) type IndexFileKey = (Vec, i32, String); + +pub(crate) fn index_file_key(entry: &IndexManifestEntry) -> IndexFileKey { + ( + entry.partition.clone(), + entry.bucket, + entry.index_file.file_name.clone(), + ) +} + +/// An automatic round fixes one source snapshot for every index definition. +/// Capturing an empty view also validates builder configuration without I/O. +pub(super) async fn build_snapshot( + table: &Table, + captured: Option<&Option>, +) -> Result> { + match captured { + Some(snapshot) => Ok(snapshot.clone()), + None => table.snapshot_manager().get_latest_snapshot().await, + } +} + /// Validate the table that owns global row IDs. Procedure options only tune /// index generation; they cannot enable row tracking or change bucket layout. /// Data evolution and `global-index.enabled` govern reads, not index creation. @@ -184,6 +207,7 @@ pub(crate) async fn indexed_row_ranges( index_type: &str, index_field_id: i32, extra_field_ids: Option<&[i32]>, + replaced: &HashSet, ) -> Result> { let Some(index_manifest_name) = index_manifest_name else { return Ok(Vec::new()); @@ -196,7 +220,10 @@ pub(crate) async fn indexed_row_ranges( let entries = IndexManifest::read(table.file_io(), &path).await?; let mut ranges = Vec::new(); for entry in entries { - if entry.kind != FileKind::Add || entry.index_file.index_type != index_type { + if entry.kind != FileKind::Add + || entry.index_file.index_type != index_type + || replaced.contains(&index_file_key(&entry)) + { continue; } let Some(meta) = entry.index_file.global_index_meta else { @@ -225,6 +252,7 @@ pub(crate) async fn validate_existing_index_overlap( index_field_id: i32, extra_field_ids: Option<&[i32]>, planned: &[RowRange], + replaced: &HashSet, ) -> Result<()> { let Some(index_manifest_name) = index_manifest_name else { return Ok(()); @@ -236,7 +264,10 @@ pub(crate) async fn validate_existing_index_overlap( ); let entries = IndexManifest::read(table.file_io(), &path).await?; for entry in entries { - if entry.kind != FileKind::Add || entry.index_file.index_type != index_type { + if entry.kind != FileKind::Add + || entry.index_file.index_type != index_type + || replaced.contains(&index_file_key(&entry)) + { continue; } let Some(meta) = entry.index_file.global_index_meta else { diff --git a/crates/paimon/src/table/global_index_options.rs b/crates/paimon/src/table/global_index_options.rs new file mode 100644 index 000000000..14f8cb18f --- /dev/null +++ b/crates/paimon/src/table/global_index_options.rs @@ -0,0 +1,207 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared field declarations and native options for global-index maintenance and raw reads. + +use crate::table::global_index_build_common::{resolve_index_fields, IndexColumns}; +use crate::table::global_index_types::normalize_global_index_type; +use crate::table::{GlobalIndexBuildBuilder, Table}; +use crate::Result; +use std::collections::{HashMap, HashSet}; + +pub(super) struct Definition { + pub(super) column: String, + pub(super) index_type: String, + pub(super) options: HashMap, +} + +impl Definition { + pub(super) fn builder<'a>(&self, table: &'a Table) -> GlobalIndexBuildBuilder<'a> { + let mut builder = table.new_global_index_build_builder(); + builder + .with_index_column(&self.column) + .with_index_type(&self.index_type) + .with_options(self.options.clone()); + builder + } +} + +pub(super) fn definitions(table: &Table) -> Result> { + let options = table.schema().options(); + let mut owners = HashMap::new(); + for family in ["btree", "bitmap", "multivalue", "fm", "full-text", "vector"] { + let key = format!("global-index.{family}.index-columns"); + let Some(columns) = options.get(&key) else { + continue; + }; + let mut unique = HashSet::new(); + for column in columns.split(',').map(str::trim) { + if column.is_empty() || !unique.insert(column) { + return Err(crate::Error::ConfigInvalid { + message: format!("{key} contains an empty or duplicate column '{column}'"), + }); + } + if owners.insert(column, family).is_some() { + return Err(crate::Error::ConfigInvalid { + message: format!("Column '{column}' can own at most one automatically maintained global index"), + }); + } + resolve_index_fields(table, &IndexColumns::Columns(vec![column.into()]), "btree")?; + } + } + let mut result = Vec::with_capacity(owners.len()); + for field in table.schema().fields() { + let column = field.name(); + let Some(&family) = owners.get(column) else { + continue; + }; + let index_type = if family == "vector" { + vector_index_type(options, column)?.ok_or_else(|| crate::Error::ConfigInvalid { + message: format!("fields.{column}.global-index.vector.index-type must specify a supported vector index type"), + })? + } else { + family + }; + result.push(Definition { + column: column.into(), + index_type: index_type.into(), + options: field_index_options(options, column, family, index_type)?, + }); + } + Ok(result) +} + +pub(super) fn field_index_options( + table_options: &HashMap, + column: &str, + family: &str, + index_type: &str, +) -> Result> { + let field_prefix = format!("fields.{column}.global-index.{family}."); + let native_prefix = match family { + "vector" | "full-text" => format!("{index_type}."), + _ => format!("{family}-index."), + }; + let mut options = HashMap::new(); + let mut insert = |key: &str, value: String| -> Result<()> { + if key.trim().is_empty() { + return Err(crate::Error::ConfigInvalid { + message: format!("{field_prefix}index-options contains an empty option key"), + }); + } + let qualified = if key.starts_with(&native_prefix) + || key.starts_with("fields.") + || key.starts_with("global-index.") + || key.starts_with("sorted-index.") + || (family == "vector" && matches!(key, "index.type" | "vindex.build.granule.enabled")) + { + key.into() + } else { + format!("{native_prefix}{key}") + }; + if options + .get(&qualified) + .or_else(|| table_options.get(&qualified)) + .is_some_and(|previous| previous != &value) + { + return Err(crate::Error::ConfigInvalid { + message: format!("Conflicting field index options for {column}: {qualified}"), + }); + } + options.insert(qualified, value); + Ok(()) + }; + let options_key = format!("{field_prefix}index-options"); + if let Some(raw) = table_options + .get(&options_key) + .filter(|raw| !raw.trim().is_empty()) + { + let parsed: serde_json::Map = serde_json::from_str(raw) + .map_err(|_| crate::Error::ConfigInvalid { + message: format!("{options_key} must be a JSON object of option key-value pairs"), + })?; + for (key, value) in parsed { + let value = match value { + serde_json::Value::String(value) => value, + serde_json::Value::Number(_) | serde_json::Value::Bool(_) => value.to_string(), + _ => { + return Err(crate::Error::ConfigInvalid { + message: format!( + "{options_key} contains an invalid option value for {key}" + ), + }) + } + }; + insert(&key, value)?; + } + } + if family == "vector" { + let metric_key = format!("{field_prefix}distance-metric"); + let metric = table_options + .get(&metric_key) + .map(String::as_str) + .unwrap_or("inner_product"); + crate::vindex::pkvector::metric::VectorSearchMetric::parse(metric)?; + options.insert( + format!("{native_prefix}metric"), + metric.to_ascii_lowercase().replace('-', "_"), + ); + } + Ok(options) +} + +pub(super) fn vector_index_type( + options: &HashMap, + column: &str, +) -> Result> { + let key = format!("fields.{column}.global-index.vector.index-type"); + match options.get(&key) { + Some(value) => normalize_global_index_type(value.trim()) + .filter(|kind| crate::vindex::is_vindex_index_type(kind)) + .map(Some) + .ok_or_else(|| crate::Error::ConfigInvalid { + message: format!("{key} must specify a supported vector index type"), + }), + None => Ok(None), + } +} + +pub(super) fn declares_column( + options: &HashMap, + column: &str, + family: &str, +) -> bool { + options + .get(&format!("global-index.{family}.index-columns")) + .is_some_and(|columns| columns.split(',').any(|name| name.trim() == column)) +} + +pub(super) fn declared_options( + options: &HashMap, + column: &str, + family: &str, + index_type: &str, +) -> Result> { + let prefix = format!("fields.{column}.global-index.{family}."); + if declares_column(options, column, family) + || options.keys().any(|key| key.starts_with(&prefix)) + { + field_index_options(options, column, family, index_type) + } else { + Ok(HashMap::new()) + } +} diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index b82a9c9df..2557a51f1 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -84,6 +84,7 @@ pub(crate) mod global_index_build_common; mod global_index_build_plan; mod global_index_drop_builder; mod global_index_freshness; +mod global_index_options; mod global_index_result; pub(crate) mod global_index_scanner; mod global_index_search_filter; diff --git a/crates/paimon/src/table/search_scan_split.rs b/crates/paimon/src/table/search_scan_split.rs index 6e2e520eb..8b3497f91 100644 --- a/crates/paimon/src/table/search_scan_split.rs +++ b/crates/paimon/src/table/search_scan_split.rs @@ -88,8 +88,15 @@ pub(super) async fn global_search_splits( .iter() .filter(|entry| entry.kind == FileKind::Add && belongs(&entry.index_file)) .collect::>(); - // Full-text admits raw fallback only after finding its index definition. - if full_text && selected.is_empty() { + // A declaration supplies the definition even before its first payload is published. + if full_text + && selected.is_empty() + && !super::global_index_options::declares_column( + table.schema().options(), + column, + "full-text", + ) + { return Ok(vec![]); } let row_filter = filter.and_then(|filter| { @@ -200,8 +207,12 @@ pub(super) async fn global_search_splits( .values() .flat_map(|files| files.iter()) .next() - .map(|file| file.index_type.clone()) - .or_else(|| configured_vector_index_type(table.schema().options(), options, column)); + .map(|file| file.index_type.clone()); + let index_type = match index_type { + Some(kind) => Some(kind), + None if full_text => None, + None => configured_vector_index_type(table.schema().options(), options, column)?, + }; result.push(SearchScanSplit::Raw { scalar_index_files: intersecting_scalar(&raw_ranges), row_ranges: raw_ranges, diff --git a/crates/paimon/src/table/sorted_global_index_build_builder.rs b/crates/paimon/src/table/sorted_global_index_build_builder.rs index 8d6be1858..821316453 100644 --- a/crates/paimon/src/table/sorted_global_index_build_builder.rs +++ b/crates/paimon/src/table/sorted_global_index_build_builder.rs @@ -70,6 +70,9 @@ pub(super) struct SortedGlobalIndexBuildBuilder<'a> { index_type: String, options: HashMap, partition_filter: Option, + pub(super) snapshot: Option>, + pub(super) replaced_index_files: + std::collections::HashSet, } impl<'a> SortedGlobalIndexBuildBuilder<'a> { @@ -80,6 +83,8 @@ impl<'a> SortedGlobalIndexBuildBuilder<'a> { index_type: BTREE_GLOBAL_INDEX_TYPE.to_string(), options: HashMap::new(), partition_filter: None, + snapshot: None, + replaced_index_files: Default::default(), } } @@ -178,7 +183,10 @@ impl<'a> SortedGlobalIndexBuildBuilder<'a> { .collect::>(); let extra_field_ids = (!extra_field_ids.is_empty()).then_some(extra_field_ids); - let Some(snapshot) = self.table.snapshot_manager().get_latest_snapshot().await? else { + let Some(snapshot) = + super::global_index_build_common::build_snapshot(self.table, self.snapshot.as_ref()) + .await? + else { return Ok((None, vec![])); }; @@ -197,6 +205,7 @@ impl<'a> SortedGlobalIndexBuildBuilder<'a> { index_type, index_field.id(), extra_field_ids.as_deref(), + &self.replaced_index_files, ) .await?; @@ -224,6 +233,7 @@ impl<'a> SortedGlobalIndexBuildBuilder<'a> { .iter() .map(|shard| RowRange::new(shard.row_range_start, shard.row_range_end)) .collect::>(), + &self.replaced_index_files, ) .await?; diff --git a/crates/paimon/src/table/table_commit.rs b/crates/paimon/src/table/table_commit.rs index fd6511190..2d47c4c33 100644 --- a/crates/paimon/src/table/table_commit.rs +++ b/crates/paimon/src/table/table_commit.rs @@ -50,6 +50,9 @@ const BATCH_COMMIT_IDENTIFIER: i64 = i64::MAX; const CHECK_ROLLING_RECORD_COUNT: usize = 1000; mod auto_compact; +mod auto_index_build; +#[cfg(test)] +mod auto_index_build_tests; mod global_index_source_check; mod row_tracking; @@ -395,6 +398,9 @@ impl TableCommit { pending.push((id, messages)); } else if self .auto_compaction_enabled(messages.iter().any(|m| !m.new_files.is_empty()))? + || self + .auto_index_build_enabled(messages.iter().any(|m| !m.new_files.is_empty())) + .await? { let kind = direct_commit_kind( &self.messages_to_entries(&messages), @@ -410,7 +416,7 @@ impl TableCommit { } let count = pending.len(); for (id, kind) in recovered { - self.recover_compact_after_write(id, &kind).await?; + self.recover_maintenance_after_write(id, &kind).await?; } for (id, messages) in pending { self.filter_and_commit_with_identifier(messages, id).await?; @@ -447,11 +453,13 @@ impl TableCommit { return Ok(super::CompactMetrics::default()); } let prepared = super::compact::prepare_results(&self.table, &results)?; + let maintain_indexes = self.auto_index_build_enabled(true).await?; + let planned_snapshot_id = prepared.snapshot_id; let mut commit = self.clone(); // One stable identity per plan and task subset, allowing repeated plans // and idempotent retry without consuming the caller's checkpoint sequence. commit.commit_user = format!("{}-compact-{}", self.commit_user, prepared.commit_suffix); - commit + let outcome = commit .try_commit( CommitEntriesPlan::Compact { entries: prepared.entries, @@ -466,6 +474,26 @@ impl TableCommit { true, ) .await?; + if maintain_indexes { + let written = match outcome { + CommitOutcome::Published(snapshot) => Some(*snapshot), + CommitOutcome::Filtered => { + let latest = commit.snapshot_manager.get_latest_snapshot().await?; + commit + .find_duplicate_commit( + planned_snapshot_id.saturating_add(1), + &latest, + BATCH_COMMIT_IDENTIFIER, + &CommitKind::COMPACT, + ) + .await? + } + CommitOutcome::Empty => None, + }; + if let Some(snapshot) = written { + commit.index_after_write(&snapshot).await?; + } + } Ok(prepared.metrics) } @@ -533,7 +561,7 @@ impl TableCommit { let check_from_snapshot = Self::check_from_snapshot(&commit_messages)?; // Never delete CommitMessage files on failure. A previous attempt may // have published them even when the caller received an exception. - self.commit_with_auto_compact( + self.commit_with_maintenance( CommitEntriesPlan::Direct { entries, changelog_entries, @@ -580,7 +608,7 @@ impl TableCommit { let changelog_entries = self.messages_to_changelog_entries(&commit_messages); let new_index_entries = self.messages_to_index_entries(&commit_messages); let check_from_snapshot = Self::check_from_snapshot(&commit_messages)?; - self.commit_with_auto_compact( + self.commit_with_maintenance( CommitEntriesPlan::Direct { entries, changelog_entries, @@ -689,7 +717,7 @@ impl TableCommit { let check_from_snapshot = Self::check_from_snapshot(&commit_messages)?; - self.commit_with_auto_compact( + self.commit_with_maintenance( CommitEntriesPlan::Overwrite { partition_filter, new_entries, diff --git a/crates/paimon/src/table/table_commit/auto_compact.rs b/crates/paimon/src/table/table_commit/auto_compact.rs index 7301e964a..a59a8d286 100644 --- a/crates/paimon/src/table/table_commit/auto_compact.rs +++ b/crates/paimon/src/table/table_commit/auto_compact.rs @@ -65,7 +65,7 @@ impl TableCommit { Ok(()) } - pub(super) async fn commit_with_auto_compact( + pub(super) async fn commit_with_maintenance( &self, plan: CommitEntriesPlan, expected_snapshot_id: Option, @@ -77,8 +77,9 @@ impl TableCommit { CommitEntriesPlan::Overwrite { new_entries, .. } => new_entries, CommitEntriesPlan::Compact { .. } => unreachable!(), }; - let enabled = - self.auto_compaction_enabled(entries.iter().any(|e| *e.kind() == FileKind::Add))?; + let has_new_files = entries.iter().any(|e| *e.kind() == FileKind::Add); + let enabled = self.auto_compaction_enabled(has_new_files)?; + let index_enabled = self.auto_index_build_enabled(has_new_files).await?; let buckets = enabled.then(|| self.written_buckets(entries)); let kind = plan.commit_kind_hint(); let outcome = self @@ -89,12 +90,18 @@ impl TableCommit { filter_committed, ) .await?; - match (outcome, buckets) { - (CommitOutcome::Published(written), Some(buckets)) => { - self.compact_after_write(&written, buckets).await + match outcome { + CommitOutcome::Published(written) => { + if let Some(buckets) = buckets { + self.compact_after_write(&written, buckets).await?; + } + if index_enabled { + self.index_after_write(&written).await?; + } + Ok(()) } - (CommitOutcome::Filtered, Some(_)) => { - self.recover_compact_after_write(commit_identifier, &kind) + CommitOutcome::Filtered if enabled || index_enabled => { + self.recover_maintenance_after_write(commit_identifier, &kind) .await } _ => Ok(()), @@ -132,14 +139,14 @@ impl TableCommit { ) } - pub(super) async fn recover_compact_after_write( + pub(super) async fn recover_maintenance_after_write( &self, identifier: i64, kind: &CommitKind, ) -> Result<()> { // Only recovery reconstructs ownership from persisted metadata. Its // messages can reference source files already compacted or expired. - let recovered = async { + let recovered: Result<()> = async { let latest = self.snapshot_manager.get_latest_snapshot().await?; let earliest = self .snapshot_manager @@ -150,30 +157,38 @@ impl TableCommit { .find_duplicate_commit(earliest, &latest, identifier, kind) .await? else { - return Ok(None); + return Ok(()); }; - let mut compact = self.clone(); - compact.commit_user = self.auto_compact_user(&written); - if compact - .is_committed_identifier_since(&latest, written.id(), Some(written.id() + 1)) - .await? - { - return Ok(None); + // Completed compaction must not suppress an interrupted index round. + // Completion checks in each phase happen before reading its inputs. + if self.auto_compaction_enabled(true)? { + let mut compact = self.clone(); + compact.commit_user = self.auto_compact_user(&written); + let latest = self.snapshot_manager.get_latest_snapshot().await?; + if !compact + .is_committed_identifier_since(&latest, written.id(), Some(written.id() + 1)) + .await? + { + let entries = self.read_delta_entries(None, &written).await?; + self.compact_after_write(&written, self.written_buckets(&entries)) + .await?; + } } - let buckets = self.written_buckets(&self.read_delta_entries(None, &written).await?); - Ok(Some((written, buckets))) - } - .await - .map_err(|error| auto_compaction_error(identifier, error))?; - if let Some((written, buckets)) = recovered { - self.compact_after_write(&written, buckets).await?; + if self.auto_index_build_enabled(true).await? { + self.index_after_write(&written).await?; + } + Ok(()) } - Ok(()) + .await; + recovered.map_err(|error| crate::Error::UnexpectedError { + message: format!("Data commit {identifier} succeeded, but automatic maintenance recovery failed; retry with the same commit user and identifier"), + source: Some(Box::new(error)), + }) } - async fn auto_compaction_table(&self, snapshot: Option<&Snapshot>) -> Result
{ + pub(super) async fn post_commit_table(&self, snapshot: Option<&Snapshot>) -> Result
{ // A historical writer can publish old-schema files against a newer - // schema. Compaction must read and write the current field layout. + // schema. Post-commit maintenance must use the current field layout. let current_schema = if let Some(env) = &self.table.rest_env { let response = env.current_table_identity_checked().await?; let schema_id = response @@ -264,7 +279,7 @@ impl TableCommit { } else { latest.as_ref() }; - let table = self.auto_compaction_table(snapshot).await?; + let table = self.post_commit_table(snapshot).await?; let mut compact = Self::new(table, commit_user); compact.snapshot_commit = self.snapshot_commit.clone(); compact.snapshot_properties = self.snapshot_properties.clone(); diff --git a/crates/paimon/src/table/table_commit/auto_index_build.rs b/crates/paimon/src/table/table_commit/auto_index_build.rs new file mode 100644 index 000000000..0b70396f2 --- /dev/null +++ b/crates/paimon/src/table/table_commit/auto_index_build.rs @@ -0,0 +1,185 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::table::global_index_options::{definitions, Definition}; +use sha2::{Digest, Sha256}; + +mod planning; + +impl TableCommit { + pub(super) async fn auto_index_build_enabled(&self, has_new_files: bool) -> Result { + if !has_new_files || self.table.schema().core_options().write_only()? { + return Ok(false); + } + let definitions = definitions(&self.table)?; + for definition in &definitions { + definition.builder(&self.table).validate().await?; + } + Ok(!definitions.is_empty()) + } + + fn auto_index_build_user(&self, written: &Snapshot) -> String { + let mut identity = Sha256::new(); + identity.update(self.table.location().as_bytes()); + identity.update(self.table.branch().as_bytes()); + identity.update(written.delta_manifest_list().as_bytes()); + format!( + "{}-auto-index-build-{}", + self.commit_user, + hex::encode(identity.finalize()) + ) + } + + pub(super) async fn index_after_write(&self, written: &Snapshot) -> Result<()> { + self.try_index_after_write(written) + .await + .map_err(|error| auto_index_build_error(written.commit_identifier(), error)) + } + + async fn try_index_after_write(&self, written: &Snapshot) -> Result<()> { + let commit_user = self.auto_index_build_user(written); + let start = current_time_millis(); + let mut retries = 0u64; + let mut publication_error = None; + loop { + let latest = self.snapshot_manager.get_latest_snapshot().await?; + let snapshot = latest + .as_ref() + .filter(|s| s.id() >= written.id()) + .ok_or_else(|| crate::Error::DataInvalid { + message: "Cannot build automatic indexes after rollback below the data commit" + .into(), + source: None, + })?; + let table = self.post_commit_table(Some(snapshot)).await?; + let mut commit = Self::new(table.clone(), commit_user.clone()); + commit.snapshot_commit = self.snapshot_commit.clone(); + commit.snapshot_properties = self.snapshot_properties.clone(); + commit.commit_callbacks = self.commit_callbacks.clone(); + // Check completion before the retry budget or touching old input files. + // This also recovers a successful publication whose response was lost. + if commit + .is_committed_identifier_since(&latest, written.id(), Some(written.id() + 1)) + .await? + { + return Ok(()); + } + if retries > u64::from(self.commit_max_retries) + || (retries > 0 && current_time_millis() - start > self.commit_timeout_ms) + { + return Err(crate::Error::DataInvalid { + message: format!("Automatic index build exhausted the commit retry budget after {retries} conflicts"), + source: publication_error.map(|error| Box::new(error) as _), + }); + } + if table.schema().core_options().write_only()? { + return Ok(()); + } + let definitions = definitions(&table)?; + if definitions.is_empty() { + return Ok(()); + } + let data = crate::table::snapshot_manifest::read_snapshot_entries( + &self.snapshot_manager, + snapshot, + None, + ) + .await?; + let indexes = match snapshot.index_manifest() { + Some(name) => { + IndexManifest::read(table.file_io(), &self.snapshot_manager.manifest_path(name)) + .await? + } + None => vec![], + }; + let mut messages = Vec::new(); + let prepared: Result<()> = async { + for definition in definitions { + let plan = planning::plan(&table, &definition, &data, &indexes).await?; + let built = definition + .builder(&table) + .build_for_maintenance(snapshot, plan.replaced) + .await?; + messages.extend(built); + messages.extend(plan.deletions); + } + Ok(()) + } + .await; + if let Err(error) = prepared { + // No files in this round have been submitted to a publisher. + // Abort only generated outputs; retired payloads remain readable. + let _ = commit.abort(&messages).await; + return Err(error); + } + if messages.is_empty() { + return Ok(()); + } + let current = self.snapshot_manager.get_latest_snapshot().await?; + if !current + .as_ref() + .is_some_and(|s| s.has_same_read_metadata(snapshot)) + || self.latest_schema_id().await? != table.schema().id() + { + let _ = commit.abort(&messages).await; + } else { + let mut plan = CommitEntriesPlan::Direct { + entries: vec![], + changelog_entries: vec![], + new_index_entries: commit.messages_to_index_entries(&messages), + check_from_snapshot: None, + }; + let resolved = match commit.resolve_commit(&mut plan, &latest, None, false).await { + Ok(resolved) => resolved, + Err(error) => { + let _ = commit.abort(&messages).await; + return Err(error); + } + }; + // One atomic attempt: a definite CAS conflict requires a fresh + // coverage plan, rather than rebasing overlapping index files. + match commit + .try_commit_once(resolved, &latest, written.id()) + .await? + { + CommitAttemptResult::Success(_) => return Ok(()), + CommitAttemptResult::Retry(state) => match state.publication_error { + None => { + let _ = commit.abort(&messages).await; + } + Some(error) => { + // Publication is uncertain. Retain every submitted + // file, and probe the stable identity on the next pass. + publication_error = Some(error); + } + }, + } + } + self.commit_retry_wait(retries.min(u64::from(u32::MAX)) as u32) + .await; + retries += 1; + } + } +} + +pub(super) fn auto_index_build_error(identifier: i64, error: crate::Error) -> crate::Error { + crate::Error::UnexpectedError { + message: format!("Data commit {identifier} succeeded, but automatic index build failed; retry through the corresponding recovery API with the same commit user and identifier"), + source: Some(Box::new(error)), + } +} diff --git a/crates/paimon/src/table/table_commit/auto_index_build/planning.rs b/crates/paimon/src/table/table_commit/auto_index_build/planning.rs new file mode 100644 index 000000000..ed2d16d01 --- /dev/null +++ b/crates/paimon/src/table/table_commit/auto_index_build/planning.rs @@ -0,0 +1,199 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Compare desired global row-ID units with their current index payloads. +//! Like Java PrimaryKeyIndexLevels, rebuild changed units and retire their old +//! payloads atomically. Global indexes use row-ID shards rather than PK levels: +//! a physical compaction preserving logical row IDs does not invalidate them. + +use super::*; +use crate::table::global_index_build_common::{ + index_file_key, resolve_index_fields, IndexColumns, IndexFileKey, +}; +use crate::table::{merge_row_ranges, RowRange}; +use std::collections::BTreeMap; + +type UnitKey = (Vec, i32, i64); + +pub(super) struct MaintenancePlan { + pub replaced: HashSet, + pub deletions: Vec, +} + +fn partition_key(bytes: &[u8]) -> Vec { + if super::super::is_empty_partition(bytes) { + vec![] + } else { + bytes.to_vec() + } +} + +pub(super) async fn plan( + table: &Table, + definition: &Definition, + data: &[ManifestEntry], + indexes: &[IndexManifestEntry], +) -> Result { + let mut options = table.schema().options().clone(); + options.extend(definition.options.clone()); + let core = CoreOptions::new(&options); + let width = if matches!( + definition.index_type.as_str(), + "btree" | "bitmap" | "multivalue" | "fm" + ) { + core.sorted_index_records_per_range()? + } else { + core.global_index_row_count_per_shard()? + }; + let fields = resolve_index_fields( + table, + &IndexColumns::Columns(vec![definition.column.clone()]), + &definition.index_type, + )?; + let field_id = fields[0].id(); + let vector_metric = if crate::vindex::is_vindex_index_type(&definition.index_type) { + let options = crate::vindex::VindexVectorIndexOptions::new( + table.schema().options(), + &definition.options, + &definition.index_type, + fields[0], + )?; + Some(crate::vindex::pkvector::metric::VectorSearchMetric::parse( + &options.native_options["metric"], + )?) + } else { + None + }; + let mut desired: HashMap> = HashMap::new(); + for entry in data { + if *entry.kind() != FileKind::Add { + continue; + } + let Some((start, end)) = entry.file().row_id_range() else { + return Err(crate::Error::DataInvalid { + message: "Automatic index maintenance requires assigned source row IDs".into(), + source: None, + }); + }; + if start < 0 || end < start { + return Err(crate::Error::DataInvalid { + message: "Invalid source row-ID range for automatic index maintenance".into(), + source: None, + }); + } + for shard in start / width..=end / width { + let from = shard.saturating_mul(width); + let to = from.saturating_add(width - 1); + desired + .entry((partition_key(entry.partition()), entry.bucket(), shard)) + .or_default() + .push(RowRange::new(start.max(from), end.min(to))); + } + } + let desired = desired + .into_iter() + .map(|(key, ranges)| (key, merge_row_ranges(ranges))) + .collect::>(); + let selected = indexes + .iter() + .filter(|entry| { + entry.kind == FileKind::Add + && (entry.index_file.index_type == definition.index_type + || vector_metric.is_some() + && crate::vindex::is_vindex_index_type(&entry.index_file.index_type)) + && entry + .index_file + .global_index_meta + .as_ref() + .is_some_and(|meta| { + meta.index_field_id == field_id + && meta.extra_field_ids.as_ref().is_none_or(Vec::is_empty) + }) + }) + .collect::>(); + let mut current: HashMap> = HashMap::new(); + let mut replaced = HashSet::new(); + for entry in &selected { + let meta = entry.index_file.global_index_meta.as_ref().unwrap(); + if let Some(metric) = vector_metric { + // A vector column owns one algorithm and scoring metric across all shards. + // Coverage equality cannot make a previous definition compatible. + let compatible = entry.index_file.index_type == definition.index_type + && crate::table::vector_search_common::read_index_vector_metric( + table.file_io(), + table.location(), + entry, + ) + .await? + .is_some_and(|persisted| { + crate::vindex::pkvector::metric::VectorSearchMetric::from_vindex(persisted) + == metric + }); + if !compatible { + replaced.insert(index_file_key(entry)); + continue; + } + } + if meta.row_range_start < 0 || meta.row_range_end < meta.row_range_start { + return Err(crate::Error::DataInvalid { + message: "Invalid global index row-ID range during maintenance".into(), + source: None, + }); + } + let shard = meta.row_range_start / width; + if shard != meta.row_range_end / width { + // A shard-size change can invalidate a former unit's boundaries. + replaced.insert(index_file_key(entry)); + } else { + current + .entry((partition_key(&entry.partition), entry.bucket, shard)) + .or_default() + .push(entry); + } + } + for (key, files) in current { + let coverage = merge_row_ranges( + files + .iter() + .map(|entry| { + let meta = entry.index_file.global_index_meta.as_ref().unwrap(); + RowRange::new(meta.row_range_start, meta.row_range_end) + }) + .collect(), + ); + let wanted = desired.get(&key).map(Vec::as_slice).unwrap_or(&[]); + if coverage.as_slice() != wanted || files.len() != wanted.len() { + replaced.extend(files.iter().map(|entry| index_file_key(entry))); + } + } + let mut deletions: BTreeMap<(Vec, i32), CommitMessage> = BTreeMap::new(); + for entry in selected { + if replaced.contains(&index_file_key(entry)) { + deletions + .entry((entry.partition.clone(), entry.bucket)) + .or_insert_with(|| { + CommitMessage::new(entry.partition.clone(), entry.bucket, vec![]) + }) + .deleted_index_files + .push(entry.index_file.clone()); + } + } + Ok(MaintenancePlan { + replaced, + deletions: deletions.into_values().collect(), + }) +} diff --git a/crates/paimon/src/table/table_commit/auto_index_build_tests.rs b/crates/paimon/src/table/table_commit/auto_index_build_tests.rs new file mode 100644 index 000000000..ae9696d17 --- /dev/null +++ b/crates/paimon/src/table/table_commit/auto_index_build_tests.rs @@ -0,0 +1,1163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::catalog::Identifier; +use crate::io::FileIOBuilder; +use crate::spec::{IntType, Schema as TableSchemaBuilder, TableSchema}; +use arrow_array::{Int32Array, RecordBatch}; +use futures::TryStreamExt; +use std::sync::atomic::{AtomicBool, Ordering}; + +mod search; + +async fn table(enabled: Option, compact: bool) -> Table { + let mut schema = TableSchemaBuilder::builder() + .column("id", DataType::Int(IntType::new())) + .column("value", DataType::Int(IntType::new())) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("global-index.btree.index-columns", "id") + .option("global-index.bitmap.index-columns", "value") + .option("sorted-index.records-per-range", "2") + .option( + "compaction.min.file-num", + if compact { "2" } else { "1000" }, + ) + .option("commit.min-retry-wait", "0 ms") + .option("commit.max-retry-wait", "0 ms"); + if let Some(enabled) = enabled { + schema = schema.option("write-only", (!enabled).to_string()); + } + let location = format!("memory:/auto-index-build/{}", uuid::Uuid::new_v4()); + let table = Table::new( + FileIOBuilder::new("memory").build().unwrap(), + Identifier::new("default", "auto_index"), + location, + TableSchema::new(0, &schema.build().unwrap()), + None, + ); + table + .file_io() + .mkdirs(&format!("{}/schema", table.location())) + .await + .unwrap(); + table + .file_io() + .mkdirs(&format!("{}/snapshot", table.location())) + .await + .unwrap(); + table + .file_io() + .new_output(&table.schema_manager().schema_path(0)) + .unwrap() + .write(serde_json::to_vec(table.schema()).unwrap().into()) + .await + .unwrap(); + table +} + +async fn messages(table: &Table, ids: Vec) -> Vec { + let values = ids.iter().map(|id| id * 10).collect::>(); + let batch = RecordBatch::try_from_iter(vec![ + ( + "id", + Arc::new(Int32Array::from(ids)) as arrow_array::ArrayRef, + ), + ( + "value", + Arc::new(Int32Array::from(values)) as arrow_array::ArrayRef, + ), + ]) + .unwrap(); + let builder = table.new_write_builder(); + let mut write = builder.new_write().unwrap(); + write.write_arrow_batch(&batch).await.unwrap(); + write.prepare_commit().await.unwrap() +} + +async fn latest(table: &Table) -> Snapshot { + table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .unwrap() +} + +async fn indexes(table: &Table) -> Vec { + let snapshot = latest(table).await; + match snapshot.index_manifest() { + None => vec![], + Some(name) => IndexManifest::read( + table.file_io(), + &table.snapshot_manager().manifest_path(name), + ) + .await + .unwrap(), + } +} + +async fn assert_complete(table: &Table) { + for (column, kind) in [("id", "btree"), ("value", "bitmap")] { + assert!(table + .new_global_index_build_builder() + .with_index_column(column) + .with_index_type(kind) + .build() + .await + .unwrap() + .is_empty()); + } + let entries = indexes(table).await; + assert!(!entries.is_empty()); + for entry in entries { + let file = entry.index_file; + let path = file + .external_path + .unwrap_or_else(|| format!("{}/index/{}", table.location(), file.file_name)); + assert!( + table.file_io().exists(&path).await.unwrap(), + "published index file must survive recovery" + ); + } +} + +#[tokio::test] +async fn write_only_controls_automatic_maintenance_and_defaults_to_enabled() { + for enabled in [None, Some(false), Some(true)] { + let table = table(enabled, false).await; + TableCommit::new(table.clone(), "writer".into()) + .commit(messages(&table, vec![1, 2]).await) + .await + .unwrap(); + assert_eq!( + latest(&table).await.id(), + if enabled != Some(false) { 2 } else { 1 } + ); + if enabled != Some(false) { + assert_complete(&table).await; + } else { + assert!(indexes(&table).await.is_empty()); + } + } +} + +#[tokio::test] +async fn write_only_ignores_declarations_and_absent_declarations_are_a_noop() { + let original = table(Some(false), false).await; + let disabled = original.copy_with_options(HashMap::from([( + "global-index.btree.index-columns".into(), + "id,id".into(), + )])); + TableCommit::new(disabled.clone(), "disabled".into()) + .commit(messages(&disabled, vec![1]).await) + .await + .unwrap(); + assert!(indexes(&disabled).await.is_empty()); + for (key, value) in [ + ("write-only", "invalid"), + ("row-tracking.enabled", "false"), + ("bucket", "1"), + ] { + let original = table(Some(true), false).await; + let table = original.copy_with_options(HashMap::from([(key.into(), value.into())])); + let error = TableCommit::new(table.clone(), "invalid".into()) + .commit(messages(&original, vec![1]).await) + .await + .unwrap_err(); + assert!(error.to_string().contains(key) || error.to_string().contains("unaware-bucket")); + assert!(table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .is_none()); + } + let original = table(Some(true), false).await; + let mut options = original.schema().options().clone(); + options.remove("global-index.btree.index-columns"); + options.remove("global-index.bitmap.index-columns"); + let missing = original + .copy_with_resolved_schema( + original.schema().copy_with_replaced_options(options), + original.branch(), + ) + .unwrap(); + TableCommit::new(missing.clone(), "missing".into()) + .commit(messages(&missing, vec![1]).await) + .await + .unwrap(); + assert_eq!(latest(&missing).await.id(), 1); + assert!(indexes(&missing).await.is_empty()); +} + +#[tokio::test] +async fn data_evolution_and_query_index_switches_do_not_gate_automatic_builds() { + for (evolution, query_indexes) in [(false, false), (false, true), (true, false)] { + let table = table(Some(true), false) + .await + .copy_with_options(HashMap::from([ + ("data-evolution.enabled".into(), evolution.to_string()), + ("global-index.enabled".into(), query_indexes.to_string()), + ])); + TableCommit::new(table.clone(), "writer".into()) + .commit(messages(&table, vec![1, 2]).await) + .await + .unwrap(); + assert_eq!(latest(&table).await.id(), 2); + assert_complete(&table).await; + } +} + +#[tokio::test] +async fn append_only_adds_missing_ranges_and_empty_commits_do_nothing() { + let table = table(Some(true), false).await; + let commit = TableCommit::new(table.clone(), "writer".into()); + let first = messages(&table, vec![1, 2]).await; + commit + .commit_with_identifier(first.clone(), 7) + .await + .unwrap(); + let original = indexes(&table).await; + commit + .commit_with_identifier(messages(&table, vec![3, 4]).await, 8) + .await + .unwrap(); + let all = indexes(&table).await; + assert_eq!(all.len(), original.len() * 2); + assert!(original.iter().all(|entry| all.contains(entry))); + assert_complete(&table).await; + let mut read = table.new_read_builder(); + read.with_filter( + crate::spec::PredicateBuilder::new(table.schema().fields()) + .equal("id", Datum::Int(3)) + .unwrap(), + ); + let plan = read.new_scan().plan().await.unwrap(); + let batches: Vec = read + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + let found = batches + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect::>(); + assert_eq!(found, vec![3], "queries must see newly indexed rows"); + let before = latest(&table).await.id(); + commit.commit(vec![]).await.unwrap(); + assert_eq!(commit.filter_and_commit(vec![(7, first)]).await.unwrap(), 0); + assert_eq!(latest(&table).await.id(), before); +} + +#[tokio::test] +async fn changed_shards_replace_fragmented_indexes_and_preserve_historical_payloads() { + let table = table(Some(true), false) + .await + .copy_with_options(HashMap::from([( + "sorted-index.records-per-range".into(), + "4".into(), + )])); + let commit = TableCommit::new(table.clone(), "writer".into()); + commit + .commit_with_identifier(messages(&table, vec![1]).await, 6) + .await + .unwrap(); + let previous = indexes(&table).await; + let input = messages(&table, vec![2]).await; + commit + .commit_with_identifier(input.clone(), 7) + .await + .unwrap(); + let current = indexes(&table).await; + assert_eq!(current.len(), 2, "one payload per field and row-ID shard"); + for entry in ¤t { + assert!(!previous.contains(entry)); + let meta = entry.index_file.global_index_meta.as_ref().unwrap(); + assert_eq!((meta.row_range_start, meta.row_range_end), (0, 1)); + } + for entry in previous { + assert!( + table + .file_io() + .exists(&format!( + "{}/index/{}", + table.location(), + entry.index_file.file_name + )) + .await + .unwrap(), + "retired payloads must remain readable by historical snapshots" + ); + } + commit + .filter_and_commit_with_identifier(input, 7) + .await + .unwrap(); + assert_eq!(latest(&table).await.id(), 4); + assert_complete(&table).await; +} + +#[tokio::test] +async fn manual_compaction_maintains_indexes_and_recovers_after_index_publication_failure() { + let original = table(Some(false), true).await; + let seed = TableCommit::new(original.clone(), "seed".into()); + seed.commit(messages(&original, vec![1]).await) + .await + .unwrap(); + seed.commit(messages(&original, vec![2]).await) + .await + .unwrap(); + let table = original.copy_with_options(HashMap::from([ + ("write-only".into(), "false".into()), + ("commit.max-retries".into(), "0".into()), + ])); + let builder = table.new_compact_builder(); + let plan = builder.new_planner().unwrap().plan().await.unwrap(); + assert!(!plan.tasks().is_empty()); + let executor = builder.new_executor().unwrap(); + let mut results = Vec::new(); + for task in plan.tasks() { + results.push(executor.execute(task).await.unwrap()); + } + let mut commit = TableCommit::new(table.clone(), "compact-operator".into()); + commit.snapshot_commit = Arc::new(InjectIndexPublication { + table: table.clone(), + mode: Publication::FailBeforePublish, + fired: AtomicBool::new(false), + }); + assert!(commit + .commit_compact(results.clone()) + .await + .unwrap_err() + .to_string() + .contains("automatic index build failed")); + let compact = latest(&table).await; + assert_eq!(compact.commit_kind(), &CommitKind::COMPACT); + let recovery = TableCommit::new(table.clone(), "compact-operator".into()); + for _ in 0..2 { + recovery.commit_compact(results.clone()).await.unwrap(); + assert_eq!(latest(&table).await.id(), compact.id() + 1); + assert_complete(&table).await; + } +} + +#[tokio::test] +async fn allowed_column_updates_rebuild_affected_indexes_from_current_values() { + use arrow_array::Int64Array; + let table = table(Some(true), false) + .await + .copy_with_options(HashMap::from([( + "global-index.column-update-action".into(), + "DROP_PARTITION_INDEX".into(), + )])); + let commit = TableCommit::new(table.clone(), "writer".into()); + commit + .commit_with_identifier(messages(&table, vec![1, 2]).await, 6) + .await + .unwrap(); + let previous = indexes(&table).await; + let builder = table.new_write_builder(); + let mut writer = builder + .new_data_evolution_writer(vec!["value".into()]) + .unwrap(); + writer + .add_matched_batch( + RecordBatch::try_from_iter(vec![ + ( + "_ROW_ID", + Arc::new(Int64Array::from(vec![0])) as arrow_array::ArrayRef, + ), + ( + "value", + Arc::new(Int32Array::from(vec![99])) as arrow_array::ArrayRef, + ), + ]) + .unwrap(), + ) + .unwrap(); + let input = writer.prepare_commit().await.unwrap(); + commit + .commit_with_identifier(input.clone(), 7) + .await + .unwrap(); + assert_complete(&table).await; + let current = indexes(&table).await; + assert!(current + .iter() + .filter(|entry| entry.index_file.index_type == "bitmap") + .all(|entry| !previous.contains(entry))); + let mut read = table.new_read_builder(); + read.with_filter( + crate::spec::PredicateBuilder::new(table.schema().fields()) + .equal("value", Datum::Int(99)) + .unwrap(), + ); + let plan = read.new_scan().plan().await.unwrap(); + let batches: Vec = read + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 1); + let before = latest(&table).await.id(); + commit + .filter_and_commit_with_identifier(input, 7) + .await + .unwrap(); + assert_eq!(latest(&table).await.id(), before); +} + +#[tokio::test] +async fn column_lists_and_java_style_scalar_options_are_resolved_per_field() { + let original = table(Some(true), false).await; + let mut options = original.schema().options().clone(); + options.remove("global-index.bitmap.index-columns"); + options.insert( + "global-index.btree.index-columns".into(), + " value, id ".into(), + ); + options.insert( + "fields.id.global-index.btree.index-options".into(), + r#"{"file-version":2,"block-size":1024,"bloom-filter.enabled":true}"#.into(), + ); + options.insert( + "fields.value.global-index.btree.index-options".into(), + r#"{"file-version":1,"block-size":2048}"#.into(), + ); + let table = original + .copy_with_resolved_schema( + original.schema().copy_with_replaced_options(options), + original.branch(), + ) + .unwrap(); + TableCommit::new(table.clone(), "writer".into()) + .commit(messages(&table, vec![1, 2]).await) + .await + .unwrap(); + let files = indexes(&table).await; + assert_eq!(files.len(), 2); + assert!(files + .iter() + .all(|entry| entry.index_file.index_type == "btree")); + let ids = files + .iter() + .map(|entry| { + entry + .index_file + .global_index_meta + .as_ref() + .unwrap() + .index_field_id + }) + .collect::>(); + assert_eq!(ids, HashSet::from([0, 1])); + for entry in &files { + let file = &entry.index_file; + let bytes = table + .file_io() + .new_input(&format!("{}/index/{}", table.location(), file.file_name)) + .unwrap() + .read() + .await + .unwrap(); + let footer = crate::btree::BTreeFileFooter::read_footer( + &bytes[bytes.len() - crate::btree::BTreeFileFooter::ENCODED_LENGTH..], + ) + .unwrap(); + let field_id = file.global_index_meta.as_ref().unwrap().index_field_id; + assert_eq!(footer.version, if field_id == 0 { 2 } else { 1 }); + assert_eq!(footer.bloom_filter_handle.is_some(), field_id == 0); + } + for column in ["id", "value"] { + assert!(table + .new_global_index_build_builder() + .with_index_column(column) + .build() + .await + .unwrap() + .is_empty()); + } +} + +#[tokio::test] +async fn vector_definitions_require_a_supported_algorithm_and_metric() { + use crate::spec::{ArrayType, FloatType}; + let schema = TableSchemaBuilder::builder() + .column( + "embedding", + DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), + ) + .option("row-tracking.enabled", "true") + .option("global-index.vector.index-columns", "embedding") + .build() + .unwrap(); + let original = Table::new( + FileIOBuilder::new("memory").build().unwrap(), + Identifier::new("default", "vector_options"), + "memory:/vector_options".into(), + TableSchema::new(0, &schema), + None, + ); + assert!(TableCommit::new(original.clone(), "writer".into()) + .auto_index_build_enabled(true) + .await + .unwrap_err() + .to_string() + .contains("fields.embedding.global-index.vector.index-type")); + for (key, value) in [ + ("fields.embedding.global-index.vector.index-type", "btree"), + ( + "fields.embedding.global-index.vector.distance-metric", + "not-a-metric", + ), + ( + "fields.embedding.global-index.vector.index-options", + r#"{"dimension":0}"#, + ), + ( + "fields.embedding.global-index.vector.index-options", + r#"{"dimension":[]}"#, + ), + ] { + let table = original + .copy_with_options(HashMap::from([( + "fields.embedding.global-index.vector.index-type".into(), + "ivf-flat".into(), + )])) + .copy_with_options(HashMap::from([(key.into(), value.into())])); + assert!( + TableCommit::new(table, "invalid".into()) + .auto_index_build_enabled(true) + .await + .is_err(), + "{key}={value}" + ); + } +} + +#[tokio::test] +async fn declarations_are_validated_before_publishing_data() { + for (key, value) in [ + ("global-index.btree.index-columns", ""), + ("global-index.btree.index-columns", "missing"), + ("global-index.btree.index-columns", "id,id"), + ("global-index.btree.index-columns", "id,"), + ("global-index.bitmap.index-columns", "id"), + ("fields.id.global-index.btree.index-options", "[]"), + ("fields.id.global-index.btree.index-options", "not-json"), + ( + "fields.id.global-index.btree.index-options", + r#"{"sorted-index.records-per-range":0}"#, + ), + ( + "fields.id.global-index.btree.index-options", + r#"{"compression":null}"#, + ), + ("fields.id.global-index.btree.index-options", r#"{"":1}"#), + ( + "fields.id.global-index.btree.index-options", + r#"{"block-size":"not-a-size"}"#, + ), + ] { + let table = table(Some(true), false) + .await + .copy_with_options(HashMap::from([(key.into(), value.into())])); + let result = TableCommit::new(table.clone(), "invalid".into()) + .commit(messages(&table, vec![1]).await) + .await; + assert!(result.is_err(), "{key}={value}"); + assert!(table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .is_none()); + } +} + +struct FailDataCallbackOnce(AtomicBool); + +#[async_trait::async_trait] +impl CommitCallback for FailDataCallbackOnce { + async fn call(&self, context: &CommitCallbackContext) -> Result<()> { + if !context.commit_entries.is_empty() && !self.0.swap(true, Ordering::SeqCst) { + return Err(crate::Error::UnexpectedError { + message: "injected data callback failure".into(), + source: None, + }); + } + Ok(()) + } +} + +#[tokio::test] +async fn recovery_finishes_indexing_after_data_publication_for_append_and_overwrite() { + for overwrite in [false, true] { + let table = table(Some(true), false).await; + let input = messages(&table, vec![1, 2]).await; + let mut commit = TableCommit::new(table.clone(), "recover".into()); + commit + .add_commit_callback(Arc::new(FailDataCallbackOnce(AtomicBool::new(false)))) + .unwrap(); + let result = if overwrite { + commit + .overwrite_with_identifier(input.clone(), None, 7) + .await + } else { + commit.commit_with_identifier(input.clone(), 7).await + }; + assert!(result + .unwrap_err() + .to_string() + .contains("injected data callback failure")); + assert_eq!(latest(&table).await.id(), 1); + for _ in 0..2 { + if overwrite { + commit + .overwrite_with_identifier(input.clone(), None, 7) + .await + .unwrap(); + } else { + assert_eq!( + commit + .filter_and_commit(vec![(7, input.clone())]) + .await + .unwrap(), + 0 + ); + } + assert_eq!(latest(&table).await.id(), 2); + assert_complete(&table).await; + } + } +} + +enum Publication { + Conflict, + LoseResponse, + FailBeforePublish, +} + +struct InjectIndexPublication { + table: Table, + mode: Publication, + fired: AtomicBool, +} + +#[async_trait::async_trait] +impl SnapshotCommit for InjectIndexPublication { + async fn commit( + &self, + _base: Option<&str>, + snapshot: &Snapshot, + _branch: &str, + _statistics: &[PartitionStatistics], + ) -> Result { + if snapshot.commit_user().contains("-auto-index-build-") + && !self.fired.swap(true, Ordering::SeqCst) + { + match self.mode { + Publication::Conflict => { + // A concurrent writer finishes the exact same coverage first. + let other = TableCommit::new(self.table.clone(), "competing-writer".into()); + other.commit(messages(&self.table, vec![99]).await).await?; + return Ok(false); + } + Publication::LoseResponse => { + assert!( + self.table + .snapshot_manager() + .commit_snapshot(snapshot) + .await? + ); + return Err(crate::Error::UnexpectedError { + message: "lost index response".into(), + source: None, + }); + } + Publication::FailBeforePublish => { + return Err(crate::Error::UnexpectedError { + message: "index publisher unavailable".into(), + source: None, + }) + } + } + } + self.table + .snapshot_manager() + .commit_snapshot(snapshot) + .await + } +} + +#[tokio::test] +async fn definite_conflict_replans_coverage_and_preserves_competing_indexes() { + let table = table(Some(true), false).await; + let mut commit = TableCommit::new(table.clone(), "writer".into()); + commit.snapshot_commit = Arc::new(InjectIndexPublication { + table: table.clone(), + mode: Publication::Conflict, + fired: AtomicBool::new(false), + }); + commit + .commit_with_identifier(messages(&table, vec![1, 2]).await, 7) + .await + .unwrap(); + assert_eq!(latest(&table).await.id(), 3); + assert_complete(&table).await; + assert_eq!( + table + .file_io() + .list_status(&format!("{}/index", table.location())) + .await + .unwrap() + .iter() + .filter(|file| !file.is_dir) + .count(), + indexes(&table).await.len(), + "a definite conflict must clean up only its unpublished index files", + ); +} + +#[tokio::test] +async fn conflicting_replacement_replans_instead_of_deleting_a_competitors_payloads() { + let table = table(Some(true), false).await; + TableCommit::new(table.clone(), "seed".into()) + .commit(messages(&table, vec![1]).await) + .await + .unwrap(); + let historical = indexes(&table).await; + let mut commit = TableCommit::new(table.clone(), "writer".into()); + commit.snapshot_commit = Arc::new(InjectIndexPublication { + table: table.clone(), + mode: Publication::Conflict, + fired: AtomicBool::new(false), + }); + commit + .commit_with_identifier(messages(&table, vec![2]).await, 7) + .await + .unwrap(); + assert_eq!(latest(&table).await.id(), 5); + assert_complete(&table).await; + let current = indexes(&table).await; + assert_eq!(current.len(), 4); + assert!(historical.iter().all(|entry| !current.contains(entry))); + let physical = table + .file_io() + .list_status(&format!("{}/index", table.location())) + .await + .unwrap(); + assert_eq!( + physical.iter().filter(|file| !file.is_dir).count(), + current.len() + historical.len(), + "discard only this round's private files and retain historical payloads" + ); +} + +#[tokio::test] +async fn lost_index_response_is_recovered_without_deleting_published_files() { + let table = table(Some(true), false).await; + let mut commit = TableCommit::new(table.clone(), "writer".into()); + commit.snapshot_commit = Arc::new(InjectIndexPublication { + table: table.clone(), + mode: Publication::LoseResponse, + fired: AtomicBool::new(false), + }); + let input = messages(&table, vec![1, 2]).await; + commit + .commit_with_identifier(input.clone(), 7) + .await + .unwrap(); + commit + .filter_and_commit_with_identifier(input, 7) + .await + .unwrap(); + assert_eq!(latest(&table).await.id(), 2); + assert_complete(&table).await; +} + +#[tokio::test] +async fn exhausted_index_retries_preserve_data_and_recover_with_the_same_identifier() { + let table = table(Some(true), false) + .await + .copy_with_options(HashMap::from([("commit.max-retries".into(), "0".into())])); + let input = messages(&table, vec![1, 2]).await; + let mut commit = TableCommit::new(table.clone(), "writer".into()); + commit.snapshot_commit = Arc::new(InjectIndexPublication { + table: table.clone(), + mode: Publication::FailBeforePublish, + fired: AtomicBool::new(false), + }); + let error = commit + .commit_with_identifier(input.clone(), 7) + .await + .unwrap_err(); + assert!(error.to_string().contains("Data commit 7 succeeded")); + assert!(error.to_string().contains("index publisher unavailable")); + assert_eq!(latest(&table).await.id(), 1); + let recovery = TableCommit::new(table.clone(), "writer".into()); + recovery + .filter_and_commit_with_identifier(input.clone(), 7) + .await + .unwrap(); + recovery + .filter_and_commit_with_identifier(input, 7) + .await + .unwrap(); + assert_eq!(latest(&table).await.id(), 2); + assert_complete(&table).await; +} + +#[tokio::test] +async fn recovery_resumes_indexing_after_compaction_already_completed() { + let table = table(Some(true), true) + .await + .copy_with_options(HashMap::from([("commit.max-retries".into(), "0".into())])); + TableCommit::new(table.clone(), "seed".into()) + .commit(messages(&table, vec![1]).await) + .await + .unwrap(); + let input = messages(&table, vec![2]).await; + let mut commit = TableCommit::new(table.clone(), "writer".into()); + commit.snapshot_commit = Arc::new(InjectIndexPublication { + table: table.clone(), + mode: Publication::FailBeforePublish, + fired: AtomicBool::new(false), + }); + assert!(commit + .commit_with_identifier(input.clone(), 7) + .await + .unwrap_err() + .to_string() + .contains("automatic index build failed")); + let compact = latest(&table).await; + assert_eq!(compact.commit_kind(), &CommitKind::COMPACT); + let recovery = TableCommit::new(table.clone(), "writer".into()); + for _ in 0..2 { + recovery + .filter_and_commit_with_identifier(input.clone(), 7) + .await + .unwrap(); + assert_eq!(latest(&table).await.id(), compact.id() + 1); + assert_complete(&table).await; + } +} + +struct RacingRecoveries { + table: Table, + ready: tokio::sync::Barrier, +} + +#[async_trait::async_trait] +impl SnapshotCommit for RacingRecoveries { + async fn commit( + &self, + _base: Option<&str>, + snapshot: &Snapshot, + _branch: &str, + _statistics: &[PartitionStatistics], + ) -> Result { + self.ready.wait().await; + self.table + .snapshot_manager() + .commit_snapshot(snapshot) + .await + } +} + +#[tokio::test] +async fn simultaneous_recoveries_publish_one_index_snapshot() { + let table = table(Some(true), false).await; + let input = messages(&table, vec![1, 2]).await; + let mut original = TableCommit::new(table.clone(), "writer".into()); + original + .add_commit_callback(Arc::new(FailDataCallbackOnce(AtomicBool::new(false)))) + .unwrap(); + assert!(original + .commit_with_identifier(input.clone(), 7) + .await + .is_err()); + let mut recovery = TableCommit::new(table.clone(), "writer".into()); + recovery.snapshot_commit = Arc::new(RacingRecoveries { + table: table.clone(), + ready: tokio::sync::Barrier::new(2), + }); + let (first, second) = tokio::time::timeout(std::time::Duration::from_secs(10), async { + tokio::join!( + recovery.filter_and_commit_with_identifier(input.clone(), 7), + recovery.filter_and_commit_with_identifier(input.clone(), 7), + ) + }) + .await + .expect("recoveries must converge after one CAS conflict"); + first.unwrap(); + second.unwrap(); + assert_eq!(latest(&table).await.id(), 2); + assert_complete(&table).await; + assert_eq!( + table + .file_io() + .list_status(&format!("{}/index", table.location())) + .await + .unwrap() + .iter() + .filter(|file| !file.is_dir) + .count(), + indexes(&table).await.len(), + "the losing recovery must leave no private output behind", + ); +} + +#[tokio::test] +async fn captured_builds_ignore_later_appends_and_validation_creates_no_files() { + let table = table(Some(false), false).await; + let commit = TableCommit::new(table.clone(), "writer".into()); + commit + .commit(messages(&table, vec![1, 2]).await) + .await + .unwrap(); + let source = latest(&table).await; + commit + .commit(messages(&table, vec![3, 4]).await) + .await + .unwrap(); + let mut builder = table.new_global_index_build_builder(); + builder.with_index_column("id"); + builder.validate().await.unwrap(); + assert_eq!(latest(&table).await.id(), 2); + assert!(indexes(&table).await.is_empty()); + let built = builder + .build_for_maintenance(&source, Default::default()) + .await + .unwrap(); + assert_eq!(built.len(), 1); + let meta = built[0].new_index_files[0] + .global_index_meta + .as_ref() + .unwrap(); + assert_eq!((meta.row_range_start, meta.row_range_end), (0, 1)); + commit.abort(&built).await.unwrap(); +} + +#[tokio::test] +async fn overwrite_replaces_coverage_without_resurrecting_old_rows() { + let table = table(Some(true), false).await; + let commit = TableCommit::new(table.clone(), "writer".into()); + commit + .commit_with_identifier(messages(&table, vec![1, 2]).await, 6) + .await + .unwrap(); + let old_files = indexes(&table).await; + let input = messages(&table, vec![3, 4]).await; + for _ in 0..2 { + commit + .overwrite_with_identifier(input.clone(), None, 7) + .await + .unwrap(); + assert_complete(&table).await; + let current = indexes(&table).await; + assert!(current.iter().all(|entry| !old_files.contains(entry))); + assert_eq!(latest(&table).await.id(), 4); + } +} + +#[tokio::test] +async fn automatic_indexes_preserve_branch_reference_write_restrictions() { + let table = table(Some(true), false).await; + TableCommit::new(table.clone(), "main".into()) + .commit(messages(&table, vec![1, 2]).await) + .await + .unwrap(); + let main_files = indexes(&table).await; + crate::table::branch_manager::BranchManager::new( + table.file_io().clone(), + table.location().into(), + ) + .create_branch("dev") + .await + .unwrap(); + let branch = table + .copy_with_resolved_schema(table.schema().clone(), "dev") + .unwrap(); + let commit = TableCommit::new(branch.clone(), "branch".into()); + let input = messages(&branch, vec![3, 4]).await; + let error = commit + .commit_with_identifier(input.clone(), 7) + .await + .unwrap_err(); + assert!(error + .to_string() + .contains("Writing to Paimon branch 'dev' is not supported")); + let error = commit + .filter_and_commit_with_identifier(input, 7) + .await + .unwrap_err(); + assert!(error + .to_string() + .contains("Writing to Paimon branch 'dev' is not supported")); + assert_eq!(latest(&table).await.id(), 2); + assert_eq!(indexes(&table).await, main_files); + assert!(branch + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .is_none()); +} + +#[tokio::test] +async fn native_builders_share_automatic_incremental_and_captured_snapshot_contracts() { + use crate::spec::{ArrayType, FloatType, VarCharType}; + use arrow_array::builder::{Float32Builder, ListBuilder}; + use arrow_array::{ArrayRef, StringArray}; + + let original = table(Some(true), false).await; + let mut options = original.schema().options().clone(); + options.remove("global-index.btree.index-columns"); + options.remove("global-index.bitmap.index-columns"); + let schema = TableSchemaBuilder::builder() + .column("id", DataType::Int(IntType::new())) + .column("text", DataType::VarChar(VarCharType::string_type())) + .column( + "embedding", + DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), + ) + .options(options) + .option("global-index.full-text.index-columns", " text ") + .option( + "fields.text.global-index.full-text.index-options", + r#"{"lower-case":true}"#, + ) + .option("global-index.vector.index-columns", "embedding") + .option( + "fields.embedding.global-index.vector.index-type", + "ivf-flat", + ) + .option( + "fields.embedding.global-index.vector.index-options", + r#"{"dimension":2,"nlist":1}"#, + ) + .build() + .unwrap(); + let table = original + .copy_with_resolved_schema(TableSchema::new(0, &schema), original.branch()) + .unwrap(); + table + .file_io() + .new_output(&table.schema_manager().schema_path(0)) + .unwrap() + .write(serde_json::to_vec(table.schema()).unwrap().into()) + .await + .unwrap(); + let mut vectors = ListBuilder::new(Float32Builder::new()); + for _ in 0..4 { + vectors.values().append_value(1.); + vectors.values().append_value(0.); + vectors.append(true); + } + let batch = RecordBatch::try_from_iter(vec![ + ( + "id", + Arc::new(Int32Array::from(vec![1, 2, 3, 4])) as ArrayRef, + ), + ( + "text", + Arc::new(StringArray::from(vec!["paimon index"; 4])) as ArrayRef, + ), + ("embedding", Arc::new(vectors.finish()) as ArrayRef), + ]) + .unwrap(); + let commit = TableCommit::new(table.clone(), "native".into()); + for _ in 0..2 { + let builder = table.new_write_builder(); + let mut write = builder.new_write().unwrap(); + write.write_arrow_batch(&batch).await.unwrap(); + commit + .commit(write.prepare_commit().await.unwrap()) + .await + .unwrap(); + } + for (column, kind) in [("text", "full-text"), ("embedding", "ivf-flat")] { + let options = if kind == "full-text" { + HashMap::new() + } else { + HashMap::from([ + ("ivf-flat.dimension".into(), "2".into()), + ("ivf-flat.nlist".into(), "1".into()), + ]) + }; + let mut builder = table.new_global_index_build_builder(); + builder + .with_index_column(column) + .with_index_type(kind) + .with_options(options); + assert!(builder.build().await.unwrap().is_empty()); + // Snapshot 1 precedes both rounds; the later append must not enter this build. + let source = table.snapshot_manager().get_snapshot(1).await.unwrap(); + let built = builder + .build_for_maintenance(&source, Default::default()) + .await + .unwrap(); + assert_eq!(built.len(), 1, "{kind}"); + let meta = built[0].new_index_files[0] + .global_index_meta + .as_ref() + .unwrap(); + assert_eq!((meta.row_range_start, meta.row_range_end), (0, 3), "{kind}"); + commit.abort(&built).await.unwrap(); + } + assert_eq!(latest(&table).await.id(), 4); + let files = indexes(&table).await; + assert_eq!( + files.len(), + 2, + "each native index should have one current shard after maintenance" + ); + let full_text = files + .iter() + .find(|entry| entry.index_file.index_type == "full-text") + .unwrap(); + let options: serde_json::Value = serde_json::from_slice( + full_text + .index_file + .global_index_meta + .as_ref() + .unwrap() + .index_meta + .as_ref() + .unwrap(), + ) + .unwrap(); + assert_eq!(options["lower-case"], "true"); +} diff --git a/crates/paimon/src/table/table_commit/auto_index_build_tests/search.rs b/crates/paimon/src/table/table_commit/auto_index_build_tests/search.rs new file mode 100644 index 000000000..a31343ba9 --- /dev/null +++ b/crates/paimon/src/table/table_commit/auto_index_build_tests/search.rs @@ -0,0 +1,493 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::catalog::{Catalog, FileSystemCatalog}; +use crate::spec::{ArrayType, FloatType, SchemaChange, VarCharType}; +use crate::Options; +use arrow_array::builder::{Float32Builder, ListBuilder}; +use arrow_array::{ArrayRef, StringArray}; + +#[tokio::test] +async fn manual_builds_and_planned_shards_share_declarations_and_preserve_explicit_overrides() { + for override_metric in [None, Some("l2")] { + let (_catalog, table) = search_table(&[ + ("write-only", "true"), + ("global-index.vector.index-columns", "embedding"), + ( + "fields.embedding.global-index.vector.index-type", + "ivf-flat", + ), + ( + "fields.embedding.global-index.vector.distance-metric", + "cosine", + ), + ( + "fields.embedding.global-index.vector.index-options", + r#"{"dimension":2,"nlist":1}"#, + ), + ("fields.embedding.distance.metric", "l2"), + ("global-index.full-text.index-columns", "text"), + ( + "fields.text.global-index.full-text.index-options", + r#"{"lower-case":false}"#, + ), + ]) + .await; + let commit = TableCommit::new(table.clone(), "writer".into()); + commit + .commit(search_messages(&table, &[[100., 1.], [1., 1.]]).await) + .await + .unwrap(); + assert!(indexes(&table).await.is_empty()); + let before = vector_scores(&table, "full").await; + let mut vector = table.new_global_index_build_builder(); + vector + .with_index_column("embedding") + .with_index_type("ivf-flat"); + if let Some(metric) = override_metric { + // Native aliases in explicit builder options override qualified declarations. + vector.with_options(HashMap::from([("metric".into(), metric.into())])); + assert_eq!(vector.execute().await.unwrap(), 1); + } else { + let plan = vector.plan().await.unwrap(); + let mut prepared = Vec::new(); + for shard in plan.shards() { + prepared.extend(vector.build_shard(&shard).await.unwrap()); + } + assert_eq!(prepared.len(), 1); + commit.commit(prepared).await.unwrap(); + } + let mut text = table.new_global_index_build_builder(); + text.with_index_column("text").with_index_type("full-text"); + if override_metric.is_some() { + text.with_options(HashMap::from([( + "full-text.lower-case".into(), + "true".into(), + )])); + } + assert_eq!(text.execute().await.unwrap(), 1); + for mode in ["full", "detail"] { + let after = vector_scores(&table, mode).await; + if override_metric.is_none() { + assert_eq!(after.row_ids, before.row_ids); + for (raw, indexed) in before.scores.iter().zip(&after.scores) { + assert!((raw - indexed).abs() < 1e-5); + } + } else { + assert_eq!(after.row_ids, vec![1, 0]); + assert!((after.scores[0] - 0.5).abs() < 1e-5); + } + let view = table.copy_with_options(HashMap::from([( + "full-text-index.search-mode".into(), + mode.into(), + )])); + let result = view + .new_full_text_search_builder() + .with_text_column("text") + .with_query_text("foo") + .with_limit(10) + .execute() + .await + .unwrap(); + let actual = result + .row_ids() + .unwrap() + .row_ids + .iter() + .copied() + .collect::>(); + let expected = if override_metric.is_some() { + vec![0, 1] + } else { + vec![1] + }; + assert_eq!(actual, expected.into_iter().collect()); + } + } +} + +async fn search_table(options: &[(&str, &str)]) -> (FileSystemCatalog, Table) { + let catalog = FileSystemCatalog::new(Options::from_map(HashMap::from([( + "warehouse".into(), + format!("memory:/auto-index-search/{}", uuid::Uuid::new_v4()), + )]))) + .unwrap(); + catalog + .create_database("default", false, HashMap::new()) + .await + .unwrap(); + let mut schema = TableSchemaBuilder::builder() + .column("id", DataType::Int(IntType::new())) + .column("text", DataType::VarChar(VarCharType::string_type())) + .column("folded_text", DataType::VarChar(VarCharType::string_type())) + .column( + "embedding", + DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))), + ) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("global-index.enabled", "true") + .option("global-index.row-count-per-shard", "2") + .option("compaction.min.file-num", "1000") + .option("commit.min-retry-wait", "0 ms") + .option("commit.max-retry-wait", "0 ms"); + for (key, value) in options { + schema = schema.option(*key, *value); + } + let id = Identifier::new("default", "search"); + catalog + .create_table(&id, schema.build().unwrap(), false) + .await + .unwrap(); + let table = catalog.get_table(&id).await.unwrap(); + (catalog, table) +} + +async fn search_messages(table: &Table, vectors: &[[f32; 2]]) -> Vec { + let mut builder = ListBuilder::new(Float32Builder::new()); + for vector in vectors { + builder.values().append_slice(vector); + builder.append(true); + } + let texts = (0..vectors.len()) + .map(|i| if i % 2 == 0 { "FOO" } else { "foo" }) + .collect::>(); + let batch = RecordBatch::try_from_iter(vec![ + ( + "id", + Arc::new(Int32Array::from( + (0..vectors.len() as i32).collect::>(), + )) as ArrayRef, + ), + ( + "text", + Arc::new(StringArray::from(texts.clone())) as ArrayRef, + ), + ( + "folded_text", + Arc::new(StringArray::from(texts)) as ArrayRef, + ), + ("embedding", Arc::new(builder.finish()) as ArrayRef), + ]) + .unwrap(); + let builder = table.new_write_builder(); + let mut writer = builder.new_write().unwrap(); + writer.write_arrow_batch(&batch).await.unwrap(); + writer.prepare_commit().await.unwrap() +} + +async fn vector_scores(table: &Table, mode: &str) -> crate::vector_search::ScoredRowIds { + table + .new_vector_search_builder() + .with_vector_column("embedding") + .with_query_vector(vec![1., 0.]) + .with_limit(4) + .with_options(HashMap::from([( + "vector-index.search-mode".into(), + mode.into(), + )])) + .execute() + .await + .unwrap() + .row_ids() + .unwrap() + .clone() +} + +#[tokio::test] +async fn full_and_detail_vector_fallback_share_declared_metrics_with_indexes() { + for metric in [None, Some("cosine"), Some("l2")] { + let mut options = vec![ + ("global-index.vector.index-columns", "embedding"), + ( + "fields.embedding.global-index.vector.index-type", + "ivf-flat", + ), + ( + "fields.embedding.global-index.vector.index-options", + r#"{"dimension":2,"nlist":1}"#, + ), + // A declared metric (including its default) has the same build-over-table + // precedence as explicit native builder options. + ("fields.embedding.distance.metric", "l2"), + ("fields.embedding.index-type", "diskann"), + ]; + if let Some(metric) = metric { + options.push(( + "fields.embedding.global-index.vector.distance-metric", + metric, + )); + } + let (_catalog, table) = search_table(&options).await; + TableCommit::new(table.clone(), "writer".into()) + .commit(search_messages(&table, &[[100., 1.], [1., 1.]]).await) + .await + .unwrap(); + let data = table.snapshot_manager().get_snapshot(1).await.unwrap(); + let raw = table.copy_with_pinned_snapshot(Some(&data)); + for mode in ["full", "detail"] { + let before = vector_scores(&raw, mode).await; + let after = vector_scores(&table, mode).await; + let expected = if metric == Some("l2") { + vec![1, 0] + } else { + vec![0, 1] + }; + assert_eq!(before.row_ids, expected, "{metric:?} {mode}: raw ranking"); + assert_eq!( + after.row_ids, expected, + "{metric:?} {mode}: indexed ranking" + ); + for (raw, indexed) in before.scores.iter().zip(&after.scores) { + assert!( + (raw - indexed).abs() < 1e-5, + "{metric:?} {mode}: {raw} != {indexed}" + ); + } + let expected_score = match metric { + None => 100., + Some("cosine") => 100. / 10001_f32.sqrt(), + Some("l2") => 0.5, + _ => unreachable!(), + }; + assert!((before.scores[0] - expected_score).abs() < 1e-5); + } + // Explicit query metrics retain precedence over the table declaration. + let overridden = raw + .new_vector_search_builder() + .with_vector_column("embedding") + .with_query_vector(vec![1., 0.]) + .with_limit(1) + .with_options(HashMap::from([ + ("vector-index.search-mode".into(), "full".into()), + ("metric".into(), "l2".into()), + ])) + .execute() + .await + .unwrap(); + assert_eq!(overridden.row_ids().unwrap().row_ids, vec![1]); + } +} + +#[tokio::test] +async fn full_and_detail_full_text_fallback_share_per_column_options_in_mixed_coverage() { + let (_catalog, table) = search_table(&[ + ("global-index.full-text.index-columns", "text,folded_text"), + ( + "fields.text.global-index.full-text.index-options", + r#"{"lower-case":false}"#, + ), + ( + "fields.folded_text.global-index.full-text.index-options", + r#"{"lower-case":true}"#, + ), + ]) + .await; + let commit = TableCommit::new(table.clone(), "writer".into()); + for _ in 0..2 { + commit + .commit(search_messages(&table, &[[1., 0.], [1., 0.]]).await) + .await + .unwrap(); + } + for snapshot_id in 1..=4 { + let snapshot = table + .snapshot_manager() + .get_snapshot(snapshot_id) + .await + .unwrap(); + let view = table.copy_with_pinned_snapshot(Some(&snapshot)); + let count = if snapshot_id <= 2 { 2 } else { 4 }; + for mode in ["full", "detail"] { + let view = view.copy_with_options(HashMap::from([( + "full-text-index.search-mode".into(), + mode.into(), + )])); + for column in ["text", "folded_text"] { + let result = view + .new_full_text_search_builder() + .with_text_column(column) + .with_query_text("foo") + .with_limit(10) + .execute() + .await + .unwrap(); + let actual = result + .row_ids() + .unwrap() + .row_ids + .iter() + .copied() + .collect::>(); + let expected = (0..count) + .filter(|id| column == "folded_text" || id % 2 == 1) + .collect::>(); + assert_eq!( + actual, expected, + "snapshot={snapshot_id} mode={mode} column={column}" + ); + } + } + } +} + +#[tokio::test] +async fn vector_algorithm_and_metric_changes_atomically_replace_incompatible_payloads() { + for (key, value, expected_type, expected_metric) in [ + ( + "fields.embedding.global-index.vector.index-type", + "ivf-sq", + "ivf-sq", + "inner_product", + ), + ( + "fields.embedding.global-index.vector.distance-metric", + "cosine", + "ivf-flat", + "cosine", + ), + ] { + let (catalog, table) = search_table(&[ + ("global-index.vector.index-columns", "embedding"), + ( + "fields.embedding.global-index.vector.index-type", + "ivf-flat", + ), + ( + "fields.embedding.global-index.vector.index-options", + r#"{"dimension":2,"nlist":1}"#, + ), + ]) + .await; + TableCommit::new(table.clone(), "seed".into()) + .commit(search_messages(&table, &[[100., 1.], [1., 1.]]).await) + .await + .unwrap(); + let historical = indexes(&table).await; + let snapshot = latest(&table).await; + catalog + .alter_table( + table.identifier(), + vec![SchemaChange::set_option(key.into(), value.into())], + false, + ) + .await + .unwrap(); + let changed = catalog.get_table(table.identifier()).await.unwrap(); + let input = search_messages(&changed, &[[2., 0.], [0., 1.]]).await; + let commit = TableCommit::new(changed.clone(), "writer".into()); + let mut interrupted = TableCommit::new( + changed.copy_with_options(HashMap::from([("commit.max-retries".into(), "0".into())])), + "writer".into(), + ); + interrupted.snapshot_commit = Arc::new(InjectIndexPublication { + table: changed.clone(), + mode: Publication::FailBeforePublish, + fired: AtomicBool::new(false), + }); + assert!(interrupted + .commit_with_identifier(input.clone(), 7) + .await + .is_err()); + assert_eq!(latest(&changed).await.id(), 3); + assert_eq!( + indexes(&changed).await, + historical, + "failed replacement must retain all previous references" + ); + commit + .filter_and_commit_with_identifier(input.clone(), 7) + .await + .unwrap(); + assert_eq!( + latest(&changed).await.id(), + 4, + "replace every incompatible shard in one publication" + ); + let current = indexes(&changed).await; + assert_eq!(current.len(), 2); + for entry in ¤t { + assert_eq!(entry.index_file.index_type, expected_type); + assert!( + !historical.contains(entry), + "every incompatible full shard must be replaced" + ); + let native: HashMap = serde_json::from_slice( + entry + .index_file + .global_index_meta + .as_ref() + .unwrap() + .index_meta + .as_ref() + .unwrap(), + ) + .unwrap(); + assert_eq!(native["metric"], expected_metric); + } + for mode in ["full", "detail"] { + let scores = vector_scores(&changed, mode).await; + assert_eq!(scores.row_ids.len(), 4); + if expected_metric == "cosine" { + assert_eq!(scores.row_ids, vec![2, 0, 1, 3]); + assert!((scores.scores[0] - 1.).abs() < 1e-5); + } + assert_eq!( + vector_scores(&table.copy_with_pinned_snapshot(Some(&snapshot)), mode) + .await + .row_ids + .len(), + 2 + ); + } + // The data snapshot retains the previous definition until the one replacement commit. + let pending = changed.snapshot_manager().get_snapshot(3).await.unwrap(); + let pending = changed.copy_with_pinned_snapshot(Some(&pending)); + let pending_snapshot = pending.travel_snapshot().unwrap(); + let entries = IndexManifest::read( + changed.file_io(), + &changed + .snapshot_manager() + .manifest_path(pending_snapshot.index_manifest().unwrap()), + ) + .await + .unwrap(); + assert_eq!(entries, historical); + for entry in historical { + assert!(changed + .file_io() + .exists(&format!( + "{}/index/{}", + changed.location(), + entry.index_file.file_name + )) + .await + .unwrap()); + } + let completed = latest(&changed).await.id(); + commit + .filter_and_commit_with_identifier(input, 7) + .await + .unwrap(); + assert_eq!( + latest(&changed).await.id(), + completed, + "recovery must not publish another replacement" + ); + } +} diff --git a/crates/paimon/src/table/vector_search_common.rs b/crates/paimon/src/table/vector_search_common.rs index 93eaea74c..5d5adf3fa 100644 --- a/crates/paimon/src/table/vector_search_common.rs +++ b/crates/paimon/src/table/vector_search_common.rs @@ -17,9 +17,11 @@ //! Shared vector-search option validation, index I/O helpers, and result ordering. +use crate::io::{FileIO, FileRead}; use crate::spec::{ CoreOptions, DataField, FileKind, IndexManifestEntry, Predicate, ROW_ID_FIELD_NAME, }; +use crate::table::index_file_path::IndexFileLocation; use crate::table::pk_vector_position_read::{PKEY_VECTOR_POSITION_COLUMN, SEARCH_SCORE_COLUMN}; use crate::table::read_builder::resolve_projected_fields; use crate::table::Table; @@ -27,10 +29,75 @@ use crate::vindex::is_vindex_index_type; use crate::vindex::range_reader::RangeIoStats; use arrow_array::{Int64Array, RecordBatch}; use arrow_select::interleave::interleave_record_batch; +use paimon_vindex_core::diskann_io::DISKANN_HEADER_SIZE; +use paimon_vindex_core::distance::MetricType; +use paimon_vindex_core::index::VectorIndexReader as VIndexReader; use std::collections::HashMap; +use std::io::Cursor; pub(super) use super::partition_filter::validate_partition_filter; +/// Read persisted scoring semantics for both raw queries and definition compatibility checks. +pub(super) async fn read_index_vector_metric( + file_io: &FileIO, + table_path: &str, + entry: &IndexManifestEntry, +) -> crate::Result> { + let Some(global_meta) = entry.index_file.global_index_meta.as_ref() else { + return Ok(None); + }; + if !is_vindex_index_type(&entry.index_file.index_type) { + return Ok(None); + } + if let Some(index_meta) = global_meta.index_meta.as_ref() { + if let Ok(options) = serde_json::from_slice::>(index_meta) { + if let Some(metric) = options.get("metric") { + let metric = match normalize_metric(metric).as_str() { + "l2" => Some(MetricType::L2), + "cosine" => Some(MetricType::Cosine), + "inner_product" => Some(MetricType::InnerProduct), + _ => None, + }; + if metric.is_some() { + return Ok(metric); + } + } + } + } + let path = IndexFileLocation::Global { table_path }.resolve( + &entry.index_file.file_name, + entry.index_file.external_path.as_deref(), + ); + let input = file_io.new_input(&path)?; + let read_error = |e| crate::Error::DataInvalid { + message: format!( + "Failed to read vindex index file '{}' for vector metric: {}", + entry.index_file.file_name, e + ), + source: Some(Box::new(e)), + }; + let header_size = if entry.index_file.file_size > 0 { + (entry.index_file.file_size as u64).min(DISKANN_HEADER_SIZE as u64) + } else { + input + .metadata() + .await + .map_err(&read_error)? + .size + .min(DISKANN_HEADER_SIZE as u64) + }; + let file_reader = input.reader().await.map_err(&read_error)?; + let bytes = file_reader.read(0..header_size).await.map_err(read_error)?; + let reader = VIndexReader::open(Cursor::new(bytes)).map_err(|e| crate::Error::DataInvalid { + message: format!( + "Failed to open paimon-vindex-core reader for vector metric: {}", + e + ), + source: Some(Box::new(e)), + })?; + Ok(Some(reader.metadata().metric)) +} + pub(super) fn add_filter(current: &mut Option, filter: Predicate) { *current = Some(match current.take() { Some(previous) => Predicate::and(vec![previous, filter]), @@ -78,16 +145,26 @@ pub(super) fn configured_vector_index_type( table_options: &HashMap, query_options: &HashMap, column: &str, -) -> Option { - [ +) -> crate::Result> { + let keys = [ "index_type".to_string(), "index-type".into(), "vector.index-type".into(), format!("fields.{column}.index-type"), - ] - .iter() - .find_map(|key| query_options.get(key).or_else(|| table_options.get(key))) - .map(|value| value.trim().to_ascii_lowercase()) + ]; + if let Some(value) = keys.iter().find_map(|key| query_options.get(key)) { + return Ok(Some(value.trim().to_ascii_lowercase())); + } + if let Some(kind) = super::global_index_options::vector_index_type(query_options, column)? { + return Ok(Some(kind.into())); + } + if let Some(kind) = super::global_index_options::vector_index_type(table_options, column)? { + return Ok(Some(kind.into())); + } + Ok(keys + .iter() + .find_map(|key| table_options.get(key)) + .map(|value| value.trim().to_ascii_lowercase())) } pub(super) fn current_tokio_runtime_handle() -> crate::Result { diff --git a/crates/paimon/src/table/vindex_index_build_builder.rs b/crates/paimon/src/table/vindex_index_build_builder.rs index b85a2539b..e3503ae1e 100644 --- a/crates/paimon/src/table/vindex_index_build_builder.rs +++ b/crates/paimon/src/table/vindex_index_build_builder.rs @@ -39,6 +39,9 @@ pub(super) struct VindexIndexBuildBuilder<'a> { index_type: String, options: HashMap, pub(super) partition_filter: Option, + pub(super) snapshot: Option>, + pub(super) replaced_index_files: + std::collections::HashSet, } impl<'a> VindexIndexBuildBuilder<'a> { @@ -49,6 +52,8 @@ impl<'a> VindexIndexBuildBuilder<'a> { index_type: index_type.to_string(), options: HashMap::new(), partition_filter: None, + snapshot: None, + replaced_index_files: Default::default(), } } @@ -83,7 +88,14 @@ impl<'a> VindexIndexBuildBuilder<'a> { source: None, })?; + let defaults = super::global_index_options::declared_options( + self.table.schema().options(), + index_column, + "vector", + &self.index_type, + )?; let mut merged_options = self.table.schema().options().clone(); + merged_options.extend(defaults.clone()); merged_options.extend(self.options.clone()); let core_options = CoreOptions::new(&merged_options); super::global_index_build_common::validate_build_table(self.table)?; @@ -92,8 +104,9 @@ impl<'a> VindexIndexBuildBuilder<'a> { let index_field = find_index_field(self.table, index_column)?; validate_vector_field(index_field)?; - let options = VindexVectorIndexOptions::new( + let options = VindexVectorIndexOptions::new_with_defaults( self.table.schema().options(), + &defaults, &self.options, &self.index_type, index_field, @@ -102,8 +115,10 @@ impl<'a> VindexIndexBuildBuilder<'a> { options.dimension() as u64, "vindex dimension is too large for Rust builder", )?; - let snapshot_manager = self.table.snapshot_manager(); - let Some(snapshot) = snapshot_manager.get_latest_snapshot().await? else { + let Some(snapshot) = + super::global_index_build_common::build_snapshot(self.table, self.snapshot.as_ref()) + .await? + else { return Ok(GlobalIndexBuildPlan::new( self.table, None, @@ -129,6 +144,7 @@ impl<'a> VindexIndexBuildBuilder<'a> { &self.index_type, index_field.id(), None, // single-column build; no extra fields today + &self.replaced_index_files, ) .await?; let shards = plan_vindex_shards( @@ -153,6 +169,7 @@ impl<'a> VindexIndexBuildBuilder<'a> { .iter() .map(|shard| RowRange::new(shard.row_range_start, shard.row_range_end)) .collect::>(), + &self.replaced_index_files, ) .await?; } @@ -180,8 +197,15 @@ impl<'a> VindexIndexBuildBuilder<'a> { })?; let index_field = find_index_field(self.table, index_column)?; validate_vector_field(index_field)?; - let options = VindexVectorIndexOptions::new( + let defaults = super::global_index_options::declared_options( + self.table.schema().options(), + index_column, + "vector", + &self.index_type, + )?; + let options = VindexVectorIndexOptions::new_with_defaults( self.table.schema().options(), + &defaults, &self.options, &self.index_type, index_field, @@ -221,6 +245,14 @@ impl<'a> VindexIndexBuildBuilder<'a> { )> { let plan = self.plan().await?; let mut options = self.table.schema().options().clone(); + if let Some(column) = &self.index_column { + options.extend(super::global_index_options::declared_options( + self.table.schema().options(), + column, + "vector", + &self.index_type, + )?); + } options.extend(self.options.clone()); let parallelism = CoreOptions::new(&options).global_index_build_parallelism()?; let prepared = super::global_index_build_common::preparation::prepare_shards( diff --git a/crates/paimon/src/vindex/mod.rs b/crates/paimon/src/vindex/mod.rs index f9c6b3355..a93b4e671 100644 --- a/crates/paimon/src/vindex/mod.rs +++ b/crates/paimon/src/vindex/mod.rs @@ -122,33 +122,29 @@ impl VindexVectorIndexOptions { index_type: &str, field: &DataField, ) -> crate::Result { - let native_index_type = - native_index_type(index_type).ok_or_else(|| crate::Error::DataInvalid { - message: format!("Unsupported vindex index type: {index_type}"), - source: None, - })?; - - validate_user_option_keys(user_options, index_type, field.name())?; - validate_index_type_option(user_options, native_index_type)?; - let mut native_options = HashMap::new(); - // Source precedence is build > table; within each source, field > index - // > native keys. Table options follow Java; explicit builder options - // also accept the documented Rust/SQL native aliases. - collect_native_options( - &mut native_options, + Self::new_with_defaults( table_options, + &HashMap::new(), + user_options, index_type, - field.name(), - false, - )?; - collect_native_options( - &mut native_options, + field, + ) + } + + pub(crate) fn new_with_defaults( + table_options: &HashMap, + defaults: &HashMap, + user_options: &HashMap, + index_type: &str, + field: &DataField, + ) -> crate::Result { + let mut native_options = Self::resolve_native_options( + table_options, + defaults, user_options, index_type, field.name(), - true, )?; - native_options.insert("index.type".into(), native_index_type.into()); let dimension = match field.data_type() { DataType::Vector(vector) => vector.length().to_string(), _ => native_options @@ -157,10 +153,6 @@ impl VindexVectorIndexOptions { .unwrap_or_else(|| DEFAULT_DIMENSION.into()), }; native_options.insert("dimension".into(), dimension); - let metric = native_options - .entry("metric".into()) - .or_insert_with(|| DEFAULT_METRIC.into()); - *metric = normalize_metric(metric); // Like Java, compile the native configuration only when training begins. // A synthetic count can reject budgets satisfied by the real vector count. @@ -173,9 +165,16 @@ impl VindexVectorIndexOptions { message: "vindex vector dimension must be a positive integer".into(), source: None, })?; - let train_sample_ratio = - resolve_train_sample_ratio(table_options, user_options, index_type, field.name())?; - let granule_build_enabled = resolve_granule_build_enabled(table_options, user_options)?; + let mut default_options = table_options.clone(); + default_options.extend(defaults.clone()); + let train_sample_ratio = resolve_train_sample_ratio( + table_options, + defaults, + user_options, + index_type, + field.name(), + )?; + let granule_build_enabled = resolve_granule_build_enabled(&default_options, user_options)?; Ok(Self { dimension, native_options, @@ -184,6 +183,46 @@ impl VindexVectorIndexOptions { }) } + /// Resolve the same option precedence and metric defaults for builders and raw queries. + pub(crate) fn resolve_native_options( + table_options: &HashMap, + defaults: &HashMap, + user_options: &HashMap, + index_type: &str, + column: &str, + ) -> crate::Result> { + let native_index_type = + native_index_type(index_type).ok_or_else(|| crate::Error::DataInvalid { + message: format!("Unsupported vindex index type: {index_type}"), + source: None, + })?; + + validate_user_option_keys(user_options, index_type, column)?; + validate_index_type_option(user_options, native_index_type)?; + validate_user_option_keys(defaults, index_type, column)?; + validate_index_type_option(defaults, native_index_type)?; + let mut native_options = HashMap::new(); + // Source precedence is build > declarations > table; within each source, field > index + // > native keys. Table options follow Java; explicit builder options + // also accept the documented Rust/SQL native aliases. + collect_native_options( + &mut native_options, + table_options, + index_type, + column, + false, + )?; + collect_native_options(&mut native_options, defaults, index_type, column, true)?; + collect_native_options(&mut native_options, user_options, index_type, column, true)?; + native_options.insert("index.type".into(), native_index_type.into()); + let metric = native_options + .entry("metric".into()) + .or_insert_with(|| DEFAULT_METRIC.into()); + *metric = normalize_metric(metric); + + Ok(native_options) + } + pub(crate) fn needs_vector_count(&self) -> bool { self.native_options .get("index.type") @@ -418,12 +457,13 @@ fn resolve_granule_build_enabled( fn resolve_train_sample_ratio( table_options: &HashMap, + defaults: &HashMap, user_options: &HashMap, index_type: &str, field_name: &str, ) -> crate::Result { let mut value = None; - for options in [user_options, table_options] { + for options in [user_options, defaults, table_options] { for key in [ format!("fields.{field_name}.train.sample-ratio"), format!("{index_type}.train.sample-ratio"), @@ -892,6 +932,31 @@ mod tests { .unwrap(); assert_eq!(options.train_sample_ratio, 0.1); + let declared = VindexVectorIndexOptions::new_with_defaults( + &table_options, + &user_options, + &HashMap::new(), + IVF_FLAT_IDENTIFIER, + &array_float_field(), + ) + .unwrap(); + assert_eq!( + declared.train_sample_ratio, 0.1, + "declarations override legacy field aliases" + ); + let explicit = VindexVectorIndexOptions::new_with_defaults( + &table_options, + &user_options, + &HashMap::from([("ivf-flat.train.sample-ratio".into(), "0.75".into())]), + IVF_FLAT_IDENTIFIER, + &array_float_field(), + ) + .unwrap(); + assert_eq!( + explicit.train_sample_ratio, 0.75, + "explicit options override declarations" + ); + let field_user_options = HashMap::from([ ("ivf-flat.train.sample-ratio".to_string(), "0.5".to_string()), ( diff --git a/crates/paimon/tests/rest_catalog/auto_compact_review.rs b/crates/paimon/tests/rest_catalog/auto_compact_review.rs index 2f6a8ee41..6acce0cdd 100644 --- a/crates/paimon/tests/rest_catalog/auto_compact_review.rs +++ b/crates/paimon/tests/rest_catalog/auto_compact_review.rs @@ -31,6 +31,65 @@ fn schema(evolved: bool) -> Schema { builder.build().unwrap() } +#[tokio::test] +async fn stale_rest_committer_auto_indexes_with_catalog_schema_and_recovers() { + let ctx = setup_catalog(vec!["default"]).await; + let temp = tempfile::tempdir().unwrap(); + let name = "rest_auto_index"; + let index_schema = |evolved: bool| { + let mut builder = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("value", DataType::Int(IntType::new())) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("write-only", "false") + .option("global-index.btree.index-columns", "id"); + if evolved { + builder = builder.column("extra", DataType::Int(IntType::new())); + } + builder.build().unwrap() + }; + ctx.server.add_table_with_schema( + "default", + name, + index_schema(false), + temp.path().to_str().unwrap(), + ); + let identifier = Identifier::new("default", name); + let stale = ctx.catalog.get_table(&identifier).await.unwrap(); + ctx.server + .set_table_schema_id("default", name, index_schema(true), 1); + let current = ctx.catalog.get_table(&identifier).await.unwrap(); + let commit = stale.new_write_builder().new_commit(); + let messages = stage(¤t, 1, true).await; + commit + .commit_with_identifier(messages.clone(), 7) + .await + .unwrap(); + commit + .filter_and_commit_with_identifier(messages, 7) + .await + .unwrap(); + let snapshot = current + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .unwrap(); + assert_eq!(snapshot.id(), 2); + assert_eq!(snapshot.schema_id(), 1); + assert!(snapshot.index_manifest().is_some()); + assert!(current.schema_manager().latest().await.unwrap().is_none()); + assert!(current + .new_global_index_build_builder() + .with_index_column("id") + .build() + .await + .unwrap() + .is_empty()); + assert_eq!(read_rows(¤t).await, vec![(1, 10, Some(100))]); +} + async fn catalog_only_tables(name: &str) -> (TestContext, tempfile::TempDir, Table, Table) { let ctx = setup_catalog(vec!["default"]).await; let temp = tempfile::tempdir().unwrap(); diff --git a/docs/src/compaction.md b/docs/src/compaction.md index 35973c7ed..11c1637bb 100644 --- a/docs/src/compaction.md +++ b/docs/src/compaction.md @@ -92,6 +92,99 @@ commit or a completed automatic `COMPACT` snapshot. A single round can leave additional work for a later write or a dedicated compaction job. This option does not schedule background work or expire snapshots. +## Automatic global index maintenance + +With `write-only=false` (the default), writes synchronously maintain declared +global indexes after automatic compaction. An empty compaction plan still allows +index maintenance. `write-only=true` leaves both jobs to dedicated operators. +Rust, Python, and DataFusion writes share this behavior. Tables without index +declarations retain their existing write behavior; no separate index switch is +required and no background job is scheduled. + +Index options follow the primary-key index convention: a comma-separated column +list for each family and optional per-column parameters. Supported families are +`btree`, `bitmap`, `multivalue`, `fm`, `full-text`, and `vector`: + +```rust +let options = std::collections::HashMap::from([ + ("row-tracking.enabled".into(), "true".into()), + ("bucket".into(), "-1".into()), + ("write-only".into(), "false".into()), + ("global-index.btree.index-columns".into(), "id,age".into()), + ("global-index.full-text.index-columns".into(), "text".into()), + ( + "fields.text.global-index.full-text.index-options".into(), + r#"{"lower-case":true}"#.into(), + ), + ("global-index.vector.index-columns".into(), "embedding".into()), + ("fields.embedding.global-index.vector.index-type".into(), "diskann".into()), + ("fields.embedding.global-index.vector.distance-metric".into(), "cosine".into()), + ( + "fields.embedding.global-index.vector.index-options".into(), + r#"{"dimension":768}"#.into(), + ), +]); +``` + +Maintenance reuses the global-index builders and their table, column-type, +and native writer restrictions. Tables must have row tracking enabled, no +primary key, and `bucket=-1`. Data evolution and the query-side +`global-index.enabled` option are not prerequisites for building indexes. +Full-text builds additionally reject deletion-vector tables. Declared columns +are trimmed; empty names, duplicates, unknown fields, and assigning multiple +families to one column are rejected before data publication. Each listed column +has its own index. Use the explicit build API for composite BTree indexes. +Update the lists and per-column option keys together with schema renames or drops. +`write-only=true` skips automatic index configuration validation. + +Per-column parameters use +`fields..global-index..index-options`, a JSON object containing +string, boolean, or numeric values. Short keys are qualified with the native +writer prefix, such as `block-size` to `btree-index.block-size`. Fully qualified +writer keys and shared shard options are accepted. Conflicting values are +rejected, matching primary-key option resolution. Vector columns additionally +require `fields..global-index.vector.index-type`; the optional +`distance-metric` defaults to `inner_product`. +Explicit single-column builds also use these declarations as defaults; explicit +builder options take precedence over them. +FULL/DETAIL raw fallback uses the same per-column parameters as automatic +builds, including vector metric defaults and full-text tokenization. A declared +full-text column can use raw fallback before its first index is published. + +Each round captures the current schema and one source snapshot. Like Java +primary-key index maintenance, it compares the desired sources with current +index units and replaces changed units atomically. Global units use partition, +bucket, and row-ID shard boundaries instead of primary-key LSM levels. Valid +units are retained, missing ranges are built, and partial or fragmented shards +are rebuilt from current rows. A physical compaction that preserves row IDs and +values can retain existing global payloads. Additions and retired references +publish together; old files remain available to historical snapshots. +Manual `commit_compact` also maintains declared indexes when `write-only=false`. +An empty maintenance plan publishes no snapshot. Empty data commits, index-only +commits, and truncation do not trigger a separate round. Indexed-column updates +retain `global-index.column-update-action`: use `DROP_PARTITION_INDEX` to allow +the data update and automatically rebuild the affected coverage. Changing a +vector algorithm or distance metric retires all incompatible payloads for that +column and publishes their replacements together. Other writer parameters alone +do not rebuild unchanged coverage; changing shard boundaries can regroup it. +This synchronous mode has no minimum-row or time threshold. + +A concurrent snapshot or schema change causes private outputs to be discarded +and coverage to be replanned, bounded by `commit.max-retries` and +`commit.timeout`. A stable identity derived from the originating data commit +makes repeated recovery idempotent. Uncertain publication outcomes retain +submitted files and check for an already published index snapshot before +retrying. Data, compaction, and index publication are separate transactions: +global row IDs must be assigned before indexing. An index failure leaves the +data committed and returns an error preserving the +cause. Retry append with `filter_and_commit_with_identifier` or +`filter_and_commit`, or overwrite with `overwrite_with_identifier`, using the +same commit user and identifier. Recovery resumes indexing even when automatic +compaction already completed. For an interrupted manual compaction, retry the +same prepared results and commit user through `commit_compact`. + +## Compaction planning + Ordinary append and PK compaction use `target-file-size` as the output size target. For unaware append tables (`bucket=-1`), task selection follows the batch packing rules in Java's [`AppendCompactCoordinator`](https://github.com/apache/paimon/blob/master/paimon-core/src/main/java/org/apache/paimon/append/AppendCompactCoordinator.java): diff --git a/docs/src/sql.md b/docs/src/sql.md index 6d99357a2..87ce2caa9 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -1344,7 +1344,13 @@ CALL sys.rollback_to_timestamp(table => 'paimon.my_db.my_table', timestamp => 12 ### create_global_index -Build and commit a global index for a table column: +Build and commit a global index for a table column. + +To maintain indexes synchronously after writes, declare columns through options +such as `global-index.btree.index-columns='id'`. Maintenance runs by default +when `write-only=false`; see +[Automatic global index maintenance](compaction.md#automatic-global-index-maintenance) +for the declaration format and recovery behavior. ```sql CALL sys.create_global_index( @@ -1490,16 +1496,17 @@ threads. Prepared commit messages retain shard plan order. On a build failure, all started shards finish before private outputs are cleaned. Files in messages already returned to a caller are retained, including after a failed commit. -The current global-index builders require a row-tracking data-evolution table -with global indexes enabled. They do not support primary-key tables or tables -with deletion vectors enabled: +Global-index builders require a non-primary-key table with row tracking enabled +and unaware buckets (`bucket=-1`). Data evolution and query-side global-index +optimization may be disabled. Full-text builds additionally reject tables with +deletion vectors enabled: ```sql CREATE TABLE paimon.my_db.items ( id INT, embedding ARRAY ) WITH ( - 'bucket' = '1', + 'bucket' = '-1', 'row-tracking.enabled' = 'true', 'data-evolution.enabled' = 'true', 'global-index.enabled' = 'true', @@ -1559,7 +1566,7 @@ CREATE TABLE paimon.my_db.image_items ( id INT, embedding ARRAY ) WITH ( - 'bucket' = '1', + 'bucket' = '-1', 'row-tracking.enabled' = 'true', 'data-evolution.enabled' = 'true', 'global-index.enabled' = 'true', @@ -2011,7 +2018,7 @@ CREATE TABLE paimon.my_db.items ( id INT, embedding ARRAY ) WITH ( - 'bucket' = '1', + 'bucket' = '-1', 'row-tracking.enabled' = 'true', 'data-evolution.enabled' = 'true', 'global-index.enabled' = 'true', @@ -2957,15 +2964,19 @@ catalog/FileIO. See [Lookup changelog producer](compaction.md#lookup-changelog-p ### Global Index Options Set these options when building global indexes with -`CALL sys.create_global_index`. The current DataFusion builders require -row-tracking and data evolution, and reject primary-key tables and tables with -deletion vectors enabled. +`CALL sys.create_global_index` or automatic builds. Builders require row tracking, +`bucket=-1`, and no primary key. Data evolution and query-side global-index +optimization may be disabled. Full-text builds reject deletion-vector tables. | Option | Default | Description | |---|---:|---| | `row-tracking.enabled` | `false` | Enables stable row ids required by global index files. | | `data-evolution.enabled` | `false` | Enables row-id-aware table evolution and partial-column writes. | | `global-index.enabled` | `true` | Enables global index metadata and global-index-aware reads. | +| `global-index..index-columns` | unset | Comma-separated columns maintained automatically with `write-only=false`. Families: `btree`, `bitmap`, `multivalue`, `fm`, `full-text`, `vector`. Each column owns one index. | +| `fields..global-index..index-options` | unset | Per-column JSON object of native writer options, following primary-key index option resolution. | +| `fields..global-index.vector.index-type` | required for declared vector columns | Vector algorithm, such as `diskann` or `ivf-flat`. Changes replace incompatible payloads atomically during maintenance. | +| `fields..global-index.vector.distance-metric` | `inner_product` | Vector distance metric for builds and FULL/DETAIL raw fallback: `inner_product`, `cosine`, or `l2`. Changes replace incompatible payloads atomically during maintenance. | | `global-index.row-count-per-shard` | `100000` | Maximum row count per vector global-index shard. | | `sorted-index.records-per-range` | `100000` | Maximum row count per BTree, bitmap, multivalue, or FM global-index file range; falls back to legacy `btree-index.records-per-range`. | | `btree-index.block-size` | `64kb` | Target BTree data-block size. | From b35e095efaf57984b41cd4a2eef247355f579efe Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 31 Oct 2026 10:42:57 +0800 Subject: [PATCH 2/4] refactor(table): rename automatic index module to auto_index --- crates/paimon/src/table/table_commit.rs | 8 ++++---- crates/paimon/src/table/table_commit/auto_compact.rs | 4 ++-- .../{auto_index_build.rs => auto_index.rs} | 10 +++++----- .../{auto_index_build => auto_index}/planning.rs | 0 .../{auto_index_build_tests.rs => auto_index_tests.rs} | 4 ++-- .../search.rs | 0 6 files changed, 13 insertions(+), 13 deletions(-) rename crates/paimon/src/table/table_commit/{auto_index_build.rs => auto_index.rs} (94%) rename crates/paimon/src/table/table_commit/{auto_index_build => auto_index}/planning.rs (100%) rename crates/paimon/src/table/table_commit/{auto_index_build_tests.rs => auto_index_tests.rs} (99%) rename crates/paimon/src/table/table_commit/{auto_index_build_tests => auto_index_tests}/search.rs (100%) diff --git a/crates/paimon/src/table/table_commit.rs b/crates/paimon/src/table/table_commit.rs index 2d47c4c33..e37bb2e6d 100644 --- a/crates/paimon/src/table/table_commit.rs +++ b/crates/paimon/src/table/table_commit.rs @@ -50,9 +50,9 @@ const BATCH_COMMIT_IDENTIFIER: i64 = i64::MAX; const CHECK_ROLLING_RECORD_COUNT: usize = 1000; mod auto_compact; -mod auto_index_build; +mod auto_index; #[cfg(test)] -mod auto_index_build_tests; +mod auto_index_tests; mod global_index_source_check; mod row_tracking; @@ -399,7 +399,7 @@ impl TableCommit { } else if self .auto_compaction_enabled(messages.iter().any(|m| !m.new_files.is_empty()))? || self - .auto_index_build_enabled(messages.iter().any(|m| !m.new_files.is_empty())) + .auto_index_enabled(messages.iter().any(|m| !m.new_files.is_empty())) .await? { let kind = direct_commit_kind( @@ -453,7 +453,7 @@ impl TableCommit { return Ok(super::CompactMetrics::default()); } let prepared = super::compact::prepare_results(&self.table, &results)?; - let maintain_indexes = self.auto_index_build_enabled(true).await?; + let maintain_indexes = self.auto_index_enabled(true).await?; let planned_snapshot_id = prepared.snapshot_id; let mut commit = self.clone(); // One stable identity per plan and task subset, allowing repeated plans diff --git a/crates/paimon/src/table/table_commit/auto_compact.rs b/crates/paimon/src/table/table_commit/auto_compact.rs index a59a8d286..19c6a847e 100644 --- a/crates/paimon/src/table/table_commit/auto_compact.rs +++ b/crates/paimon/src/table/table_commit/auto_compact.rs @@ -79,7 +79,7 @@ impl TableCommit { }; let has_new_files = entries.iter().any(|e| *e.kind() == FileKind::Add); let enabled = self.auto_compaction_enabled(has_new_files)?; - let index_enabled = self.auto_index_build_enabled(has_new_files).await?; + let index_enabled = self.auto_index_enabled(has_new_files).await?; let buckets = enabled.then(|| self.written_buckets(entries)); let kind = plan.commit_kind_hint(); let outcome = self @@ -174,7 +174,7 @@ impl TableCommit { .await?; } } - if self.auto_index_build_enabled(true).await? { + if self.auto_index_enabled(true).await? { self.index_after_write(&written).await?; } Ok(()) diff --git a/crates/paimon/src/table/table_commit/auto_index_build.rs b/crates/paimon/src/table/table_commit/auto_index.rs similarity index 94% rename from crates/paimon/src/table/table_commit/auto_index_build.rs rename to crates/paimon/src/table/table_commit/auto_index.rs index 0b70396f2..c3fe71e19 100644 --- a/crates/paimon/src/table/table_commit/auto_index_build.rs +++ b/crates/paimon/src/table/table_commit/auto_index.rs @@ -22,7 +22,7 @@ use sha2::{Digest, Sha256}; mod planning; impl TableCommit { - pub(super) async fn auto_index_build_enabled(&self, has_new_files: bool) -> Result { + pub(super) async fn auto_index_enabled(&self, has_new_files: bool) -> Result { if !has_new_files || self.table.schema().core_options().write_only()? { return Ok(false); } @@ -33,7 +33,7 @@ impl TableCommit { Ok(!definitions.is_empty()) } - fn auto_index_build_user(&self, written: &Snapshot) -> String { + fn auto_index_user(&self, written: &Snapshot) -> String { let mut identity = Sha256::new(); identity.update(self.table.location().as_bytes()); identity.update(self.table.branch().as_bytes()); @@ -48,11 +48,11 @@ impl TableCommit { pub(super) async fn index_after_write(&self, written: &Snapshot) -> Result<()> { self.try_index_after_write(written) .await - .map_err(|error| auto_index_build_error(written.commit_identifier(), error)) + .map_err(|error| auto_index_error(written.commit_identifier(), error)) } async fn try_index_after_write(&self, written: &Snapshot) -> Result<()> { - let commit_user = self.auto_index_build_user(written); + let commit_user = self.auto_index_user(written); let start = current_time_millis(); let mut retries = 0u64; let mut publication_error = None; @@ -177,7 +177,7 @@ impl TableCommit { } } -pub(super) fn auto_index_build_error(identifier: i64, error: crate::Error) -> crate::Error { +pub(super) fn auto_index_error(identifier: i64, error: crate::Error) -> crate::Error { crate::Error::UnexpectedError { message: format!("Data commit {identifier} succeeded, but automatic index build failed; retry through the corresponding recovery API with the same commit user and identifier"), source: Some(Box::new(error)), diff --git a/crates/paimon/src/table/table_commit/auto_index_build/planning.rs b/crates/paimon/src/table/table_commit/auto_index/planning.rs similarity index 100% rename from crates/paimon/src/table/table_commit/auto_index_build/planning.rs rename to crates/paimon/src/table/table_commit/auto_index/planning.rs diff --git a/crates/paimon/src/table/table_commit/auto_index_build_tests.rs b/crates/paimon/src/table/table_commit/auto_index_tests.rs similarity index 99% rename from crates/paimon/src/table/table_commit/auto_index_build_tests.rs rename to crates/paimon/src/table/table_commit/auto_index_tests.rs index ae9696d17..effcf2c04 100644 --- a/crates/paimon/src/table/table_commit/auto_index_build_tests.rs +++ b/crates/paimon/src/table/table_commit/auto_index_tests.rs @@ -530,7 +530,7 @@ async fn vector_definitions_require_a_supported_algorithm_and_metric() { None, ); assert!(TableCommit::new(original.clone(), "writer".into()) - .auto_index_build_enabled(true) + .auto_index_enabled(true) .await .unwrap_err() .to_string() @@ -558,7 +558,7 @@ async fn vector_definitions_require_a_supported_algorithm_and_metric() { .copy_with_options(HashMap::from([(key.into(), value.into())])); assert!( TableCommit::new(table, "invalid".into()) - .auto_index_build_enabled(true) + .auto_index_enabled(true) .await .is_err(), "{key}={value}" diff --git a/crates/paimon/src/table/table_commit/auto_index_build_tests/search.rs b/crates/paimon/src/table/table_commit/auto_index_tests/search.rs similarity index 100% rename from crates/paimon/src/table/table_commit/auto_index_build_tests/search.rs rename to crates/paimon/src/table/table_commit/auto_index_tests/search.rs From 01b6ccf7dcc8f0349c4f1fe2f6e705efef86f98b Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 31 Oct 2026 11:14:49 +0800 Subject: [PATCH 3/4] refactor(table): isolate index replacement planning in auto_index --- .../table/full_text_index_build_builder.rs | 33 ++-- .../src/table/global_index_build_builder.rs | 31 +--- .../src/table/global_index_build_common.rs | 122 ++++++------ .../sorted_global_index_build_builder.rs | 33 ++-- .../src/table/table_commit/auto_index.rs | 14 +- .../table/table_commit/auto_index/planning.rs | 19 +- .../table/table_commit/auto_index_tests.rs | 13 +- .../table_commit/auto_index_tests/search.rs | 175 ++++++++++++++++++ .../src/table/vindex_index_build_builder.rs | 35 ++-- 9 files changed, 324 insertions(+), 151 deletions(-) diff --git a/crates/paimon/src/table/full_text_index_build_builder.rs b/crates/paimon/src/table/full_text_index_build_builder.rs index c8598867b..96045b373 100644 --- a/crates/paimon/src/table/full_text_index_build_builder.rs +++ b/crates/paimon/src/table/full_text_index_build_builder.rs @@ -53,9 +53,7 @@ pub(super) struct FullTextIndexBuildBuilder<'a> { index_column: Option, options: HashMap, pub(super) partition_filter: Option, - pub(super) snapshot: Option>, - pub(super) replaced_index_files: - std::collections::HashSet, + pub(super) view: Option, } impl<'a> FullTextIndexBuildBuilder<'a> { @@ -65,8 +63,7 @@ impl<'a> FullTextIndexBuildBuilder<'a> { index_column: None, options: HashMap::new(), partition_filter: None, - snapshot: None, - replaced_index_files: Default::default(), + view: None, } } @@ -115,10 +112,12 @@ impl<'a> FullTextIndexBuildBuilder<'a> { full_text_config(&native_options)?; let index_meta = serialize_index_meta(&native_options)?; - let Some(snapshot) = - super::global_index_build_common::build_snapshot(self.table, self.snapshot.as_ref()) - .await? - else { + let view = super::global_index_build_common::GlobalIndexBuildView::resolve( + self.table, + self.view.as_ref(), + ) + .await?; + let Some(snapshot) = view.snapshot() else { return Ok((None, vec![])); }; @@ -129,17 +128,14 @@ impl<'a> FullTextIndexBuildBuilder<'a> { let manifest_entries = read_builder .new_scan() .with_scan_all_files() - .plan_manifest_entries(&snapshot) + .plan_manifest_entries(snapshot) .await?; let indexed = indexed_row_ranges( - self.table, - snapshot.index_manifest(), + view.index_entries(), FULL_TEXT_GLOBAL_INDEX_TYPE, index_field.id(), None, - &self.replaced_index_files, - ) - .await?; + ); let shards = plan_vector_index_shards( self.table.location(), self.table.schema().partition_keys(), @@ -156,8 +152,7 @@ impl<'a> FullTextIndexBuildBuilder<'a> { } validate_existing_index_overlap( - self.table, - snapshot.index_manifest(), + view.index_entries(), FULL_TEXT_GLOBAL_INDEX_TYPE, index_field.id(), None, @@ -165,9 +160,7 @@ impl<'a> FullTextIndexBuildBuilder<'a> { .iter() .map(|shard| RowRange::new(shard.row_range_start, shard.row_range_end)) .collect::>(), - &self.replaced_index_files, - ) - .await?; + )?; let prepared = super::global_index_build_common::preparation::prepare_shards( self.table, diff --git a/crates/paimon/src/table/global_index_build_builder.rs b/crates/paimon/src/table/global_index_build_builder.rs index d19c9c0a7..120f2de8b 100644 --- a/crates/paimon/src/table/global_index_build_builder.rs +++ b/crates/paimon/src/table/global_index_build_builder.rs @@ -19,7 +19,7 @@ use std::collections::HashMap; use std::time::Instant; use super::full_text_index_build_builder::FullTextIndexBuildBuilder; -use super::global_index_build_common::{resolve_index_fields, IndexColumns}; +use super::global_index_build_common::{resolve_index_fields, GlobalIndexBuildView, IndexColumns}; use super::global_index_build_plan::{GlobalIndexBuildPlan, GlobalIndexBuildShard}; use super::global_index_types::{ normalize_global_index_type, normalize_queryable_global_index_type, BTREE_GLOBAL_INDEX_TYPE, @@ -211,32 +211,24 @@ impl<'a> GlobalIndexBuildBuilder<'a> { } async fn prepare(&self) -> Result { - self.prepare_at_snapshot(None, Default::default()).await + self.prepare_with_view(None).await } /// Check the complete writer configuration without scanning or creating files. pub(crate) async fn validate(&self) -> Result<()> { - self.prepare_at_snapshot(Some(None), Default::default()) + self.prepare_with_view(Some(GlobalIndexBuildView::default())) .await .map(|_| ()) } - pub(crate) async fn build_for_maintenance( + pub(crate) async fn build_with_view( &self, - snapshot: &crate::spec::Snapshot, - replaced: std::collections::HashSet, + view: GlobalIndexBuildView, ) -> Result> { - Ok(self - .prepare_at_snapshot(Some(Some(snapshot.clone())), replaced) - .await? - .messages) + Ok(self.prepare_with_view(Some(view)).await?.messages) } - async fn prepare_at_snapshot( - &self, - snapshot: Option>, - replaced: std::collections::HashSet, - ) -> Result { + async fn prepare_with_view(&self, view: Option) -> Result { CoreOptions::new(self.table.schema().options()).ensure_read_authorized()?; self.table.ensure_not_branch_reference_for_write()?; let index_type = normalize_global_index_type(self.index_type.trim()).ok_or_else(|| { @@ -253,8 +245,7 @@ impl<'a> GlobalIndexBuildBuilder<'a> { let table = self.configured_table(&columns, index_type)?; if normalize_queryable_global_index_type(index_type).is_some() { let mut builder = SortedGlobalIndexBuildBuilder::new(&table); - builder.snapshot = snapshot; - builder.replaced_index_files = replaced; + builder.view = view; builder .with_index_columns(&columns) .with_index_type(index_type) @@ -266,8 +257,7 @@ impl<'a> GlobalIndexBuildBuilder<'a> { } if index_type == FULL_TEXT_GLOBAL_INDEX_TYPE { let mut builder = FullTextIndexBuildBuilder::new(&table); - builder.snapshot = snapshot; - builder.replaced_index_files = replaced; + builder.view = view; builder .with_index_column(columns[0]) .with_options(self.options.clone()); @@ -275,8 +265,7 @@ impl<'a> GlobalIndexBuildBuilder<'a> { return Ok(builder.prepare().await?.into()); } let mut builder = VindexIndexBuildBuilder::new(&table, index_type); - builder.snapshot = snapshot; - builder.replaced_index_files = replaced; + builder.view = view; builder .with_index_column(columns[0]) .with_options(self.options.clone()); diff --git a/crates/paimon/src/table/global_index_build_common.rs b/crates/paimon/src/table/global_index_build_common.rs index ca8e8c20f..0a937d7f3 100644 --- a/crates/paimon/src/table/global_index_build_common.rs +++ b/crates/paimon/src/table/global_index_build_common.rs @@ -30,8 +30,8 @@ use crate::spec::{DataField, FileKind, IndexManifest, IndexManifestEntry, Snapsh use crate::table::{merge_row_ranges, RowRange, Table}; use crate::{Error, Result}; use bytes::Bytes; -use std::collections::HashSet; use std::path::Path; +use std::sync::Arc; use tokio::io::AsyncReadExt; pub(crate) mod preparation; @@ -39,25 +39,63 @@ pub(crate) mod vector; const COPY_BUFFER_SIZE: usize = 1024 * 1024; -pub(crate) type IndexFileKey = (Vec, i32, String); - -pub(crate) fn index_file_key(entry: &IndexManifestEntry) -> IndexFileKey { - ( - entry.partition.clone(), - entry.bucket, - entry.index_file.file_name.clone(), - ) +/// One source snapshot and its effective index coverage. Builders use this same +/// immutable view for both gap planning and overlap validation. +#[derive(Clone, Default)] +pub(crate) struct GlobalIndexBuildView { + snapshot: Option, + index_entries: Arc<[IndexManifestEntry]>, } -/// An automatic round fixes one source snapshot for every index definition. -/// Capturing an empty view also validates builder configuration without I/O. -pub(super) async fn build_snapshot( - table: &Table, - captured: Option<&Option>, -) -> Result> { - match captured { - Some(snapshot) => Ok(snapshot.clone()), - None => table.snapshot_manager().get_latest_snapshot().await, +impl GlobalIndexBuildView { + pub(crate) async fn capture(table: &Table, snapshot: Option) -> Result { + let Some(snapshot) = snapshot else { + return Ok(Self::default()); + }; + let entries = match snapshot.index_manifest() { + Some(name) => { + IndexManifest::read( + table.file_io(), + &table.snapshot_manager().manifest_path(name), + ) + .await? + } + None => vec![], + }; + Ok(Self { + snapshot: Some(snapshot), + index_entries: entries.into(), + }) + } + + /// An empty captured view validates configuration without reading metadata. + pub(super) async fn resolve(table: &Table, captured: Option<&Self>) -> Result { + match captured { + Some(view) => Ok(view.clone()), + None => { + Self::capture(table, table.snapshot_manager().get_latest_snapshot().await?).await + } + } + } + + pub(crate) fn snapshot(&self) -> Option<&Snapshot> { + self.snapshot.as_ref() + } + + pub(crate) fn index_entries(&self) -> &[IndexManifestEntry] { + &self.index_entries + } + + pub(crate) fn filter_indexes(&self, mut keep: impl FnMut(&IndexManifestEntry) -> bool) -> Self { + Self { + snapshot: self.snapshot.clone(), + index_entries: self + .index_entries + .iter() + .filter(|entry| keep(entry)) + .cloned() + .collect(), + } } } @@ -201,32 +239,18 @@ fn ranges_overlap(a_start: i64, a_end: i64, b_start: i64, b_end: i64) -> bool { /// Row ranges already covered by `index_type` global-index files for /// `index_field_id` (and matching `extra_field_ids`). Mirrors Java /// `GlobalIndexBuilderUtils.indexedRowRanges`. -pub(crate) async fn indexed_row_ranges( - table: &Table, - index_manifest_name: Option<&str>, +pub(crate) fn indexed_row_ranges( + entries: &[IndexManifestEntry], index_type: &str, index_field_id: i32, extra_field_ids: Option<&[i32]>, - replaced: &HashSet, -) -> Result> { - let Some(index_manifest_name) = index_manifest_name else { - return Ok(Vec::new()); - }; - let path = format!( - "{}/manifest/{}", - table.location().trim_end_matches('/'), - index_manifest_name - ); - let entries = IndexManifest::read(table.file_io(), &path).await?; +) -> Vec { let mut ranges = Vec::new(); for entry in entries { - if entry.kind != FileKind::Add - || entry.index_file.index_type != index_type - || replaced.contains(&index_file_key(&entry)) - { + if entry.kind != FileKind::Add || entry.index_file.index_type != index_type { continue; } - let Some(meta) = entry.index_file.global_index_meta else { + let Some(meta) = &entry.index_file.global_index_meta else { continue; }; if meta.index_field_id != index_field_id @@ -236,7 +260,7 @@ pub(crate) async fn indexed_row_ranges( } ranges.push(RowRange::new(meta.row_range_start, meta.row_range_end)); } - Ok(merge_row_ranges(ranges)) + merge_row_ranges(ranges) } /// Guard against building a global index over a row range that an existing index @@ -245,32 +269,18 @@ pub(crate) async fn indexed_row_ranges( /// `indexed_row_ranges` and Java `GlobalIndexIdentifier` -- so a different index /// type (or different `extra_field_ids`) on the same field coexists. Two files /// of the same identity with overlapping ranges are still rejected. -pub(crate) async fn validate_existing_index_overlap( - table: &Table, - index_manifest_name: Option<&str>, +pub(crate) fn validate_existing_index_overlap( + entries: &[IndexManifestEntry], index_type: &str, index_field_id: i32, extra_field_ids: Option<&[i32]>, planned: &[RowRange], - replaced: &HashSet, ) -> Result<()> { - let Some(index_manifest_name) = index_manifest_name else { - return Ok(()); - }; - let path = format!( - "{}/manifest/{}", - table.location().trim_end_matches('/'), - index_manifest_name - ); - let entries = IndexManifest::read(table.file_io(), &path).await?; for entry in entries { - if entry.kind != FileKind::Add - || entry.index_file.index_type != index_type - || replaced.contains(&index_file_key(&entry)) - { + if entry.kind != FileKind::Add || entry.index_file.index_type != index_type { continue; } - let Some(meta) = entry.index_file.global_index_meta else { + let Some(meta) = &entry.index_file.global_index_meta else { continue; }; if meta.index_field_id != index_field_id diff --git a/crates/paimon/src/table/sorted_global_index_build_builder.rs b/crates/paimon/src/table/sorted_global_index_build_builder.rs index 821316453..bd386d170 100644 --- a/crates/paimon/src/table/sorted_global_index_build_builder.rs +++ b/crates/paimon/src/table/sorted_global_index_build_builder.rs @@ -70,9 +70,7 @@ pub(super) struct SortedGlobalIndexBuildBuilder<'a> { index_type: String, options: HashMap, partition_filter: Option, - pub(super) snapshot: Option>, - pub(super) replaced_index_files: - std::collections::HashSet, + pub(super) view: Option, } impl<'a> SortedGlobalIndexBuildBuilder<'a> { @@ -83,8 +81,7 @@ impl<'a> SortedGlobalIndexBuildBuilder<'a> { index_type: BTREE_GLOBAL_INDEX_TYPE.to_string(), options: HashMap::new(), partition_filter: None, - snapshot: None, - replaced_index_files: Default::default(), + view: None, } } @@ -183,10 +180,12 @@ impl<'a> SortedGlobalIndexBuildBuilder<'a> { .collect::>(); let extra_field_ids = (!extra_field_ids.is_empty()).then_some(extra_field_ids); - let Some(snapshot) = - super::global_index_build_common::build_snapshot(self.table, self.snapshot.as_ref()) - .await? - else { + let view = super::global_index_build_common::GlobalIndexBuildView::resolve( + self.table, + self.view.as_ref(), + ) + .await?; + let Some(snapshot) = view.snapshot() else { return Ok((None, vec![])); }; @@ -197,17 +196,14 @@ impl<'a> SortedGlobalIndexBuildBuilder<'a> { let manifest_entries = read_builder .new_scan() .with_scan_all_files() - .plan_manifest_entries(&snapshot) + .plan_manifest_entries(snapshot) .await?; let indexed = crate::table::global_index_build_common::indexed_row_ranges( - self.table, - snapshot.index_manifest(), + view.index_entries(), index_type, index_field.id(), extra_field_ids.as_deref(), - &self.replaced_index_files, - ) - .await?; + ); let shards = plan_sorted_index_shards( self.table.location(), @@ -224,8 +220,7 @@ impl<'a> SortedGlobalIndexBuildBuilder<'a> { } crate::table::global_index_build_common::validate_existing_index_overlap( - self.table, - snapshot.index_manifest(), + view.index_entries(), index_type, index_field.id(), extra_field_ids.as_deref(), @@ -233,9 +228,7 @@ impl<'a> SortedGlobalIndexBuildBuilder<'a> { .iter() .map(|shard| RowRange::new(shard.row_range_start, shard.row_range_end)) .collect::>(), - &self.replaced_index_files, - ) - .await?; + )?; let commit = self.new_commit(); let shard_count = shards.len(); diff --git a/crates/paimon/src/table/table_commit/auto_index.rs b/crates/paimon/src/table/table_commit/auto_index.rs index c3fe71e19..511f0051a 100644 --- a/crates/paimon/src/table/table_commit/auto_index.rs +++ b/crates/paimon/src/table/table_commit/auto_index.rs @@ -16,6 +16,7 @@ // under the License. use super::*; +use crate::table::global_index_build_common::GlobalIndexBuildView; use crate::table::global_index_options::{definitions, Definition}; use sha2::{Digest, Sha256}; @@ -100,20 +101,15 @@ impl TableCommit { None, ) .await?; - let indexes = match snapshot.index_manifest() { - Some(name) => { - IndexManifest::read(table.file_io(), &self.snapshot_manager.manifest_path(name)) - .await? - } - None => vec![], - }; + let view = GlobalIndexBuildView::capture(&table, Some(snapshot.clone())).await?; let mut messages = Vec::new(); let prepared: Result<()> = async { for definition in definitions { - let plan = planning::plan(&table, &definition, &data, &indexes).await?; + let plan = + planning::plan(&table, &definition, &data, view.index_entries()).await?; let built = definition .builder(&table) - .build_for_maintenance(snapshot, plan.replaced) + .build_with_view(plan.build_view(&view)) .await?; messages.extend(built); messages.extend(plan.deletions); diff --git a/crates/paimon/src/table/table_commit/auto_index/planning.rs b/crates/paimon/src/table/table_commit/auto_index/planning.rs index ed2d16d01..3e2f24669 100644 --- a/crates/paimon/src/table/table_commit/auto_index/planning.rs +++ b/crates/paimon/src/table/table_commit/auto_index/planning.rs @@ -22,18 +22,33 @@ use super::*; use crate::table::global_index_build_common::{ - index_file_key, resolve_index_fields, IndexColumns, IndexFileKey, + resolve_index_fields, GlobalIndexBuildView, IndexColumns, }; use crate::table::{merge_row_ranges, RowRange}; use std::collections::BTreeMap; type UnitKey = (Vec, i32, i64); +type IndexFileKey = (Vec, i32, String); + +fn index_file_key(entry: &IndexManifestEntry) -> IndexFileKey { + ( + entry.partition.clone(), + entry.bucket, + entry.index_file.file_name.clone(), + ) +} pub(super) struct MaintenancePlan { - pub replaced: HashSet, + replaced: HashSet, pub deletions: Vec, } +impl MaintenancePlan { + pub(super) fn build_view(&self, source: &GlobalIndexBuildView) -> GlobalIndexBuildView { + source.filter_indexes(|entry| !self.replaced.contains(&index_file_key(entry))) + } +} + fn partition_key(bytes: &[u8]) -> Vec { if super::super::is_empty_partition(bytes) { vec![] diff --git a/crates/paimon/src/table/table_commit/auto_index_tests.rs b/crates/paimon/src/table/table_commit/auto_index_tests.rs index effcf2c04..101d677f7 100644 --- a/crates/paimon/src/table/table_commit/auto_index_tests.rs +++ b/crates/paimon/src/table/table_commit/auto_index_tests.rs @@ -19,6 +19,7 @@ use super::*; use crate::catalog::Identifier; use crate::io::FileIOBuilder; use crate::spec::{IntType, Schema as TableSchemaBuilder, TableSchema}; +use crate::table::global_index_build_common::GlobalIndexBuildView; use arrow_array::{Int32Array, RecordBatch}; use futures::TryStreamExt; use std::sync::atomic::{AtomicBool, Ordering}; @@ -958,7 +959,11 @@ async fn captured_builds_ignore_later_appends_and_validation_creates_no_files() assert_eq!(latest(&table).await.id(), 2); assert!(indexes(&table).await.is_empty()); let built = builder - .build_for_maintenance(&source, Default::default()) + .build_with_view( + GlobalIndexBuildView::capture(&table, Some(source)) + .await + .unwrap(), + ) .await .unwrap(); assert_eq!(built.len(), 1); @@ -1126,7 +1131,11 @@ async fn native_builders_share_automatic_incremental_and_captured_snapshot_contr // Snapshot 1 precedes both rounds; the later append must not enter this build. let source = table.snapshot_manager().get_snapshot(1).await.unwrap(); let built = builder - .build_for_maintenance(&source, Default::default()) + .build_with_view( + GlobalIndexBuildView::capture(&table, Some(source)) + .await + .unwrap(), + ) .await .unwrap(); assert_eq!(built.len(), 1, "{kind}"); diff --git a/crates/paimon/src/table/table_commit/auto_index_tests/search.rs b/crates/paimon/src/table/table_commit/auto_index_tests/search.rs index a31343ba9..9681bb404 100644 --- a/crates/paimon/src/table/table_commit/auto_index_tests/search.rs +++ b/crates/paimon/src/table/table_commit/auto_index_tests/search.rs @@ -22,6 +22,181 @@ use crate::Options; use arrow_array::builder::{Float32Builder, ListBuilder}; use arrow_array::{ArrayRef, StringArray}; +#[tokio::test] +async fn filtered_build_views_preserve_snapshot_and_unrelated_index_coverage() { + use crate::table::global_index_build_common::{ + indexed_row_ranges, validate_existing_index_overlap, GlobalIndexBuildView, + }; + + let (_catalog, table) = search_table(&[ + ("global-index.btree.index-columns", "id"), + ("sorted-index.records-per-range", "2"), + ("global-index.full-text.index-columns", "text"), + ("global-index.vector.index-columns", "embedding"), + ( + "fields.embedding.global-index.vector.index-type", + "ivf-flat", + ), + ( + "fields.embedding.global-index.vector.index-options", + r#"{"dimension":2,"nlist":1}"#, + ), + ]) + .await; + let commit = TableCommit::new(table.clone(), "writer".into()); + commit + .commit(search_messages(&table, &[[1., 0.], [2., 0.], [3., 0.], [4., 0.]]).await) + .await + .unwrap(); + let captured = latest(&table).await; + let source = GlobalIndexBuildView::capture(&table, Some(captured.clone())) + .await + .unwrap(); + assert_eq!(source.index_entries().len(), 6); + + commit + .commit(search_messages(&table, &[[5., 0.], [6., 0.]]).await) + .await + .unwrap(); + let current = indexes(&table).await; + let current_snapshot = latest(&table).await.id(); + assert_eq!(current.len(), 9); + + for (column, kind) in [ + ("id", "btree"), + ("text", "full-text"), + ("embedding", "ivf-flat"), + ] { + let field_id = table + .schema() + .fields() + .iter() + .find(|field| field.name() == column) + .unwrap() + .id(); + let retired = source + .index_entries() + .iter() + .find(|entry| { + entry.index_file.index_type == kind + && entry + .index_file + .global_index_meta + .as_ref() + .is_some_and(|meta| { + meta.index_field_id == field_id && meta.row_range_start == 0 + }) + }) + .unwrap(); + let filtered = source + .filter_indexes(|entry| entry.index_file.file_name != retired.index_file.file_name); + assert_eq!( + source.index_entries().len(), + 6, + "filtering must not alter the source view" + ); + assert_eq!(filtered.index_entries().len(), 5); + assert_eq!(filtered.snapshot().unwrap().id(), captured.id()); + assert_eq!( + indexed_row_ranges(filtered.index_entries(), kind, field_id, None), + vec![crate::table::RowRange::new(2, 3)], + ); + + let replacement = [crate::table::RowRange::new(0, 1)]; + assert!(validate_existing_index_overlap( + source.index_entries(), + kind, + field_id, + None, + &replacement + ) + .is_err()); + validate_existing_index_overlap( + filtered.index_entries(), + kind, + field_id, + None, + &replacement, + ) + .unwrap(); + assert!( + validate_existing_index_overlap( + filtered.index_entries(), + kind, + field_id, + None, + &[crate::table::RowRange::new(2, 3)], + ) + .is_err(), + "retained coverage must still reject overlap" + ); + + let mut builder = table.new_global_index_build_builder(); + builder.with_index_column(column).with_index_type(kind); + assert!(builder + .build_with_view(source.clone()) + .await + .unwrap() + .is_empty()); + let built = builder.build_with_view(filtered.clone()).await.unwrap(); + assert_eq!( + built.len(), + 1, + "{kind}: rebuild only the gap in the captured view" + ); + assert!( + built[0].deleted_index_files.is_empty(), + "builders do not decide which old references to retire" + ); + let meta = built[0].new_index_files[0] + .global_index_meta + .as_ref() + .unwrap(); + assert_eq!((meta.row_range_start, meta.row_range_end), (0, 1), "{kind}"); + assert_eq!( + crate::spec::DataEvolutionIndexSourceMeta::deserialize( + meta.source_meta.as_deref().unwrap() + ) + .unwrap() + .scan_snapshot_id(), + captured.id(), + ); + + let (other_column, other_type) = if kind == "btree" { + ("text", "full-text") + } else { + ("id", "btree") + }; + assert!( + table + .new_global_index_build_builder() + .with_index_column(other_column) + .with_index_type(other_type) + .build_with_view(filtered) + .await + .unwrap() + .is_empty(), + "unrelated index definitions must keep their coverage" + ); + assert_eq!(latest(&table).await.id(), current_snapshot); + assert_eq!( + indexes(&table).await, + current, + "a build view must not change published references" + ); + assert!(table + .file_io() + .exists(&format!( + "{}/index/{}", + table.location(), + retired.index_file.file_name + )) + .await + .unwrap()); + commit.abort(&built).await.unwrap(); + } +} + #[tokio::test] async fn manual_builds_and_planned_shards_share_declarations_and_preserve_explicit_overrides() { for override_metric in [None, Some("l2")] { diff --git a/crates/paimon/src/table/vindex_index_build_builder.rs b/crates/paimon/src/table/vindex_index_build_builder.rs index e3503ae1e..3b7210944 100644 --- a/crates/paimon/src/table/vindex_index_build_builder.rs +++ b/crates/paimon/src/table/vindex_index_build_builder.rs @@ -39,9 +39,7 @@ pub(super) struct VindexIndexBuildBuilder<'a> { index_type: String, options: HashMap, pub(super) partition_filter: Option, - pub(super) snapshot: Option>, - pub(super) replaced_index_files: - std::collections::HashSet, + pub(super) view: Option, } impl<'a> VindexIndexBuildBuilder<'a> { @@ -52,8 +50,7 @@ impl<'a> VindexIndexBuildBuilder<'a> { index_type: index_type.to_string(), options: HashMap::new(), partition_filter: None, - snapshot: None, - replaced_index_files: Default::default(), + view: None, } } @@ -115,10 +112,12 @@ impl<'a> VindexIndexBuildBuilder<'a> { options.dimension() as u64, "vindex dimension is too large for Rust builder", )?; - let Some(snapshot) = - super::global_index_build_common::build_snapshot(self.table, self.snapshot.as_ref()) - .await? - else { + let view = super::global_index_build_common::GlobalIndexBuildView::resolve( + self.table, + self.view.as_ref(), + ) + .await?; + let Some(snapshot) = view.snapshot() else { return Ok(GlobalIndexBuildPlan::new( self.table, None, @@ -136,17 +135,14 @@ impl<'a> VindexIndexBuildBuilder<'a> { let manifest_entries = read_builder .new_scan() .with_scan_all_files() - .plan_manifest_entries(&snapshot) + .plan_manifest_entries(snapshot) .await?; let indexed = crate::table::global_index_build_common::indexed_row_ranges( - self.table, - snapshot.index_manifest(), + view.index_entries(), &self.index_type, index_field.id(), None, // single-column build; no extra fields today - &self.replaced_index_files, - ) - .await?; + ); let shards = plan_vindex_shards( self.table.location(), self.table.schema().partition_keys(), @@ -160,8 +156,7 @@ impl<'a> VindexIndexBuildBuilder<'a> { if !shards.is_empty() { crate::table::global_index_build_common::validate_existing_index_overlap( - self.table, - snapshot.index_manifest(), + view.index_entries(), &self.index_type, index_field.id(), None, @@ -169,14 +164,12 @@ impl<'a> VindexIndexBuildBuilder<'a> { .iter() .map(|shard| RowRange::new(shard.row_range_start, shard.row_range_end)) .collect::>(), - &self.replaced_index_files, - ) - .await?; + )?; } Ok(GlobalIndexBuildPlan::new( self.table, - Some(snapshot), + Some(snapshot.clone()), index_column.into(), self.index_type.clone(), self.options.clone(), From f5e01a07be1906118ac2ec98ab11d250901130ae Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 31 Oct 2026 11:32:37 +0800 Subject: [PATCH 4/4] refactor(table): rename auto_compact module to auto_maintenance --- crates/paimon/src/table/table_commit.rs | 2 +- .../table/table_commit/{auto_compact.rs => auto_maintenance.rs} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename crates/paimon/src/table/table_commit/{auto_compact.rs => auto_maintenance.rs} (100%) diff --git a/crates/paimon/src/table/table_commit.rs b/crates/paimon/src/table/table_commit.rs index e37bb2e6d..0f3de339d 100644 --- a/crates/paimon/src/table/table_commit.rs +++ b/crates/paimon/src/table/table_commit.rs @@ -49,10 +49,10 @@ const BATCH_COMMIT_IDENTIFIER: i64 = i64::MAX; /// Java RollingFileWriter.CHECK_ROLLING_RECORD_CNT. const CHECK_ROLLING_RECORD_COUNT: usize = 1000; -mod auto_compact; mod auto_index; #[cfg(test)] mod auto_index_tests; +mod auto_maintenance; mod global_index_source_check; mod row_tracking; diff --git a/crates/paimon/src/table/table_commit/auto_compact.rs b/crates/paimon/src/table/table_commit/auto_maintenance.rs similarity index 100% rename from crates/paimon/src/table/table_commit/auto_compact.rs rename to crates/paimon/src/table/table_commit/auto_maintenance.rs