From 4fad44b28836ed1c1494ee1ba92f05f03c1bedff Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Mon, 5 Oct 2026 20:48:05 +0800 Subject: [PATCH 01/18] feat(datafusion): support the paimon_incremental_query table function --- .../datafusion/src/incremental_query.rs | 365 ++++++++++++++++++ crates/integrations/datafusion/src/lib.rs | 1 + .../datafusion/src/sql_context.rs | 5 + .../datafusion/tests/incremental_query.rs | 201 ++++++++++ 4 files changed, 572 insertions(+) create mode 100644 crates/integrations/datafusion/src/incremental_query.rs create mode 100644 crates/integrations/datafusion/tests/incremental_query.rs diff --git a/crates/integrations/datafusion/src/incremental_query.rs b/crates/integrations/datafusion/src/incremental_query.rs new file mode 100644 index 000000000..420b96df0 --- /dev/null +++ b/crates/integrations/datafusion/src/incremental_query.rs @@ -0,0 +1,365 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! The `paimon_incremental_query` table function reads a snapshot-id range +//! `(start_exclusive, end_inclusive]` as a batch. Snapshot planning stays in +//! paimon core; this only parses arguments and streams the core plan's output. +//! +//! ```sql +//! SELECT * FROM paimon_incremental_query('db.t', 0, 5); +//! SELECT * FROM paimon_incremental_query('db.t', 0, 5, 'delta'); +//! SELECT * FROM paimon_incremental_query('db.t$audit_log', 0, 5, 'changelog'); +//! ``` +//! +//! A `$audit_log` table-name suffix prepends the `rowkind` audit column (and the +//! optional `_SEQUENCE_NUMBER`), mirroring the core audit-log read. + +use std::fmt; +use std::sync::Arc; + +use async_trait::async_trait; +use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; +use datafusion::arrow::record_batch::RecordBatch; +use datafusion::catalog::{Session, TableFunctionImpl}; +use datafusion::common::stats::Precision; +use datafusion::common::{internal_err, project_schema, Statistics}; +use datafusion::datasource::{TableProvider, TableType}; +use datafusion::error::{DataFusionError, Result as DFResult}; +use datafusion::execution::{SendableRecordBatchStream, TaskContext}; +use datafusion::logical_expr::Expr; +use datafusion::physical_expr::EquivalenceProperties; +use datafusion::physical_plan::empty::EmptyExec; +use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, +}; +use datafusion::prelude::SessionContext; +use futures::{stream, TryStreamExt}; +use paimon::catalog::Catalog; +use paimon::table::{AuditLogTable, IncrementalScanMode, Table}; + +use crate::error::to_datafusion_error; +use crate::runtime::{await_with_runtime, block_on_with_runtime}; +use crate::table::datafusion_arrow_schema; +use crate::table_function_args::{ + extract_int_literal, extract_string_literal, parse_table_identifier, +}; +use crate::table_loader::load_data_table_for_read; + +const FUNCTION_NAME: &str = "paimon_incremental_query"; +const AUDIT_LOG_SUFFIX: &str = "$audit_log"; + +/// Registers `paimon_incremental_query` against `catalog`. +pub(crate) fn register_incremental_query( + ctx: &SessionContext, + catalog: Arc, + default_database: &str, +) { + ctx.register_udtf( + FUNCTION_NAME, + Arc::new(IncrementalQueryFunction { + catalog, + default_database: default_database.to_string(), + }), + ); +} + +pub(crate) struct IncrementalQueryFunction { + catalog: Arc, + default_database: String, +} + +impl fmt::Debug for IncrementalQueryFunction { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("IncrementalQueryFunction") + .field("default_database", &self.default_database) + .finish() + } +} + +/// Parses the optional mode argument. `Auto` resolves from `changelog-producer` +/// during planning; `Diff` is handled by core's diff planning. +fn parse_mode(raw: &str) -> DFResult { + match raw.to_ascii_lowercase().as_str() { + "auto" => Ok(IncrementalScanMode::Auto), + "delta" => Ok(IncrementalScanMode::Delta), + "changelog" => Ok(IncrementalScanMode::Changelog), + "diff" => Ok(IncrementalScanMode::Diff), + other => Err(DataFusionError::Plan(format!( + "{FUNCTION_NAME}: unknown scan mode '{other}', expected auto, delta, changelog, or diff" + ))), + } +} + +impl TableFunctionImpl for IncrementalQueryFunction { + fn call(&self, args: &[Expr]) -> DFResult> { + if args.len() < 3 || args.len() > 4 { + return Err(DataFusionError::Plan(format!( + "{FUNCTION_NAME} requires 3 or 4 arguments: (table_name, start_snapshot_exclusive, end_snapshot_inclusive [, mode])" + ))); + } + let table_name = extract_string_literal(FUNCTION_NAME, &args[0], "table_name")?; + let start_exclusive = + extract_int_literal(FUNCTION_NAME, &args[1], "start_snapshot_exclusive")?; + let end_inclusive = extract_int_literal(FUNCTION_NAME, &args[2], "end_snapshot_inclusive")?; + let mode = match args.get(3) { + Some(expr) => parse_mode(&extract_string_literal(FUNCTION_NAME, expr, "mode")?)?, + None => IncrementalScanMode::Auto, + }; + if end_inclusive < start_exclusive { + return Err(DataFusionError::Plan(format!( + "{FUNCTION_NAME}: end_snapshot_inclusive ({end_inclusive}) must be >= start_snapshot_exclusive ({start_exclusive})" + ))); + } + + // `db.t$audit_log` selects the audit-log projection of `db.t`. + let (base_name, audit_log) = match table_name.strip_suffix(AUDIT_LOG_SUFFIX) { + Some(base) => (base.to_string(), true), + None => (table_name.clone(), false), + }; + let identifier = parse_table_identifier(FUNCTION_NAME, &base_name, &self.default_database)?; + + let catalog = Arc::clone(&self.catalog); + let table = block_on_with_runtime( + async move { load_data_table_for_read(&catalog, &identifier, FUNCTION_NAME).await }, + "paimon_incremental_query: catalog access thread panicked", + )?; + + Ok(Arc::new(IncrementalQueryTableProvider::try_new( + table, + audit_log, + mode, + start_exclusive, + end_inclusive, + )?)) + } +} +#[derive(Debug, Clone)] +struct IncrementalQueryTableProvider { + table: Table, + audit_log: bool, + mode: IncrementalScanMode, + start_exclusive: i64, + end_inclusive: i64, + schema: ArrowSchemaRef, +} + +impl IncrementalQueryTableProvider { + fn try_new( + table: Table, + audit_log: bool, + mode: IncrementalScanMode, + start_exclusive: i64, + end_inclusive: i64, + ) -> DFResult { + // Audit-log output leads with `rowkind` (+ optional `_SEQUENCE_NUMBER`); + // the plain projection is the table's own fields. + let fields = if audit_log { + AuditLogTable::new(table.clone()) + .fields() + .map_err(to_datafusion_error)? + } else { + table.schema().fields().to_vec() + }; + // Core streams raw Arrow (`Utf8`, not `Utf8View`), so declare the schema + // the reader actually yields rather than the view-forced scan schema. + let schema = datafusion_arrow_schema(&fields, false)?; + Ok(Self { + table, + audit_log, + mode, + start_exclusive, + end_inclusive, + schema, + }) + } +} + +#[async_trait] +impl TableProvider for IncrementalQueryTableProvider { + fn schema(&self) -> ArrowSchemaRef { + Arc::clone(&self.schema) + } + + fn table_type(&self) -> TableType { + TableType::Base + } + + async fn scan( + &self, + _state: &dyn Session, + projection: Option<&Vec>, + _filters: &[Expr], + limit: Option, + ) -> DFResult> { + let projected_schema = project_schema(&self.schema, projection)?; + // An outer `LIMIT 0` needs no rows. + if limit == Some(0) { + return Ok(Arc::new(EmptyExec::new(projected_schema))); + } + Ok(Arc::new(IncrementalQueryExec::new( + self.table.clone(), + self.audit_log, + self.mode, + self.start_exclusive, + self.end_inclusive, + projection.cloned(), + projected_schema, + ))) + } +} +/// Execution-time plan: builds the incremental plan in paimon core and streams +/// its Arrow output when polled, so planning and `EXPLAIN` stay cheap. +#[derive(Debug, Clone)] +struct IncrementalQueryExec { + table: Table, + audit_log: bool, + mode: IncrementalScanMode, + start_exclusive: i64, + end_inclusive: i64, + projection: Option>, + output_schema: ArrowSchemaRef, + plan_properties: Arc, +} + +impl IncrementalQueryExec { + fn new( + table: Table, + audit_log: bool, + mode: IncrementalScanMode, + start_exclusive: i64, + end_inclusive: i64, + projection: Option>, + output_schema: ArrowSchemaRef, + ) -> Self { + let plan_properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(output_schema.clone()), + Partitioning::UnknownPartitioning(1), + EmissionType::Incremental, + Boundedness::Bounded, + )); + Self { + table, + audit_log, + mode, + start_exclusive, + end_inclusive, + projection, + output_schema, + plan_properties, + } + } + + async fn compute_batches(&self) -> DFResult> { + // Core owns snapshot planning; its stream is `'static`, so it outlives + // the transient read builder it is built from. + let batches = await_with_runtime(async { + let read_builder = self.table.new_read_builder(); + let scan = read_builder.new_incremental_scan( + self.mode, + self.start_exclusive, + self.end_inclusive, + ); + let plan = scan.plan().await.map_err(to_datafusion_error)?; + let read = read_builder.new_read().map_err(to_datafusion_error)?; + let mut stream = if self.audit_log { + read.to_audit_log_arrow(&plan) + } else { + read.to_incremental_arrow(&plan) + } + .map_err(to_datafusion_error)?; + let mut batches = Vec::new(); + while let Some(batch) = stream.try_next().await.map_err(to_datafusion_error)? { + batches.push(batch); + } + Ok::<_, DataFusionError>(batches) + }) + .await?; + + match &self.projection { + Some(indices) => batches + .iter() + .map(|batch| batch.project(indices).map_err(DataFusionError::from)) + .collect(), + None => Ok(batches), + } + } +} +impl DisplayAs for IncrementalQueryExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "IncrementalQueryExec: mode={:?}, range=({}, {}], audit_log={}", + self.mode, self.start_exclusive, self.end_inclusive, self.audit_log + ) + } +} + +impl ExecutionPlan for IncrementalQueryExec { + fn name(&self) -> &str { + "IncrementalQueryExec" + } + + fn properties(&self) -> &Arc { + &self.plan_properties + } + + fn children(&self) -> Vec<&Arc> { + vec![] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DFResult> { + if !children.is_empty() { + return internal_err!("IncrementalQueryExec is a leaf and takes no children"); + } + Ok(self) + } + + fn execute( + &self, + partition: usize, + _context: Arc, + ) -> DFResult { + if partition != 0 { + return internal_err!( + "IncrementalQueryExec has a single partition, got partition {partition}" + ); + } + let exec = self.clone(); + let stream = stream::once(async move { + let batches = exec.compute_batches().await?; + Ok::<_, DataFusionError>(stream::iter(batches.into_iter().map(Ok))) + }) + .try_flatten(); + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.output_schema.clone(), + stream, + ))) + } + + fn partition_statistics(&self, _partition: Option) -> DFResult> { + Ok(Arc::new(Statistics { + num_rows: Precision::Absent, + total_byte_size: Precision::Absent, + column_statistics: Statistics::unknown_column(&self.output_schema), + })) + } +} diff --git a/crates/integrations/datafusion/src/lib.rs b/crates/integrations/datafusion/src/lib.rs index 7043303e0..bb17cb732 100644 --- a/crates/integrations/datafusion/src/lib.rs +++ b/crates/integrations/datafusion/src/lib.rs @@ -50,6 +50,7 @@ mod format_table_truncate; #[cfg(feature = "fulltext")] mod full_text_search; mod hybrid_search; +mod incremental_query; mod lateral_vector_search; mod memory; mod merge_into; diff --git a/crates/integrations/datafusion/src/sql_context.rs b/crates/integrations/datafusion/src/sql_context.rs index f5f27276b..0236417be 100644 --- a/crates/integrations/datafusion/src/sql_context.rs +++ b/crates/integrations/datafusion/src/sql_context.rs @@ -3842,6 +3842,11 @@ fn register_table_functions( #[cfg(feature = "fulltext")] crate::full_text_search::register_full_text_search(ctx, Arc::clone(catalog), default_database); crate::hybrid_search::register_hybrid_search(ctx, Arc::clone(catalog), default_database); + crate::incremental_query::register_incremental_query( + ctx, + Arc::clone(catalog), + default_database, + ); } #[cfg(test)] diff --git a/crates/integrations/datafusion/tests/incremental_query.rs b/crates/integrations/datafusion/tests/incremental_query.rs new file mode 100644 index 000000000..8510d4b80 --- /dev/null +++ b/crates/integrations/datafusion/tests/incremental_query.rs @@ -0,0 +1,201 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Integration tests for the `paimon_incremental_query` table function. + +use std::sync::Arc; + +use datafusion::arrow::array::RecordBatch; +use paimon::{CatalogOptions, FileSystemCatalog, Options}; +use paimon_datafusion::SQLContext; +use tempfile::TempDir; + +async fn run(ctx: &SQLContext, sql: &str) -> Vec { + ctx.sql(sql).await.unwrap().collect().await.unwrap() +} + +fn total_rows(batches: &[RecordBatch]) -> usize { + batches.iter().map(RecordBatch::num_rows).sum() +} + +/// Append-only table with one row per snapshot: snapshot 1 = (1,'a'), +/// snapshot 2 = (2,'b'), snapshot 3 = (3,'c'). +async fn setup_three_snapshots() -> (TempDir, SQLContext) { + let temp_dir = TempDir::new().expect("temp dir"); + let warehouse = format!("file://{}", temp_dir.path().display()); + let mut options = Options::new(); + options.set(CatalogOptions::WAREHOUSE, warehouse); + let catalog = Arc::new(FileSystemCatalog::new(options).unwrap()); + let mut ctx = SQLContext::new(); + ctx.register_catalog("paimon", catalog).await.unwrap(); + + run(&ctx, "CREATE TABLE paimon.default.t (id INT, name STRING)").await; + run(&ctx, "INSERT INTO paimon.default.t VALUES (1, 'a')").await; + run(&ctx, "INSERT INTO paimon.default.t VALUES (2, 'b')").await; + run(&ctx, "INSERT INTO paimon.default.t VALUES (3, 'c')").await; + (temp_dir, ctx) +} +/// Collects the `id` column across batches into a sorted vector. +fn sorted_ids(batches: &[RecordBatch]) -> Vec { + use datafusion::arrow::array::{Array, Int32Array}; + let mut ids = Vec::new(); + for batch in batches { + let col = batch + .column_by_name("id") + .expect("id column") + .as_any() + .downcast_ref::() + .expect("id is Int32"); + for i in 0..col.len() { + ids.push(col.value(i)); + } + } + ids.sort_unstable(); + ids +} + +#[tokio::test] +async fn test_auto_range_is_start_exclusive_end_inclusive() { + let (_tmp, ctx) = setup_three_snapshots().await; + + // (1, 3]: snapshots 2 and 3 only — start is exclusive, end inclusive. + let batches = run( + &ctx, + "SELECT * FROM paimon_incremental_query('default.t', 1, 3)", + ) + .await; + assert_eq!(total_rows(&batches), 2); + assert_eq!(sorted_ids(&batches), vec![2, 3]); + + // (0, 3]: every snapshot. + let all = run( + &ctx, + "SELECT * FROM paimon_incremental_query('default.t', 0, 3)", + ) + .await; + assert_eq!(sorted_ids(&all), vec![1, 2, 3]); + + // (3, 3]: empty range. + let empty = run( + &ctx, + "SELECT * FROM paimon_incremental_query('default.t', 3, 3)", + ) + .await; + assert_eq!(total_rows(&empty), 0); +} + +#[tokio::test] +async fn test_explicit_delta_mode_matches_auto() { + let (_tmp, ctx) = setup_three_snapshots().await; + let batches = run( + &ctx, + "SELECT * FROM paimon_incremental_query('default.t', 0, 2, 'delta')", + ) + .await; + assert_eq!(sorted_ids(&batches), vec![1, 2]); +} + +#[tokio::test] +async fn test_projection_selects_single_column() { + let (_tmp, ctx) = setup_three_snapshots().await; + let batches = run( + &ctx, + "SELECT id FROM paimon_incremental_query('default.t', 0, 3) ORDER BY id", + ) + .await; + assert_eq!(batches[0].num_columns(), 1); + assert_eq!(sorted_ids(&batches), vec![1, 2, 3]); +} +#[tokio::test] +async fn test_audit_log_suffix_prepends_rowkind() { + use datafusion::arrow::array::{Array, StringArray}; + let (_tmp, ctx) = setup_three_snapshots().await; + let batches = run( + &ctx, + "SELECT * FROM paimon_incremental_query('default.t$audit_log', 0, 3, 'delta')", + ) + .await; + assert_eq!(total_rows(&batches), 3); + let batch = &batches[0]; + assert_eq!( + batch.schema().field(0).name(), + "rowkind", + "audit-log output must lead with the rowkind column" + ); + let rowkind = batch + .column(0) + .as_any() + .downcast_ref::() + .expect("rowkind is Utf8"); + // Append-only rows are all inserts. + for i in 0..rowkind.len() { + assert_eq!(rowkind.value(i), "+I"); + } +} + +async fn expect_error(ctx: &SQLContext, sql: &str) -> String { + match ctx.sql(sql).await { + Err(e) => e.to_string(), + Ok(df) => df + .collect() + .await + .expect_err("expected the query to fail") + .to_string(), + } +} + +#[tokio::test] +async fn test_rejects_unknown_mode() { + let (_tmp, ctx) = setup_three_snapshots().await; + let err = expect_error( + &ctx, + "SELECT * FROM paimon_incremental_query('default.t', 0, 1, 'bogus')", + ) + .await; + assert!( + err.contains("unknown scan mode"), + "error must name the bad mode: {err}" + ); +} + +#[tokio::test] +async fn test_rejects_bad_arity() { + let (_tmp, ctx) = setup_three_snapshots().await; + let err = expect_error( + &ctx, + "SELECT * FROM paimon_incremental_query('default.t', 0)", + ) + .await; + assert!( + err.contains("requires 3 or 4 arguments"), + "error must explain the arity: {err}" + ); +} + +#[tokio::test] +async fn test_rejects_end_before_start() { + let (_tmp, ctx) = setup_three_snapshots().await; + let err = expect_error( + &ctx, + "SELECT * FROM paimon_incremental_query('default.t', 5, 2)", + ) + .await; + assert!( + err.contains("must be >="), + "error must explain the range order: {err}" + ); +} From 95681813d97441e5e8d12d27ee6186fae06dc8d1 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Thu, 15 Oct 2026 19:43:22 +0800 Subject: [PATCH 02/18] fix: stream incremental reads and decode them as ordinary reads `IncrementalQueryExec` collected the whole snapshot range into a `Vec` before emitting a batch, decoded every column even for a narrow projection, and ignored the executing `TaskContext`, so the DataFusion memory pool never saw the reader's working memory. Forward core's batch stream lazily, push the projection into the read (`$audit_log` columns are mapped by position, one read field per projected column), attach the task's reader resources, and rebuild each batch against the declared (projected) schema, as the ordinary scan does. Core's `to_incremental_arrow` decoded each planned file raw, so a data-evolution BLOB or VECTOR column file came back as rows of its own and a primary-key BLOB column as its stored descriptor. Decode those files as `to_arrow` does, read limit included, and plan a data-evolution Delta range as one batch, as Java's `incremental-between` scan does, so a column update merges into the rows it updates when both are in the range. Document `paimon_incremental_query` in the SQL guide. --- .../datafusion/src/incremental_query.rs | 154 +++++-- .../datafusion/src/physical_plan/scan.rs | 5 +- .../datafusion/tests/incremental_query.rs | 405 +++++++++++++++++- crates/paimon/src/table/table_read.rs | 41 +- .../tests/incremental_batch_scan_test.rs | 291 +++++++++++++ crates/paimon/tests/partition_filter_test.rs | 35 ++ docs/src/sql.md | 34 +- 7 files changed, 908 insertions(+), 57 deletions(-) diff --git a/crates/integrations/datafusion/src/incremental_query.rs b/crates/integrations/datafusion/src/incremental_query.rs index 420b96df0..cdf2f0ab0 100644 --- a/crates/integrations/datafusion/src/incremental_query.rs +++ b/crates/integrations/datafusion/src/incremental_query.rs @@ -17,7 +17,8 @@ //! The `paimon_incremental_query` table function reads a snapshot-id range //! `(start_exclusive, end_inclusive]` as a batch. Snapshot planning stays in -//! paimon core; this only parses arguments and streams the core plan's output. +//! paimon core; this parses arguments, picks the core planner that matches +//! Java's `incremental-between` for the table, and streams its output. //! //! ```sql //! SELECT * FROM paimon_incremental_query('db.t', 0, 5); @@ -33,7 +34,6 @@ use std::sync::Arc; use async_trait::async_trait; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; -use datafusion::arrow::record_batch::RecordBatch; use datafusion::catalog::{Session, TableFunctionImpl}; use datafusion::common::stats::Precision; use datafusion::common::{internal_err, project_schema, Statistics}; @@ -49,11 +49,18 @@ use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, }; use datafusion::prelude::SessionContext; -use futures::{stream, TryStreamExt}; +use futures::{stream, StreamExt, TryStreamExt}; use paimon::catalog::Catalog; -use paimon::table::{AuditLogTable, IncrementalScanMode, Table}; +use paimon::resource::ResourceContext; +use paimon::spec::{CoreOptions, DataField}; +use paimon::table::{ + ArrowRecordBatchStream, AuditLogTable, IncrementalPlan, IncrementalScanMode, IncrementalSplit, + Table, +}; use crate::error::to_datafusion_error; +use crate::memory::reader_resources; +use crate::physical_plan::scan::to_datafusion_batch; use crate::runtime::{await_with_runtime, block_on_with_runtime}; use crate::table::datafusion_arrow_schema; use crate::table_function_args::{ @@ -156,6 +163,7 @@ struct IncrementalQueryTableProvider { mode: IncrementalScanMode, start_exclusive: i64, end_inclusive: i64, + fields: Vec, schema: ArrowSchemaRef, } @@ -185,9 +193,29 @@ impl IncrementalQueryTableProvider { mode, start_exclusive, end_inclusive, + fields, schema, }) } + + /// Splits a projection into the table fields core reads and the column of + /// core's batches each output column comes from. Core's audit-log batches + /// lead with the same audit columns as `fields`, then the read fields. + fn read_projection(&self, indices: &[usize]) -> ReadProjection { + let audit_columns = self.fields.len() - self.table.schema().fields().len(); + let mut read_type = Vec::new(); + let columns = indices + .iter() + .map(|&index| { + if index < audit_columns { + return index; + } + read_type.push(self.fields[index].clone()); + audit_columns + read_type.len() - 1 + }) + .collect(); + ReadProjection { read_type, columns } + } } #[async_trait] @@ -218,11 +246,20 @@ impl TableProvider for IncrementalQueryTableProvider { self.mode, self.start_exclusive, self.end_inclusive, - projection.cloned(), + projection.map(|indices| self.read_projection(indices)), projected_schema, ))) } } + +/// The table fields to read, and per output column the column of core's +/// batches it comes from. +#[derive(Debug, Clone)] +struct ReadProjection { + read_type: Vec, + columns: Vec, +} + /// Execution-time plan: builds the incremental plan in paimon core and streams /// its Arrow output when polled, so planning and `EXPLAIN` stay cheap. #[derive(Debug, Clone)] @@ -232,7 +269,7 @@ struct IncrementalQueryExec { mode: IncrementalScanMode, start_exclusive: i64, end_inclusive: i64, - projection: Option>, + projection: Option, output_schema: ArrowSchemaRef, plan_properties: Arc, } @@ -244,7 +281,7 @@ impl IncrementalQueryExec { mode: IncrementalScanMode, start_exclusive: i64, end_inclusive: i64, - projection: Option>, + projection: Option, output_schema: ArrowSchemaRef, ) -> Self { let plan_properties = Arc::new(PlanProperties::new( @@ -265,39 +302,60 @@ impl IncrementalQueryExec { } } - async fn compute_batches(&self) -> DFResult> { - // Core owns snapshot planning; its stream is `'static`, so it outlives - // the transient read builder it is built from. - let batches = await_with_runtime(async { - let read_builder = self.table.new_read_builder(); - let scan = read_builder.new_incremental_scan( - self.mode, - self.start_exclusive, - self.end_inclusive, - ); - let plan = scan.plan().await.map_err(to_datafusion_error)?; - let read = read_builder.new_read().map_err(to_datafusion_error)?; - let mut stream = if self.audit_log { - read.to_audit_log_arrow(&plan) - } else { - read.to_incremental_arrow(&plan) - } - .map_err(to_datafusion_error)?; - let mut batches = Vec::new(); - while let Some(batch) = stream.try_next().await.map_err(to_datafusion_error)? { - batches.push(batch); - } - Ok::<_, DataFusionError>(batches) - }) - .await?; - - match &self.projection { - Some(indices) => batches - .iter() - .map(|batch| batch.project(indices).map_err(DataFusionError::from)) - .collect(), - None => Ok(batches), + /// Plans the range in paimon core and opens its batch stream. Reader memory + /// comes from the executing task's pool, as in the ordinary scan. The + /// projection is pushed into the read: Delta and Changelog decode the + /// projected fields, while Diff decodes full rows to compare them and + /// projects its output. The stream is `'static`, so it outlives the + /// transient read builder it is built from. + async fn open(&self, resources: ResourceContext) -> DFResult { + let mut read_builder = self.table.new_read_builder(); + read_builder.with_resources(resources); + if let Some(projection) = &self.projection { + read_builder.with_read_type(projection.read_type.clone()); } + let scan = + read_builder.new_incremental_scan(self.mode, self.start_exclusive, self.end_inclusive); + let plan = if self.plans_range_as_one_batch() { + let combined = scan + .plan_combined_delta() + .await + .map_err(to_datafusion_error)?; + IncrementalPlan::try_new( + IncrementalScanMode::Delta, + combined + .into_splits() + .into_iter() + .map(IncrementalSplit::Data) + .collect(), + ) + } else { + scan.plan().await + } + .map_err(to_datafusion_error)?; + let read = read_builder.new_read().map_err(to_datafusion_error)?; + if self.audit_log { + read.to_audit_log_arrow(&plan) + } else { + read.to_incremental_arrow(&plan) + } + .map_err(to_datafusion_error) + } + + /// Java's `incremental-between` scan plans a data-evolution Delta range as + /// one batch, so a column update merges into the rows it updates when they + /// were written in the same range; planned per commit, it would come back + /// as rows of its own. `Auto` is Delta here, since such a table has no + /// primary key and so no changelog producer. An empty range is left to + /// `plan`; within the available snapshots it returns no splits, as Java's + /// empty-range scanner does. + fn plans_range_as_one_batch(&self) -> bool { + CoreOptions::new(self.table.schema().options()).data_evolution_enabled() + && matches!( + self.mode, + IncrementalScanMode::Auto | IncrementalScanMode::Delta + ) + && self.start_exclusive < self.end_inclusive } } impl DisplayAs for IncrementalQueryExec { @@ -336,7 +394,7 @@ impl ExecutionPlan for IncrementalQueryExec { fn execute( &self, partition: usize, - _context: Arc, + context: Arc, ) -> DFResult { if partition != 0 { return internal_err!( @@ -345,8 +403,20 @@ impl ExecutionPlan for IncrementalQueryExec { } let exec = self.clone(); let stream = stream::once(async move { - let batches = exec.compute_batches().await?; - Ok::<_, DataFusionError>(stream::iter(batches.into_iter().map(Ok))) + let resources = reader_resources(&context, partition).map_err(to_datafusion_error)?; + let batches = await_with_runtime(exec.open(resources)).await?; + // Batches are forwarded as core emits them, projected, and rebuilt + // against the declared schema as the ordinary scan does. + let columns = exec.projection.map(|projection| projection.columns); + let schema = exec.output_schema; + Ok::<_, DataFusionError>(batches.map(move |batch| { + let batch = batch.map_err(to_datafusion_error)?; + let batch = match &columns { + Some(columns) => batch.project(columns)?, + None => batch, + }; + to_datafusion_batch(batch, &schema) + })) }) .try_flatten(); Ok(Box::pin(RecordBatchStreamAdapter::new( diff --git a/crates/integrations/datafusion/src/physical_plan/scan.rs b/crates/integrations/datafusion/src/physical_plan/scan.rs index 0a9eb5ab6..e3ec6d115 100644 --- a/crates/integrations/datafusion/src/physical_plan/scan.rs +++ b/crates/integrations/datafusion/src/physical_plan/scan.rs @@ -59,7 +59,10 @@ use paimon::DataSplit; use crate::error::to_datafusion_error; use crate::filter_pushdown::scalar_to_datum; -fn to_datafusion_batch(batch: RecordBatch, schema: &ArrowSchemaRef) -> DFResult { +pub(crate) fn to_datafusion_batch( + batch: RecordBatch, + schema: &ArrowSchemaRef, +) -> DFResult { if batch.num_columns() != schema.fields().len() { return Err(datafusion::error::DataFusionError::Execution(format!( "Paimon reader returned {} columns for DataFusion schema with {} fields", diff --git a/crates/integrations/datafusion/tests/incremental_query.rs b/crates/integrations/datafusion/tests/incremental_query.rs index 8510d4b80..e55da99a0 100644 --- a/crates/integrations/datafusion/tests/incremental_query.rs +++ b/crates/integrations/datafusion/tests/incremental_query.rs @@ -20,6 +20,11 @@ use std::sync::Arc; use datafusion::arrow::array::RecordBatch; +use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryPool}; +use datafusion::execution::runtime_env::RuntimeEnvBuilder; +use futures::StreamExt; +use paimon::catalog::{Catalog, Identifier}; +use paimon::table::{IncrementalScanMode, IncrementalSplit}; use paimon::{CatalogOptions, FileSystemCatalog, Options}; use paimon_datafusion::SQLContext; use tempfile::TempDir; @@ -32,16 +37,45 @@ fn total_rows(batches: &[RecordBatch]) -> usize { batches.iter().map(RecordBatch::num_rows).sum() } +fn catalog(warehouse: &TempDir) -> Arc { + let mut options = Options::new(); + options.set( + CatalogOptions::WAREHOUSE, + format!("file://{}", warehouse.path().display()), + ); + Arc::new(FileSystemCatalog::new(options).unwrap()) +} + +async fn context(warehouse: &TempDir) -> SQLContext { + let mut ctx = SQLContext::new(); + ctx.register_catalog("paimon", catalog(warehouse)) + .await + .unwrap(); + ctx +} + +/// A session over `warehouse` whose memory pool admits `limit` bytes. +async fn context_with_memory_limit( + warehouse: &TempDir, + limit: usize, +) -> (SQLContext, Arc) { + let pool = Arc::new(GreedyMemoryPool::new(limit)); + let runtime_env = RuntimeEnvBuilder::new() + .with_memory_pool(pool.clone()) + .build_arc() + .unwrap(); + let mut ctx = SQLContext::builder().with_runtime_env(runtime_env).build(); + ctx.register_catalog("paimon", catalog(warehouse)) + .await + .unwrap(); + (ctx, pool) +} + /// Append-only table with one row per snapshot: snapshot 1 = (1,'a'), /// snapshot 2 = (2,'b'), snapshot 3 = (3,'c'). async fn setup_three_snapshots() -> (TempDir, SQLContext) { let temp_dir = TempDir::new().expect("temp dir"); - let warehouse = format!("file://{}", temp_dir.path().display()); - let mut options = Options::new(); - options.set(CatalogOptions::WAREHOUSE, warehouse); - let catalog = Arc::new(FileSystemCatalog::new(options).unwrap()); - let mut ctx = SQLContext::new(); - ctx.register_catalog("paimon", catalog).await.unwrap(); + let ctx = context(&temp_dir).await; run(&ctx, "CREATE TABLE paimon.default.t (id INT, name STRING)").await; run(&ctx, "INSERT INTO paimon.default.t VALUES (1, 'a')").await; @@ -199,3 +233,362 @@ async fn test_rejects_end_before_start() { "error must explain the range order: {err}" ); } + +#[tokio::test] +async fn test_reads_reserve_memory_from_the_session_pool() { + let (tmp, _ctx) = setup_three_snapshots().await; + let (ctx, _pool) = context_with_memory_limit(&tmp, 0).await; + // A pool that admits nothing refuses the ordinary scan, and must refuse + // the incremental read the same way. + for sql in [ + "SELECT * FROM paimon.default.t", + "SELECT * FROM paimon_incremental_query('default.t', 0, 3)", + ] { + let err = expect_error(&ctx, sql).await; + assert!(err.contains("Resources exhausted"), "{sql}: {err}"); + } +} + +#[tokio::test] +async fn test_projection_skips_unselected_columns() { + let tmp = TempDir::new().unwrap(); + let ctx = context(&tmp).await; + run( + &ctx, + "CREATE TABLE paimon.default.wide (id INT, payload STRING)", + ) + .await; + run( + &ctx, + "INSERT INTO paimon.default.wide VALUES (1, repeat('x', 1048576)), (2, repeat('y', 1048576))", + ) + .await; + + // The pool admits the `id` column but not the 1 MiB payloads, so only a + // read that never decodes `payload` can succeed. + let (ctx, pool) = context_with_memory_limit(&tmp, 64 * 1024).await; + let ids = run( + &ctx, + "SELECT id FROM paimon_incremental_query('default.wide', 0, 1)", + ) + .await; + assert_eq!(sorted_ids(&ids), vec![1, 2]); + // The reader releases its reservation; emitted batches belong to the consumer. + assert_eq!(pool.reserved(), 0); + let err = expect_error( + &ctx, + "SELECT * FROM paimon_incremental_query('default.wide', 0, 1)", + ) + .await; + assert!(err.contains("Resources exhausted"), "{err}"); +} + +#[tokio::test] +async fn test_batches_stream_before_the_whole_range_is_read() { + let (tmp, ctx) = setup_three_snapshots().await; + // Lose the range's last data file. Reading the whole range before emitting + // anything would fail before the first batch. + let table = catalog(&tmp) + .get_table(&Identifier::new("default", "t")) + .await + .unwrap(); + let plan = table + .new_read_builder() + .new_incremental_scan(IncrementalScanMode::Delta, 0, 3) + .plan() + .await + .unwrap(); + let Some(IncrementalSplit::Data(split)) = plan.splits().last() else { + panic!("expected a data split"); + }; + let file = split.data_files().last().expect("a data file"); + table + .file_io() + .delete_file(&format!("{}/{}", split.bucket_path(), file.file_name)) + .await + .unwrap(); + + let mut stream = ctx + .sql("SELECT * FROM paimon_incremental_query('default.t', 0, 3)") + .await + .unwrap() + .execute_stream() + .await + .unwrap(); + assert!(stream.next().await.unwrap().is_ok()); + let mut rest = Vec::new(); + while let Some(batch) = stream.next().await { + rest.push(batch); + } + assert!( + rest.iter().any(Result::is_err), + "the missing file still fails the read" + ); +} + +#[tokio::test] +async fn test_audit_log_projection_keeps_column_offsets() { + use datafusion::arrow::array::{Array, Int32Array, Int8Array, StringArray}; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + // With sequence numbers, core's audit batches lead with two audit columns, + // so a projected data column sits two positions further right. + let tmp = TempDir::new().unwrap(); + let ctx = context(&tmp).await; + run( + &ctx, + "CREATE TABLE paimon.default.pk (id INT NOT NULL, name STRING, PRIMARY KEY (id)) WITH ( + 'bucket' = '1', + 'changelog-producer' = 'input', + 'table-read.sequence-number.enabled' = 'true' + )", + ) + .await; + run( + &ctx, + "INSERT INTO paimon.default.pk VALUES (1, 'a'), (2, 'b')", + ) + .await; + // Snapshot 2 updates id 1 and deletes id 2. + let table = catalog(&tmp) + .get_table(&Identifier::new("default", "pk")) + .await + .unwrap(); + let changes = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true), + Field::new("_VALUE_KIND", DataType::Int8, false), + ])), + vec![ + Arc::new(Int32Array::from(vec![1, 1, 2])), + Arc::new(StringArray::from(vec!["a", "a2", "b"])), + // -U, +U, -D + Arc::new(Int8Array::from(vec![1, 2, 3])), + ], + ) + .unwrap(); + let builder = table.new_write_builder(); + let mut write = builder.new_write().unwrap(); + write.write_arrow_batch(&changes).await.unwrap(); + let messages = write.prepare_commit().await.unwrap(); + builder.new_commit().commit(messages).await.unwrap(); + + for mode in ["changelog", "diff"] { + let batches = run( + &ctx, + &format!( + "SELECT name, rowkind, id FROM paimon_incremental_query('default.pk$audit_log', 1, 2, '{mode}') ORDER BY id, rowkind" + ), + ) + .await; + let mut rows = Vec::new(); + for batch in &batches { + let name = batch.column(0).as_any().downcast_ref::(); + let rowkind = batch.column(1).as_any().downcast_ref::(); + let id = batch.column(2).as_any().downcast_ref::(); + let (name, rowkind, id) = (name.unwrap(), rowkind.unwrap(), id.unwrap()); + for row in 0..batch.num_rows() { + rows.push(( + name.value(row).to_string(), + rowkind.value(row).to_string(), + id.value(row), + )); + } + } + let expected = [("a2", "+U", 1), ("a", "-U", 1), ("b", "-D", 2)] + .map(|(name, rowkind, id)| (name.to_string(), rowkind.to_string(), id)); + assert_eq!(rows, expected, "{mode}"); + + // Skipping `id` makes `name` the only read field, so its column comes + // right after the audit columns rather than at its table position. + let batches = run( + &ctx, + &format!( + "SELECT rowkind, name FROM paimon_incremental_query('default.pk$audit_log', 1, 2, '{mode}') ORDER BY name, rowkind" + ), + ) + .await; + let mut rows = Vec::new(); + for batch in &batches { + let rowkind = batch.column(0).as_any().downcast_ref::(); + let name = batch.column(1).as_any().downcast_ref::(); + let (rowkind, name) = (rowkind.unwrap(), name.unwrap()); + for row in 0..batch.num_rows() { + rows.push((rowkind.value(row).to_string(), name.value(row).to_string())); + } + } + let expected = [("-U", "a"), ("+U", "a2"), ("-D", "b")] + .map(|(rowkind, name)| (rowkind.to_string(), name.to_string())); + assert_eq!(rows, expected, "{mode}"); + } +} + +#[tokio::test] +async fn test_batches_match_the_declared_schema() { + // `DataFrame::cache` keeps the batches only if the plan declares their schema. + let (_tmp, ctx) = setup_three_snapshots().await; + for sql in [ + "SELECT * FROM paimon_incremental_query('default.t', 0, 3)", + "SELECT name, rowkind FROM paimon_incremental_query('default.t$audit_log', 0, 3)", + ] { + let cached = ctx.sql(sql).await.unwrap().cache().await.unwrap(); + assert_eq!(total_rows(&cached.collect().await.unwrap()), 3, "{sql}"); + } +} + +#[tokio::test] +async fn test_blob_rows_match_the_ordinary_read() { + use datafusion::arrow::util::pretty::pretty_format_batches; + // BLOB values live apart from the row's other columns: in a file of their + // own on a data-evolution table, behind a stored descriptor on a + // primary-key table. Reading the range must return the ordinary read. + let tmp = TempDir::new().unwrap(); + let ctx = context(&tmp).await; + run( + &ctx, + "CREATE TABLE paimon.default.de_assets (id INT, picture BLOB) WITH ( + 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true' + )", + ) + .await; + run( + &ctx, + "CREATE TABLE paimon.default.pk_assets (id INT NOT NULL, picture BLOB, PRIMARY KEY (id)) + WITH ('bucket' = '1')", + ) + .await; + for table in ["de_assets", "pk_assets"] { + for values in ["(1, X'414141'), (2, X'4242')", "(3, X'43')"] { + run( + &ctx, + &format!("INSERT INTO paimon.default.{table} (id, picture) VALUES {values}"), + ) + .await; + } + let format = + |batches: Vec| pretty_format_batches(&batches).unwrap().to_string(); + let expected = format( + run( + &ctx, + &format!("SELECT id, picture FROM paimon.default.{table} ORDER BY id"), + ) + .await, + ); + for name in [table.to_string(), format!("{table}$audit_log")] { + let actual = format( + run( + &ctx, + &format!( + "SELECT id, picture FROM paimon_incremental_query('default.{name}', 0, 2) ORDER BY id" + ), + ) + .await, + ); + assert_eq!(actual, expected, "{name}"); + } + for (start, rows) in [(0, 3), (1, 1)] { + let batches = run( + &ctx, + &format!( + "SELECT count(*) AS n FROM paimon_incremental_query('default.{table}', {start}, 2)" + ), + ) + .await; + let count = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + assert_eq!(count, rows, "{table} from {start}"); + } + } +} + +#[tokio::test] +async fn test_data_evolution_range_merges_column_updates() { + use datafusion::arrow::array::{Int64Array, StringArray}; + use datafusion::arrow::util::pretty::pretty_format_batches; + // MERGE INTO on a data-evolution table writes only the updated `name` to a + // file of its own. A range that also holds the rows it updates reads both + // files together, so the update merges into them; read per commit, the + // update would come back as rows whose NOT NULL `id` is missing. + let tmp = TempDir::new().unwrap(); + let ctx = context(&tmp).await; + run( + &ctx, + "CREATE TABLE paimon.default.de (id INT NOT NULL, name STRING, value INT) WITH ( + 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true' + )", + ) + .await; + run( + &ctx, + "INSERT INTO paimon.default.de (id, name, value) VALUES (1, 'alice', 10), (2, 'bob', 20)", + ) + .await; + run( + &ctx, + "CREATE TEMPORARY TABLE paimon.default.renames AS + SELECT * FROM (VALUES (1, 'ALICE')) AS t(id, name)", + ) + .await; + run( + &ctx, + "MERGE INTO paimon.default.de t USING paimon.default.renames s ON t.id = s.id + WHEN MATCHED THEN UPDATE SET name = s.name", + ) + .await; + let format = |batches: Vec| pretty_format_batches(&batches).unwrap().to_string(); + let expected = format( + run( + &ctx, + "SELECT id, name, value FROM paimon.default.de ORDER BY id", + ) + .await, + ); + // Auto is Delta on a data-evolution table, so both read the range as one batch. + for mode in ["", ", 'delta'"] { + for name in ["de", "de$audit_log"] { + let actual = format( + run( + &ctx, + &format!( + "SELECT id, name, value FROM paimon_incremental_query('default.{name}', 0, 2{mode}) ORDER BY id" + ), + ) + .await, + ); + assert_eq!(actual, expected, "{name}{mode}"); + } + let batches = run( + &ctx, + &format!( + "SELECT rowkind FROM paimon_incremental_query('default.de$audit_log', 0, 2{mode})" + ), + ) + .await; + let kinds: Vec<&str> = batches + .iter() + .flat_map(|batch| { + let kinds = batch.column(0).as_any().downcast_ref::(); + kinds.unwrap().iter().map(Option::unwrap) + }) + .collect(); + assert_eq!(kinds, ["+I", "+I"], "{mode}"); + } + // An empty range plans nothing, as on other tables. + for (start, end, rows) in [(0, 2, 2), (0, 0, 0)] { + let batches = run( + &ctx, + &format!( + "SELECT count(*) AS n FROM paimon_incremental_query('default.de', {start}, {end})" + ), + ) + .await; + let count = batches[0].column(0).as_any().downcast_ref::(); + assert_eq!(count.unwrap().value(0), rows, "({start}, {end}]"); + } +} diff --git a/crates/paimon/src/table/table_read.rs b/crates/paimon/src/table/table_read.rs index 4ae7b7112..951f868a7 100644 --- a/crates/paimon/src/table/table_read.rs +++ b/crates/paimon/src/table/table_read.rs @@ -466,10 +466,21 @@ impl<'a> PaimonTableRead<'a> { } } } - // Delta / Changelog rows are read as-is from planned files (no full-table - // merge against historical base versions). + // Delta / Changelog rows are read as-is from planned files, without a merge + // against historical base versions. Files are still decoded as an ordinary + // read decodes them: primary-key BLOB descriptors resolve to payloads, and + // data-evolution files in a split that hold column slices of the same rows + // merge by row id. let data_splits = self.matching_splits(&data_splits)?; - self.new_data_file_reader()?.read(&data_splits) + let core_options = self.table.schema().core_options(); + if !self.table.schema().primary_keys().is_empty() { + return self + .resolving_primary_key_blobs(&core_options, |read| read.read_raw(&data_splits)); + } + if core_options.data_evolution_enabled() { + return self.read_with_evolution(&data_splits, &core_options); + } + self.read_raw(&data_splits) } fn to_incremental_diff_arrow( @@ -518,6 +529,14 @@ impl<'a> PaimonTableRead<'a> { ) -> crate::Result { match plan.mode() { IncrementalScanMode::Diff => self.audit_diff_stream(plan), + // A data-evolution table has no primary key, so its rows are inserts, + // read as `to_incremental_arrow` reads them. + IncrementalScanMode::Delta + if self.table.schema().core_options().data_evolution_enabled() => + { + let schema = audit_schema_for_read_type(&self.read_type, false)?; + prepend_insert_row_kind_stream(self.to_incremental_arrow(plan)?, schema) + } IncrementalScanMode::Delta => { self.audit_raw_stream(plan, !self.table.schema().primary_keys().is_empty()) } @@ -1519,6 +1538,18 @@ impl<'a> PaimonTableRead<'a> { &self, data_splits: &[DataSplit], core_options: &CoreOptions<'_>, + ) -> crate::Result { + self.resolving_primary_key_blobs(core_options, |read| { + read.read_pk(data_splits, core_options) + }) + } + + /// Runs `read` over the stored BLOB descriptors and resolves primary-key + /// managed BLOB payloads in its output. + fn resolving_primary_key_blobs( + &self, + core_options: &CoreOptions<'_>, + read: impl FnOnce(&Self) -> crate::Result, ) -> crate::Result { use super::managed_blob_reader::{resolve_primary_key_blob_stream, ManagedBlobReadPlan}; @@ -1531,7 +1562,7 @@ impl<'a> PaimonTableRead<'a> { let mut inner = self.clone(); inner.read_type = plan.scan_fields().to_vec(); inner.data_predicates.clear(); - let stream = inner.read_pk(data_splits, core_options)?; + let stream = read(&inner)?; return Ok(plan.finish( stream, core_options, @@ -1541,7 +1572,7 @@ impl<'a> PaimonTableRead<'a> { )); } - let stream = self.read_pk(data_splits, core_options)?; + let stream = read(self)?; Ok(resolve_primary_key_blob_stream( stream, self.read_type(), diff --git a/crates/paimon/tests/incremental_batch_scan_test.rs b/crates/paimon/tests/incremental_batch_scan_test.rs index 193afb123..60eb2846f 100644 --- a/crates/paimon/tests/incremental_batch_scan_test.rs +++ b/crates/paimon/tests/incremental_batch_scan_test.rs @@ -1811,3 +1811,294 @@ async fn delta_keeps_physical_retractions_and_changelog_row_kinds() { } assert!(read_current_pairs(&table).await.is_empty()); } + +type BlobRow = (Option, Option>); + +/// `(id, picture)` of every row, sorted, located by column name so a leading +/// `rowkind` column is skipped. +fn blob_rows(batches: &[RecordBatch]) -> Vec { + use arrow_array::LargeBinaryArray; + let mut rows = Vec::new(); + for batch in batches { + let ids = batch.column_by_name("id").unwrap(); + let ids = ids.as_any().downcast_ref::().unwrap(); + let pictures = batch.column_by_name("picture").unwrap(); + let pictures = pictures + .as_any() + .downcast_ref::() + .unwrap(); + for row in 0..batch.num_rows() { + rows.push(( + ids.is_valid(row).then(|| ids.value(row)), + pictures.is_valid(row).then(|| pictures.value(row).to_vec()), + )); + } + } + rows.sort(); + rows +} + +/// Reads `(start, end]` per commit, as rows and as audit rows. +async fn read_delta_both_ways( + table: &paimon::table::Table, + start: i64, + end: i64, +) -> (Vec, Vec) { + let builder = table.new_read_builder(); + let plan = builder + .new_incremental_scan(IncrementalScanMode::Delta, start, end) + .plan() + .await + .unwrap(); + let read = builder.new_read().unwrap(); + let rows = read + .to_incremental_arrow(&plan) + .unwrap() + .try_collect() + .await + .unwrap(); + let audit = read + .to_audit_log_arrow(&plan) + .unwrap() + .try_collect() + .await + .unwrap(); + (rows, audit) +} + +/// A data-evolution commit writes BLOB and dedicated VECTOR columns to files of +/// their own beside the row's other columns. Each commit must still read back +/// as whole rows, as in an ordinary read. +#[tokio::test] +async fn delta_merges_data_evolution_column_files_into_rows() { + use arrow_array::builder::{FixedSizeListBuilder, Float32Builder}; + use arrow_array::{ArrayRef, FixedSizeListArray, Float32Array, LargeBinaryArray, StringArray}; + use arrow_schema::{DataType as ArrowDataType, Field, Schema as ArrowSchema}; + use paimon::spec::{BlobType, DataType, FloatType, IntType, Schema, TableSchema, VectorType}; + use std::sync::Arc; + + let path = "memory:/incremental_batch/data_evolution_files"; + let vector = VectorType::try_new(true, 2, DataType::Float(FloatType::new())).unwrap(); + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("picture", DataType::Blob(BlobType::new())) + .column("embedding", DataType::Vector(vector)) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("vector.file.format", "parquet") + .build() + .unwrap(); + let (io, table) = memory_table(path, TableSchema::new(0, &schema)); + setup_dirs(&io, path).await; + persist_table_schema(&io, path, table.schema()).await; + let element = Arc::new(Field::new("element", ArrowDataType::Float32, true)); + let batch = |ids: Vec, pictures: Vec<&[u8]>, embeddings: Vec<[f32; 2]>| { + let mut builder = + FixedSizeListBuilder::new(Float32Builder::new(), 2).with_field(element.clone()); + for embedding in embeddings { + builder.values().append_slice(&embedding); + builder.append(true); + } + RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + Field::new("id", ArrowDataType::Int32, true), + Field::new("picture", ArrowDataType::LargeBinary, true), + Field::new( + "embedding", + ArrowDataType::FixedSizeList(element.clone(), 2), + true, + ), + ])), + vec![ + Arc::new(Int32Array::from(ids)) as ArrayRef, + Arc::new(LargeBinaryArray::from(pictures)), + Arc::new(builder.finish()), + ], + ) + .unwrap() + }; + write_batch( + &table, + &batch( + vec![1, 2], + vec![b"aaa", b"bb"], + vec![[1.0, 0.0], [0.0, 1.0]], + ), + ) + .await; + write_batch(&table, &batch(vec![3], vec![b"c"], vec![[0.5, 0.5]])).await; + + let embeddings = |batches: &[RecordBatch]| { + let mut rows = Vec::new(); + for batch in batches { + let ids = batch.column_by_name("id").unwrap(); + let ids = ids.as_any().downcast_ref::().unwrap(); + let lists = batch.column_by_name("embedding").unwrap(); + let lists = lists.as_any().downcast_ref::().unwrap(); + for row in 0..batch.num_rows() { + let values = lists.is_valid(row).then(|| { + let values = lists.value(row); + let values = values.as_any().downcast_ref::().unwrap(); + values.values().to_vec() + }); + rows.push((ids.is_valid(row).then(|| ids.value(row)), values)); + } + } + rows.sort_by_key(|row| row.0); + rows + }; + let picture = |id: i32, bytes: &[u8]| (Some(id), Some(bytes.to_vec())); + for (start, ids) in [(0, vec![1, 2, 3]), (1, vec![3])] { + let (rows, audit) = read_delta_both_ways(&table, start, 2).await; + let all = [picture(1, b"aaa"), picture(2, b"bb"), picture(3, b"c")]; + let expected: Vec = all + .into_iter() + .filter(|row| ids.contains(&row.0.unwrap())) + .collect(); + assert_eq!(blob_rows(&rows), expected, "rows from {start}"); + assert_eq!(blob_rows(&audit), expected, "audit rows from {start}"); + let vectors = [ + (Some(1), Some(vec![1.0, 0.0])), + (Some(2), Some(vec![0.0, 1.0])), + (Some(3), Some(vec![0.5, 0.5])), + ]; + let expected: Vec<_> = vectors + .into_iter() + .filter(|row| ids.contains(&row.0.unwrap())) + .collect(); + assert_eq!(embeddings(&rows), expected, "embeddings from {start}"); + let kinds: Vec<_> = audit + .iter() + .flat_map(|batch| { + let kinds = batch.column(0).as_any().downcast_ref::(); + kinds.unwrap().iter().map(|kind| kind.unwrap().to_owned()) + }) + .collect(); + assert_eq!(kinds, vec!["+I"; ids.len()], "row kinds from {start}"); + } +} + +/// A primary-key table keeps managed BLOB descriptors in its data files. Each +/// commit must read back with the payloads, as in an ordinary read. +#[tokio::test] +async fn delta_resolves_primary_key_blob_payloads() { + use arrow_array::{ArrayRef, LargeBinaryArray}; + use arrow_schema::{DataType as ArrowDataType, Field, Schema as ArrowSchema}; + use paimon::spec::{BlobType, DataType, IntType, Schema, TableSchema}; + use std::sync::Arc; + + let path = "memory:/incremental_batch/primary_key_blobs"; + let schema = Schema::builder() + .column("id", DataType::Int(IntType::with_nullable(false))) + .column("picture", DataType::Blob(BlobType::new())) + .primary_key(["id"]) + .option("bucket", "1") + .build() + .unwrap(); + let (io, table) = memory_table(path, TableSchema::new(0, &schema)); + setup_dirs(&io, path).await; + persist_table_schema(&io, path, table.schema()).await; + let batch = |ids: Vec, pictures: Vec<&[u8]>| { + RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + Field::new("id", ArrowDataType::Int32, false), + Field::new("picture", ArrowDataType::LargeBinary, true), + ])), + vec![ + Arc::new(Int32Array::from(ids)) as ArrayRef, + Arc::new(LargeBinaryArray::from(pictures)), + ], + ) + .unwrap() + }; + write_batch(&table, &batch(vec![1, 2], vec![b"aaa", b"bb"])).await; + write_batch(&table, &batch(vec![2], vec![b"c"])).await; + + let (rows, audit) = read_delta_both_ways(&table, 0, 2).await; + let expected = vec![ + (Some(1), Some(b"aaa".to_vec())), + (Some(2), Some(b"bb".to_vec())), + (Some(2), Some(b"c".to_vec())), + ]; + assert_eq!(blob_rows(&rows), expected); + assert_eq!(blob_rows(&audit), expected); +} + +/// Each commit reads back as a streaming read of that snapshot does, including +/// a data-evolution update that rewrites a column of existing rows. +#[tokio::test] +async fn delta_reads_each_commit_as_its_stream_follow_up() { + use arrow_array::{ArrayRef, Int64Array, LargeBinaryArray}; + use paimon::spec::{BlobType, DataType, IntType, Schema, TableSchema}; + use std::sync::Arc; + + let path = "memory:/incremental_batch/stream_follow_up"; + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("picture", DataType::Blob(BlobType::new())) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .build() + .unwrap(); + let (io, table) = memory_table(path, TableSchema::new(0, &schema)); + setup_dirs(&io, path).await; + persist_table_schema(&io, path, table.schema()).await; + let batch = |ids: Vec, pictures: Vec<&[u8]>| { + RecordBatch::try_from_iter([ + ("id", Arc::new(Int32Array::from(ids)) as ArrayRef), + ("picture", Arc::new(LargeBinaryArray::from(pictures))), + ]) + .unwrap() + }; + write_batch(&table, &batch(vec![1, 2], vec![b"aaa", b"bb"])).await; + write_batch(&table, &batch(vec![3], vec![b"c"])).await; + // Snapshot 3 rewrites `picture` of row 0. + let builder = table.new_write_builder(); + let mut update = builder + .new_data_evolution_writer(vec!["picture".into()]) + .unwrap(); + update + .add_matched_batch( + RecordBatch::try_from_iter([ + ("_ROW_ID", Arc::new(Int64Array::from(vec![0])) as ArrayRef), + ( + "picture", + Arc::new(LargeBinaryArray::from(vec![b"new".as_slice()])), + ), + ]) + .unwrap(), + ) + .unwrap(); + builder + .new_commit() + .commit(update.prepare_commit().await.unwrap()) + .await + .unwrap(); + + let read_builder = table.new_read_builder(); + let mut stream = read_builder.new_stream_scan().unwrap(); + for snapshot in 1..=3 { + stream.restore(Some(snapshot)).unwrap(); + let plan = stream.plan().await.unwrap().unwrap(); + let streamed: Vec = read_builder + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + let (rows, audit) = read_delta_both_ways(&table, snapshot - 1, snapshot).await; + assert!(!streamed.is_empty(), "snapshot {snapshot}"); + assert_eq!( + blob_rows(&rows), + blob_rows(&streamed), + "snapshot {snapshot}" + ); + assert_eq!( + blob_rows(&audit), + blob_rows(&streamed), + "snapshot {snapshot}" + ); + } +} diff --git a/crates/paimon/tests/partition_filter_test.rs b/crates/paimon/tests/partition_filter_test.rs index 0ee258e05..c55a61bf4 100644 --- a/crates/paimon/tests/partition_filter_test.rs +++ b/crates/paimon/tests/partition_filter_test.rs @@ -380,3 +380,38 @@ async fn ordinary_partition_filter_survives_diff_workers_and_audit_images() { .all(|id| *id == 1)); } } + +#[tokio::test] +async fn ordinary_partition_filter_survives_incremental_delta_reads() { + // Delta reads take a separate path per table kind, plain and audit-log + // alike; the read's partition conjuncts must restrict every one of them. + for mode in ["append", "pk", "de"] { + let table = partitioned(mode).await; + let plan = table + .new_read_builder() + .new_incremental_scan(IncrementalScanMode::Delta, 0, 1) + .plan() + .await + .unwrap(); + let mut builder = table.new_read_builder(); + builder + .with_projection(&["id"]) + .unwrap() + .with_filter(equal(&table, "p", "a")); + let read = builder.new_read().unwrap(); + for audit in [false, true] { + let stream = if audit { + read.to_audit_log_arrow(&plan) + } else { + read.to_incremental_arrow(&plan) + }; + let mut actual: Vec = Vec::new(); + for batch in stream.unwrap().try_collect::>().await.unwrap() { + let column = batch.column_by_name("id").unwrap().as_any(); + actual.extend(column.downcast_ref::().unwrap().values()); + } + actual.sort_unstable(); + assert_eq!(actual, [1, 3], "{mode} audit={audit}"); + } + } +} diff --git a/docs/src/sql.md b/docs/src/sql.md index 120b8a333..89c6d3565 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -74,9 +74,9 @@ the catalog-independent `path_to_descriptor` and `descriptor_to_string` scalar functions pre-registered. Use `register_catalog(...).await` to add one or more Paimon catalogs; registering a catalog also registers the built-in scalar function `blob_view` (alias `sys.blob_view`) and the built-in table-valued -functions (`vector_search`, `hybrid_search`, and `full_text_search` when the -`fulltext` feature is enabled) against it. It also manages session-scoped -dynamic options internally for `SET`/`RESET` support. +functions (`vector_search`, `hybrid_search`, `paimon_incremental_query`, and +`full_text_search` when the `fulltext` feature is enabled) against it. It also +manages session-scoped dynamic options internally for `SET`/`RESET` support. ### REST Catalog Views and SQL Functions @@ -2231,6 +2231,34 @@ given value (snapshots without a watermark are skipped). It is mutually exclusive with the other time-travel selectors. If no matching snapshot exists, scan planning fails. +## Incremental Query + +`paimon_incremental_query` reads the changes between two snapshots. The range +excludes the start snapshot and includes the end snapshot: + +```sql +SELECT * FROM paimon_incremental_query('my_db.my_table', 1, 5); +SELECT * FROM paimon_incremental_query('my_db.my_table$audit_log', 1, 5, 'changelog'); +``` + +The optional fourth argument selects the mode: + +- `delta` returns the rows each APPEND commit wrote, every version of a key included. + A data-evolution table's range is read as one batch, as Java's `incremental-between` + scan reads it, so a column update merges into the rows it updates when they were + written in the range. For rows written before the range it returns every row of the + files the update rewrote, unchanged rows included, with only the updated columns + set and the others NULL, NOT NULL columns included. +- `changelog` returns the changelog rows of each commit. +- `diff` compares the table at the two snapshots and returns the added and changed + rows (`$audit_log` adds `-U` and `-D` rows). It requires a primary-key table with + `merge-engine=deduplicate` and no BLOB or VARIANT columns. +- `auto`, the default, is `delta` when `changelog-producer` is `none` and + `changelog` otherwise. + +A `$audit_log` suffix puts `rowkind` first, followed by `_SEQUENCE_NUMBER` when +`table-read.sequence-number.enabled=true`. + ## Dynamic Options (SET / RESET) Use `SET` to configure session-scoped Paimon dynamic options that apply to subsequent table loads: From f8016e12449b12f34bbb27a35a7dbc11229808aa Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Wed, 21 Oct 2026 21:24:27 +0800 Subject: [PATCH 03/18] fix(datafusion): read incremental queries in the calling session `paimon_incremental_query` was bound to the catalog registered last and dropped the catalog part of a three-part name, so a qualified name could read another warehouse. It now resolves the name in the calling session as a FROM clause does and looks the catalog up in the session's catalog list. It also ignored the session's dynamic options. They now apply as they do to an ordinary read of the table, and the table is registered for BLOB descriptor reads. Options that choose another range to read (time-travel selectors, `incremental-between*`, creation-time filters, a non-default `scan.mode`) are rejected, whether the session or the table read sets them, and `$audit_log` keeps the ordinary audit-log rule. A data-evolution range of column updates without the rows they update failed with "Cannot read non-nullable field 'id' without a provider". A split of change events now reads a field no file of its group wrote as NULL, NOT NULL and nested fields included; a snapshot read keeps the check. --- crates/integrations/datafusion/src/catalog.rs | 16 + .../datafusion/src/incremental_query.rs | 124 ++++- .../datafusion/src/sql_context.rs | 6 +- .../datafusion/src/system_tables/mod.rs | 22 +- .../datafusion/src/table_function_args.rs | 35 +- .../datafusion/tests/incremental_query.rs | 426 ++++++++++++++++++ crates/paimon/src/spec/core_options.rs | 60 +++ .../paimon/src/table/data_evolution_nested.rs | 47 +- .../paimon/src/table/data_evolution_reader.rs | 65 ++- .../tests/incremental_batch_scan_test.rs | 159 +++++++ docs/src/sql.md | 25 +- 11 files changed, 923 insertions(+), 62 deletions(-) diff --git a/crates/integrations/datafusion/src/catalog.rs b/crates/integrations/datafusion/src/catalog.rs index e5093f140..e619d2170 100644 --- a/crates/integrations/datafusion/src/catalog.rs +++ b/crates/integrations/datafusion/src/catalog.rs @@ -440,6 +440,22 @@ impl PaimonCatalogProvider { self } + pub(crate) fn catalog(&self) -> &Arc { + &self.catalog + } + + /// The session's dynamic options as they stand now. + pub(crate) fn session_options(&self) -> HashMap { + self.dynamic_options + .read() + .unwrap_or_else(|e| e.into_inner()) + .clone() + } + + pub(crate) fn blob_reader_registry(&self) -> &BlobReaderRegistry { + &self.blob_reader_registry + } + /// Register an engine for a table type the Paimon reader cannot serve /// (e.g. [`PaimonTableType::IcebergTable`]); everything else takes the /// Paimon path unchanged. Kept inside the provider so the registered diff --git a/crates/integrations/datafusion/src/incremental_query.rs b/crates/integrations/datafusion/src/incremental_query.rs index cdf2f0ab0..3ac1a40d6 100644 --- a/crates/integrations/datafusion/src/incremental_query.rs +++ b/crates/integrations/datafusion/src/incremental_query.rs @@ -28,13 +28,18 @@ //! //! A `$audit_log` table-name suffix prepends the `rowkind` audit column (and the //! optional `_SEQUENCE_NUMBER`), mirroring the core audit-log read. +//! +//! The table name resolves as in a `FROM` clause, in the calling session, and +//! the session's dynamic options apply as they do to an ordinary read of the +//! table, except those that choose another range to read. +use std::collections::HashMap; use std::fmt; use std::sync::Arc; use async_trait::async_trait; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; -use datafusion::catalog::{Session, TableFunctionImpl}; +use datafusion::catalog::{CatalogProviderList, Session, TableFunctionArgs, TableFunctionImpl}; use datafusion::common::stats::Precision; use datafusion::common::{internal_err, project_schema, Statistics}; use datafusion::datasource::{TableProvider, TableType}; @@ -50,7 +55,7 @@ use datafusion::physical_plan::{ }; use datafusion::prelude::SessionContext; use futures::{stream, StreamExt, TryStreamExt}; -use paimon::catalog::Catalog; +use paimon::catalog::DEFAULT_MAIN_BRANCH; use paimon::resource::ResourceContext; use paimon::spec::{CoreOptions, DataField}; use paimon::table::{ @@ -58,44 +63,41 @@ use paimon::table::{ Table, }; +use crate::catalog::PaimonCatalogProvider; use crate::error::to_datafusion_error; use crate::memory::reader_resources; use crate::physical_plan::scan::to_datafusion_batch; use crate::runtime::{await_with_runtime, block_on_with_runtime}; +use crate::system_tables::ensure_audit_log_dynamic_options; use crate::table::datafusion_arrow_schema; use crate::table_function_args::{ - extract_int_literal, extract_string_literal, parse_table_identifier, + extract_int_literal, extract_string_literal, resolve_table_argument, }; use crate::table_loader::load_data_table_for_read; const FUNCTION_NAME: &str = "paimon_incremental_query"; const AUDIT_LOG_SUFFIX: &str = "$audit_log"; -/// Registers `paimon_incremental_query` against `catalog`. -pub(crate) fn register_incremental_query( - ctx: &SessionContext, - catalog: Arc, - default_database: &str, -) { +/// Registers `paimon_incremental_query`, which looks its table up among the +/// catalogs of `ctx` when it is called. +pub(crate) fn register_incremental_query(ctx: &SessionContext) { ctx.register_udtf( FUNCTION_NAME, Arc::new(IncrementalQueryFunction { - catalog, - default_database: default_database.to_string(), + catalogs: Arc::clone(ctx.state().catalog_list()), }), ); } pub(crate) struct IncrementalQueryFunction { - catalog: Arc, - default_database: String, + /// The catalog list `register_catalog` adds to, as ordinary queries see it. + catalogs: Arc, } impl fmt::Debug for IncrementalQueryFunction { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("IncrementalQueryFunction") - .field("default_database", &self.default_database) - .finish() + .finish_non_exhaustive() } } @@ -114,17 +116,19 @@ fn parse_mode(raw: &str) -> DFResult { } impl TableFunctionImpl for IncrementalQueryFunction { - fn call(&self, args: &[Expr]) -> DFResult> { - if args.len() < 3 || args.len() > 4 { + fn call_with_args(&self, args: TableFunctionArgs) -> DFResult> { + let exprs = args.exprs(); + if exprs.len() < 3 || exprs.len() > 4 { return Err(DataFusionError::Plan(format!( "{FUNCTION_NAME} requires 3 or 4 arguments: (table_name, start_snapshot_exclusive, end_snapshot_inclusive [, mode])" ))); } - let table_name = extract_string_literal(FUNCTION_NAME, &args[0], "table_name")?; + let table_name = extract_string_literal(FUNCTION_NAME, &exprs[0], "table_name")?; let start_exclusive = - extract_int_literal(FUNCTION_NAME, &args[1], "start_snapshot_exclusive")?; - let end_inclusive = extract_int_literal(FUNCTION_NAME, &args[2], "end_snapshot_inclusive")?; - let mode = match args.get(3) { + extract_int_literal(FUNCTION_NAME, &exprs[1], "start_snapshot_exclusive")?; + let end_inclusive = + extract_int_literal(FUNCTION_NAME, &exprs[2], "end_snapshot_inclusive")?; + let mode = match exprs.get(3) { Some(expr) => parse_mode(&extract_string_literal(FUNCTION_NAME, expr, "mode")?)?, None => IncrementalScanMode::Auto, }; @@ -139,13 +143,69 @@ impl TableFunctionImpl for IncrementalQueryFunction { Some(base) => (base.to_string(), true), None => (table_name.clone(), false), }; - let identifier = parse_table_identifier(FUNCTION_NAME, &base_name, &self.default_database)?; + // The calling session resolves the name, so `USE` applies here as it + // does to a `FROM` clause. + let (catalog_name, identifier) = resolve_table_argument( + FUNCTION_NAME, + &base_name, + &args.session().config_options().catalog, + )?; + let registered = self + .catalogs + .catalog(&catalog_name) + .ok_or_else(|| DataFusionError::Plan(format!("Unknown catalog '{catalog_name}'")))?; + let provider = registered + .downcast_ref::() + .ok_or_else(|| { + DataFusionError::Plan(format!("Catalog '{catalog_name}' is not a Paimon catalog")) + })?; - let catalog = Arc::clone(&self.catalog); + // Session options apply as they do to an ordinary read of the table; + // those choosing another range are refused before anything is read. + let session_options = provider.session_options(); + ensure_no_range_options(&session_options, |key| { + format!( + "{FUNCTION_NAME} reads the range its arguments give and cannot run while \ + scan option '{key}' is set; RESET 'paimon.{key}' first" + ) + })?; + if audit_log { + ensure_audit_log_dynamic_options(&session_options)?; + } + let catalog = Arc::clone(provider.catalog()); let table = block_on_with_runtime( - async move { load_data_table_for_read(&catalog, &identifier, FUNCTION_NAME).await }, + async move { + let table = load_data_table_for_read(&catalog, &identifier, FUNCTION_NAME).await?; + let table = if session_options.is_empty() { + table + } else { + table + .copy_with_time_travel(session_options) + .await + .map_err(to_datafusion_error)? + }; + // Checked on the table read: a session `branch` reads that + // branch, with that branch's options. + ensure_no_range_options(table.schema().options(), |key| { + let read = match table.branch() { + DEFAULT_MAIN_BRANCH => format!("table '{}'", identifier.full_name()), + branch => { + format!("branch '{branch}' of table '{}'", identifier.full_name()) + } + }; + format!( + "{FUNCTION_NAME} reads the range its arguments give and cannot run on \ + {read}, which sets scan option '{key}'" + ) + })?; + Ok::<_, DataFusionError>(table) + }, "paimon_incremental_query: catalog access thread panicked", )?; + // As an ordinary read does, so BLOB descriptors it returns can be read. + provider + .blob_reader_registry() + .register_if_absent(table.location().to_string(), table.file_io().clone()); Ok(Arc::new(IncrementalQueryTableProvider::try_new( table, @@ -156,6 +216,22 @@ impl TableFunctionImpl for IncrementalQueryFunction { )?)) } } + +/// Fails with `message(key)` when `options` choose what to read, which the +/// function's arguments do. +fn ensure_no_range_options( + options: &HashMap, + message: impl FnOnce(&str) -> String, +) -> DFResult<()> { + match CoreOptions::new(options) + .configured_scan_range_options() + .first() + { + Some(key) => Err(DataFusionError::Plan(message(key))), + None => Ok(()), + } +} + #[derive(Debug, Clone)] struct IncrementalQueryTableProvider { table: Table, diff --git a/crates/integrations/datafusion/src/sql_context.rs b/crates/integrations/datafusion/src/sql_context.rs index 0f2f25d1c..f7dc7588a 100644 --- a/crates/integrations/datafusion/src/sql_context.rs +++ b/crates/integrations/datafusion/src/sql_context.rs @@ -170,6 +170,7 @@ impl SQLContextBuilder { let ctx = SessionContext::new_with_state(state); crate::blob_descriptor_functions::register_blob_descriptor_functions(&ctx); crate::variant_functions::register_variant_functions(&ctx); + crate::incremental_query::register_incremental_query(&ctx); SQLContext { ctx, catalogs: HashMap::new(), @@ -4005,11 +4006,6 @@ fn register_table_functions( #[cfg(feature = "fulltext")] crate::full_text_search::register_full_text_search(ctx, Arc::clone(catalog), default_database); crate::hybrid_search::register_hybrid_search(ctx, Arc::clone(catalog), default_database); - crate::incremental_query::register_incremental_query( - ctx, - Arc::clone(catalog), - default_database, - ); } #[cfg(test)] diff --git a/crates/integrations/datafusion/src/system_tables/mod.rs b/crates/integrations/datafusion/src/system_tables/mod.rs index 6b7d36d2c..dbd474b6f 100644 --- a/crates/integrations/datafusion/src/system_tables/mod.rs +++ b/crates/integrations/datafusion/src/system_tables/mod.rs @@ -145,6 +145,19 @@ pub(crate) fn provider_for_table( .map(Some) } +/// Rejects session options a `$audit_log` read does not take. +pub(crate) fn ensure_audit_log_dynamic_options( + dynamic_options: &HashMap, +) -> DFResult<()> { + if paimon::spec::CoreOptions::new(dynamic_options).table_read_sequence_number_enabled() { + return Err(DataFusionError::Plan( + "table-read.sequence-number.enabled is not supported by dynamic options for $audit_log" + .to_string(), + )); + } + Ok(()) +} + /// Builds a system-table provider from a loaded Paimon table. pub(crate) async fn load( catalog: Arc, @@ -157,13 +170,8 @@ pub(crate) async fn load( if !is_registered(&system_name) { return Ok(None); } - if system_name.eq_ignore_ascii_case("audit_log") - && paimon::spec::CoreOptions::new(&dynamic_options).table_read_sequence_number_enabled() - { - return Err(DataFusionError::Plan( - "table-read.sequence-number.enabled is not supported by dynamic options for $audit_log" - .to_string(), - )); + if system_name.eq_ignore_ascii_case("audit_log") { + ensure_audit_log_dynamic_options(&dynamic_options)?; } paimon::spec::CoreOptions::new(&dynamic_options) .ensure_read_authorized() diff --git a/crates/integrations/datafusion/src/table_function_args.rs b/crates/integrations/datafusion/src/table_function_args.rs index c0a6c833e..4f7f1d5a4 100644 --- a/crates/integrations/datafusion/src/table_function_args.rs +++ b/crates/integrations/datafusion/src/table_function_args.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use datafusion::common::config::CatalogOptions; use datafusion::common::ScalarValue; use datafusion::error::{DataFusionError, Result as DFResult}; use datafusion::logical_expr::Expr; @@ -70,11 +71,35 @@ pub(crate) fn parse_table_identifier( name: &str, default_database: &str, ) -> DFResult { - let parts: Vec<&str> = name.split('.').collect(); - match parts.len() { - 1 => Ok(Identifier::new(default_database, parts[0])), - 2 => Ok(Identifier::new(parts[0], parts[1])), - 3 => Ok(Identifier::new(parts[1], parts[2])), + let (_, database, table) = split_table_name(function_name, name)?; + Ok(Identifier::new(database.unwrap_or(default_database), table)) +} + +/// Resolves a table-name argument the way a `FROM` clause resolves a table +/// name: `table` and `database.table` take the session's current catalog (and +/// database), and `catalog.database.table` names the catalog. Returns the +/// catalog's name and the table's identifier in that catalog. +pub(crate) fn resolve_table_argument( + function_name: &str, + name: &str, + session: &CatalogOptions, +) -> DFResult<(String, Identifier)> { + let (catalog, database, table) = split_table_name(function_name, name)?; + Ok(( + catalog.unwrap_or(&session.default_catalog).to_string(), + Identifier::new(database.unwrap_or(&session.default_schema), table), + )) +} + +/// Splits `table`, `database.table` or `catalog.database.table`. +fn split_table_name<'a>( + function_name: &str, + name: &'a str, +) -> DFResult<(Option<&'a str>, Option<&'a str>, &'a str)> { + match name.split('.').collect::>().as_slice() { + [table] => Ok((None, None, *table)), + [database, table] => Ok((None, Some(*database), *table)), + [catalog, database, table] => Ok((Some(*catalog), Some(*database), *table)), _ => Err(DataFusionError::Plan(format!( "{function_name}: invalid table name '{name}', expected 'table', 'database.table', or 'catalog.database.table'" ))), diff --git a/crates/integrations/datafusion/tests/incremental_query.rs b/crates/integrations/datafusion/tests/incremental_query.rs index e55da99a0..5c2dfd3e9 100644 --- a/crates/integrations/datafusion/tests/incremental_query.rs +++ b/crates/integrations/datafusion/tests/incremental_query.rs @@ -592,3 +592,429 @@ async fn test_data_evolution_range_merges_column_updates() { assert_eq!(count.unwrap().value(0), rows, "({start}, {end}]"); } } + +#[tokio::test] +async fn test_data_evolution_range_of_column_updates_reads_unwritten_columns_as_null() { + use datafusion::arrow::array::{Array, Int32Array, StringArray}; + // Snapshot 2 updates `name` and snapshot 3 updates `value`, so (1, 3] + // holds two column files of the same rows and no file with `id`. + let tmp = TempDir::new().unwrap(); + let ctx = context(&tmp).await; + run( + &ctx, + "CREATE TABLE paimon.default.de (id INT NOT NULL, name STRING, value INT) WITH ( + 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true' + )", + ) + .await; + run( + &ctx, + "INSERT INTO paimon.default.de (id, name, value) VALUES (1, 'alice', 10), (2, 'bob', 20)", + ) + .await; + for (source, values, column) in [ + ("renames", "(1, 'ALICE')", "name"), + ("revalues", "(2, 99)", "value"), + ] { + run( + &ctx, + &format!( + "CREATE TEMPORARY TABLE paimon.default.{source} AS + SELECT * FROM (VALUES {values}) AS t(id, {column})" + ), + ) + .await; + run( + &ctx, + &format!( + "MERGE INTO paimon.default.de t USING paimon.default.{source} s ON t.id = s.id + WHEN MATCHED THEN UPDATE SET {column} = s.{column}" + ), + ) + .await; + } + for mode in ["", ", 'delta'"] { + for name in ["de", "de$audit_log"] { + let batches = run( + &ctx, + &format!( + "SELECT id, name, value FROM paimon_incremental_query('default.{name}', 1, 3{mode})" + ), + ) + .await; + let mut rows = Vec::new(); + for batch in &batches { + let id = batch.column(0).as_any().downcast_ref::(); + let name = batch.column(1).as_any().downcast_ref::(); + let value = batch.column(2).as_any().downcast_ref::(); + let (id, name, value) = (id.unwrap(), name.unwrap(), value.unwrap()); + for row in 0..batch.num_rows() { + rows.push(( + id.is_valid(row).then(|| id.value(row)), + name.value(row).to_string(), + value.value(row), + )); + } + } + rows.sort(); + let expected = [ + (None, "ALICE".to_string(), 10), + (None, "bob".to_string(), 99), + ]; + assert_eq!(rows, expected, "{name}{mode}"); + } + } +} + +/// Asserts that, for `name`, the function reads the rows a `FROM` clause in +/// the same session reads. +async fn assert_reads_as_from(ctx: &SQLContext, name: &str) { + let expected = sorted_ids(&run(ctx, &format!("SELECT id FROM {name}")).await); + let actual = sorted_ids( + &run( + ctx, + &format!("SELECT id FROM paimon_incremental_query('{name}', 0, 1)"), + ) + .await, + ); + assert_eq!(actual, expected, "{name}"); +} + +#[tokio::test] +async fn test_table_name_resolves_as_in_a_from_clause() { + use datafusion::catalog::MemoryCatalogProvider; + // Two warehouses with a `default.t` each, and a database in cat_a named + // like the catalog cat_b. + let (a, b) = (TempDir::new().unwrap(), TempDir::new().unwrap()); + let mut ctx = SQLContext::new(); + ctx.register_catalog("cat_a", catalog(&a)).await.unwrap(); + ctx.register_catalog("cat_b", catalog(&b)).await.unwrap(); + run(&ctx, "CREATE SCHEMA cat_a.cat_b").await; + for (table, id) in [ + ("cat_a.default.t", 1), + ("cat_b.default.t", 2), + ("cat_a.cat_b.t", 3), + ] { + run(&ctx, &format!("CREATE TABLE {table} (id INT)")).await; + run(&ctx, &format!("INSERT INTO {table} VALUES ({id})")).await; + } + + // cat_a, registered first, is the current catalog. + for name in [ + "cat_a.default.t", + "cat_b.default.t", + "default.t", + "t", + "cat_b.t", + ] { + assert_reads_as_from(&ctx, name).await; + } + run(&ctx, "USE cat_b.default").await; + for name in ["t", "default.t", "cat_a.default.t"] { + assert_reads_as_from(&ctx, name).await; + } + run(&ctx, "USE cat_a.cat_b").await; + assert_reads_as_from(&ctx, "t").await; + + let err = expect_error( + &ctx, + "SELECT * FROM paimon_incremental_query('nope.default.t', 0, 1)", + ) + .await; + assert!(err.contains("Unknown catalog 'nope'"), "{err}"); + ctx.ctx() + .register_catalog("mem", Arc::new(MemoryCatalogProvider::new())); + let err = expect_error( + &ctx, + "SELECT * FROM paimon_incremental_query('mem.public.t', 0, 1)", + ) + .await; + assert!( + err.contains("Catalog 'mem' is not a Paimon catalog"), + "{err}" + ); +} +#[tokio::test] +async fn test_session_read_options_apply_as_in_an_ordinary_read() { + use datafusion::arrow::util::pretty::pretty_format_batches; + let tmp = TempDir::new().unwrap(); + let ctx = context(&tmp).await; + run( + &ctx, + "CREATE TABLE paimon.default.de_assets (id INT, picture BLOB) WITH ( + 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true' + )", + ) + .await; + run( + &ctx, + "CREATE TABLE paimon.default.pk_assets (id INT NOT NULL, picture BLOB, PRIMARY KEY (id)) + WITH ('bucket' = '1')", + ) + .await; + let format = |batches: Vec| pretty_format_batches(&batches).unwrap().to_string(); + for table in ["de_assets", "pk_assets"] { + run( + &ctx, + &format!("INSERT INTO paimon.default.{table} (id, picture) VALUES (1, X'414141')"), + ) + .await; + let ordinary = format!("SELECT id, picture FROM paimon.default.{table}"); + let payloads = format(run(&ctx, &ordinary).await); + run(&ctx, "SET 'paimon.blob-as-descriptor' = 'true'").await; + let descriptors = format(run(&ctx, &ordinary).await); + assert_ne!( + descriptors, payloads, + "{table}: the option changes the read" + ); + for (setting, expected) in [ + ("SET 'paimon.blob-as-descriptor' = 'true'", &descriptors), + ("RESET 'paimon.blob-as-descriptor'", &payloads), + ] { + run(&ctx, setting).await; + for name in [table.to_string(), format!("{table}$audit_log")] { + let sql = format!( + "SELECT id, picture FROM paimon_incremental_query('default.{name}', 0, 1)" + ); + assert_eq!( + &format(run(&ctx, &sql).await), + expected, + "{name}: {setting}" + ); + } + } + } + + // Descriptors the function returns can be read back through the session, + // as those of an ordinary read can. + let fresh = context(&tmp).await; + run(&fresh, "SET 'paimon.blob-as-descriptor' = 'true'").await; + run( + &fresh, + "SELECT * FROM paimon_incremental_query('default.pk_assets', 0, 1)", + ) + .await; + let location = catalog(&tmp) + .get_table(&Identifier::new("default", "pk_assets")) + .await + .unwrap() + .location() + .to_string(); + assert!(fresh + .blob_reader_registry() + .resolve(&format!("{location}/bucket-0/data.blob")) + .is_some()); + + run(&ctx, "CREATE TABLE paimon.default.t (id INT)").await; + run( + &ctx, + "INSERT INTO paimon.default.t VALUES (1), (2), (3), (4), (5)", + ) + .await; + run(&ctx, "SET 'paimon.read.batch-size' = '2'").await; + let sizes = |batches: Vec| { + batches + .iter() + .map(RecordBatch::num_rows) + .filter(|rows| *rows > 0) + .collect::>() + }; + let ordinary = sizes(run(&ctx, "SELECT id FROM paimon.default.t").await); + assert_eq!(ordinary, [2, 2, 1]); + let incremental = sizes( + run( + &ctx, + "SELECT id FROM paimon_incremental_query('default.t', 0, 1)", + ) + .await, + ); + assert_eq!(incremental, ordinary); +} + +#[tokio::test] +async fn test_options_that_choose_another_range_are_refused() { + let (_tmp, ctx) = setup_three_snapshots().await; + for (key, value) in [ + ("scan.snapshot-id", "1"), + // A tag that does not exist: the option is refused before any lookup. + ("scan.tag-name", "not_a_tag"), + ("scan.version", "1"), + ("scan.timestamp-millis", "0"), + ("scan.timestamp", "2024-01-01 00:00:00"), + ("scan.watermark", "0"), + ("incremental-between", "1,2"), + ("incremental-between-timestamp", "0,1"), + ("incremental-between-scan-mode", "delta"), + ("scan.file-creation-time-millis", "0"), + ("scan.creation-time-millis", "0"), + ("scan.mode", "latest"), + ] { + run(&ctx, &format!("SET 'paimon.{key}' = '{value}'")).await; + for name in ["t", "t$audit_log"] { + let err = expect_error( + &ctx, + &format!("SELECT * FROM paimon_incremental_query('default.{name}', 1, 3)"), + ) + .await; + assert!( + err.contains(&format!( + "scan option '{key}' is set; RESET 'paimon.{key}' first" + )), + "{key} on {name}: {err}" + ); + } + run(&ctx, &format!("RESET 'paimon.{key}'")).await; + } + run(&ctx, "SET 'paimon.scan.mode' = 'default'").await; + let batches = run( + &ctx, + "SELECT * FROM paimon_incremental_query('default.t', 1, 3)", + ) + .await; + assert_eq!(sorted_ids(&batches), vec![2, 3]); + run(&ctx, "RESET 'paimon.scan.mode'").await; + + // The table's own options choose no range either. + run( + &ctx, + "CREATE TABLE paimon.default.latest (id INT) WITH ('scan.mode' = 'latest')", + ) + .await; + run(&ctx, "INSERT INTO paimon.default.latest VALUES (1)").await; + let err = expect_error( + &ctx, + "SELECT * FROM paimon_incremental_query('default.latest', 0, 1)", + ) + .await; + assert!( + err.contains("table 'default.latest', which sets scan option 'scan.mode'"), + "{err}" + ); + // A stored time-travel selector is refused too, with or without other + // session options. + run(&ctx, "CREATE TABLE paimon.default.tagged (id INT)").await; + run(&ctx, "INSERT INTO paimon.default.tagged VALUES (1)").await; + run( + &ctx, + "ALTER TABLE paimon.default.tagged SET TBLPROPERTIES('scan.tag-name' = 'not_a_tag')", + ) + .await; + for setting in [None, Some("read.batch-size")] { + if let Some(key) = setting { + run(&ctx, &format!("SET 'paimon.{key}' = '2'")).await; + } + let err = expect_error( + &ctx, + "SELECT * FROM paimon_incremental_query('default.tagged', 0, 1)", + ) + .await; + assert!( + err.contains("table 'default.tagged', which sets scan option 'scan.tag-name'"), + "{setting:?}: {err}" + ); + if let Some(key) = setting { + run(&ctx, &format!("RESET 'paimon.{key}'")).await; + } + } +} + +#[tokio::test] +async fn test_range_options_are_checked_on_the_branch_read() { + use paimon::table::{BranchManager, SnapshotManager, TagManager}; + // A session `branch` reads that branch, so its options decide: `on_branch` + // stores `scan.mode=latest` only on b1, `on_main` only on the main branch. + let tmp = TempDir::new().unwrap(); + let ctx = context(&tmp).await; + let warehouse = catalog(&tmp); + for (table, created, altered) in [ + ("on_branch", "latest", "default"), + ("on_main", "default", "latest"), + ] { + run( + &ctx, + &format!( + "CREATE TABLE paimon.default.{table} (id INT) WITH ('scan.mode' = '{created}')" + ), + ) + .await; + run( + &ctx, + &format!("INSERT INTO paimon.default.{table} VALUES (1)"), + ) + .await; + let loaded = warehouse + .get_table(&Identifier::new("default", table)) + .await + .unwrap(); + let (io, location) = (loaded.file_io().clone(), loaded.location().to_string()); + let snapshot = SnapshotManager::new(io.clone(), location.clone()) + .get_latest_snapshot() + .await + .unwrap() + .unwrap(); + TagManager::new(io.clone(), location.clone()) + .create("base", &snapshot) + .await + .unwrap(); + BranchManager::new(io, location) + .create_branch_from_tag("b1", "base") + .await + .unwrap(); + run( + &ctx, + &format!( + "ALTER TABLE paimon.default.{table} SET TBLPROPERTIES('scan.mode' = '{altered}')" + ), + ) + .await; + } + + let query = + |table: &str| format!("SELECT id FROM paimon_incremental_query('default.{table}', 0, 1)"); + assert_eq!(sorted_ids(&run(&ctx, &query("on_branch")).await), vec![1]); + let err = expect_error(&ctx, &query("on_main")).await; + assert!( + err.contains("table 'default.on_main', which sets scan option 'scan.mode'"), + "{err}" + ); + run(&ctx, "SET 'paimon.branch' = 'b1'").await; + let err = expect_error(&ctx, &query("on_branch")).await; + assert!( + err.contains( + "branch 'b1' of table 'default.on_branch', which sets scan option 'scan.mode'" + ), + "{err}" + ); + assert_eq!(sorted_ids(&run(&ctx, &query("on_main")).await), vec![1]); +} + +#[tokio::test] +async fn test_audit_log_refuses_the_session_options_its_ordinary_read_refuses() { + let tmp = TempDir::new().unwrap(); + let ctx = context(&tmp).await; + run( + &ctx, + "CREATE TABLE paimon.default.pk (id INT NOT NULL, PRIMARY KEY (id)) WITH ('bucket' = '1')", + ) + .await; + run(&ctx, "INSERT INTO paimon.default.pk VALUES (1)").await; + run( + &ctx, + "SET 'paimon.table-read.sequence-number.enabled' = 'true'", + ) + .await; + for sql in [ + "SELECT * FROM paimon.default.pk$audit_log", + "SELECT * FROM paimon_incremental_query('default.pk$audit_log', 0, 1)", + ] { + let err = expect_error(&ctx, sql).await; + assert!( + err.contains( + "table-read.sequence-number.enabled is not supported by dynamic options for $audit_log" + ), + "{sql}: {err}" + ); + } +} diff --git a/crates/paimon/src/spec/core_options.rs b/crates/paimon/src/spec/core_options.rs index c98509752..9c1b2ff50 100644 --- a/crates/paimon/src/spec/core_options.rs +++ b/crates/paimon/src/spec/core_options.rs @@ -1391,6 +1391,34 @@ impl<'a> CoreOptions<'a> { selectors } + /// Options that choose which snapshots or files a scan reads: the + /// time-travel selectors, an incremental range or timestamp window, a file + /// creation-time filter, and a `scan.mode` other than `default`. A read + /// whose range its caller sets, such as an incremental query, cannot also + /// honor them. + pub fn configured_scan_range_options(&self) -> Vec<&'static str> { + let mut options = self.configured_time_travel_selectors(); + options.extend( + [ + INCREMENTAL_BETWEEN_OPTION, + INCREMENTAL_BETWEEN_TIMESTAMP_OPTION, + INCREMENTAL_BETWEEN_SCAN_MODE_OPTION, + "scan.file-creation-time-millis", + "scan.creation-time-millis", + ] + .into_iter() + .filter(|key| self.options.contains_key(*key)), + ); + if self + .options + .get(SCAN_MODE_OPTION) + .is_some_and(|mode| !mode.eq_ignore_ascii_case("default")) + { + options.push(SCAN_MODE_OPTION); + } + options + } + /// Validates and normalizes the internal time-travel selector. /// /// This is the semantic owner for selector mutual exclusion and strict @@ -3786,6 +3814,38 @@ mod tests { .is_err()); } + #[test] + fn test_scan_range_options_list_each_snapshot_or_file_selector() { + for (key, value) in [ + (SCAN_SNAPSHOT_ID_OPTION, "1"), + (SCAN_TAG_NAME_OPTION, "t"), + (SCAN_VERSION_OPTION, "1"), + (SCAN_TIMESTAMP_MILLIS_OPTION, "1"), + (SCAN_TIMESTAMP_OPTION, "2024-01-01 00:00:00"), + (SCAN_WATERMARK_OPTION, "1"), + (INCREMENTAL_BETWEEN_OPTION, "1,2"), + (INCREMENTAL_BETWEEN_TIMESTAMP_OPTION, "1,2"), + (INCREMENTAL_BETWEEN_SCAN_MODE_OPTION, "delta"), + ("scan.file-creation-time-millis", "1"), + ("scan.creation-time-millis", "1"), + (SCAN_MODE_OPTION, "latest"), + ] { + let options = HashMap::from([(key.to_string(), value.to_string())]); + assert_eq!( + CoreOptions::new(&options).configured_scan_range_options(), + vec![key], + "{key}" + ); + } + let options = HashMap::from([ + (SCAN_MODE_OPTION.to_string(), "DEFAULT".to_string()), + ("read.batch-size".to_string(), "2".to_string()), + ]); + assert!(CoreOptions::new(&options) + .configured_scan_range_options() + .is_empty()); + } + #[test] fn test_watermark_conflicts_with_other_selectors() { let options = HashMap::from([ diff --git a/crates/paimon/src/table/data_evolution_nested.rs b/crates/paimon/src/table/data_evolution_nested.rs index c75203f64..9254b1c52 100644 --- a/crates/paimon/src/table/data_evolution_nested.rs +++ b/crates/paimon/src/table/data_evolution_nested.rs @@ -43,10 +43,13 @@ pub(super) struct NestedSelection { pub whole_source: Option, } -/// Return `None` only when no file supplies any leaf of this ROW. +/// Return `None` only when no file supplies any leaf of this ROW. In change +/// events (`change_events`), a child no file wrote reads as NULL even when it +/// is NOT NULL. pub(super) fn select_nested_row( read_field: &DataField, sources: &[PhysicalRowSource<'_>], + change_events: bool, ) -> Result> { let DataType::Row(read_row) = read_field.data_type() else { return Err(Error::DataInvalid { @@ -123,7 +126,7 @@ pub(super) fn select_nested_row( ), }); } - if providers.is_empty() && !child.data_type().is_nullable() { + if providers.is_empty() && !change_events && !child.data_type().is_nullable() { return Err(Error::DataInvalid { message: format!( "Cannot read non-nullable nested field '{}.{}' without a provider", @@ -516,7 +519,7 @@ mod tests { fields: &whole, }]; assert_eq!( - select_nested_row(&requested, &sources) + select_nested_row(&requested, &sources, false) .unwrap() .unwrap() .whole_source, @@ -535,7 +538,9 @@ mod tests { fields: &older, }, ]; - let selection = select_nested_row(&requested, &sources).unwrap().unwrap(); + let selection = select_nested_row(&requested, &sources, false) + .unwrap() + .unwrap(); assert_eq!(selection.children, vec![Some(1), Some(1)]); assert_eq!(selection.whole_source, None); let old_request = source_read_field(&requested, ¤t, 1, &older, &selection).unwrap(); @@ -559,7 +564,7 @@ mod tests { fields: &base, }, ]; - let selected = select_nested_row(&row(vec![name(), age()]), &sources) + let selected = select_nested_row(&row(vec![name(), age()]), &sources, false) .unwrap() .unwrap(); assert_eq!(selected.anchors, vec![0, 1]); @@ -581,7 +586,7 @@ mod tests { fields: &base, }, ]; - let selected = select_nested_row(&row(vec![name()]), &sources) + let selected = select_nested_row(&row(vec![name()]), &sources, false) .unwrap() .unwrap(); assert_eq!(selected.anchors, vec![0, 1]); @@ -605,7 +610,9 @@ mod tests { fields: &base_file, }, ]; - let selection = select_nested_row(&requested, &sources).unwrap().unwrap(); + let selection = select_nested_row(&requested, &sources, false) + .unwrap() + .unwrap(); let anchor = source_read_field(&requested, ¤t, 0, &age_file, &selection).unwrap(); let provider = source_read_field(&requested, ¤t, 1, &base_file, &selection).unwrap(); @@ -631,7 +638,9 @@ mod tests { source_index: 0, fields: &physical, }]; - let selection = select_nested_row(&requested, &sources).unwrap().unwrap(); + let selection = select_nested_row(&requested, &sources, false) + .unwrap() + .unwrap(); let read = source_read_field(&requested, ¤t, 0, &physical, &selection).unwrap(); let DataType::Row(read_row) = read.data_type() else { panic!("read field must be ROW") @@ -656,7 +665,7 @@ mod tests { fields: &older, }, ]; - let selected = select_nested_row(&row(vec![name(), age()]), &sources) + let selected = select_nested_row(&row(vec![name(), age()]), &sources, false) .unwrap() .unwrap(); assert_eq!(selected.anchors, vec![0]); @@ -683,7 +692,7 @@ mod tests { }, ]; assert!(matches!( - select_nested_row(&read, &sources), + select_nested_row(&read, &sources, false), Err(Error::Unsupported { message }) if message.contains("profile.address") )); } @@ -697,11 +706,27 @@ mod tests { fields: &available, }]; assert!(matches!( - select_nested_row(&row(vec![required]), &sources), + select_nested_row(&row(vec![required]), &sources, false), Err(Error::DataInvalid { message, .. }) if message.contains("profile.required") )); } + #[test] + fn change_events_read_a_missing_non_nullable_subfield_as_null() { + let required = field(4, "required", DataType::Int(IntType::with_nullable(false))); + let available = vec![row(vec![name()])]; + let sources = [PhysicalRowSource { + source_index: 0, + fields: &available, + }]; + let selected = select_nested_row(&row(vec![name(), required]), &sources, true) + .unwrap() + .unwrap(); + assert_eq!(selected.anchors, vec![0]); + assert_eq!(selected.children, vec![Some(0), None]); + assert_eq!(selected.whole_source, None); + } + fn struct_batch( names: Vec>, ages: Vec>, diff --git a/crates/paimon/src/table/data_evolution_reader.rs b/crates/paimon/src/table/data_evolution_reader.rs index 6b0bff0c7..20696535f 100644 --- a/crates/paimon/src/table/data_evolution_reader.rs +++ b/crates/paimon/src/table/data_evolution_reader.rs @@ -1142,6 +1142,7 @@ impl DataEvolutionReader { partition_fields: &partition_fields, row_id_pushdown: row_ranges.is_some(), nested_enabled, + change_events: split.is_streaming(), }, )?; @@ -2518,6 +2519,11 @@ struct SourcePlanReadContext<'a> { partition_fields: &'a [DataField], row_id_pushdown: bool, nested_enabled: bool, + /// The split holds change events rather than a materialized snapshot + /// ([`DataSplit::is_streaming`]). A field no file of the group wrote was + /// not written in the planned range, so it reads as NULL, NOT NULL fields + /// included. + change_events: bool, } #[cfg(test)] @@ -2537,6 +2543,7 @@ fn build_source_plan( partition_fields: &[], row_id_pushdown: false, nested_enabled: false, + change_events: false, }, ) } @@ -2553,6 +2560,7 @@ fn build_source_plan_with_row_id_pushdown( partition_fields, row_id_pushdown, nested_enabled, + change_events, } = context; let mut sources = Vec::new(); let mut normal_providers: HashMap = HashMap::new(); // field_id -> source_idx @@ -2697,9 +2705,11 @@ fn build_source_plan_with_row_id_pushdown( }, ) .collect::>(); - if let Some(selection) = - super::data_evolution_nested::select_nested_row(field, &physical_sources)? - { + if let Some(selection) = super::data_evolution_nested::select_nested_row( + field, + &physical_sources, + change_events, + )? { if selection.whole_source.is_none() { let current_field = table_fields .iter() @@ -2771,6 +2781,7 @@ fn build_source_plan_with_row_id_pushdown( column_plan.push(Some((source_idx, field_offset))); nested_plan.push(None); } else if !field.data_type().is_nullable() + && !change_events && !partition_fields .iter() .any(|partition| partition.id() == field.id()) @@ -4602,6 +4613,7 @@ mod tests { partition_fields: &[], row_id_pushdown: true, nested_enabled: false, + change_events: false, }, ) .unwrap_err(); @@ -4610,6 +4622,52 @@ mod tests { if message.contains("non-nullable field 'payload'"))); } + #[test] + fn change_events_read_a_field_no_file_wrote_as_null() { + // Two column updates of one file, neither of which wrote `id`. + let files = vec![ + data_file("a-update.parquet", 0, 4, 2, Some(vec!["a"])), + data_file("b-update.parquet", 0, 4, 3, Some(vec!["b"])), + ]; + let prepared_group = PreparedMergeGroup::new(&files).unwrap(); + // Newest first: the `b` update, then the `a` update. + let file_infos = vec![resolved_info(vec![3]), resolved_info(vec![2])]; + let read_type = vec![ + DataField::new( + 1, + "id".to_string(), + DataType::Int(IntType::with_nullable(false)), + ), + DataField::new(2, "a".to_string(), DataType::Int(IntType::new())), + DataField::new(3, "b".to_string(), DataType::Int(IntType::new())), + ]; + let plan = |change_events| { + build_source_plan_with_row_id_pushdown( + &prepared_group, + &file_infos, + &read_type, + &HashSet::new(), + SourcePlanReadContext { + table_fields: &read_type, + partition_fields: &[], + row_id_pushdown: false, + nested_enabled: false, + change_events, + }, + ) + }; + + assert!( + matches!(plan(false), Err(Error::DataInvalid { message, .. }) + if message.contains("non-nullable field 'id'")) + ); + let source_plan = plan(true).unwrap(); + assert_eq!( + source_plan.column_plan, + vec![None, Some((1, 0)), Some((0, 0))] + ); + } + #[test] fn test_build_source_plan_validates_complete_range_only_for_read_blob_source() { let files = vec![ @@ -4694,6 +4752,7 @@ mod tests { partition_fields: &[], row_id_pushdown: false, nested_enabled: false, + change_events: false, }, ) .unwrap(); diff --git a/crates/paimon/tests/incremental_batch_scan_test.rs b/crates/paimon/tests/incremental_batch_scan_test.rs index 60eb2846f..d4a4b87e4 100644 --- a/crates/paimon/tests/incremental_batch_scan_test.rs +++ b/crates/paimon/tests/incremental_batch_scan_test.rs @@ -2102,3 +2102,162 @@ async fn delta_reads_each_commit_as_its_stream_follow_up() { ); } } + +/// A range of two column updates of the same rows, without the rows they +/// update: every reader of the combined plan returns the updated columns and +/// reads the column no file of the range wrote as NULL, NOT NULL `id` +/// included. +#[tokio::test] +async fn combined_delta_reads_the_columns_a_range_did_not_write_as_null() { + use arrow_array::{ArrayRef, Int64Array, StringArray}; + use paimon::spec::{DataType, IntType, Schema, TableSchema, VarCharType}; + use paimon::table::{IncrementalPlan, IncrementalSplit}; + use std::collections::HashMap; + use std::sync::Arc; + + let path = "memory:/incremental_batch/column_updates_only"; + let schema = Schema::builder() + .column("id", DataType::Int(IntType::with_nullable(false))) + .column("name", DataType::VarChar(VarCharType::string_type())) + .column("value", DataType::Int(IntType::new())) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .build() + .unwrap(); + let (io, table) = memory_table(path, TableSchema::new(0, &schema)); + setup_dirs(&io, path).await; + persist_table_schema(&io, path, table.schema()).await; + let rows = RecordBatch::try_from_iter_with_nullable([ + ( + "id", + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + false, + ), + ( + "name", + Arc::new(StringArray::from(vec!["alice", "bob"])), + true, + ), + ("value", Arc::new(Int32Array::from(vec![10, 20])), true), + ]) + .unwrap(); + write_batch(&table, &rows).await; + // Snapshot 2 rewrites `name` of row 0 and snapshot 3 `value` of row 1. + for (column, row_id, values) in [ + ( + "name", + 0, + Arc::new(StringArray::from(vec!["ALICE"])) as ArrayRef, + ), + ("value", 1, Arc::new(Int32Array::from(vec![99]))), + ] { + // Distinct commit times, for the timestamp window below. + std::thread::sleep(std::time::Duration::from_millis(5)); + let builder = table.new_write_builder(); + let mut update = builder + .new_data_evolution_writer(vec![column.into()]) + .unwrap(); + update + .add_matched_batch( + RecordBatch::try_from_iter([ + ( + "_ROW_ID", + Arc::new(Int64Array::from(vec![row_id])) as ArrayRef, + ), + (column, values), + ]) + .unwrap(), + ) + .unwrap(); + builder + .new_commit() + .commit(update.prepare_commit().await.unwrap()) + .await + .unwrap(); + } + type Row = (Option, Option, Option); + let collect = |batches: &[RecordBatch]| { + let mut rows: Vec = Vec::new(); + for batch in batches { + let column = |name: &str| batch.column_by_name(name).unwrap().clone(); + let (id, name, value) = (column("id"), column("name"), column("value")); + let id = id.as_any().downcast_ref::().unwrap(); + let name = name.as_any().downcast_ref::().unwrap(); + let value = value.as_any().downcast_ref::().unwrap(); + for row in 0..batch.num_rows() { + rows.push(( + id.is_valid(row).then(|| id.value(row)), + name.is_valid(row).then(|| name.value(row).to_string()), + value.is_valid(row).then(|| value.value(row)), + )); + } + } + rows.sort(); + rows + }; + let updates: Vec = vec![ + (None, Some("ALICE".into()), Some(10)), + (None, Some("bob".into()), Some(99)), + ]; + + let builder = table.new_read_builder(); + let splits = builder + .new_incremental_scan(IncrementalScanMode::Delta, 1, 3) + .plan_combined_delta() + .await + .unwrap() + .into_splits(); + let read = builder.new_read().unwrap(); + let batches: Vec = read.to_arrow(&splits).unwrap().try_collect().await.unwrap(); + assert_eq!(collect(&batches), updates, "to_arrow"); + let plan = IncrementalPlan::try_new( + IncrementalScanMode::Delta, + splits.into_iter().map(IncrementalSplit::Data).collect(), + ) + .unwrap(); + for (reader, stream) in [ + ("to_incremental_arrow", read.to_incremental_arrow(&plan)), + ("to_audit_log_arrow", read.to_audit_log_arrow(&plan)), + ] { + let batches: Vec = stream.unwrap().try_collect().await.unwrap(); + assert_eq!(collect(&batches), updates, "{reader}"); + } + + // An ordinary scan given the same range as a timestamp window plans it + // the same way. + let snapshots = table.snapshot_manager(); + let start = snapshots.get_snapshot(1).await.unwrap().time_millis(); + let end = snapshots.get_snapshot(3).await.unwrap().time_millis(); + let windowed = table.copy_with_options(HashMap::from([( + "incremental-between-timestamp".to_string(), + format!("{start},{end}"), + )])); + let builder = windowed.new_read_builder(); + let plan = builder.new_scan().plan().await.unwrap(); + let batches: Vec = builder + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(collect(&batches), updates, "incremental-between-timestamp"); + + // The table itself still reads whole rows. + let builder = table.new_read_builder(); + let plan = builder.new_scan().plan().await.unwrap(); + let batches: Vec = builder + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + let current: Vec = vec![ + (Some(1), Some("ALICE".into()), Some(10)), + (Some(2), Some("bob".into()), Some(99)), + ]; + assert_eq!(collect(&batches), current, "snapshot read"); +} diff --git a/docs/src/sql.md b/docs/src/sql.md index 957c872f9..f921ecf8f 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -69,14 +69,15 @@ async fn example() -> Result<(), Box> { } ``` -`SQLContext::new` creates a session context with the Paimon relation planner and +`SQLContext::new` creates a session context with the Paimon relation planner, the catalog-independent `path_to_descriptor` and `descriptor_to_string` scalar -functions pre-registered. Use `register_catalog(...).await` to add one or more +functions and the `paimon_incremental_query` table-valued function +pre-registered. Use `register_catalog(...).await` to add one or more Paimon catalogs; registering a catalog also registers the built-in scalar function `blob_view` (alias `sys.blob_view`) and the built-in table-valued -functions (`vector_search`, `hybrid_search`, `paimon_incremental_query`, and -`full_text_search` when the `fulltext` feature is enabled) against it. It also -manages session-scoped dynamic options internally for `SET`/`RESET` support. +functions (`vector_search`, `hybrid_search`, and `full_text_search` when the +`fulltext` feature is enabled) against it. It also manages session-scoped +dynamic options internally for `SET`/`RESET` support. ### REST Catalog Views and SQL Functions @@ -2326,8 +2327,9 @@ The optional fourth argument selects the mode: A data-evolution table's range is read as one batch, as Java's `incremental-between` scan reads it, so a column update merges into the rows it updates when they were written in the range. For rows written before the range it returns every row of the - files the update rewrote, unchanged rows included, with only the updated columns - set and the others NULL, NOT NULL columns included. + files the updates rewrote, unchanged rows included, with only the updated columns + set and the others NULL, NOT NULL columns included. (Java's reader instead fails + when two such updates of the same rows leave a NOT NULL column unwritten.) - `changelog` returns the changelog rows of each commit. - `diff` compares the table at the two snapshots and returns the added and changed rows (`$audit_log` adds `-U` and `-D` rows). It requires a primary-key table with @@ -2338,6 +2340,15 @@ The optional fourth argument selects the mode: A `$audit_log` suffix puts `rowkind` first, followed by `_SEQUENCE_NUMBER` when `table-read.sequence-number.enabled=true`. +The table name resolves as in a `FROM` clause of the same session: `my_table` and +`my_db.my_table` use the current catalog (and database), and +`my_catalog.my_db.my_table` names the catalog. Dynamic options set with +`SET 'paimon.…'` apply as they do to a query on the table, except options that +choose another range to read (time-travel selectors, `incremental-between*`, file +creation-time filters and a `scan.mode` other than `default`), which are rejected +whether the session or the table read sets them. With `SET 'paimon.branch'`, the +table read is that branch. + ## Dynamic Options (SET / RESET) Use `SET` to configure session-scoped Paimon dynamic options that apply to subsequent table loads: From 8f26c02d8ad449f8a89e6eb273a6d5da58614a4e Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Fri, 23 Oct 2026 17:23:35 +0800 Subject: [PATCH 04/18] fix(core): decode change events of BLOB/VECTOR rewrites file by file `to_incremental_arrow` merged every data-evolution row-id group of a range, so a range that rewrote only VECTOR columns, such as recomputed embeddings, failed with "requires at least one normal data file", as did one batch of two BLOB rewrites. Base decoded them file by file. Change events now merge a group only when its files are column slices of one row range: its normal files cover the same rows, its BLOB and VECTOR files lie within them, and each VECTOR field is written in one layout. Other groups are decoded file by file again, and incremental plans keep groups whole, so every projection reads the same rows. A BLOB file read alone honors `blob-as-descriptor`, and change events skip the managed-BLOB preselection, whose row ids can name rows of several files. Merged change events read the rows no BLOB or VECTOR file of the range wrote as NULL, NOT NULL fields included. The BLOB and video readers build the nullable read schema, so a NOT NULL field reads a placeholder, a row an update left unwritten, as NULL. --- .../datafusion/tests/incremental_query.rs | 190 +++++ crates/paimon/src/arrow/format/blob.rs | 61 +- crates/paimon/src/arrow/format/video.rs | 15 +- .../paimon/src/table/data_evolution_reader.rs | 299 ++++++- crates/paimon/src/table/source.rs | 65 ++ crates/paimon/src/table/table_scan.rs | 5 +- .../tests/incremental_batch_scan_test.rs | 772 ++++++++++++++++++ docs/src/sql.md | 18 +- 8 files changed, 1353 insertions(+), 72 deletions(-) diff --git a/crates/integrations/datafusion/tests/incremental_query.rs b/crates/integrations/datafusion/tests/incremental_query.rs index 5c2dfd3e9..c673d63ba 100644 --- a/crates/integrations/datafusion/tests/incremental_query.rs +++ b/crates/integrations/datafusion/tests/incremental_query.rs @@ -735,6 +735,7 @@ async fn test_table_name_resolves_as_in_a_from_clause() { "{err}" ); } + #[tokio::test] async fn test_session_read_options_apply_as_in_an_ordinary_read() { use datafusion::arrow::util::pretty::pretty_format_batches; @@ -1018,3 +1019,192 @@ async fn test_audit_log_refuses_the_session_options_its_ordinary_read_refuses() ); } } + +/// A range that only rewrites a VECTOR column of rows written before it, as a +/// recomputed embedding does, is read as the files it wrote: the rewritten +/// vectors, with the columns the range did not write NULL. A range with the +/// insert merges the rewrite into the inserted rows. DataFusion always pushes a +/// projection, so `SELECT *` and a single column take the projected path. +#[tokio::test] +async fn test_range_of_vector_rewrites_reads_the_rewritten_vectors() { + use datafusion::arrow::array::{ + Array, ArrayRef, FixedSizeListArray, Float32Array, Int32Array, Int64Array, StringArray, + }; + use datafusion::arrow::datatypes::{DataType as ArrowDataType, Field, Schema as ArrowSchema}; + use paimon::spec::{DataType, FloatType, IntType, Schema, VarCharType, VectorType}; + + let tmp = TempDir::new().unwrap(); + let ctx = context(&tmp).await; + let warehouse = catalog(&tmp); + let identifier = Identifier::new("default", "vectors"); + let vector = VectorType::try_new(true, 2, DataType::Float(FloatType::new())).unwrap(); + let schema = Schema::builder() + .column("id", DataType::Int(IntType::with_nullable(false))) + .column("name", DataType::VarChar(VarCharType::string_type())) + .column("embedding", DataType::Vector(vector)) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("vector.file.format", "parquet") + .build() + .unwrap(); + warehouse + .create_database("default", true, std::collections::HashMap::new()) + .await + .unwrap(); + warehouse + .create_table(&identifier, schema, false) + .await + .unwrap(); + let element = Arc::new(Field::new("element", ArrowDataType::Float32, true)); + let embedding_field = Field::new( + "embedding", + ArrowDataType::FixedSizeList(element.clone(), 2), + true, + ); + let embeddings = |values: &[[f32; 2]]| -> ArrayRef { + use datafusion::arrow::array::builder::{FixedSizeListBuilder, Float32Builder}; + let mut builder = + FixedSizeListBuilder::new(Float32Builder::new(), 2).with_field(element.clone()); + for value in values { + builder.values().append_slice(value); + builder.append(true); + } + Arc::new(builder.finish()) + }; + let table = warehouse.get_table(&identifier).await.unwrap(); + let rows = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + Field::new("id", ArrowDataType::Int32, false), + Field::new("name", ArrowDataType::Utf8, true), + embedding_field.clone(), + ])), + vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + Arc::new(StringArray::from(vec!["alice", "bob"])), + embeddings(&[[1.0, 0.0], [0.0, 1.0]]), + ], + ) + .unwrap(); + let builder = table.new_write_builder(); + let mut writer = builder.new_write().unwrap(); + writer.write_arrow_batch(&rows).await.unwrap(); + builder + .new_commit() + .commit(writer.prepare_commit().await.unwrap()) + .await + .unwrap(); + // Snapshots 2 and 3 recompute the embedding of row 0, then of row 1. + for (row_id, value) in [(0, [9.0, 9.0]), (1, [7.0, 7.0])] { + let mut update = builder + .new_data_evolution_writer(vec!["embedding".to_string()]) + .unwrap(); + update + .add_matched_batch( + RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + Field::new("_ROW_ID", ArrowDataType::Int64, false), + embedding_field.clone(), + ])), + vec![ + Arc::new(Int64Array::from(vec![row_id])), + embeddings(&[value]), + ], + ) + .unwrap(), + ) + .unwrap(); + builder + .new_commit() + .commit(update.prepare_commit().await.unwrap()) + .await + .unwrap(); + } + + type Row = (Option, Option, Option>); + let collect = |batches: &[RecordBatch]| { + let mut rows: Vec = Vec::new(); + for batch in batches { + let column = |name: &str| batch.column_by_name(name).map(|c| c.as_any()); + let ids = column("id").map(|c| c.downcast_ref::().unwrap()); + // SQL reads strings as Utf8View; core streams Utf8. + let names = batch.column_by_name("name").map(|names| { + datafusion::arrow::compute::cast(names, &ArrowDataType::Utf8).unwrap() + }); + let names = names + .as_ref() + .map(|c| c.as_any().downcast_ref::().unwrap()); + let vectors = + column("embedding").map(|c| c.downcast_ref::().unwrap()); + for row in 0..batch.num_rows() { + let vector = vectors.filter(|v| v.is_valid(row)).map(|v| { + let values = v.value(row); + let values = values.as_any().downcast_ref::().unwrap(); + values.values().iter().map(|value| *value as i32).collect() + }); + rows.push(( + ids.filter(|ids| ids.is_valid(row)) + .map(|ids| ids.value(row)), + names + .filter(|n| n.is_valid(row)) + .map(|n| n.value(row).to_string()), + vector, + )); + } + } + rows.sort(); + rows + }; + let vector = |values: [i32; 2]| -> Row { (None, None, Some(values.to_vec())) }; + let count = |batches: &[RecordBatch]| { + let counts = batches[0].column(0).as_any().downcast_ref::(); + counts.unwrap().value(0) + }; + for mode in ["", ", 'delta'"] { + for name in ["vectors", "vectors$audit_log"] { + let query = |columns: &str, start, end| { + format!( + "SELECT {columns} FROM paimon_incremental_query('default.{name}', {start}, {end}{mode})" + ) + }; + let label = format!("{name}{mode}"); + for (start, end, expected) in [ + (1, 2, vec![vector([0, 1]), vector([9, 9])]), + (2, 3, vec![vector([7, 7]), vector([9, 9])]), + ( + 1, + 3, + vec![ + vector([0, 1]), + vector([7, 7]), + vector([9, 9]), + vector([9, 9]), + ], + ), + // With the insert, the first rewrite merges into its rows. + ( + 0, + 2, + vec![ + (Some(1), Some("alice".into()), Some(vec![9, 9])), + (Some(2), Some("bob".into()), Some(vec![0, 1])), + ], + ), + ] { + let label = format!("{label} ({start},{end}]"); + let rows = collect(&run(&ctx, &query("*", start, end)).await); + assert_eq!(rows, expected, "{label} *"); + let ids = collect(&run(&ctx, &query("id", start, end)).await); + let expected_ids: Vec = expected.iter().map(|r| (r.0, None, None)).collect(); + assert_eq!(ids, expected_ids, "{label} id"); + let count = count(&run(&ctx, &query("count(*)", start, end)).await); + assert_eq!(count, expected.len() as i64, "{label} count"); + } + } + } + let rows = collect(&run(&ctx, "SELECT * FROM paimon.default.vectors").await); + let current: Vec = vec![ + (Some(1), Some("alice".into()), Some(vec![9, 9])), + (Some(2), Some("bob".into()), Some(vec![7, 7])), + ]; + assert_eq!(rows, current, "ordinary read"); +} diff --git a/crates/paimon/src/arrow/format/blob.rs b/crates/paimon/src/arrow/format/blob.rs index 608938db3..2df7f0620 100644 --- a/crates/paimon/src/arrow/format/blob.rs +++ b/crates/paimon/src/arrow/format/blob.rs @@ -22,7 +22,7 @@ use super::delta_varint::{ use super::delta_varint::{decode_varint, encode_varint}; use super::metadata_cache::FileMetadataCache; use super::{FilePredicates, FormatFileReader}; -use crate::arrow::build_target_arrow_schema; +use crate::arrow::build_read_arrow_schema; use crate::io::{BlobIndexCacheContext, FileRead}; use crate::spec::{BlobDescriptor, DataField, DataType}; use crate::table::{ArrowRecordBatchStream, RowRange}; @@ -223,7 +223,10 @@ impl FormatFileReader for BlobFormatReader { } let field_kind = validate_read_fields(read_fields)?; - let target_schema = build_target_arrow_schema(read_fields)?; + // A placeholder, a row an update left unwritten, reads as NULL, so + // the batch takes the nullable schema of the table readers: a NOT NULL + // field read as change events can hold it. + let target_schema = build_read_arrow_schema(read_fields)?; let batch_size = batch_size.unwrap_or(DEFAULT_BATCH_SIZE); let blob_reader = IndexedBlobReader::open_with_parallelism( reader, @@ -3677,34 +3680,36 @@ mod tests { #[tokio::test] async fn test_blob_reader_treats_java_placeholders_as_null() { - let read_fields = vec![DataField::new( - 0, - "payload".to_string(), - DataType::Blob(BlobType::new()), - )]; let file_bytes = load_blob_fixture("blob-placeholder.blob"); + // A NOT NULL field reads them as NULL too. + for blob_type in [BlobType::new(), BlobType::with_nullable(false)] { + let read_fields = vec![DataField::new( + 0, + "payload".to_string(), + DataType::Blob(blob_type), + )]; + let batches = BlobFormatReader::new(String::new(), false) + .read_batch_stream( + Box::new(BytesFileRead(Bytes::from(file_bytes.clone()))), + file_bytes.len() as u64, + &read_fields, + None, + Some(2), + None, + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); - let batches = BlobFormatReader::new(String::new(), false) - .read_batch_stream( - Box::new(BytesFileRead(Bytes::from(file_bytes.clone()))), - file_bytes.len() as u64, - &read_fields, - None, - Some(2), - None, - ) - .await - .unwrap() - .try_collect::>() - .await - .unwrap(); - - assert_eq!(batches.len(), 2); - assert_eq!(collect_binary_values(&batches[0]), vec![None, None]); - assert_eq!( - collect_binary_values(&batches[1]), - vec![Some(b"latest-3".to_vec()), None] - ); + assert_eq!(batches.len(), 2); + assert_eq!(collect_binary_values(&batches[0]), vec![None, None]); + assert_eq!( + collect_binary_values(&batches[1]), + vec![Some(b"latest-3".to_vec()), None] + ); + } } #[tokio::test] diff --git a/crates/paimon/src/arrow/format/video.rs b/crates/paimon/src/arrow/format/video.rs index 944ff6a20..c5c9ff793 100644 --- a/crates/paimon/src/arrow/format/video.rs +++ b/crates/paimon/src/arrow/format/video.rs @@ -21,7 +21,7 @@ use super::blob::{build_blob_batch, BlobReadValue}; use super::delta_varint::decode_delta_varints; use super::{FilePredicates, FormatFileReader}; -use crate::arrow::build_target_arrow_schema; +use crate::arrow::build_read_arrow_schema; use crate::io::FileRead; use crate::spec::{DataField, DataType, VideoFrameDescriptor}; use crate::table::{ArrowRecordBatchStream, RowRange}; @@ -271,7 +271,9 @@ impl FormatFileReader for VideoFormatReader { { return Err(corrupt("video format only supports one scalar BLOB field")); } - let schema = build_target_arrow_schema(read_fields)?; + // NULLs and placeholders read as NULL, also in a NOT NULL field, as + // in the BLOB reader. + let schema = build_read_arrow_schema(read_fields)?; let indexed = IndexedVideoReader::open(reader.as_ref(), file_size, self.path.clone()).await?; let mut positions: Box + Send> = match row_selection { @@ -386,7 +388,14 @@ mod tests { let bytes = java_fixture(); let format = VideoFormatReader::new("file:/data.video".into()); let field = DataField::new(0, "video".into(), DataType::Blob(BlobType::new())); - for fields in [vec![field], vec![]] { + // Rows 2 and 3 are a NULL and a placeholder, read as NULL even in a + // NOT NULL field. + let not_null = DataField::new( + 0, + "video".into(), + DataType::Blob(BlobType::with_nullable(false)), + ); + for fields in [vec![field], vec![not_null], vec![]] { let stream = format .read_batch_stream( Box::new(BytesFileRead(bytes.clone())), diff --git a/crates/paimon/src/table/data_evolution_reader.rs b/crates/paimon/src/table/data_evolution_reader.rs index 20696535f..7fe2978f6 100644 --- a/crates/paimon/src/table/data_evolution_reader.rs +++ b/crates/paimon/src/table/data_evolution_reader.rs @@ -38,7 +38,7 @@ use crate::spec::{ }; use crate::table::dedicated_format_file_writer::is_blob_or_video_file_name; use crate::table::schema_manager::SchemaManager; -use crate::table::source::any_range_overlaps_file; +use crate::table::source::{any_range_overlaps_file, is_data_evolution_column_slice_group}; use crate::table::stats_filter::group_by_overlapping_row_id; use crate::table::{ArrowRecordBatchStream, RESTEnv, RowRange}; use crate::{DataSplit, Error}; @@ -267,6 +267,29 @@ impl DataEvolutionReader { } } + /// Whether a row-id group of change events merges into rows: its files + /// are column slices of one row range, and each VECTOR field is written + /// by files of one layout. Other groups are read file by file. + async fn merges_change_events(&self, files: &[DataFileMeta]) -> crate::Result { + if !is_data_evolution_column_slice_group(files) { + return Ok(false); + } + if !files + .iter() + .any(|file| is_vector_store_file_name(&file.file_name)) + { + return Ok(true); + } + let (infos, _) = load_file_infos( + &self.schema_manager, + self.table_schema_id, + &self.table_fields, + files, + ) + .await?; + Ok(!vector_field_has_two_layouts(files, &infos)) + } + /// Managed BLOB payloads are fetched by the format reader, before the /// residual predicate. A LIMIT with a predicate on ordinary columns must /// therefore select matching row IDs before opening those payloads. @@ -294,6 +317,10 @@ impl DataEvolutionReader { .iter() .all(|file| file.first_row_id.is_some()) }) + // Change events may read a row-id group file by file, where a row + // id names a row of each file, so selecting by row id would + // return rows of files the predicate did not match. + || splits.iter().any(DataSplit::is_streaming) { return false; } @@ -525,7 +552,12 @@ impl DataEvolutionReader { if remaining == Some(0) { break 'splits; } - if is_raw_convertible(&files) { + // Change events merge a group only as column slices of + // one row range (`merges_change_events`). Any other group, + // such as a VECTOR rewrite alone, is read file by file. + if is_raw_convertible(&files) + || (split.is_streaming() && !self.merges_change_events(&files).await?) + { for file_meta in files { if remaining == Some(0) { break 'splits; @@ -634,6 +666,11 @@ impl DataEvolutionReader { let mut row_id_offset: usize = 0; let mut stream = raw_file_reader.clone() + // A BLOB file of change events read on its own + // yields what the merged sources yield. + .with_blob_as_descriptor( + split.is_streaming() && self.blob_as_descriptor, + ) .with_nested_field_enabled( CoreOptions::new(&self.table_options) .data_evolution_nested_field_enabled() @@ -1224,6 +1261,7 @@ impl DataEvolutionReader { mosaic_prefetch, read_timing.clone(), anchor_deletion_vector.as_ref(), + split.is_streaming(), ) .map(Some) } @@ -1857,6 +1895,7 @@ fn open_source_stream( mosaic_prefetch: MosaicPrefetchOptions, read_timing: Option>, anchor_deletion_vector: Option<&DeletionVectorContext>, + change_events: bool, ) -> crate::Result { let mut row_ranges = row_ranges; if let FieldSource::BlobBunch { bunch, read_fields } = source { @@ -1866,33 +1905,42 @@ fn open_source_stream( row_ranges.as_deref(), anchor_deletion_vector.map(|context| context.deletion_vector.as_ref()), )?; - if let Some(selected_ranges) = selected_ranges.as_deref() { - let uncovered_ranges = - crate::table::source::exclude_row_ranges(selected_ranges, bunch.logical_ranges()); - if !uncovered_ranges.is_empty() { - if read_fields + // Change events read the rows of the group that no BLOB file of the + // range wrote as NULL, NOT NULL fields included. + let uncovered_ranges = match selected_ranges.as_deref() { + Some(selected_ranges) => { + crate::table::source::exclude_row_ranges(selected_ranges, bunch.logical_ranges()) + } + None if change_events => crate::table::source::exclude_row_ranges( + &[bunch.expected_range()?], + bunch.logical_ranges(), + ), + None => Vec::new(), + }; + if !uncovered_ranges.is_empty() { + if !change_events + && read_fields .iter() .any(|field| !field.data_type().is_nullable()) - { - return Err(Error::DataInvalid { - message: format!( - "Cannot NULL-fill uncovered ranges {uncovered_ranges:?} for non-nullable BLOB field" - ), - source: None, - }); - } - return blob_fallback::read( - split, - bunch.clone(), - read_fields.clone(), - row_ranges, - batch_size, - file_io, - blob_as_descriptor, - blob_parallelism, - anchor_deletion_vector.cloned(), - ); + { + return Err(Error::DataInvalid { + message: format!( + "Cannot NULL-fill uncovered ranges {uncovered_ranges:?} for non-nullable BLOB field" + ), + source: None, + }); } + return blob_fallback::read( + split, + bunch.clone(), + read_fields.clone(), + row_ranges, + batch_size, + file_io, + blob_as_descriptor, + blob_parallelism, + anchor_deletion_vector.cloned(), + ); } // A single sequence group has no fallback work, so keep per-file lazy streaming. @@ -1967,6 +2015,17 @@ fn open_source_stream( row_ranges.as_deref(), anchor_deletion_vector.map(|context| context.deletion_vector.as_ref()), )?; + // Change events read the rows of the group that no VECTOR file of + // the range wrote as NULL, NOT NULL fields included. + let selected_ranges = match selected_ranges { + None if change_events && bunch.row_count() < bunch.expected_row_count => { + Some(vec![RowRange::new( + bunch.expected_first_row_id, + bunch.expected_first_row_id + bunch.expected_row_count - 1, + )]) + } + selected_ranges => selected_ranges, + }; let files = match selected_ranges.as_deref() { Some([]) => Vec::new(), Some(ranges) => { @@ -1978,10 +2037,11 @@ fn open_source_stream( let uncovered_ranges = crate::table::source::exclude_row_ranges(ranges, &covered_ranges); if !uncovered_ranges.is_empty() { - if source - .read_fields() - .iter() - .any(|field| !field.data_type().is_nullable()) + if !change_events + && source + .read_fields() + .iter() + .any(|field| !field.data_type().is_nullable()) { return Err(Error::DataInvalid { message: format!( @@ -2460,6 +2520,42 @@ fn resolve_field_ids(file: &DataFileMeta, fields: &[DataField]) -> crate::Result } } +/// The key of the vector bunch a vector file joins: its schema, format and +/// normalized write columns. +fn vector_bunch_key( + file: &DataFileMeta, + normalized_write_cols: &[String], +) -> (i64, String, Vec) { + ( + file.schema_id, + vector_format_suffix(&file.file_name), + normalized_write_cols.to_vec(), + ) +} + +/// Whether some VECTOR field is written by files of two vector bunch keys, +/// such as a rewrite after `ALTER TABLE`; the source plan cannot combine +/// their bunches. +fn vector_field_has_two_layouts(files: &[DataFileMeta], infos: &[ResolvedFileInfo]) -> bool { + let mut layouts = HashMap::new(); + for (file, info) in files.iter().zip(infos) { + let Some(columns) = info + .normalized_write_cols + .as_deref() + .filter(|_| is_vector_store_file_name(&file.file_name)) + else { + continue; + }; + let layout = vector_bunch_key(file, columns); + for field_id in &info.field_ids { + if layouts.entry(*field_id).or_insert_with(|| layout.clone()) != &layout { + return true; + } + } + } + false +} + /// Lowercased final filename extension, used as a vector bunch's format identifier. /// `"data.vector.parquet" -> "parquet"`, `"emb-1.vector.vortex" -> "vortex"`. fn vector_format_suffix(file_name: &str) -> String { @@ -2520,8 +2616,9 @@ struct SourcePlanReadContext<'a> { row_id_pushdown: bool, nested_enabled: bool, /// The split holds change events rather than a materialized snapshot - /// ([`DataSplit::is_streaming`]). A field no file of the group wrote was - /// not written in the planned range, so it reads as NULL, NOT NULL fields + /// ([`DataSplit::is_streaming`]). A field no file of the group wrote, or a + /// row of the group no BLOB or VECTOR file of its field wrote, was not + /// written in the planned range, so it reads as NULL, NOT NULL fields /// included. change_events: bool, } @@ -2614,7 +2711,7 @@ fn build_source_plan_with_row_id_pushdown( source: None, })?; let format_suffix = vector_format_suffix(&file.file_name); - let key = (file.schema_id, format_suffix.clone(), normalized.clone()); + let key = vector_bunch_key(file, &normalized); let source_idx = if let Some(&existing_idx) = vector_bunch_indices.get(&key) { existing_idx @@ -2628,7 +2725,8 @@ fn build_source_plan_with_row_id_pushdown( format_suffix, normalized.clone(), ) - .with_row_id_pushdown(row_id_pushdown), + // Row ranges and change events may leave gaps between files. + .with_row_id_pushdown(row_id_pushdown || change_events), data_fields: info.data_fields.clone(), read_fields: Vec::new(), }); @@ -2799,13 +2897,17 @@ fn build_source_plan_with_row_id_pushdown( } } + // A range's BLOB and VECTOR files may cover only some rows of the change + // events it merges into: a BLOB update writes the rows it updates, and a + // VECTOR update before a compaction merged its normal file with others + // covers the rows of the file it updated. The reader NULL-fills the rest. for source in &mut sources { if let FieldSource::BlobBunch { bunch, read_fields, .. } = source { bunch.finalize()?; - if !read_fields.is_empty() && !row_id_pushdown { + if !read_fields.is_empty() && !row_id_pushdown && !change_events { bunch.validate_logical_range()?; } } @@ -2818,6 +2920,7 @@ fn build_source_plan_with_row_id_pushdown( { if !read_fields.is_empty() && !row_id_pushdown + && !change_events && bunch.row_count() != prepared_group.logical_row_count { return Err(Error::DataInvalid { @@ -6390,6 +6493,132 @@ mod tests { assert_eq!(row, expected.len(), "row count mismatch"); } + /// Change events merge BLOB and VECTOR files that cover only some rows of + /// the normal file of their range, with gaps before, between and after + /// the VECTOR files, and read the rows they do not cover as NULL, NOT NULL + /// fields included. Java's compaction leaves such groups: it merges normal + /// files but not their VECTOR files, so a later update of the merged rows + /// writes a normal file wider than a VECTOR file of the range. A snapshot + /// read of the same files still fails. + #[tokio::test] + async fn change_events_null_fill_rows_no_blob_or_vector_file_wrote() { + let tempdir = tempdir().unwrap(); + let bucket_dir = tempdir.path().join("bucket-0"); + fs::create_dir_all(&bucket_dir).unwrap(); + let normal_path = bucket_dir.join("data.parquet"); + write_int_parquet_file( + &normal_path, + vec![("id", vec![1, 2, 3, 4, 5, 6, 7, 8])], + None, + ); + let size = |path: &Path| path.metadata().unwrap().len() as i64; + // Rows 2-3 and 5 have vectors, leaving gaps before, between and after + // them; row 1 has a picture. + let mut files = vec![data_file_meta_with_path( + "data.parquet", + 0, + 8, + 1, + size(&normal_path), + Some(vec!["id"]), + )]; + for (name, first_row_id, values) in [ + ("emb-2.vector.parquet", 2, vec![[3.0, 3.0], [4.0, 4.0]]), + ("emb-5.vector.parquet", 5, vec![[6.0, 6.0]]), + ] { + let path = bucket_dir.join(name); + let rows: Vec<_> = values.iter().map(|value| Some(value.to_vec())).collect(); + write_fixed_size_list_parquet(&path, "embedding", 2, &rows); + files.push(data_file_meta_with_path( + name, + first_row_id, + rows.len() as i64, + 1, + size(&path), + Some(vec!["embedding"]), + )); + } + let blob_path = bucket_dir.join("payload-1.blob"); + write_blob_file(&blob_path, &[Some(&b"one"[..])]); + files.push(data_file_meta_with_path( + "payload-1.blob", + 1, + 1, + 1, + size(&blob_path), + Some(vec!["payload"]), + )); + + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column( + "embedding", + DataType::Vector( + VectorType::try_new(false, 2, DataType::Float(FloatType::new())).unwrap(), + ), + ) + .column("payload", DataType::Blob(BlobType::with_nullable(false))) + .option("data-evolution.enabled", "true") + .option("row-tracking.enabled", "true") + .build() + .unwrap(); + let table = Table::new( + FileIOBuilder::new("file").build().unwrap(), + Identifier::new("default", "change_event_gaps_t"), + local_file_path(tempdir.path()), + TableSchema::new(0, &schema), + None, + ); + let split = |streaming| { + DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path(local_file_path(&bucket_dir)) + .with_total_buckets(1) + .with_data_files(files.clone()) + .with_streaming(streaming) + .build() + .unwrap() + }; + let read = TableRead::new(&table, table.schema().fields().to_vec(), Vec::new()); + + let batches = read + .to_arrow(&[split(true)]) + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!( + collect_int_values(&batches, "id"), + vec![1, 2, 3, 4, 5, 6, 7, 8] + ); + let vectors = [ + None, + None, + Some(vec![3.0, 3.0]), + Some(vec![4.0, 4.0]), + None, + Some(vec![6.0, 6.0]), + None, + None, + ]; + assert_fixed_size_list(&batches, "embedding", 2, &vectors); + let mut payloads = vec![None; 8]; + payloads[1] = Some(b"one".to_vec()); + assert_eq!(collect_binary_values(&batches, "payload"), payloads); + + let snapshot = read + .to_arrow(&[split(false)]) + .unwrap() + .try_collect::>() + .await; + assert!( + snapshot.is_err(), + "a snapshot read keeps the coverage checks" + ); + } + /// (1) Provider priority: the normal data file ALSO advertises the embedding write_col, /// but the dedicated `.vector.parquet` file must win. #[tokio::test] diff --git a/crates/paimon/src/table/source.rs b/crates/paimon/src/table/source.rs index 01b292aa1..ae355f13e 100644 --- a/crates/paimon/src/table/source.rs +++ b/crates/paimon/src/table/source.rs @@ -34,6 +34,23 @@ pub(crate) fn is_data_evolution_normal_file(file: &DataFileMeta) -> bool { && !is_vector_store_file_name(&file.file_name) } +/// Whether the files of a data-evolution row-id group are column slices of +/// one row range: the group has a normal data file, every normal file covers +/// the same row ids, and every BLOB or VECTOR file lies within them. +pub(crate) fn is_data_evolution_column_slice_group(files: &[DataFileMeta]) -> bool { + let mut normal_files = files + .iter() + .filter(|file| is_data_evolution_normal_file(file)); + let Some((first, last)) = normal_files.next().and_then(DataFileMeta::row_id_range) else { + return false; + }; + normal_files.all(|file| file.row_id_range() == Some((first, last))) + && files.iter().all(|file| { + file.row_id_range() + .is_some_and(|(from, to)| from >= first && to <= last) + }) +} + pub(crate) fn data_evolution_anchor_file(files: &[DataFileMeta]) -> crate::Result<&DataFileMeta> { files .iter() @@ -1830,6 +1847,54 @@ mod tests { .unwrap() } + #[test] + fn column_slice_groups_have_a_normal_file_covering_every_file() { + let meta = |name: &str, first_row_id, row_count| file(name, row_count, Some(first_row_id)); + for (files, expected) in [ + (vec![meta("data.parquet", 0, 4)], true), + ( + vec![ + meta("data.parquet", 0, 4), + meta("update.parquet", 0, 4), + meta("emb.vector.parquet", 0, 4), + meta("pic.blob", 1, 2), + ], + true, + ), + // Dedicated files only, as a range that rewrites only them holds. + (vec![meta("emb.vector.parquet", 0, 4)], false), + ( + vec![meta("emb.vector.parquet", 0, 4), meta("pic.blob", 0, 1)], + false, + ), + // A dedicated file reaching past the normal files' rows. + ( + vec![meta("data.parquet", 0, 2), meta("emb.vector.parquet", 0, 4)], + false, + ), + // Normal files of different row ranges. + ( + vec![meta("data.parquet", 0, 4), meta("update.parquet", 2, 2)], + false, + ), + ] { + let names = files + .iter() + .map(|f| f.file_name.clone()) + .collect::>(); + assert_eq!( + is_data_evolution_column_slice_group(&files), + expected, + "{names:?}" + ); + } + assert!(!is_data_evolution_column_slice_group(&[file( + "legacy.parquet", + 4, + None + )])); + } + #[test] fn data_split_clone_shares_planned_metadata() { let split = DataSplitBuilder::new() diff --git a/crates/paimon/src/table/table_scan.rs b/crates/paimon/src/table/table_scan.rs index 119e88eb6..60e97e2bb 100644 --- a/crates/paimon/src/table/table_scan.rs +++ b/crates/paimon/src/table/table_scan.rs @@ -3168,7 +3168,10 @@ impl<'a> PaimonTableScan<'a> { }; let row_id_groups = if let Some(read_field_ids) = data_evolution_read_field_ids { - if read_field_ids.is_empty() { + // Change events keep each group whole, as Java's + // `incremental-between` scan does: a group read file by + // file must return the same rows whatever the projection. + if read_field_ids.is_empty() || self.is_streaming() { row_id_groups } else { let mut pruned = Vec::with_capacity(row_id_groups.len()); diff --git a/crates/paimon/tests/incremental_batch_scan_test.rs b/crates/paimon/tests/incremental_batch_scan_test.rs index d4a4b87e4..7c0627921 100644 --- a/crates/paimon/tests/incremental_batch_scan_test.rs +++ b/crates/paimon/tests/incremental_batch_scan_test.rs @@ -2261,3 +2261,775 @@ async fn combined_delta_reads_the_columns_a_range_did_not_write_as_null() { ]; assert_eq!(collect(&batches), current, "snapshot read"); } + +/// `(id, name, picture, embedding)` of a row of an `evolved_table`. Embedding +/// values are whole numbers, kept as `i32` so that rows sort. +type EvolvedRow = ( + Option, + Option, + Option>, + Option>, +); + +/// An `EvolvedRow` whose picture holds the bytes of `picture`. +fn evolved_row( + id: Option, + name: Option<&str>, + picture: Option<&str>, + embedding: Option<[i32; 2]>, +) -> EvolvedRow { + ( + id, + name.map(str::to_string), + picture.map(|picture| picture.as_bytes().to_vec()), + embedding.map(|values| values.to_vec()), + ) +} + +fn embedding_field(name: &str) -> arrow_schema::Field { + use arrow_schema::{DataType as ArrowDataType, Field}; + let element = std::sync::Arc::new(Field::new("element", ArrowDataType::Float32, true)); + Field::new(name, ArrowDataType::FixedSizeList(element, 2), true) +} + +fn embedding_array(values: &[[f32; 2]]) -> arrow_array::ArrayRef { + use arrow_array::builder::{FixedSizeListBuilder, Float32Builder}; + use arrow_schema::{DataType as ArrowDataType, Field}; + let element = std::sync::Arc::new(Field::new("element", ArrowDataType::Float32, true)); + let mut builder = FixedSizeListBuilder::new(Float32Builder::new(), 2).with_field(element); + for value in values { + builder.values().append_slice(value); + builder.append(true); + } + std::sync::Arc::new(builder.finish()) +} + +/// Rows `(id, name, picture, embedding)` to append to an `evolved_table`. +fn evolved_batch( + ids: Vec, + names: Vec<&str>, + pictures: Vec<&[u8]>, + embeddings: &[[f32; 2]], +) -> RecordBatch { + use arrow_array::{ArrayRef, LargeBinaryArray, StringArray}; + use arrow_schema::{DataType as ArrowDataType, Field, Schema as ArrowSchema}; + RecordBatch::try_new( + std::sync::Arc::new(ArrowSchema::new(vec![ + Field::new("id", ArrowDataType::Int32, false), + Field::new("name", ArrowDataType::Utf8, true), + Field::new("picture", ArrowDataType::LargeBinary, true), + embedding_field("embedding"), + ])), + vec![ + std::sync::Arc::new(Int32Array::from(ids)) as ArrayRef, + std::sync::Arc::new(StringArray::from(names)), + std::sync::Arc::new(LargeBinaryArray::from(pictures)), + embedding_array(embeddings), + ], + ) + .unwrap() +} + +/// A data-evolution table `(id INT NOT NULL, name STRING, picture BLOB, +/// embedding VECTOR)`, which keeps the VECTOR column in +/// `.vector.parquet` files and whose snapshot 1 writes `alice` and `bob`, row +/// ids 0 and 1. +async fn evolved_table(path: &str, options: &[(&str, &str)]) -> paimon::table::Table { + evolved_table_with_picture(path, options, paimon::spec::BlobType::new()).await +} + +/// An `evolved_table` whose `picture` column has type `picture`. +async fn evolved_table_with_picture( + path: &str, + options: &[(&str, &str)], + picture: paimon::spec::BlobType, +) -> paimon::table::Table { + use paimon::spec::{ + DataType, FloatType, IntType, Schema, TableSchema, VarCharType, VectorType, + }; + let vector = VectorType::try_new(true, 2, DataType::Float(FloatType::new())).unwrap(); + let mut schema = Schema::builder() + .column("id", DataType::Int(IntType::with_nullable(false))) + .column("name", DataType::VarChar(VarCharType::string_type())) + .column("picture", DataType::Blob(picture)) + .column("embedding", DataType::Vector(vector)) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("vector.file.format", "parquet"); + for (key, value) in options { + schema = schema.option(*key, *value); + } + let (io, table) = memory_table(path, TableSchema::new(0, &schema.build().unwrap())); + setup_dirs(&io, path).await; + persist_table_schema(&io, path, table.schema()).await; + let rows = vec![b"aaa".as_slice(), b"bb".as_slice()]; + let batch = evolved_batch( + vec![1, 2], + vec!["alice", "bob"], + rows, + &[[1.0, 0.0], [0.0, 1.0]], + ); + write_batch(&table, &batch).await; + table +} + +/// Commits one data-evolution update of `columns` for the row with `row_id`. +async fn rewrite( + table: &paimon::table::Table, + row_id: i64, + columns: Vec<(arrow_schema::Field, arrow_array::ArrayRef)>, +) { + use arrow_array::{ArrayRef, Int64Array}; + use arrow_schema::{DataType as ArrowDataType, Field, Schema as ArrowSchema}; + let names = columns + .iter() + .map(|(field, _)| field.name().clone()) + .collect(); + let mut fields = vec![Field::new("_ROW_ID", ArrowDataType::Int64, false)]; + let mut arrays = vec![std::sync::Arc::new(Int64Array::from(vec![row_id])) as ArrayRef]; + for (field, array) in columns { + fields.push(field); + arrays.push(array); + } + let batch = + RecordBatch::try_new(std::sync::Arc::new(ArrowSchema::new(fields)), arrays).unwrap(); + let builder = table.new_write_builder(); + let mut writer = builder.new_data_evolution_writer(names).unwrap(); + writer.add_matched_batch(batch).unwrap(); + let messages = writer.prepare_commit().await.unwrap(); + builder.new_commit().commit(messages).await.unwrap(); +} + +fn new_embedding(value: [f32; 2]) -> (arrow_schema::Field, arrow_array::ArrayRef) { + (embedding_field("embedding"), embedding_array(&[value])) +} + +fn new_picture(value: &[u8]) -> (arrow_schema::Field, arrow_array::ArrayRef) { + use arrow_array::LargeBinaryArray; + use arrow_schema::{DataType as ArrowDataType, Field}; + let field = Field::new("picture", ArrowDataType::LargeBinary, true); + ( + field, + std::sync::Arc::new(LargeBinaryArray::from(vec![value])), + ) +} + +fn new_name(value: &str) -> (arrow_schema::Field, arrow_array::ArrayRef) { + use arrow_array::StringArray; + use arrow_schema::{DataType as ArrowDataType, Field}; + let field = Field::new("name", ArrowDataType::Utf8, true); + (field, std::sync::Arc::new(StringArray::from(vec![value]))) +} + +/// The rows of `batches`, sorted; a column the read leaves out reads as `None`. +fn evolved_rows(batches: &[RecordBatch]) -> Vec { + use arrow_array::{FixedSizeListArray, Float32Array, LargeBinaryArray, StringArray}; + let mut rows = Vec::new(); + for batch in batches { + let column = |name: &str| batch.column_by_name(name).map(|column| column.as_any()); + let ids = column("id").map(|ids| ids.downcast_ref::().unwrap()); + let names = column("name").map(|names| names.downcast_ref::().unwrap()); + let pictures = column("picture").map(|p| p.downcast_ref::().unwrap()); + let embeddings = + column("embedding").map(|e| e.downcast_ref::().unwrap()); + for row in 0..batch.num_rows() { + let embedding = embeddings.filter(|e| e.is_valid(row)).map(|e| { + let values = e.value(row); + let values = values.as_any().downcast_ref::().unwrap(); + values.values().iter().map(|value| *value as i32).collect() + }); + rows.push(( + ids.filter(|ids| ids.is_valid(row)) + .map(|ids| ids.value(row)), + names + .filter(|n| n.is_valid(row)) + .map(|n| n.value(row).to_string()), + pictures + .filter(|p| p.is_valid(row)) + .map(|p| p.value(row).to_vec()), + embedding, + )); + } + } + rows.sort(); + rows +} + +/// `rows` as a read type of `columns` (every column for `None`) returns them. +fn project_evolved(rows: &[EvolvedRow], columns: Option<&[&str]>) -> Vec { + let read = |name: &str| columns.is_none_or(|columns| columns.contains(&name)); + let mut rows: Vec = rows + .iter() + .map(|(id, name, picture, embedding)| { + ( + id.filter(|_| read("id")), + name.clone().filter(|_| read("name")), + picture.clone().filter(|_| read("picture")), + embedding.clone().filter(|_| read("embedding")), + ) + }) + .collect(); + rows.sort(); + rows +} + +/// Reads `(start, end]` through a read type of `columns` (every column for +/// `None`), planned per commit and as one batch. Each plan reads alike through +/// every reader taking it; returns the rows of both plans. +async fn read_evolved_range( + table: &paimon::table::Table, + start: i64, + end: i64, + columns: Option<&[&str]>, +) -> (Vec, Vec) { + use paimon::table::{IncrementalPlan, IncrementalSplit}; + let label = format!("({start},{end}] {columns:?}"); + let mut builder = table.new_read_builder(); + if let Some(columns) = columns { + let fields = table.schema().fields().iter(); + builder.with_read_type( + fields + .filter(|f| columns.contains(&f.name())) + .cloned() + .collect(), + ); + } + let read = builder.new_read().unwrap(); + let scan = builder.new_incremental_scan(IncrementalScanMode::Delta, start, end); + let plan = scan.plan().await.unwrap(); + let rows: Vec = read + .to_incremental_arrow(&plan) + .unwrap() + .try_collect() + .await + .unwrap(); + let audit: Vec = read + .to_audit_log_arrow(&plan) + .unwrap() + .try_collect() + .await + .unwrap(); + let per_commit = evolved_rows(&rows); + assert_eq!(evolved_rows(&audit), per_commit, "{label} audit per commit"); + // A stream scan reads the splits of each commit with `to_arrow`. + let rows: Vec = read + .to_arrow(&plan.data_splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!( + evolved_rows(&rows), + per_commit, + "{label} to_arrow per commit" + ); + + let splits = scan.plan_combined_delta().await.unwrap().into_splits(); + let rows: Vec = read.to_arrow(&splits).unwrap().try_collect().await.unwrap(); + let combined = evolved_rows(&rows); + let plan = IncrementalPlan::try_new( + IncrementalScanMode::Delta, + splits.into_iter().map(IncrementalSplit::Data).collect(), + ) + .unwrap(); + let rows: Vec = read + .to_incremental_arrow(&plan) + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(evolved_rows(&rows), combined, "{label} as one batch"); + (per_commit, combined) +} + +/// The ordinary read of the table's latest snapshot. +async fn read_evolved_snapshot(table: &paimon::table::Table) -> Vec { + let builder = table.new_read_builder(); + let plan = builder.new_scan().plan().await.unwrap(); + let read = builder.new_read().unwrap(); + let rows: Vec = read + .to_arrow(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + evolved_rows(&rows) +} + +/// Ranges of data-evolution updates to rows written before them. A BLOB or +/// VECTOR file a range wrote for rows of no ordinary-column file in the range, +/// such as a VECTOR rewrite alone, is read as written, as every data-evolution +/// file was read before the files of a range were merged. Column slices of one +/// row range still merge, a row a BLOB file did not write reading NULL. Every +/// reader and read type returns the same rows, and the ordinary read is +/// unchanged. +#[tokio::test] +async fn delta_reads_rewrites_of_blob_and_vector_columns() { + let row = evolved_row; + let vector = |values: [i32; 2]| row(None, None, None, Some(values)); + let picture = |text: &str| row(None, None, Some(text), None); + let named = |name: &str| row(None, Some(name), None, None); + let alice = |embedding| row(Some(1), Some("alice"), Some("aaa"), Some(embedding)); + let bob = |embedding| row(Some(2), Some("bob"), Some("bb"), Some(embedding)); + for (suffix, options) in [ + ("", &[][..]), + ("_dv", &[("deletion-vectors.enabled", "true")][..]), + ] { + let path = |name: &str| format!("memory:/incremental_batch/rewrites_{name}{suffix}"); + // (case, table, range, rows per commit, rows as one batch, snapshot) + let mut cases = Vec::new(); + + // The reviewer's case: a VECTOR rewrite writes one `.vector.parquet`. + let table = evolved_table(&path("vector"), options).await; + rewrite(&table, 0, vec![new_embedding([9.0, 9.0])]).await; + let rewritten = vec![vector([0, 1]), vector([9, 9])]; + let current = vec![alice([9, 9]), bob([0, 1])]; + let range = (1, 2); + cases.push(( + "vector", + table.clone(), + range, + rewritten.clone(), + rewritten, + current.clone(), + )); + // Together with the insert of its rows, the range merges into rows. + let inserted = vec![alice([1, 0]), bob([0, 1]), vector([0, 1]), vector([9, 9])]; + cases.push(( + "insert, vector", + table, + (0, 2), + inserted, + current.clone(), + current, + )); + + let table = evolved_table(&path("two_vectors"), options).await; + rewrite(&table, 0, vec![new_embedding([9.0, 9.0])]).await; + rewrite(&table, 1, vec![new_embedding([7.0, 7.0])]).await; + let rewritten = vec![ + vector([0, 1]), + vector([7, 7]), + vector([9, 9]), + vector([9, 9]), + ]; + let current = vec![alice([9, 9]), bob([7, 7])]; + cases.push(( + "two vectors", + table.clone(), + (1, 3), + rewritten.clone(), + rewritten.clone(), + current.clone(), + )); + // With the insert, the latest of the two rewrites wins. + let mut inserted = vec![alice([1, 0]), bob([0, 1])]; + inserted.extend(rewritten); + cases.push(( + "insert, two vectors", + table, + (0, 3), + inserted, + current.clone(), + current, + )); + + let table = evolved_table(&path("vector_blob"), options).await; + rewrite(&table, 0, vec![new_embedding([9.0, 9.0])]).await; + rewrite(&table, 1, vec![new_picture(b"zzzz")]).await; + // The BLOB file of row 1 holds no value for row 0. + let rewritten = vec![ + row(None, None, None, None), + vector([0, 1]), + vector([9, 9]), + picture("zzzz"), + ]; + let current = vec![ + alice([9, 9]), + row(Some(2), Some("bob"), Some("zzzz"), Some([0, 1])), + ]; + cases.push(( + "vector, blob", + table, + (1, 3), + rewritten.clone(), + rewritten, + current, + )); + + // One commit rewriting both: a BLOB file of row 0, a VECTOR file of both rows. + let table = evolved_table(&path("vector_and_blob"), options).await; + rewrite( + &table, + 0, + vec![new_embedding([9.0, 9.0]), new_picture(b"zzzz")], + ) + .await; + let rewritten = vec![vector([0, 1]), vector([9, 9]), picture("zzzz")]; + let current = vec![ + row(Some(1), Some("alice"), Some("zzzz"), Some([9, 9])), + bob([0, 1]), + ]; + cases.push(( + "vector and blob", + table, + (1, 2), + rewritten.clone(), + rewritten, + current, + )); + // The name file holds every row the VECTOR file holds, so they merge. + let table = evolved_table(&path("vector_name"), options).await; + rewrite(&table, 0, vec![new_embedding([9.0, 9.0])]).await; + rewrite(&table, 1, vec![new_name("BOB")]).await; + let rewritten = vec![named("BOB"), named("alice"), vector([0, 1]), vector([9, 9])]; + let merged = vec![ + row(None, Some("BOB"), None, Some([0, 1])), + row(None, Some("alice"), None, Some([9, 9])), + ]; + let current = vec![ + alice([9, 9]), + row(Some(2), Some("BOB"), Some("bb"), Some([0, 1])), + ]; + cases.push(("vector, name", table, (1, 3), rewritten, merged, current)); + + let table = evolved_table(&path("two_blobs"), options).await; + rewrite(&table, 0, vec![new_picture(b"zzzz")]).await; + rewrite(&table, 1, vec![new_picture(b"yyyyy")]).await; + let rewritten = vec![ + row(None, None, None, None), + picture("yyyyy"), + picture("zzzz"), + ]; + let current = vec![ + row(Some(1), Some("alice"), Some("zzzz"), Some([1, 0])), + row(Some(2), Some("bob"), Some("yyyyy"), Some([0, 1])), + ]; + cases.push(( + "two blobs", + table, + (1, 3), + rewritten.clone(), + rewritten, + current, + )); + + // A BLOB file holding fewer rows than the name file: the rest read NULL. + let table = evolved_table(&path("name_blob"), options).await; + rewrite(&table, 1, vec![new_name("BOB")]).await; + rewrite(&table, 0, vec![new_picture(b"zzzz")]).await; + let rewritten = vec![named("BOB"), named("alice"), picture("zzzz")]; + let merged = vec![named("BOB"), row(None, Some("alice"), Some("zzzz"), None)]; + let current = vec![ + row(Some(1), Some("alice"), Some("zzzz"), Some([1, 0])), + row(Some(2), Some("BOB"), Some("bb"), Some([0, 1])), + ]; + cases.push(("name, blob", table, (1, 3), rewritten, merged, current)); + + // A rewrite of old rows next to an insert of new ones. + let table = evolved_table(&path("vector_insert"), options).await; + rewrite(&table, 0, vec![new_embedding([9.0, 9.0])]).await; + let pictures = vec![b"c".as_slice(), b"dd".as_slice()]; + let batch = evolved_batch( + vec![3, 4], + vec!["carol", "dave"], + pictures, + &[[3.0, 3.0], [4.0, 4.0]], + ); + write_batch(&table, &batch).await; + let carol = row(Some(3), Some("carol"), Some("c"), Some([3, 3])); + let dave = row(Some(4), Some("dave"), Some("dd"), Some([4, 4])); + let rows = vec![vector([0, 1]), vector([9, 9]), carol.clone(), dave.clone()]; + let current = vec![alice([9, 9]), bob([0, 1]), carol, dave]; + cases.push(("vector, insert", table, (1, 3), rows.clone(), rows, current)); + + for (case, table, (start, end), per_commit, combined, current) in &cases { + for columns in [ + None, + Some(&["id", "name", "picture", "embedding"][..]), + Some(&[][..]), + Some(&["id"][..]), + Some(&["embedding"][..]), + Some(&["name", "picture"][..]), + ] { + let (rows, merged) = read_evolved_range(table, *start, *end, columns).await; + let label = format!("{case}{suffix} {columns:?}"); + assert_eq!( + rows, + project_evolved(per_commit, columns), + "{label} per commit" + ); + assert_eq!( + merged, + project_evolved(combined, columns), + "{label} as one batch" + ); + } + let mut current = current.clone(); + current.sort(); + assert_eq!( + read_evolved_snapshot(table).await, + current, + "{case}{suffix} snapshot" + ); + } + } +} + +/// A VECTOR field a row-id group holds in files of two layouts, written before +/// and after a schema change (a column added or an option set) or together +/// with and apart from another VECTOR field, cannot be merged into rows. Each +/// file of such a group in a range is read as written, whatever the read type. +#[tokio::test] +async fn delta_reads_vector_files_of_two_layouts_file_by_file() { + use arrow_schema::{DataType as ArrowDataType, Field, Schema as ArrowSchema}; + use paimon::spec::{ + BlobType, DataType, FloatType, IntType, Schema, TableSchema, VarCharType, VectorType, + }; + let vector_type = || { + DataType::Vector(VectorType::try_new(true, 2, DataType::Float(FloatType::new())).unwrap()) + }; + let row = evolved_row; + let vector = |values: [i32; 2]| row(None, None, None, Some(values)); + let picture = |text: &str| row(None, None, Some(text), None); + let columns: [Option<&[&str]>; 4] = [None, Some(&[]), Some(&["id"]), Some(&["embedding"])]; + + // The rewrite follows `ADD COLUMN note` or `SET` of an option, after which + // its file has schema 1. + for add_column in [true, false] { + let path = format!("memory:/incremental_batch/two_layouts_add_column_{add_column}"); + let table = evolved_table(&path, &[]).await; + let mut schema = Schema::builder() + .column("id", DataType::Int(IntType::with_nullable(false))) + .column("name", DataType::VarChar(VarCharType::string_type())) + .column("picture", DataType::Blob(BlobType::new())) + .column("embedding", vector_type()); + let mut options = table.schema().options().clone(); + if add_column { + schema = schema.column("note", DataType::VarChar(VarCharType::string_type())); + } else { + options.insert("snapshot.num-retained.max".into(), "10".into()); + } + let table = paimon::table::Table::new( + table.file_io().clone(), + table.identifier().clone(), + path.clone(), + TableSchema::new(1, &schema.options(options).build().unwrap()), + None, + ); + persist_table_schema(table.file_io(), &path, table.schema()).await; + rewrite(&table, 0, vec![new_embedding([9.0, 9.0])]).await; + let per_commit = vec![ + row(Some(1), Some("alice"), Some("aaa"), Some([1, 0])), + row(Some(2), Some("bob"), Some("bb"), Some([0, 1])), + vector([0, 1]), + vector([9, 9]), + ]; + let by_file = vec![ + row(Some(1), Some("alice"), None, None), + row(Some(2), Some("bob"), None, None), + picture("aaa"), + picture("bb"), + vector([1, 0]), + vector([0, 1]), + vector([0, 1]), + vector([9, 9]), + ]; + for columns in columns { + let (rows, merged) = read_evolved_range(&table, 0, 2, columns).await; + let label = format!("add column: {add_column} {columns:?}"); + assert_eq!(rows, project_evolved(&per_commit, columns), "{label}"); + assert_eq!(merged, project_evolved(&by_file, columns), "{label}"); + } + // A filter applies to each row read, even when its row id also names rows + // of other files: under a LIMIT, the BLOB file's row 0 must not come back. + for (plan_combined, expected) in [ + ( + false, + vec![row(Some(1), Some("alice"), Some("aaa"), Some([1, 0]))], + ), + (true, vec![row(Some(1), Some("alice"), None, None)]), + ] { + use paimon::spec::{Datum, PredicateBuilder}; + use paimon::table::{IncrementalPlan, IncrementalSplit}; + let mut builder = table.new_read_builder(); + let predicate = PredicateBuilder::new(table.schema().fields()) + .equal("name", Datum::String("alice".into())) + .unwrap(); + builder.with_filter(predicate).with_limit(2); + let scan = builder.new_incremental_scan(IncrementalScanMode::Delta, 0, 2); + let plan = if plan_combined { + let splits = scan.plan_combined_delta().await.unwrap().into_splits(); + IncrementalPlan::try_new( + IncrementalScanMode::Delta, + splits.into_iter().map(IncrementalSplit::Data).collect(), + ) + .unwrap() + } else { + scan.plan().await.unwrap() + }; + let read = builder.new_read().unwrap(); + let batches: Vec = read + .to_incremental_arrow(&plan) + .unwrap() + .try_collect() + .await + .unwrap(); + let label = + format!("filter and limit, add column: {add_column}, combined: {plan_combined}"); + assert_eq!(evolved_rows(&batches), expected, "{label}"); + } + } + // A rewrite of one of two VECTOR fields that share a file. + let path = "memory:/incremental_batch/two_layouts_vector_columns"; + let schema = Schema::builder() + .column("id", DataType::Int(IntType::with_nullable(false))) + .column("embedding", vector_type()) + .column("embedding2", vector_type()) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("vector.file.format", "parquet") + .build() + .unwrap(); + let (io, table) = memory_table(path, TableSchema::new(0, &schema)); + setup_dirs(&io, path).await; + persist_table_schema(&io, path, table.schema()).await; + let batch = RecordBatch::try_new( + std::sync::Arc::new(ArrowSchema::new(vec![ + Field::new("id", ArrowDataType::Int32, false), + embedding_field("embedding"), + embedding_field("embedding2"), + ])), + vec![ + std::sync::Arc::new(Int32Array::from(vec![1, 2])) as arrow_array::ArrayRef, + embedding_array(&[[1.0, 0.0], [0.0, 1.0]]), + embedding_array(&[[5.0, 5.0], [6.0, 6.0]]), + ], + ) + .unwrap(); + write_batch(&table, &batch).await; + rewrite(&table, 0, vec![new_embedding([9.0, 9.0])]).await; + let per_commit = vec![ + row(Some(1), None, None, Some([1, 0])), + row(Some(2), None, None, Some([0, 1])), + vector([0, 1]), + vector([9, 9]), + ]; + let by_file = vec![ + row(Some(1), None, None, None), + row(Some(2), None, None, None), + vector([1, 0]), + vector([0, 1]), + vector([0, 1]), + vector([9, 9]), + ]; + for columns in columns { + let (rows, merged) = read_evolved_range(&table, 0, 2, columns).await; + assert_eq!( + rows, + project_evolved(&per_commit, columns), + "two fields {columns:?}" + ); + assert_eq!( + merged, + project_evolved(&by_file, columns), + "two fields {columns:?}" + ); + } +} + +/// With `blob-as-descriptor`, a range returns the descriptors of the BLOBs it +/// wrote, whether a BLOB file is read on its own or merged into rows, as the +/// ordinary read does. +#[tokio::test] +async fn delta_returns_blob_descriptors_of_rewrites() { + use paimon::spec::BlobDescriptor; + use std::collections::HashMap; + let path = "memory:/incremental_batch/rewrite_descriptors"; + let table = evolved_table(path, &[]).await; + rewrite(&table, 0, vec![new_picture(b"zzzz")]).await; + let table = table.copy_with_options(HashMap::from([( + "blob-as-descriptor".to_string(), + "true".to_string(), + )])); + // The payload length each descriptor points at, by id; rows of the BLOB + // file alone have none. + let lengths = |rows: &[EvolvedRow]| { + let mut lengths: Vec<_> = rows + .iter() + .filter_map(|(id, _, picture, _)| { + let descriptor = BlobDescriptor::deserialize(picture.as_ref()?).unwrap(); + Some((*id, descriptor.length())) + }) + .collect(); + lengths.sort(); + lengths + }; + let (rows, merged) = read_evolved_range(&table, 1, 2, None).await; + assert_eq!(lengths(&rows), [(None, 4)]); + assert_eq!(lengths(&merged), [(None, 4)]); + let (rows, merged) = read_evolved_range(&table, 0, 2, None).await; + assert_eq!(lengths(&rows), [(None, 4), (Some(1), 3), (Some(2), 2)]); + assert_eq!(lengths(&merged), [(Some(1), 4), (Some(2), 2)]); + let current = read_evolved_snapshot(&table).await; + assert_eq!(lengths(¤t), [(Some(1), 4), (Some(2), 2)]); +} + +/// A BLOB rewrite of bob's picture writes a placeholder for alice's, which a +/// range without an older picture of hers reads as NULL, also in a `BLOB NOT +/// NULL` column: alone, and merged into the rows of a name rewrite. The +/// ordinary read is unchanged. +#[tokio::test] +async fn delta_reads_pictures_a_rewrite_left_unwritten_as_null() { + use paimon::spec::BlobType; + let row = evolved_row; + for nullable in [true, false] { + let picture = BlobType::with_nullable(nullable); + let path = |name: &str| format!("memory:/incremental_batch/unwritten_{name}_{nullable}"); + let table = evolved_table_with_picture(&path("picture"), &[], picture.clone()).await; + rewrite(&table, 1, vec![new_picture(b"zz")]).await; + let rewritten = vec![ + row(None, None, None, None), + row(None, None, Some("zz"), None), + ]; + assert_eq!( + read_evolved_range(&table, 1, 2, None).await, + (rewritten.clone(), rewritten), + "picture, nullable: {nullable}" + ); + let current = vec![ + row(Some(1), Some("alice"), Some("aaa"), Some([1, 0])), + row(Some(2), Some("bob"), Some("zz"), Some([0, 1])), + ]; + assert_eq!(read_evolved_snapshot(&table).await, current); + + let table = evolved_table_with_picture(&path("name_picture"), &[], picture).await; + rewrite(&table, 1, vec![new_name("BOB")]).await; + rewrite(&table, 1, vec![new_picture(b"zz")]).await; + let (per_commit, merged) = read_evolved_range(&table, 1, 3, None).await; + let names = [ + row(None, Some("BOB"), None, None), + row(None, Some("alice"), None, None), + ]; + let pictures = [ + row(None, None, None, None), + row(None, None, Some("zz"), None), + ]; + let mut expected = [names, pictures].concat(); + expected.sort(); + assert_eq!(per_commit, expected, "name, picture, nullable: {nullable}"); + assert_eq!( + merged, + [ + row(None, Some("BOB"), Some("zz"), None), + row(None, Some("alice"), None, None), + ], + "name, picture as one batch, nullable: {nullable}" + ); + let current = vec![ + row(Some(1), Some("alice"), Some("aaa"), Some([1, 0])), + row(Some(2), Some("BOB"), Some("zz"), Some([0, 1])), + ]; + assert_eq!(read_evolved_snapshot(&table).await, current); + } +} diff --git a/docs/src/sql.md b/docs/src/sql.md index 8a1e94473..700f04c29 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -2336,11 +2336,19 @@ The optional fourth argument selects the mode: - `delta` returns the rows each APPEND commit wrote, every version of a key included. A data-evolution table's range is read as one batch, as Java's `incremental-between` - scan reads it, so a column update merges into the rows it updates when they were - written in the range. For rows written before the range it returns every row of the - files the updates rewrote, unchanged rows included, with only the updated columns - set and the others NULL, NOT NULL columns included. (Java's reader instead fails - when two such updates of the same rows leave a NOT NULL column unwritten.) + scan reads it: the files it wrote for the same rows merge into rows when they are + column slices of one row range, that is, its normal data files cover the same rows, + its BLOB and VECTOR files lie within them, and each VECTOR column has one layout. + For rows written before the range it returns every row of the files the updates + rewrote, unchanged rows included, with only the updated columns set and the others + NULL, NOT NULL columns included. (Java's reader instead fails when two such updates + of the same rows leave a NOT NULL column unwritten.) Other files are returned as + written, one row per row of each file. These include BLOB or VECTOR files alone, such + as recomputed embeddings, so two embedding updates of the same rows return both + versions where Java returns the latest; every file of rows whose VECTOR column the + range wrote in two layouts (before and after an `ALTER TABLE`, or with and without + another VECTOR column); and every file of rows whose files are no longer column + slices because a compaction in the range merged some of them. - `changelog` returns the changelog rows of each commit. - `diff` compares the table at the two snapshots and returns the added and changed rows (`$audit_log` adds `-U` and `-D` rows). It requires a primary-key table with From 60fcd30888ce299fb04be15cec88e5e8c43fa776 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Thu, 29 Oct 2026 22:01:31 +0800 Subject: [PATCH 05/18] fix(core): decode change events of a field stored two ways file by file A table can move a VECTOR or BLOB column between dedicated files and inline storage, for example by removing `vector.file.format` or setting `blob-descriptor-field`, and a later rewrite of the same rows writes the column the other way. Change events merged such a row-id group, but the source plan reads each field from one layout: a combined range lost the rewrite's new embedding, and BLOB files written before the change read as NULL. Base returned both. Change events now merge a group only when each BLOB or VECTOR field of the table is stored in one layout, a BLOB field in the layout the table now reads, and no older normal file holds such a field that the newest normal file naming it leaves out: with the write-cols optimization, a file without write columns holds only the fields its stored schema keeps inline. Other groups are decoded file by file, as base decoded them, and a BLOB file read alone is no longer parsed as descriptors or views. --- .../paimon/src/table/data_evolution_reader.rs | 364 +++++++++++++-- .../tests/incremental_batch_scan_test.rs | 419 ++++++++++++++++++ docs/src/sql.md | 30 +- 3 files changed, 776 insertions(+), 37 deletions(-) diff --git a/crates/paimon/src/table/data_evolution_reader.rs b/crates/paimon/src/table/data_evolution_reader.rs index 7fe2978f6..f802513ec 100644 --- a/crates/paimon/src/table/data_evolution_reader.rs +++ b/crates/paimon/src/table/data_evolution_reader.rs @@ -268,18 +268,26 @@ impl DataEvolutionReader { } /// Whether a row-id group of change events merges into rows: its files - /// are column slices of one row range, and each VECTOR field is written - /// by files of one layout. Other groups are read file by file. + /// are column slices of one row range, and the source plan reads each + /// field from a file that holds it, in the one layout the group stores it + /// in. Other groups are read file by file. async fn merges_change_events(&self, files: &[DataFileMeta]) -> crate::Result { if !is_data_evolution_column_slice_group(files) { return Ok(false); } - if !files + // The source plan reads a BLOB field from dedicated files only, or, + // for descriptor and view fields, from normal files only, whichever + // layout the field had when the files were written. + let dedicated_blobs: HashSet = self + .table_fields .iter() - .any(|file| is_vector_store_file_name(&file.file_name)) - { - return Ok(true); - } + .filter(|field| { + field.data_type().is_blob_file_field() + && !self.blob_descriptor_fields.contains(field.name()) + && !self.blob_view_fields.contains(field.name()) + }) + .map(DataField::id) + .collect(); let (infos, _) = load_file_infos( &self.schema_manager, self.table_schema_id, @@ -287,7 +295,130 @@ impl DataEvolutionReader { files, ) .await?; - Ok(!vector_field_has_two_layouts(files, &infos)) + let mut layouts = Vec::with_capacity(files.len()); + // Normal files with the fields their write columns name. + let mut normal_files = Vec::new(); + for (file, info) in files.iter().zip(infos) { + if is_vector_store_file_name(&file.file_name) { + let columns = info.normalized_write_cols.unwrap_or_default(); + layouts.push(( + FieldLayout::Vector(vector_bunch_key(file, &columns)), + info.field_ids, + )); + } else if is_blob_or_video_file_name(&file.file_name) { + layouts.push((FieldLayout::Blob, info.field_ids)); + } else { + normal_files.push((file, info.field_ids)); + } + } + // Only the layouts of the table's BLOB and VECTOR fields can decide: + // the merge reads any other field from normal files alone, a field + // retyped from VECTOR to text too, and does not read a field dropped + // since. So normal layouts list only those fields, while dedicated + // layouts keep every field, as the source plan rejects a field of two + // vector bunches, read or not. A normal file without write columns + // leaves out the fields its own schema stores in dedicated files; that + // schema is read only where the answer can decide: for a BLOB or + // VECTOR field that a dedicated file of the group also holds or that + // the table now stores in BLOB files. + let maybe_dedicated: HashSet = self + .table_fields + .iter() + .filter(|field| { + field.data_type().is_blob_file_field() + || matches!(field.data_type(), DataType::Vector(_)) + }) + .map(DataField::id) + .collect(); + let in_dedicated_files: HashSet = layouts + .iter() + .flat_map(|(_, field_ids)| field_ids.iter().copied()) + .collect(); + for (file, written) in &normal_files { + let decides = file.write_cols.is_none() + && written.iter().any(|id| { + maybe_dedicated.contains(id) + && (in_dedicated_files.contains(id) || dedicated_blobs.contains(id)) + }); + let held = if decides { + self.fields_without_write_cols(file).await? + } else { + written.clone() + }; + let held = held.into_iter().filter(|id| maybe_dedicated.contains(id)); + layouts.push((FieldLayout::Normal, held.collect())); + } + let blob_fields: HashSet = self + .table_fields + .iter() + .filter(|field| field.data_type().is_blob_file_field()) + .map(DataField::id) + .collect(); + let read_where_stored = layouts.iter().all(|(layout, field_ids)| { + field_ids.iter().all(|id| { + !blob_fields.contains(id) + || (*layout == FieldLayout::Blob) == dedicated_blobs.contains(id) + }) + }); + if !read_where_stored || field_has_two_layouts(&layouts) { + return Ok(false); + } + // Like the source plan, the newest normal file that writes a field, + // as its write columns say, provides it. One without write columns + // that leaves the field out would hide an older file that holds it; + // files of one schema hold such a field alike, so only files of more + // than one schema need their schemas read. Only the table's BLOB and + // VECTOR fields can be left out: a field typed otherwise now was at + // most a VECTOR before an ALTER to text, whose old values no read can + // decode. + normal_files.sort_by_key(|(file, _)| std::cmp::Reverse(file.max_sequence_number)); + let vector_provided: HashSet = layouts + .iter() + .filter(|(layout, _)| matches!(layout, FieldLayout::Vector(_))) + .flat_map(|(_, field_ids)| field_ids.iter().copied()) + .collect(); + let fields = self.table_fields.iter().map(DataField::id); + for id in fields.filter(|id| maybe_dedicated.contains(id)) { + if dedicated_blobs.contains(&id) || vector_provided.contains(&id) { + continue; + } + let mut writers = normal_files + .iter() + .filter(|(_, written)| written.contains(&id)) + .map(|(file, _)| *file); + let Some(newest) = writers.next() else { + continue; + }; + let older: Vec<_> = writers.collect(); + if newest.write_cols.is_some() + || older.iter().all(|file| file.schema_id == newest.schema_id) + || self.fields_without_write_cols(newest).await?.contains(&id) + { + continue; + } + for file in older { + if file.write_cols.is_some() + || self.fields_without_write_cols(file).await?.contains(&id) + { + return Ok(false); + } + } + } + Ok(true) + } + + /// The fields a normal file without write columns holds: those its + /// schema keeps in normal files, which with the write-cols optimization + /// leave out the fields the schema stores in dedicated files. Like Java, + /// the schema the file was written with decides, not the options of this + /// read. + async fn fields_without_write_cols(&self, file: &DataFileMeta) -> crate::Result> { + let schema = self.schema_manager.schema(file.schema_id).await?; + Ok(schema + .data_file_fields(None) + .iter() + .map(DataField::id) + .collect()) } /// Managed BLOB payloads are fetched by the format reader, before the @@ -482,7 +613,12 @@ impl DataEvolutionReader { // LIMIT reads resolve views only after the surviving batch has // been selected. Eagerly scanning every split here can fail on a // reference beyond the quota and load unrelated upstream blobs. - let mut blob_view_lookup = if resolve_blob_views && (self.limit.is_some() || splits.iter().any(|split| + // Change events resolve them per batch too when they hold BLOB or + // video files: one they read on its own holds payloads, not views. + let lazy_views = self.limit.is_some() + || splits.iter().any(|split| split.is_streaming() + && split.data_files().iter().any(|file| is_blob_or_video_file_name(&file.file_name))); + let mut blob_view_lookup = if resolve_blob_views && (lazy_views || splits.iter().any(|split| split.data_files().iter().any(|file| file.extra_files.iter().any(|name| name.ends_with(".row"))))) { Some(BlobViewLookup::default()) } else { @@ -492,6 +628,8 @@ impl DataEvolutionReader { let descriptor_fields = self.descriptor_fields_to_resolve(blob_view_lookup.is_some()); let filter_before_blob_resolution = self.can_filter_before_blob_resolution(blob_view_lookup.is_some(), &descriptor_fields); + let mut no_view_lookup = None; + let no_descriptor_fields = HashSet::new(); // A raw-convertible split consists only of independent files: every // row-id segment has a single column provider, so predicates can be @@ -577,6 +715,11 @@ impl DataEvolutionReader { .await?; let has_row_id = file_meta.first_row_id.is_some(); + // A BLOB or video file holds payloads, also for a + // field the table now stores as descriptors or views + // inline; change events read them as written. + let read_as_written = split.is_streaming() + && is_blob_or_video_file_name(&file_meta.file_name); let requested_row_ranges = if has_row_id { row_ranges.clone() } else { None }; let mut effective_row_ranges = requested_row_ranges.clone(); let mut has_bitmap_selection = false; @@ -702,10 +845,15 @@ impl DataEvolutionReader { } else { batch }; + let (lookup, fields) = if read_as_written { + (&mut no_view_lookup, &no_descriptor_fields) + } else { + (&mut blob_view_lookup, &descriptor_fields) + }; yield self.finish_wide_batch( batch, - &mut blob_view_lookup, - &descriptor_fields, + lookup, + fields, filter_before_blob_resolution, &mut remaining, ).await?; @@ -2533,22 +2681,27 @@ fn vector_bunch_key( ) } -/// Whether some VECTOR field is written by files of two vector bunch keys, -/// such as a rewrite after `ALTER TABLE`; the source plan cannot combine -/// their bunches. -fn vector_field_has_two_layouts(files: &[DataFileMeta], infos: &[ResolvedFileInfo]) -> bool { +/// Where a data-evolution file stores the fields it writes. +#[derive(PartialEq)] +enum FieldLayout { + /// Inline in a normal data file. + Normal, + /// In dedicated BLOB or video files. + Blob, + /// In dedicated vector files of one vector bunch key. + Vector((i64, String, Vec)), +} + +/// Whether some field is stored in two layouts, given each file's layout and +/// field ids: inline and in dedicated BLOB or VECTOR files, as after the +/// table changed the field's storage, or in VECTOR files of two vector bunch +/// keys, as after `ALTER TABLE`. The source plan reads such a field from one +/// layout only, so it would drop the other's values. +fn field_has_two_layouts(files: &[(FieldLayout, Vec)]) -> bool { let mut layouts = HashMap::new(); - for (file, info) in files.iter().zip(infos) { - let Some(columns) = info - .normalized_write_cols - .as_deref() - .filter(|_| is_vector_store_file_name(&file.file_name)) - else { - continue; - }; - let layout = vector_bunch_key(file, columns); - for field_id in &info.field_ids { - if layouts.entry(*field_id).or_insert_with(|| layout.clone()) != &layout { + for (layout, field_ids) in files { + for field_id in field_ids { + if *layouts.entry(*field_id).or_insert(layout) != layout { return true; } } @@ -6619,6 +6772,165 @@ mod tests { ); } + /// Change events read a row-id group file by file when a BLOB or VECTOR + /// field of it is stored both inline in a normal file and in dedicated + /// files, as after the table changed the field's storage. The source plan + /// reads such a field from one layout only, so merging the group would + /// drop the newer inline values. + #[tokio::test] + async fn change_events_read_a_field_of_two_layouts_file_by_file() { + use arrow_array::LargeBinaryArray; + use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema}; + use parquet::arrow::ArrowWriter; + + let tempdir = tempdir().unwrap(); + let bucket_dir = tempdir.path().join("bucket-0"); + fs::create_dir_all(&bucket_dir).unwrap(); + let size = |path: &Path| path.metadata().unwrap().len() as i64; + let normal_path = bucket_dir.join("data.parquet"); + write_int_parquet_file(&normal_path, vec![("id", vec![1, 2])], None); + let vectors = [Some(vec![1.0, 0.0]), Some(vec![0.0, 1.0])]; + write_fixed_size_list_parquet( + &bucket_dir.join("emb.vector.parquet"), + "embedding", + 2, + &vectors, + ); + let updated = [Some(vec![9.0, 9.0]), Some(vec![0.0, 1.0])]; + write_fixed_size_list_parquet( + &bucket_dir.join("emb-inline.parquet"), + "embedding", + 2, + &updated, + ); + write_blob_file( + &bucket_dir.join("payload.blob"), + &[Some(&b"aaa"[..]), Some(&b"bb"[..])], + ); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "payload", + ArrowDataType::LargeBinary, + true, + )])); + let payloads = LargeBinaryArray::from(vec![Some(&b"zzzz"[..]), None]); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(payloads)]).unwrap(); + let file = fs::File::create(bucket_dir.join("payload-inline.parquet")).unwrap(); + let mut writer = ArrowWriter::try_new(file, schema, None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + let meta = |name: &str, sequence, columns: Vec<&str>| { + data_file_meta_with_path( + name, + 0, + 2, + sequence, + size(&bucket_dir.join(name)), + Some(columns), + ) + }; + + let table_with = |options: &[(&str, &str)]| { + let mut schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("embedding", vector_float_type(2)) + .column("payload", DataType::Blob(BlobType::new())) + .option("data-evolution.enabled", "true") + .option("row-tracking.enabled", "true"); + for (key, value) in options { + schema = schema.option(*key, *value); + } + Table::new( + FileIOBuilder::new("file").build().unwrap(), + Identifier::new("default", "change_event_layouts_t"), + local_file_path(tempdir.path()), + TableSchema::new(0, &schema.build().unwrap()), + None, + ) + }; + let read_events = |table: Table, files: Vec| { + let split = DataSplitBuilder::new() + .with_snapshot(2) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path(local_file_path(&bucket_dir)) + .with_total_buckets(1) + .with_data_files(files) + .with_streaming(true) + .build() + .unwrap(); + async move { + let read = TableRead::new(&table, table.schema().fields().to_vec(), Vec::new()); + read.to_arrow(&[split]) + .unwrap() + .try_collect::>() + .await + .unwrap() + } + }; + + // The embeddings of the insert in a vector file, then of a rewrite of + // row 0 inline, after the table stopped using vector files. + let batches = read_events( + table_with(&[]), + vec![ + meta("data.parquet", 1, vec!["id"]), + meta("emb.vector.parquet", 1, vec!["embedding"]), + meta("emb-inline.parquet", 2, vec!["embedding"]), + ], + ) + .await; + let mut rows = vec![None, None]; + rows.extend(vectors.iter().cloned()); + rows.extend(updated.iter().cloned()); + assert_fixed_size_list(&batches, "embedding", 2, &rows); + + // The pictures of the insert in a BLOB file, then of a rewrite of + // row 0 inline, after the table started storing them inline. + let batches = read_events( + table_with(&[]), + vec![ + meta("data.parquet", 1, vec!["id"]), + meta("payload.blob", 1, vec!["payload"]), + meta("payload-inline.parquet", 2, vec!["payload"]), + ], + ) + .await; + let pictures = [ + None, + None, + Some(b"aaa".to_vec()), + Some(b"bb".to_vec()), + Some(b"zzzz".to_vec()), + None, + ]; + assert_eq!(collect_binary_values(&batches, "payload"), pictures); + + // One layout per field, but not the one the table now reads: inline + // pictures of a table that now stores them in BLOB files, and BLOB + // files of a table that now stores them inline as descriptors, read + // as written rather than as descriptors. + let batches = read_events( + table_with(&[]), + vec![ + meta("data.parquet", 1, vec!["id"]), + meta("payload-inline.parquet", 1, vec!["payload"]), + ], + ) + .await; + let pictures = [None, None, Some(b"zzzz".to_vec()), None]; + assert_eq!(collect_binary_values(&batches, "payload"), pictures); + let batches = read_events( + table_with(&[("blob-descriptor-field", "payload")]), + vec![ + meta("data.parquet", 1, vec!["id"]), + meta("payload.blob", 1, vec!["payload"]), + ], + ) + .await; + let pictures = [None, None, Some(b"aaa".to_vec()), Some(b"bb".to_vec())]; + assert_eq!(collect_binary_values(&batches, "payload"), pictures); + } + /// (1) Provider priority: the normal data file ALSO advertises the embedding write_col, /// but the dedicated `.vector.parquet` file must win. #[tokio::test] diff --git a/crates/paimon/tests/incremental_batch_scan_test.rs b/crates/paimon/tests/incremental_batch_scan_test.rs index 7c0627921..753e52e9f 100644 --- a/crates/paimon/tests/incremental_batch_scan_test.rs +++ b/crates/paimon/tests/incremental_batch_scan_test.rs @@ -2571,9 +2571,15 @@ async fn delta_reads_rewrites_of_blob_and_vector_columns() { let named = |name: &str| row(None, Some(name), None, None); let alice = |embedding| row(Some(1), Some("alice"), Some("aaa"), Some(embedding)); let bob = |embedding| row(Some(2), Some("bob"), Some("bb"), Some(embedding)); + // With the write-cols optimization, an insert's normal file omits its + // write columns although its BLOB and VECTOR columns are in other files. for (suffix, options) in [ ("", &[][..]), ("_dv", &[("deletion-vectors.enabled", "true")][..]), + ( + "_omitted_cols", + &[("data-evolution.write-cols-optimization.enabled", "true")][..], + ), ] { let path = |name: &str| format!("memory:/incremental_batch/rewrites_{name}{suffix}"); // (case, table, range, rows per commit, rows as one batch, snapshot) @@ -3033,3 +3039,416 @@ async fn delta_reads_pictures_a_rewrite_left_unwritten_as_null() { assert_eq!(read_evolved_snapshot(&table).await, current); } } + +/// A table that switches a VECTOR column between vector files and inline +/// storage writes the rows' later embeddings in the other layout. A range +/// holding both layouts of the same rows is read as written, so it returns +/// the rewrite's embedding in either direction. +#[tokio::test] +async fn delta_reads_a_vector_column_stored_in_two_ways_file_by_file() { + use arrow_schema::{DataType as ArrowDataType, Field, Schema as ArrowSchema}; + use paimon::spec::{ + DataType, FloatType, IntType, Schema, SchemaChange, TableSchema, VectorType, + }; + let row = evolved_row; + let vector = |values: [i32; 2]| row(None, None, None, Some(values)); + let vector_type = + DataType::Vector(VectorType::try_new(true, 2, DataType::Float(FloatType::new())).unwrap()); + let batch = RecordBatch::try_new( + std::sync::Arc::new(ArrowSchema::new(vec![ + Field::new("id", ArrowDataType::Int32, false), + embedding_field("embedding"), + ])), + vec![ + std::sync::Arc::new(Int32Array::from(vec![1, 2])) as arrow_array::ArrayRef, + embedding_array(&[[1.0, 0.0], [0.0, 1.0]]), + ], + ) + .unwrap(); + let columns: [Option<&[&str]>; 3] = [None, Some(&["id", "embedding"]), Some(&["embedding"])]; + for vector_files_first in [true, false] { + let path = format!("memory:/incremental_batch/two_storages_{vector_files_first}"); + let mut schema = Schema::builder() + .column("id", DataType::Int(IntType::with_nullable(false))) + .column("embedding", vector_type.clone()) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true"); + if vector_files_first { + schema = schema.option("vector.file.format", "parquet"); + } + let (io, table) = memory_table(&path, TableSchema::new(0, &schema.build().unwrap())); + setup_dirs(&io, &path).await; + persist_table_schema(&io, &path, table.schema()).await; + write_batch(&table, &batch).await; + let change = if vector_files_first { + SchemaChange::remove_option("vector.file.format".into()) + } else { + SchemaChange::set_option("vector.file.format".into(), "parquet".into()) + }; + let schema = table.schema().apply_changes(vec![change]).unwrap(); + persist_table_schema(&io, &path, &schema).await; + let table = paimon::table::Table::new(io, table.identifier().clone(), path, schema, None); + rewrite(&table, 0, vec![new_embedding([9.0, 9.0])]).await; + + let inserted = vec![ + row(Some(1), None, None, Some([1, 0])), + row(Some(2), None, None, Some([0, 1])), + ]; + let rewritten = vec![vector([9, 9]), vector([0, 1])]; + let insert_files = if vector_files_first { + let ids = [ + row(Some(1), None, None, None), + row(Some(2), None, None, None), + ]; + [ids.to_vec(), vec![vector([1, 0]), vector([0, 1])]].concat() + } else { + inserted.clone() + }; + let mut per_commit = [inserted, rewritten.clone()].concat(); + per_commit.sort(); + let mut by_file = [insert_files, rewritten].concat(); + by_file.sort(); + for columns in columns { + let label = format!("vector files first: {vector_files_first} {columns:?}"); + let (rows, merged) = read_evolved_range(&table, 0, 2, columns).await; + assert_eq!(rows, project_evolved(&per_commit, columns), "{label}"); + assert_eq!(merged, project_evolved(&by_file, columns), "{label}"); + } + } +} + +/// After the table starts storing a BLOB column inline as descriptors, the +/// BLOB files written before still hold the pictures. A range of them is read +/// as written, pictures included, rather than merged into rows that would +/// look for the pictures inline. +#[tokio::test] +async fn delta_reads_blob_files_written_before_a_storage_change_as_written() { + use paimon::spec::SchemaChange; + let row = evolved_row; + let path = "memory:/incremental_batch/blob_storage_change"; + let table = evolved_table(path, &[]).await; + let change = SchemaChange::set_option("blob-descriptor-field".into(), "picture".into()); + let schema = table.schema().apply_changes(vec![change]).unwrap(); + persist_table_schema(table.file_io(), path, &schema).await; + let table = paimon::table::Table::new( + table.file_io().clone(), + table.identifier().clone(), + path.to_string(), + schema, + None, + ); + let mut by_file = vec![ + row(Some(1), Some("alice"), None, None), + row(Some(2), Some("bob"), None, None), + row(None, None, Some("aaa"), None), + row(None, None, Some("bb"), None), + row(None, None, None, Some([1, 0])), + row(None, None, None, Some([0, 1])), + ]; + by_file.sort(); + let columns: [Option<&[&str]>; 3] = [None, Some(&["id", "picture"]), Some(&["picture"])]; + for columns in columns { + let (rows, merged) = read_evolved_range(&table, 0, 1, columns).await; + let expected = project_evolved(&by_file, columns); + assert_eq!(rows, expected, "{columns:?}"); + assert_eq!(merged, expected, "{columns:?}"); + } +} + +/// With the write-cols optimization, a file holding every column kept in +/// normal files omits its write columns. Once the VECTOR column moves to +/// vector files, such a file of an `id` rewrite holds no embeddings, while +/// the older file of the same rows holds them inline. A range of both is read +/// as written, embeddings included, also by a read that turns the +/// optimization off. +#[tokio::test] +async fn delta_reads_embeddings_a_newer_file_does_not_hold_as_written() { + use arrow_schema::{DataType as ArrowDataType, Field, Schema as ArrowSchema}; + use paimon::spec::{ + DataType, FloatType, IntType, Schema, SchemaChange, TableSchema, VectorType, + }; + let row = evolved_row; + let path = "memory:/incremental_batch/omitted_write_cols"; + let vector = VectorType::try_new(true, 2, DataType::Float(FloatType::new())).unwrap(); + let schema = Schema::builder() + .column("id", DataType::Int(IntType::with_nullable(false))) + .column("embedding", DataType::Vector(vector)) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("data-evolution.write-cols-optimization.enabled", "true") + .build() + .unwrap(); + let (io, table) = memory_table(path, TableSchema::new(0, &schema)); + setup_dirs(&io, path).await; + persist_table_schema(&io, path, table.schema()).await; + let id = Field::new("id", ArrowDataType::Int32, false); + let batch = RecordBatch::try_new( + std::sync::Arc::new(ArrowSchema::new(vec![ + id.clone(), + embedding_field("embedding"), + ])), + vec![ + std::sync::Arc::new(Int32Array::from(vec![1, 2])) as arrow_array::ArrayRef, + embedding_array(&[[1.0, 0.0], [0.0, 1.0]]), + ], + ) + .unwrap(); + write_batch(&table, &batch).await; + let change = SchemaChange::set_option("vector.file.format".into(), "parquet".into()); + let schema = table.schema().apply_changes(vec![change]).unwrap(); + persist_table_schema(&io, path, &schema).await; + let identifier = table.identifier().clone(); + let table = paimon::table::Table::new(io, identifier, path.to_string(), schema, None); + let ids = std::sync::Arc::new(Int32Array::from(vec![11])) as arrow_array::ArrayRef; + rewrite(&table, 0, vec![(id, ids)]).await; + + let mut by_file = vec![ + row(Some(1), None, None, Some([1, 0])), + row(Some(2), None, None, Some([0, 1])), + row(Some(11), None, None, None), + row(Some(2), None, None, None), + ]; + by_file.sort(); + let off = std::collections::HashMap::from([( + "data-evolution.write-cols-optimization.enabled".to_string(), + "false".to_string(), + )]); + let columns: [Option<&[&str]>; 3] = [None, Some(&["id"]), Some(&["embedding"])]; + for (optimization_off, table) in [(false, table.clone()), (true, table.copy_with_options(off))] + { + for columns in columns { + let label = format!("optimization off: {optimization_off} {columns:?}"); + let (rows, merged) = read_evolved_range(&table, 0, 2, columns).await; + let expected = project_evolved(&by_file, columns); + assert_eq!(rows, expected, "{label}"); + assert_eq!(merged, expected, "{label}"); + } + } +} + +/// A table handle built in memory can lack a schema file, which reads of +/// the files of its own schema never need. Without the write-cols +/// optimization or files of another schema, what a file without write columns +/// holds does not decide the merge, so a range of an insert, an `id` rewrite +/// and a rewrite of every column reads without one. +#[tokio::test] +async fn delta_reads_a_table_without_a_schema_file() { + use arrow_schema::{DataType as ArrowDataType, Field, Schema as ArrowSchema}; + use paimon::spec::{DataType, FloatType, IntType, Schema, TableSchema, VectorType}; + let row = evolved_row; + let path = "memory:/incremental_batch/no_schema_file"; + let vector = VectorType::try_new(true, 2, DataType::Float(FloatType::new())).unwrap(); + let schema = Schema::builder() + .column("id", DataType::Int(IntType::with_nullable(false))) + .column("embedding", DataType::Vector(vector)) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .build() + .unwrap(); + let (io, table) = memory_table(path, TableSchema::new(0, &schema)); + setup_dirs(&io, path).await; + let id = Field::new("id", ArrowDataType::Int32, false); + let ids = + |values: Vec| std::sync::Arc::new(Int32Array::from(values)) as arrow_array::ArrayRef; + let batch = RecordBatch::try_new( + std::sync::Arc::new(ArrowSchema::new(vec![ + id.clone(), + embedding_field("embedding"), + ])), + vec![ids(vec![1, 2]), embedding_array(&[[1.0, 0.0], [0.0, 1.0]])], + ) + .unwrap(); + write_batch(&table, &batch).await; + rewrite(&table, 0, vec![(id.clone(), ids(vec![11]))]).await; + rewrite( + &table, + 1, + vec![(id, ids(vec![22])), new_embedding([7.0, 7.0])], + ) + .await; + let schema_file = format!("{path}/schema/schema-0"); + assert!(!io.exists(&schema_file).await.unwrap()); + + let current = vec![ + row(Some(11), None, None, Some([1, 0])), + row(Some(22), None, None, Some([7, 7])), + ]; + let mut per_commit = vec![ + row(Some(1), None, None, Some([1, 0])), + row(Some(2), None, None, Some([0, 1])), + row(Some(11), None, None, None), + row(Some(2), None, None, None), + ]; + per_commit.extend(current.clone()); + per_commit.sort(); + assert_eq!( + read_evolved_range(&table, 0, 3, None).await, + (per_commit, current.clone()) + ); + assert_eq!(read_evolved_snapshot(&table).await, current); +} + +/// With the write-cols optimization, an insert's normal file omits its write +/// columns and leaves the BLOB or VECTOR column to dedicated files. After the +/// column is dropped, a range of the insert still merges into its rows: the +/// column is no longer read, so which file kept it decides nothing. +#[tokio::test] +async fn delta_merges_an_insert_whose_dedicated_column_was_dropped() { + use arrow_array::{ArrayRef, LargeBinaryArray}; + use arrow_schema::{DataType as ArrowDataType, Field, Schema as ArrowSchema}; + use paimon::spec::{ + BlobType, DataType, FloatType, IntType, Schema, SchemaChange, TableSchema, VectorType, + }; + let row = evolved_row; + let vector = VectorType::try_new(true, 2, DataType::Float(FloatType::new())).unwrap(); + let pictures: ArrayRef = std::sync::Arc::new(LargeBinaryArray::from(vec![ + b"aaa".as_slice(), + b"bb".as_slice(), + ])); + let columns = [ + ( + Field::new("picture", ArrowDataType::LargeBinary, true), + DataType::Blob(BlobType::new()), + pictures, + ), + ( + embedding_field("embedding"), + DataType::Vector(vector), + embedding_array(&[[1.0, 0.0], [0.0, 1.0]]), + ), + ]; + for (field, data_type, values) in columns { + let column = field.name().clone(); + let path = format!("memory:/incremental_batch/dropped_{column}"); + let mut schema = Schema::builder() + .column("id", DataType::Int(IntType::with_nullable(false))) + .column(&column, data_type) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("data-evolution.write-cols-optimization.enabled", "true"); + if column == "embedding" { + schema = schema.option("vector.file.format", "parquet"); + } + let (io, table) = memory_table(&path, TableSchema::new(0, &schema.build().unwrap())); + setup_dirs(&io, &path).await; + persist_table_schema(&io, &path, table.schema()).await; + let batch = RecordBatch::try_new( + std::sync::Arc::new(ArrowSchema::new(vec![ + Field::new("id", ArrowDataType::Int32, false), + field, + ])), + vec![ + std::sync::Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + values, + ], + ) + .unwrap(); + write_batch(&table, &batch).await; + let change = SchemaChange::drop_column(column.clone()); + let schema = table.schema().apply_changes(vec![change]).unwrap(); + persist_table_schema(&io, &path, &schema).await; + let identifier = table.identifier().clone(); + let table = paimon::table::Table::new(io, identifier, path, schema, None); + + let inserted = vec![ + row(Some(1), None, None, None), + row(Some(2), None, None, None), + ]; + for read in [None, Some(&["id"][..])] { + assert_eq!( + read_evolved_range(&table, 0, 1, read).await, + (inserted.clone(), inserted.clone()), + "dropped {column}, read {read:?}" + ); + } + assert_eq!(read_evolved_snapshot(&table).await, inserted); + } +} + +/// A VECTOR column of one dimension can change type to STRING, called `name` +/// here so that `evolved_rows` reads it as text. The vector files written +/// before hold values no read decodes as text, and a merge never opens them: +/// it reads a STRING column from normal files alone. So an insert of vector +/// files merges into its rows, and so does a later update of the column, +/// written inline, with or without the write-cols optimization. +#[tokio::test] +async fn delta_merges_rows_whose_vector_column_became_a_string() { + use arrow_array::builder::{FixedSizeListBuilder, Float32Builder}; + use arrow_array::ArrayRef; + use arrow_schema::{DataType as ArrowDataType, Field, Schema as ArrowSchema}; + use paimon::spec::{ + DataType, FloatType, IntType, Schema, SchemaChange, TableSchema, VarCharType, VectorType, + }; + let row = evolved_row; + let vector = VectorType::try_new(true, 1, DataType::Float(FloatType::new())).unwrap(); + for optimized in [false, true] { + let path = format!("memory:/incremental_batch/vector_to_string_{optimized}"); + let mut schema = Schema::builder() + .column("id", DataType::Int(IntType::with_nullable(false))) + .column("name", DataType::Vector(vector.clone())) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("vector.file.format", "parquet"); + if optimized { + schema = schema.option("data-evolution.write-cols-optimization.enabled", "true"); + } + let (io, table) = memory_table(&path, TableSchema::new(0, &schema.build().unwrap())); + setup_dirs(&io, &path).await; + persist_table_schema(&io, &path, table.schema()).await; + let element = std::sync::Arc::new(Field::new("element", ArrowDataType::Float32, true)); + let mut vectors = + FixedSizeListBuilder::new(Float32Builder::new(), 1).with_field(element.clone()); + for value in [1.0, 2.0] { + vectors.values().append_value(value); + vectors.append(true); + } + let batch = RecordBatch::try_new( + std::sync::Arc::new(ArrowSchema::new(vec![ + Field::new("id", ArrowDataType::Int32, false), + Field::new("name", ArrowDataType::FixedSizeList(element, 1), true), + ])), + vec![ + std::sync::Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + std::sync::Arc::new(vectors.finish()), + ], + ) + .unwrap(); + write_batch(&table, &batch).await; + let text = DataType::VarChar(VarCharType::string_type()); + let change = SchemaChange::update_column_type("name".into(), text); + let schema = table.schema().apply_changes(vec![change]).unwrap(); + persist_table_schema(&io, &path, &schema).await; + let identifier = table.identifier().clone(); + let table = paimon::table::Table::new(io, identifier, path, schema, None); + + let label = |read: Option<&[&str]>| format!("optimized: {optimized}, read {read:?}"); + let inserted = vec![ + row(Some(1), None, None, None), + row(Some(2), None, None, None), + ]; + for read in [None, Some(&["id"][..])] { + let expected = (inserted.clone(), inserted.clone()); + let rows = read_evolved_range(&table, 0, 1, read).await; + assert_eq!(rows, expected, "{}", label(read)); + } + rewrite(&table, 0, vec![new_name("x")]).await; + let mut per_commit = vec![ + row(None, Some("x"), None, None), + row(None, None, None, None), + ]; + per_commit.extend(inserted); + let current = vec![ + row(Some(1), Some("x"), None, None), + row(Some(2), None, None, None), + ]; + for read in [None, Some(&["id"][..]), Some(&["name"][..])] { + let expected = ( + project_evolved(&per_commit, read), + project_evolved(¤t, read), + ); + let rows = read_evolved_range(&table, 0, 2, read).await; + assert_eq!(rows, expected, "{}", label(read)); + } + assert_eq!(read_evolved_snapshot(&table).await, current); + } +} diff --git a/docs/src/sql.md b/docs/src/sql.md index 01882caef..2e6d5f271 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -2387,17 +2387,25 @@ The optional fourth argument selects the mode: A data-evolution table's range is read as one batch, as Java's `incremental-between` scan reads it: the files it wrote for the same rows merge into rows when they are column slices of one row range, that is, its normal data files cover the same rows, - its BLOB and VECTOR files lie within them, and each VECTOR column has one layout. - For rows written before the range it returns every row of the files the updates - rewrote, unchanged rows included, with only the updated columns set and the others - NULL, NOT NULL columns included. (Java's reader instead fails when two such updates - of the same rows leave a NOT NULL column unwritten.) Other files are returned as - written, one row per row of each file. These include BLOB or VECTOR files alone, such - as recomputed embeddings, so two embedding updates of the same rows return both - versions where Java returns the latest; every file of rows whose VECTOR column the - range wrote in two layouts (before and after an `ALTER TABLE`, or with and without - another VECTOR column); and every file of rows whose files are no longer column - slices because a compaction in the range merged some of them. + its BLOB and VECTOR files lie within them, and each BLOB or VECTOR column has one + layout, for BLOB the one the table now uses, and the merge would read each column + where it is stored. For rows written before the range it returns every row of the + files the updates rewrote, unchanged rows included, with only the updated columns set + and the others NULL, NOT NULL columns included. (Java's reader instead fails when two + such updates of the same rows leave a NOT NULL column unwritten.) Other files are + returned as written, one row per row of each file. These include BLOB or VECTOR files + alone, such as recomputed embeddings, so two embedding updates of the same rows return + both versions where Java returns the latest; every file of rows whose BLOB or VECTOR + column the range wrote in two layouts (inline and in dedicated files, after a change + of the column's storage, or in VECTOR files before and after an `ALTER TABLE`, or with + and without another VECTOR column), whose BLOB column it wrote in a layout the table + no longer uses, or whose BLOB or VECTOR column an older file holds inline and the + newest file writing it leaves out, written with + `data-evolution.write-cols-optimization.enabled` after the column moved to dedicated + files; and every file of rows whose files are no longer column slices because a + compaction in the range merged some of them. A column the table has since dropped, or + changed from VECTOR to a string type, keeps rows from merging only when the range + wrote it in VECTOR files of two layouts. - `changelog` returns the changelog rows of each commit. - `diff` compares the table at the two snapshots and returns the added and changed rows (`$audit_log` adds `-U` and `-D` rows). It requires a primary-key table with From 792dc5738f70bcf92f8304bbf11a1bd7c0c9839a Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sat, 31 Oct 2026 00:58:04 +0800 Subject: [PATCH 06/18] fix(core): keep change events a filter matches in one of their files Planning prunes a data-evolution row-id group by the merged stats of its newest column providers. Change events may decode a group file by file, returning each file's own rows, so a filter on a value one file wrote and a newer file of the same rows overwrote dropped the group: a filtered combined range returned none of the rows the unfiltered range returned for that value. A streaming group now also stays when one of its files may match the filter. --- crates/paimon/src/table/table_scan.rs | 26 ++++++-- .../tests/incremental_batch_scan_test.rs | 63 ++++++++++++++++++- 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/crates/paimon/src/table/table_scan.rs b/crates/paimon/src/table/table_scan.rs index 60e97e2bb..210d464da 100644 --- a/crates/paimon/src/table/table_scan.rs +++ b/crates/paimon/src/table/table_scan.rs @@ -3145,21 +3145,39 @@ impl<'a> PaimonTableScan<'a> { scan_stats.data_evolution_groups_pruned_by_row_ranges += groups_pruned_by_row_ranges; - // Filter groups by merged stats before splitting. + // Filter groups by merged stats before splitting. Change events + // may decode a group file by file, returning each file's own + // rows rather than the merged rows these stats describe, so + // such a group also stays when one of its files may match. let row_id_groups: Vec> = if self.data_predicates.is_empty() { row_id_groups } else { let before = row_id_groups.len(); let mut groups = Vec::with_capacity(before); for group in row_id_groups { - if data_evolution_group_matches_predicates_for_table( + let mut keep = data_evolution_group_matches_predicates_for_table( &self.table, &group, &self.data_predicates, &mut stats_schema_cache, ) - .await - { + .await; + if !keep && self.is_streaming() && group.len() > 1 { + for file in &group { + if data_evolution_group_matches_predicates_for_table( + &self.table, + std::slice::from_ref(file), + &self.data_predicates, + &mut stats_schema_cache, + ) + .await + { + keep = true; + break; + } + } + } + if keep { groups.push(group); } } diff --git a/crates/paimon/tests/incremental_batch_scan_test.rs b/crates/paimon/tests/incremental_batch_scan_test.rs index 753e52e9f..0b04574a6 100644 --- a/crates/paimon/tests/incremental_batch_scan_test.rs +++ b/crates/paimon/tests/incremental_batch_scan_test.rs @@ -2481,9 +2481,21 @@ async fn read_evolved_range( start: i64, end: i64, columns: Option<&[&str]>, +) -> (Vec, Vec) { + read_evolved_range_where(table, start, end, columns, None).await +} + +/// [`read_evolved_range`] with `filter` set on the read builder, so that it +/// applies when the range is planned as well as to the rows read. +async fn read_evolved_range_where( + table: &paimon::table::Table, + start: i64, + end: i64, + columns: Option<&[&str]>, + filter: Option, ) -> (Vec, Vec) { use paimon::table::{IncrementalPlan, IncrementalSplit}; - let label = format!("({start},{end}] {columns:?}"); + let label = format!("({start},{end}] {columns:?} {filter:?}"); let mut builder = table.new_read_builder(); if let Some(columns) = columns { let fields = table.schema().fields().iter(); @@ -2494,6 +2506,9 @@ async fn read_evolved_range( .collect(), ); } + if let Some(filter) = filter { + builder.with_filter(filter); + } let read = builder.new_read().unwrap(); let scan = builder.new_incremental_scan(IncrementalScanMode::Delta, start, end); let plan = scan.plan().await.unwrap(); @@ -3452,3 +3467,49 @@ async fn delta_merges_rows_whose_vector_column_became_a_string() { assert_eq!(read_evolved_snapshot(&table).await, current); } } + +/// A filter keeps every row of a range it matches. After the table stops +/// keeping the VECTOR column in vector files, a rewrite of bob's name and +/// embedding stores both inline, so the range of the insert and the rewrite is +/// read file by file and returns the insert's `bob` as well as the rewrite's +/// `BOB`. The rewrite's newer file no longer holds `bob`; planning still keeps +/// the rows' files, as reading the range and then filtering it would. +#[tokio::test] +async fn delta_filters_each_file_of_rows_read_file_by_file() { + use paimon::spec::{Datum, PredicateBuilder, SchemaChange}; + let row = evolved_row; + let path = "memory:/incremental_batch/filter_by_file"; + let table = evolved_table(path, &[]).await; + let change = SchemaChange::remove_option("vector.file.format".into()); + let schema = table.schema().apply_changes(vec![change]).unwrap(); + persist_table_schema(table.file_io(), path, &schema).await; + let table = paimon::table::Table::new( + table.file_io().clone(), + table.identifier().clone(), + path.to_string(), + schema, + None, + ); + rewrite(&table, 1, vec![new_name("BOB"), new_embedding([9.0, 9.0])]).await; + + let bob = || { + PredicateBuilder::new(table.schema().fields()) + .equal("name", Datum::String("bob".into())) + .unwrap() + }; + let (per_commit, by_file) = read_evolved_range(&table, 0, 2, None).await; + let named_bob = |rows: Vec| -> Vec { + let bob = |row: &EvolvedRow| row.1.as_deref() == Some("bob"); + rows.into_iter().filter(bob).collect() + }; + let expected = (named_bob(per_commit), named_bob(by_file)); + assert_eq!( + expected, + ( + vec![row(Some(2), Some("bob"), Some("bb"), Some([0, 1]))], + vec![row(Some(2), Some("bob"), None, None)], + ) + ); + let filtered = read_evolved_range_where(&table, 0, 2, None, Some(bob())).await; + assert_eq!(filtered, expected); +} From 597f7a5c8d8d546a588a6e8e6697d1d63cd3f03a Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sat, 31 Oct 2026 00:58:53 +0800 Subject: [PATCH 07/18] fix(datafusion): return _ROW_ID from paimon_incremental_query Without `$audit_log`, the function now returns the columns of a query on the table, `_ROW_ID` included on a data-evolution table, so a column update returned without the row's other columns can be matched to its row. The `$audit_log` suffix is matched case-insensitively, as a system-table name is in a FROM clause, and the docs state that the table name is split at dots and used as written, as in the other Paimon table functions. --- .../datafusion/src/incremental_query.rs | 81 ++++++++++------- .../datafusion/src/table_function_args.rs | 10 ++- .../datafusion/tests/incremental_query.rs | 86 +++++++++++++++++++ docs/src/sql.md | 14 +-- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/crates/integrations/datafusion/src/incremental_query.rs b/crates/integrations/datafusion/src/incremental_query.rs index 3ac1a40d6..bf05fe7b8 100644 --- a/crates/integrations/datafusion/src/incremental_query.rs +++ b/crates/integrations/datafusion/src/incremental_query.rs @@ -27,10 +27,13 @@ //! ``` //! //! A `$audit_log` table-name suffix prepends the `rowkind` audit column (and the -//! optional `_SEQUENCE_NUMBER`), mirroring the core audit-log read. +//! optional `_SEQUENCE_NUMBER`), mirroring the core audit-log read. Without it, +//! the columns are those of a query on the table, so a data-evolution table's +//! rows carry `_ROW_ID`. //! -//! The table name resolves as in a `FROM` clause, in the calling session, and -//! the session's dynamic options apply as they do to an ordinary read of the +//! The table name resolves in the calling session, whose current catalog and +//! database complete it as they complete a name in a `FROM` clause, and the +//! session's dynamic options apply as they do to an ordinary read of the //! table, except those that choose another range to read. use std::collections::HashMap; @@ -69,7 +72,7 @@ use crate::memory::reader_resources; use crate::physical_plan::scan::to_datafusion_batch; use crate::runtime::{await_with_runtime, block_on_with_runtime}; use crate::system_tables::ensure_audit_log_dynamic_options; -use crate::table::datafusion_arrow_schema; +use crate::table::{datafusion_arrow_schema, datafusion_read_fields}; use crate::table_function_args::{ extract_int_literal, extract_string_literal, resolve_table_argument, }; @@ -138,10 +141,14 @@ impl TableFunctionImpl for IncrementalQueryFunction { ))); } - // `db.t$audit_log` selects the audit-log projection of `db.t`. - let (base_name, audit_log) = match table_name.strip_suffix(AUDIT_LOG_SUFFIX) { - Some(base) => (base.to_string(), true), - None => (table_name.clone(), false), + // `db.t$audit_log` selects the audit-log projection of `db.t`; as in a + // `FROM` clause, the system-table name is case-insensitive. + let split = table_name.len().saturating_sub(AUDIT_LOG_SUFFIX.len()); + let (base_name, audit_log) = match table_name.get(split..) { + Some(suffix) if suffix.eq_ignore_ascii_case(AUDIT_LOG_SUFFIX) => { + (table_name[..split].to_string(), true) + } + _ => (table_name.clone(), false), }; // The calling session resolves the name, so `USE` applies here as it // does to a `FROM` clause. @@ -240,6 +247,9 @@ struct IncrementalQueryTableProvider { start_exclusive: i64, end_inclusive: i64, fields: Vec, + /// The leading audit columns of `fields`, which core's audit-log batches + /// put before the read fields. + audit_columns: usize, schema: ArrowSchemaRef, } @@ -251,14 +261,17 @@ impl IncrementalQueryTableProvider { start_exclusive: i64, end_inclusive: i64, ) -> DFResult { - // Audit-log output leads with `rowkind` (+ optional `_SEQUENCE_NUMBER`); - // the plain projection is the table's own fields. - let fields = if audit_log { - AuditLogTable::new(table.clone()) + // Audit-log output leads with `rowkind` (+ optional `_SEQUENCE_NUMBER`), + // as the `$audit_log` table does; the plain output has the columns of a + // query on the table, `_ROW_ID` of a data-evolution table included. + let (fields, audit_columns) = if audit_log { + let fields = AuditLogTable::new(table.clone()) .fields() - .map_err(to_datafusion_error)? + .map_err(to_datafusion_error)?; + let audit_columns = fields.len() - table.schema().fields().len(); + (fields, audit_columns) } else { - table.schema().fields().to_vec() + (datafusion_read_fields(&table), 0) }; // Core streams raw Arrow (`Utf8`, not `Utf8View`), so declare the schema // the reader actually yields rather than the view-forced scan schema. @@ -270,24 +283,24 @@ impl IncrementalQueryTableProvider { start_exclusive, end_inclusive, fields, + audit_columns, schema, }) } - /// Splits a projection into the table fields core reads and the column of + /// Splits a projection into the fields core reads and the column of /// core's batches each output column comes from. Core's audit-log batches /// lead with the same audit columns as `fields`, then the read fields. fn read_projection(&self, indices: &[usize]) -> ReadProjection { - let audit_columns = self.fields.len() - self.table.schema().fields().len(); let mut read_type = Vec::new(); let columns = indices .iter() .map(|&index| { - if index < audit_columns { + if index < self.audit_columns { return index; } read_type.push(self.fields[index].clone()); - audit_columns + read_type.len() - 1 + self.audit_columns + read_type.len() - 1 }) .collect(); ReadProjection { read_type, columns } @@ -316,20 +329,30 @@ impl TableProvider for IncrementalQueryTableProvider { if limit == Some(0) { return Ok(Arc::new(EmptyExec::new(projected_schema))); } + // Without a projection every column is read, `_ROW_ID` included, which + // core's default read type would leave out. + let all_columns: Vec; + let indices = match projection { + Some(indices) => indices, + None => { + all_columns = (0..self.fields.len()).collect(); + &all_columns + } + }; Ok(Arc::new(IncrementalQueryExec::new( self.table.clone(), self.audit_log, self.mode, self.start_exclusive, self.end_inclusive, - projection.map(|indices| self.read_projection(indices)), + self.read_projection(indices), projected_schema, ))) } } -/// The table fields to read, and per output column the column of core's -/// batches it comes from. +/// The fields to read, and per output column the column of core's batches it +/// comes from. #[derive(Debug, Clone)] struct ReadProjection { read_type: Vec, @@ -345,7 +368,7 @@ struct IncrementalQueryExec { mode: IncrementalScanMode, start_exclusive: i64, end_inclusive: i64, - projection: Option, + projection: ReadProjection, output_schema: ArrowSchemaRef, plan_properties: Arc, } @@ -357,7 +380,7 @@ impl IncrementalQueryExec { mode: IncrementalScanMode, start_exclusive: i64, end_inclusive: i64, - projection: Option, + projection: ReadProjection, output_schema: ArrowSchemaRef, ) -> Self { let plan_properties = Arc::new(PlanProperties::new( @@ -387,9 +410,7 @@ impl IncrementalQueryExec { async fn open(&self, resources: ResourceContext) -> DFResult { let mut read_builder = self.table.new_read_builder(); read_builder.with_resources(resources); - if let Some(projection) = &self.projection { - read_builder.with_read_type(projection.read_type.clone()); - } + read_builder.with_read_type(self.projection.read_type.clone()); let scan = read_builder.new_incremental_scan(self.mode, self.start_exclusive, self.end_inclusive); let plan = if self.plans_range_as_one_batch() { @@ -483,14 +504,10 @@ impl ExecutionPlan for IncrementalQueryExec { let batches = await_with_runtime(exec.open(resources)).await?; // Batches are forwarded as core emits them, projected, and rebuilt // against the declared schema as the ordinary scan does. - let columns = exec.projection.map(|projection| projection.columns); + let columns = exec.projection.columns; let schema = exec.output_schema; Ok::<_, DataFusionError>(batches.map(move |batch| { - let batch = batch.map_err(to_datafusion_error)?; - let batch = match &columns { - Some(columns) => batch.project(columns)?, - None => batch, - }; + let batch = batch.map_err(to_datafusion_error)?.project(&columns)?; to_datafusion_batch(batch, &schema) })) }) diff --git a/crates/integrations/datafusion/src/table_function_args.rs b/crates/integrations/datafusion/src/table_function_args.rs index 4f7f1d5a4..e18b1b2ff 100644 --- a/crates/integrations/datafusion/src/table_function_args.rs +++ b/crates/integrations/datafusion/src/table_function_args.rs @@ -75,10 +75,12 @@ pub(crate) fn parse_table_identifier( Ok(Identifier::new(database.unwrap_or(default_database), table)) } -/// Resolves a table-name argument the way a `FROM` clause resolves a table -/// name: `table` and `database.table` take the session's current catalog (and -/// database), and `catalog.database.table` names the catalog. Returns the -/// catalog's name and the table's identifier in that catalog. +/// Resolves a table-name argument in the calling session: `table` and +/// `database.table` take the session's current catalog (and database), as in +/// a `FROM` clause, and `catalog.database.table` names the catalog. As in +/// `parse_table_identifier`, the name is split at dots and its parts are +/// used as written. Returns the catalog's name and the table's identifier in +/// that catalog. pub(crate) fn resolve_table_argument( function_name: &str, name: &str, diff --git a/crates/integrations/datafusion/tests/incremental_query.rs b/crates/integrations/datafusion/tests/incremental_query.rs index c673d63ba..3e7208f7a 100644 --- a/crates/integrations/datafusion/tests/incremental_query.rs +++ b/crates/integrations/datafusion/tests/incremental_query.rs @@ -179,6 +179,13 @@ async fn test_audit_log_suffix_prepends_rowkind() { for i in 0..rowkind.len() { assert_eq!(rowkind.value(i), "+I"); } + // As in a `FROM` clause, the system-table name is case-insensitive. + let upper = run( + &ctx, + "SELECT * FROM paimon_incremental_query('default.t$AUDIT_LOG', 0, 3, 'delta')", + ) + .await; + assert_eq!(upper, batches); } async fn expect_error(ctx: &SQLContext, sql: &str) -> String { @@ -667,6 +674,85 @@ async fn test_data_evolution_range_of_column_updates_reads_unwritten_columns_as_ } } +#[tokio::test] +async fn test_columns_match_a_query_on_the_table() { + use datafusion::arrow::util::pretty::pretty_format_batches; + // The function returns the columns a query on the table returns, `_ROW_ID` + // included on a data-evolution table, so a column update returned without + // the row's other columns can be matched to its row. + let tmp = TempDir::new().unwrap(); + let ctx = context(&tmp).await; + run( + &ctx, + "CREATE TABLE paimon.default.de (id INT NOT NULL, name STRING) WITH ( + 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true' + )", + ) + .await; + run( + &ctx, + "INSERT INTO paimon.default.de (id, name) VALUES (1, 'alice'), (2, 'bob')", + ) + .await; + run( + &ctx, + "CREATE TEMPORARY TABLE paimon.default.renames AS + SELECT * FROM (VALUES (2, 'BOB')) AS t(id, name)", + ) + .await; + run( + &ctx, + "MERGE INTO paimon.default.de t USING paimon.default.renames s ON t.id = s.id + WHEN MATCHED THEN UPDATE SET name = s.name", + ) + .await; + for name in ["de", "de$audit_log"] { + let mut columns = Vec::new(); + for from in [ + format!("paimon.default.{name}"), + format!("paimon_incremental_query('default.{name}', 0, 2)"), + ] { + let frame = ctx.sql(&format!("SELECT * FROM {from}")).await.unwrap(); + let fields = frame.schema().fields().iter(); + columns.push(fields.map(|field| field.name().clone()).collect::>()); + } + assert_eq!(columns[1], columns[0], "{name}"); + } + let format = |batches: Vec| pretty_format_batches(&batches).unwrap().to_string(); + let read = |from: &str| format!(r#"SELECT "_ROW_ID", id, name FROM {from} ORDER BY "_ROW_ID""#); + let table = format(run(&ctx, &read("paimon.default.de")).await); + assert_eq!( + table, + [ + "+---------+----+-------+", + "| _ROW_ID | id | name |", + "+---------+----+-------+", + "| 0 | 1 | alice |", + "| 1 | 2 | BOB |", + "+---------+----+-------+", + ] + .join("\n") + ); + // The update rewrote `name` of both rows, so read alone they come back + // without `id`; read with the insert, they are the rows of the table. + let update = read("paimon_incremental_query('default.de', 1, 2)"); + assert_eq!( + format(run(&ctx, &update).await), + [ + "+---------+----+-------+", + "| _ROW_ID | id | name |", + "+---------+----+-------+", + "| 0 | | alice |", + "| 1 | | BOB |", + "+---------+----+-------+", + ] + .join("\n") + ); + let range = read("paimon_incremental_query('default.de', 0, 2)"); + assert_eq!(format(run(&ctx, &range).await), table); +} + /// Asserts that, for `name`, the function reads the rows a `FROM` clause in /// the same session reads. async fn assert_reads_as_from(ctx: &SQLContext, name: &str) { diff --git a/docs/src/sql.md b/docs/src/sql.md index 2e6d5f271..a85c57b35 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -2414,11 +2414,15 @@ The optional fourth argument selects the mode: `changelog` otherwise. A `$audit_log` suffix puts `rowkind` first, followed by `_SEQUENCE_NUMBER` when -`table-read.sequence-number.enabled=true`. - -The table name resolves as in a `FROM` clause of the same session: `my_table` and -`my_db.my_table` use the current catalog (and database), and -`my_catalog.my_db.my_table` names the catalog. Dynamic options set with +`table-read.sequence-number.enabled=true`. Without it the columns are those of a +query on the table, `_ROW_ID` included on a data-evolution table, so a column update +returned without the row's other columns can be matched to its row. + +The table name resolves in the calling session: `my_table` and `my_db.my_table` +use the current catalog (and database), as in a `FROM` clause, and +`my_catalog.my_db.my_table` names the catalog. As in the other Paimon table +functions, the name is split at dots, and the catalog, database and table names are +used as written, without SQL quoting or case folding. Dynamic options set with `SET 'paimon.…'` apply as they do to a query on the table, except options that choose another range to read (time-travel selectors, `incremental-between*`, file creation-time filters and a `scan.mode` other than `default`), which are rejected From 7d4701d1ae3632b7de1d21a0b4aa531ba08d16b3 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sat, 31 Oct 2026 03:22:00 +0800 Subject: [PATCH 08/18] fix: count the rows change events return An exact row count of a data-evolution split assumes merged rows. The scan reported it as exact statistics, from which `COUNT(*)` is answered, and the partition-count rewrite answered `COUNT(*)` from the manifests of the snapshot read. Change events may decode a row-id group file by file, so an incremental window over a field stored two ways, or over a compacted range a later update rewrote, counted fewer rows than it returned. The rewrite also counted a whole snapshot for the window of an append table. A streaming split that is not raw convertible now gives no merged row count, and the rewrite skips incremental windows. The docs add that the compaction a write runs after its commit can keep rows from merging. --- .../src/partition_count_pushdown.rs | 8 +- .../datafusion/tests/incremental_query.rs | 96 +++++++++++++++++++ crates/paimon/src/spec/core_options.rs | 2 +- crates/paimon/src/table/source.rs | 7 ++ .../tests/incremental_batch_scan_test.rs | 38 ++++++++ docs/src/sql.md | 3 +- 6 files changed, 150 insertions(+), 4 deletions(-) diff --git a/crates/integrations/datafusion/src/partition_count_pushdown.rs b/crates/integrations/datafusion/src/partition_count_pushdown.rs index f7fc0cbc0..3d425631e 100644 --- a/crates/integrations/datafusion/src/partition_count_pushdown.rs +++ b/crates/integrations/datafusion/src/partition_count_pushdown.rs @@ -134,9 +134,13 @@ fn rewrite_aggregate(aggregate: &Aggregate) -> DFResult> { let table = paimon.table(); let table_schema = table.schema(); // Manifest row counts of a primary-key table are physical: several versions - // of a key count separately until they are merged at read time. - if CoreOptions::new(table_schema.options()).is_format_table() + // of a key count separately until they are merged at read time. Manifests + // count a snapshot's rows, while an incremental window reads the rows its + // commits wrote. + let options = CoreOptions::new(table_schema.options()); + if options.is_format_table() || !table_schema.primary_keys().is_empty() + || !matches!(options.incremental_timestamp_window(), Ok(None)) { return Ok(None); } diff --git a/crates/integrations/datafusion/tests/incremental_query.rs b/crates/integrations/datafusion/tests/incremental_query.rs index 3e7208f7a..4ad6d99e7 100644 --- a/crates/integrations/datafusion/tests/incremental_query.rs +++ b/crates/integrations/datafusion/tests/incremental_query.rs @@ -753,6 +753,102 @@ async fn test_columns_match_a_query_on_the_table() { assert_eq!(format(run(&ctx, &range).await), table); } +#[tokio::test] +async fn test_a_read_of_the_range_as_its_window_counts_the_rows_it_returns() { + use datafusion::arrow::array::Int64Array; + use datafusion::prelude::SessionContext; + use paimon_datafusion::PaimonTableProvider; + // Two inserts, a compaction of their files and an update of every row: + // the update covers the compacted file's rows, not those of one insert, + // so the range is read file by file, eight rows. A read of the table with + // the range as its incremental window returns them too, and its scan's + // row count, from which `COUNT(*)` is answered, counts them. + let tmp = TempDir::new().unwrap(); + let ctx = context(&tmp).await; + run( + &ctx, + "CREATE TABLE paimon.default.de (id INT, v INT) WITH ( + 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true', + 'compaction.min.file-num' = '2', + 'write-only' = 'true' + )", + ) + .await; + for values in ["(1, 1), (2, 2)", "(3, 3), (4, 4)"] { + run( + &ctx, + &format!("INSERT INTO paimon.default.de (id, v) VALUES {values}"), + ) + .await; + } + let warehouse = catalog(&tmp); + let identifier = Identifier::new("default", "de"); + let table = warehouse.get_table(&identifier).await.unwrap(); + let compaction = table.new_compact_builder(); + let plan = compaction.new_planner().unwrap().plan().await.unwrap(); + let executor = compaction.new_executor().unwrap(); + let mut results = Vec::new(); + for task in plan.tasks() { + results.push(executor.execute(task).await.unwrap()); + } + let commit = compaction.new_commit().unwrap(); + commit.commit_compact(results).await.unwrap(); + run(&ctx, "UPDATE paimon.default.de SET v = 50").await; + + let latest = table.snapshot_manager().get_latest_snapshot().await; + let latest = latest.unwrap().unwrap().id(); + // (rows of `SELECT *`, the `COUNT(*)`) from the batches of both queries. + let read = |all: Vec, count: Vec| { + let count = count[0].column(0).as_any().downcast_ref::(); + (total_rows(&all), count.unwrap().value(0) as usize) + }; + let queries = |from: &str| { + [ + format!("SELECT * FROM {from}"), + format!("SELECT COUNT(*) FROM {from}"), + ] + }; + let [all, count] = queries(&format!( + "paimon_incremental_query('default.de', 0, {latest})" + )); + let function = read(run(&ctx, &all).await, run(&ctx, &count).await); + assert_eq!(function, (8, 8), "function"); + + // The same range as the incremental window of a query on the table: set + // for the session, where `COUNT(*)` is not answered from the manifests, + // and on the table a bare session reads, whose scan counts the rows. + let window = "0,9999999999999"; + run( + &ctx, + &format!("SET 'paimon.incremental-between-timestamp' = '{window}'"), + ) + .await; + let [all, count] = queries("paimon.default.de"); + let set = read(run(&ctx, &all).await, run(&ctx, &count).await); + assert_eq!(set, (8, 8), "SET"); + + let options = std::collections::HashMap::from([( + "incremental-between-timestamp".to_string(), + window.to_string(), + )]); + let table = warehouse.get_table(&identifier).await.unwrap(); + let table = table.copy_with_time_travel(options).await.unwrap(); + let session = SessionContext::new(); + let provider = PaimonTableProvider::try_new(table).unwrap(); + session + .register_table("changes", Arc::new(provider)) + .unwrap(); + let [all, count] = queries("changes"); + let session = &session; + let collect = |sql: String| async move { + let frame = session.sql(&sql).await.unwrap(); + frame.collect().await.unwrap() + }; + let provider = read(collect(all).await, collect(count).await); + assert_eq!(provider, (8, 8), "provider"); +} + /// Asserts that, for `name`, the function reads the rows a `FROM` clause in /// the same session reads. async fn assert_reads_as_from(ctx: &SQLContext, name: &str) { diff --git a/crates/paimon/src/spec/core_options.rs b/crates/paimon/src/spec/core_options.rs index 0c927af26..acb6b0c37 100644 --- a/crates/paimon/src/spec/core_options.rs +++ b/crates/paimon/src/spec/core_options.rs @@ -651,7 +651,7 @@ impl<'a> CoreOptions<'a> { /// The numeric timestamp window supported by PyPaimon. Snapshot resolution /// and ordering checks belong to the batch scanner, as in Java. - pub(crate) fn incremental_timestamp_window(&self) -> crate::Result> { + pub fn incremental_timestamp_window(&self) -> crate::Result> { let Some(raw) = self.options.get(INCREMENTAL_BETWEEN_TIMESTAMP_OPTION) else { return Ok(None); }; diff --git a/crates/paimon/src/table/source.rs b/crates/paimon/src/table/source.rs index ae355f13e..462130ce3 100644 --- a/crates/paimon/src/table/source.rs +++ b/crates/paimon/src/table/source.rs @@ -775,6 +775,10 @@ impl DataSplit { /// Also `None` under a query-auth row filter, which drops rows the count /// includes, as Java `QueryAuthSplit` does. /// + /// Also `None` for change events of a split that is not raw convertible: + /// they may decode a row-id group file by file, returning each file's own + /// rows rather than the merged rows counted here. + /// /// Reference: [DataSplit.mergedRowCount()](https://github.com/apache/paimon/blob/release-1.3/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java#L133) pub fn merged_row_count(&self) -> Option { if self @@ -792,6 +796,9 @@ impl DataSplit { { return None; } + if self.is_streaming && !self.raw_convertible { + return None; + } if let Some(ranges) = &self.row_ranges { return Some(ranges.iter().map(RowRange::count).sum()); } diff --git a/crates/paimon/tests/incremental_batch_scan_test.rs b/crates/paimon/tests/incremental_batch_scan_test.rs index 0b04574a6..75fc59c13 100644 --- a/crates/paimon/tests/incremental_batch_scan_test.rs +++ b/crates/paimon/tests/incremental_batch_scan_test.rs @@ -3513,3 +3513,41 @@ async fn delta_filters_each_file_of_rows_read_file_by_file() { let filtered = read_evolved_range_where(&table, 0, 2, None, Some(bob())).await; assert_eq!(filtered, expected); } + +/// A split that gives a merged row count reads that many rows. After the +/// VECTOR column moved inline, the insert and the rewrite of bob's name and +/// embedding are read file by file: eight rows, one per row of each of their +/// four files, where merged rows would be two. Their split gives no count. +#[tokio::test] +async fn combined_delta_row_counts_match_the_rows_read() { + use paimon::spec::SchemaChange; + let path = "memory:/incremental_batch/row_counts"; + let table = evolved_table(path, &[]).await; + let change = SchemaChange::remove_option("vector.file.format".into()); + let schema = table.schema().apply_changes(vec![change]).unwrap(); + persist_table_schema(table.file_io(), path, &schema).await; + let table = paimon::table::Table::new( + table.file_io().clone(), + table.identifier().clone(), + path.to_string(), + schema, + None, + ); + rewrite(&table, 1, vec![new_name("BOB"), new_embedding([9.0, 9.0])]).await; + + let builder = table.new_read_builder(); + let read = builder.new_read().unwrap(); + let scan = builder.new_incremental_scan(IncrementalScanMode::Delta, 0, 2); + let mut counts = Vec::new(); + for split in scan.plan_combined_delta().await.unwrap().into_splits() { + let batches: Vec = read + .to_arrow(std::slice::from_ref(&split)) + .unwrap() + .try_collect() + .await + .unwrap(); + let rows = batches.iter().map(RecordBatch::num_rows).sum::() as i64; + counts.push((split.merged_row_count(), rows)); + } + assert_eq!(counts, [(None, 8)]); +} diff --git a/docs/src/sql.md b/docs/src/sql.md index a85c57b35..27505fd79 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -2403,7 +2403,8 @@ The optional fourth argument selects the mode: newest file writing it leaves out, written with `data-evolution.write-cols-optimization.enabled` after the column moved to dedicated files; and every file of rows whose files are no longer column slices because a - compaction in the range merged some of them. A column the table has since dropped, or + compaction in the range, such as the one a write runs after its commit unless + `write-only=true`, merged some of them. A column the table has since dropped, or changed from VECTOR to a string type, keeps rows from merging only when the range wrote it in VECTOR files of two layouts. - `changelog` returns the changelog rows of each commit. From 5b93ddeaf5df7f077ca5a3fe466398bce927aa60 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sat, 31 Oct 2026 05:15:55 +0800 Subject: [PATCH 09/18] fix(core): stop a limit at the rows change events return at least The previous commit withdrew the merged row count from change events that may decode a row-id group file by file, and with it the early stop of a limit: `LIMIT 1` over a range of a table whose inserts write BLOB or VECTOR files planned every split. Such a group returns at least its merged rows, as each of its files returns its own, so the limit now plans on that lower bound, while the merged row count stays exact. The docs add that a lookup or full-compaction changelog lands in a later COMPACT snapshot, that a row read file by file is matched to the table's rows by `_ROW_ID`, and that row-position selections and chunk shuffle of a data-evolution window count each row once while a group read file by file returns the selected rows of each of its files. --- crates/paimon/src/table/read_builder.rs | 2 +- crates/paimon/src/table/source.rs | 14 ++- crates/paimon/src/table/table_scan.rs | 10 +- .../tests/incremental_batch_scan_test.rs | 98 ++++++++++++++++--- docs/src/python-binding.md | 10 +- docs/src/sql.md | 15 ++- 6 files changed, 121 insertions(+), 28 deletions(-) diff --git a/crates/paimon/src/table/read_builder.rs b/crates/paimon/src/table/read_builder.rs index 590ebe2b5..56e84c0db 100644 --- a/crates/paimon/src/table/read_builder.rs +++ b/crates/paimon/src/table/read_builder.rs @@ -541,7 +541,7 @@ impl<'a> PaimonReadBuilder<'a> { /// Configure a row limit for scan planning and reads. /// /// This allows paimon-core scan planning to generate fewer splits when the - /// current scan state keeps split-level `merged_row_count()` conservative. + /// current scan state keeps split-level row counts conservative. /// /// A [`TableRead`] created by this builder also enforces the final limit /// across splits after filtering and row selection. Data-evolution reads diff --git a/crates/paimon/src/table/source.rs b/crates/paimon/src/table/source.rs index 462130ce3..36fdd6a66 100644 --- a/crates/paimon/src/table/source.rs +++ b/crates/paimon/src/table/source.rs @@ -781,6 +781,17 @@ impl DataSplit { /// /// Reference: [DataSplit.mergedRowCount()](https://github.com/apache/paimon/blob/release-1.3/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java#L133) pub fn merged_row_count(&self) -> Option { + if self.is_streaming && !self.raw_convertible { + return None; + } + self.row_count_lower_bound() + } + + /// A lower bound of the rows a read of this split returns, computed as + /// [`Self::merged_row_count`] is, but also for change events that may + /// decode a row-id group file by file: its files then return their own + /// rows, at least as many as the merged ones. A limit can be planned on it. + pub(crate) fn row_count_lower_bound(&self) -> Option { if self .query_auth_grant .as_ref() @@ -796,9 +807,6 @@ impl DataSplit { { return None; } - if self.is_streaming && !self.raw_convertible { - return None; - } if let Some(ranges) = &self.row_ranges { return Some(ranges.iter().map(RowRange::count).sum()); } diff --git a/crates/paimon/src/table/table_scan.rs b/crates/paimon/src/table/table_scan.rs index 210d464da..a17957a95 100644 --- a/crates/paimon/src/table/table_scan.rs +++ b/crates/paimon/src/table/table_scan.rs @@ -731,8 +731,8 @@ fn merge_manifest_entries(mut entries: Vec) -> Vec entries } -/// Whether scan-owned pruning still preserves `merged_row_count()` as a safe -/// row-count hint. +/// Whether scan-owned pruning still preserves `row_count_lower_bound()` as a +/// safe row-count hint. /// /// Data predicates and row ranges can reduce rows within a split after planning, /// so split-level row counts stop being a conservative bound for final rows. @@ -775,10 +775,10 @@ impl LimitPushdownAccumulator { return true; } - if let Some(merged_count) = split.merged_row_count() { + if let Some(count) = split.row_count_lower_bound() { self.fallback_splits.push(split.clone()); self.limited_splits.push(split); - self.scanned_row_count += merged_count; + self.scanned_row_count += count; // Compare without a lossy signed cast: `limit` is `usize`, so // `limit as i64` turns any value above `i64::MAX` (e.g. a C caller // passing `SIZE_MAX`) negative, which would early-stop after the @@ -2347,7 +2347,7 @@ impl<'a> PaimonTableScan<'a> { /// Apply a limit-pushdown hint to the generated splits. /// /// Mirrors Java `DataTableBatchScan#applyPushDownLimit`: splits whose - /// `merged_row_count()` is unknown (for example merge-needed PK splits or + /// `row_count_lower_bound()` is unknown (for example merge-needed PK splits or /// unknown deletion cardinality) are skipped — they contribute an unknown /// number of rows, so they cannot help satisfy the limit. Pruning is /// committed only once the accumulated known row count reaches the limit; diff --git a/crates/paimon/tests/incremental_batch_scan_test.rs b/crates/paimon/tests/incremental_batch_scan_test.rs index 75fc59c13..3de0055c5 100644 --- a/crates/paimon/tests/incremental_batch_scan_test.rs +++ b/crates/paimon/tests/incremental_batch_scan_test.rs @@ -3514,13 +3514,40 @@ async fn delta_filters_each_file_of_rows_read_file_by_file() { assert_eq!(filtered, expected); } +/// `(merged row count, rows read)` of each split of the combined Delta range +/// `(start, end]`, sorted. +async fn combined_delta_row_counts( + table: &paimon::table::Table, + start: i64, + end: i64, +) -> Vec<(Option, i64)> { + let builder = table.new_read_builder(); + let read = builder.new_read().unwrap(); + let scan = builder.new_incremental_scan(IncrementalScanMode::Delta, start, end); + let mut counts = Vec::new(); + for split in scan.plan_combined_delta().await.unwrap().into_splits() { + let batches: Vec = read + .to_arrow(std::slice::from_ref(&split)) + .unwrap() + .try_collect() + .await + .unwrap(); + let rows = batches.iter().map(RecordBatch::num_rows).sum::() as i64; + counts.push((split.merged_row_count(), rows)); + } + counts.sort(); + counts +} + /// A split that gives a merged row count reads that many rows. After the /// VECTOR column moved inline, the insert and the rewrite of bob's name and /// embedding are read file by file: eight rows, one per row of each of their -/// four files, where merged rows would be two. Their split gives no count. +/// four files, where merged rows would be two. Their split gives no count, +/// while an insert of one file, planned as a split of its own, gives its own. #[tokio::test] async fn combined_delta_row_counts_match_the_rows_read() { - use paimon::spec::SchemaChange; + use arrow_array::ArrayRef; + use paimon::spec::{DataType, IntType, Schema, SchemaChange, TableSchema}; let path = "memory:/incremental_batch/row_counts"; let table = evolved_table(path, &[]).await; let change = SchemaChange::remove_option("vector.file.format".into()); @@ -3534,20 +3561,67 @@ async fn combined_delta_row_counts_match_the_rows_read() { None, ); rewrite(&table, 1, vec![new_name("BOB"), new_embedding([9.0, 9.0])]).await; + assert_eq!(combined_delta_row_counts(&table, 0, 2).await, [(None, 8)]); - let builder = table.new_read_builder(); - let read = builder.new_read().unwrap(); - let scan = builder.new_incremental_scan(IncrementalScanMode::Delta, 0, 2); - let mut counts = Vec::new(); - for split in scan.plan_combined_delta().await.unwrap().into_splits() { - let batches: Vec = read - .to_arrow(std::slice::from_ref(&split)) + // An update of the first insert's values merges into its rows; the + // second insert is one file. + let path = "memory:/incremental_batch/row_counts_of_one_file"; + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("value", DataType::Int(IntType::new())) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .build() + .unwrap(); + let (file_io, table) = memory_table(path, TableSchema::new(0, &schema)); + setup_dirs(&file_io, path).await; + persist_table_schema(&file_io, path, table.schema()).await; + write_batch(&table, &make_batch(vec![1, 2, 3], vec![10, 20, 30])).await; + write_batch(&table, &make_batch(vec![4, 5], vec![40, 50])).await; + let value = arrow_schema::Field::new("value", arrow_schema::DataType::Int32, true); + let values = std::sync::Arc::new(Int32Array::from(vec![11])) as ArrayRef; + rewrite(&table, 0, vec![(value, values)]).await; + let counts = combined_delta_row_counts(&table, 0, 3).await; + assert_eq!(counts, [(None, 3), (Some(2), 2)]); +} + +/// A limit stops planning once the splits planned return enough rows, also +/// for change events that may decode a row-id group file by file and so give +/// no merged row count: such a group returns at least its merged rows. Each +/// insert of the table writes a normal, a BLOB and a VECTOR file. +#[tokio::test] +async fn combined_delta_limit_stops_at_the_rows_splits_return_at_least() { + let path = "memory:/incremental_batch/limit_lower_bound"; + let table = evolved_table(path, &[]).await; + for (ids, names) in [ + (vec![3, 4], vec!["carol", "dave"]), + (vec![5, 6], vec!["erin", "fred"]), + ] { + let pictures = vec![b"c".as_slice(), b"d".as_slice()]; + let batch = evolved_batch(ids, names, pictures, &[[1.0, 1.0], [2.0, 2.0]]); + write_batch(&table, &batch).await; + } + for (limit, expected) in [(None, (3, false, 6)), (Some(1), (1, true, 1))] { + let mut builder = table.new_read_builder(); + if let Some(limit) = limit { + builder.with_limit(limit); + } + let scan = builder.new_incremental_scan(IncrementalScanMode::Delta, 0, 3); + let plan = scan.plan_combined_delta().await.unwrap(); + let splits = plan.splits(); + assert!(splits + .iter() + .all(|split| split.merged_row_count().is_none())); + let batches: Vec = builder + .new_read() + .unwrap() + .to_arrow(splits) .unwrap() .try_collect() .await .unwrap(); - let rows = batches.iter().map(RecordBatch::num_rows).sum::() as i64; - counts.push((split.merged_row_count(), rows)); + let rows = batches.iter().map(RecordBatch::num_rows).sum::(); + let early = plan.stats().limit_early_stopped; + assert_eq!((splits.len(), early, rows), expected, "{limit:?}"); } - assert_eq!(counts, [(None, 8)]); } diff --git a/docs/src/python-binding.md b/docs/src/python-binding.md index 0f7eb061c..9f35774b9 100644 --- a/docs/src/python-binding.md +++ b/docs/src/python-binding.md @@ -174,7 +174,10 @@ primary keys and physical retracts remain separate rows. Readers do not merge these events into the window's final table state. The end snapshot must exist and supplies snapshot metadata, including for empty results. Snapshot deletion vectors and automatic global indexes are not applied to historical events. -Builder filters, projections, and limits still apply. +Builder filters, projections, and limits still apply. A Data Evolution range +merges its files into rows as described for +[`paimon_incremental_query`](sql.md#incremental-query); files that do not merge +are read as written, one row per row of each file. `split.is_streaming()` identifies this read contract. `split.serialize()` exports both batch and streaming splits to Java binary encoding, preserving the streaming @@ -199,7 +202,10 @@ intersect the positions after assignment, even when they exclude earlier files. Slices require `start < end`; shards require a positive count and `0 <= index < count`. Slice and shard selection are mutually exclusive and may also be applied to `new_incremental_scan` results, where positions count the -combined APPEND-delta batch. The selection is encoded in the returned splits +combined APPEND-delta batch. There positions and `with_chunk_shuffle` chunk +sizes count each Data Evolution row once, while a group read file by file +returns the selected rows from each of its files, so a selection can return +more rows than it selects. The selection is encoded in the returned splits and survives serialization and direct native reads. Deletion-vector reads accept both Java and Python Avro array-item schemas. diff --git a/docs/src/sql.md b/docs/src/sql.md index 27505fd79..ba1792764 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -2404,10 +2404,12 @@ The optional fourth argument selects the mode: `data-evolution.write-cols-optimization.enabled` after the column moved to dedicated files; and every file of rows whose files are no longer column slices because a compaction in the range, such as the one a write runs after its commit unless - `write-only=true`, merged some of them. A column the table has since dropped, or - changed from VECTOR to a string type, keeps rows from merging only when the range - wrote it in VECTOR files of two layouts. -- `changelog` returns the changelog rows of each commit. + `write-only=true`, merged some of them (Java's reader fails on such a range). A + column the table has since dropped, or changed from VECTOR to a string type, keeps + rows from merging only when the range wrote it in VECTOR files of two layouts. +- `changelog` returns the changelog rows of each commit. A `lookup` or `full-compaction` + producer writes them when it compacts, in a later COMPACT snapshot, which the range + must include. - `diff` compares the table at the two snapshots and returns the added and changed rows (`$audit_log` adds `-U` and `-D` rows). It requires a primary-key table with `merge-engine=deduplicate` and no BLOB or VARIANT columns. @@ -2417,7 +2419,10 @@ The optional fourth argument selects the mode: A `$audit_log` suffix puts `rowkind` first, followed by `_SEQUENCE_NUMBER` when `table-read.sequence-number.enabled=true`. Without it the columns are those of a query on the table, `_ROW_ID` included on a data-evolution table, so a column update -returned without the row's other columns can be matched to its row. +returned without the row's other columns can be matched to its row. A row read file +by file comes once per file that wrote it, and nothing in the output tells which is +newer; to read the rows' current values, join their `_ROW_ID`s with a query on the +table. The table name resolves in the calling session: `my_table` and `my_db.my_table` use the current catalog (and database), as in a `FROM` clause, and From d6f8693852002df0295bf05f3b52f847d02173fc Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sat, 31 Oct 2026 06:18:32 +0800 Subject: [PATCH 10/18] fix(core): chunk change events of BLOB or VECTOR files alone Chunk shuffle looked up a normal anchor file in every data-evolution row-id group, although only deletion vectors, which belong to that file, need one. Change events load no deletion vectors and read a range of BLOB or VECTOR rewrites alone file by file, but `with_chunk_shuffle` on such a range failed at planning, as it did on base. A group now needs an anchor only when it has deletion files; otherwise it is sliced over its row range, as a group with a normal file already was. A limit test now also stops at a group read file by file, and the docs say only the splitting at dots is shared with the other Paimon table functions. --- .../datafusion/src/table_function_args.rs | 4 +- crates/paimon/src/table/chunk_shuffle.rs | 55 ++++++++++------ .../tests/incremental_batch_scan_test.rs | 65 ++++++++++++++++++- docs/src/sql.md | 6 +- 4 files changed, 101 insertions(+), 29 deletions(-) diff --git a/crates/integrations/datafusion/src/table_function_args.rs b/crates/integrations/datafusion/src/table_function_args.rs index e18b1b2ff..50b3bf3ac 100644 --- a/crates/integrations/datafusion/src/table_function_args.rs +++ b/crates/integrations/datafusion/src/table_function_args.rs @@ -77,8 +77,8 @@ pub(crate) fn parse_table_identifier( /// Resolves a table-name argument in the calling session: `table` and /// `database.table` take the session's current catalog (and database), as in -/// a `FROM` clause, and `catalog.database.table` names the catalog. As in -/// `parse_table_identifier`, the name is split at dots and its parts are +/// a `FROM` clause, and `catalog.database.table` names the catalog. The name +/// is split at dots, as `parse_table_identifier` splits it, and its parts are /// used as written. Returns the catalog's name and the table's identifier in /// that catalog. pub(crate) fn resolve_table_argument( diff --git a/crates/paimon/src/table/chunk_shuffle.rs b/crates/paimon/src/table/chunk_shuffle.rs index d426f649d..cf9d640d1 100644 --- a/crates/paimon/src/table/chunk_shuffle.rs +++ b/crates/paimon/src/table/chunk_shuffle.rs @@ -366,30 +366,43 @@ async fn evolution_chunks( message: "data evolution chunk group has no row range".to_string(), source: None, })?; - let anchor = data_evolution_anchor_file(&files)?; - let anchor_deletion = deletion_by_name.get(&anchor.file_name).cloned().flatten(); - let (first_row_id, physical_count) = if anchor_deletion.is_some() { - let (anchor_from, anchor_to) = anchor.row_id_range().unwrap(); - if anchor_from > from || anchor_to < to { - return Err(crate::Error::DataInvalid { - message: format!( - "data evolution anchor range [{anchor_from}, {anchor_to}] does not contain group [{from}, {to}]" - ), - source: None, - }); - } - (anchor_from, anchor.row_count) + // Deletion vectors belong to a group's normal anchor file. A group + // without them is sliced over its row range and needs no normal file, + // as change events of BLOB or VECTOR rewrites alone have none. + let anchor = if files + .iter() + .any(|file| matches!(deletion_by_name.get(&file.file_name), Some(Some(_)))) + { + Some(data_evolution_anchor_file(&files)?) } else { - (from, to - from + 1) + None }; - let anchor_input = InputFile { - file: anchor.clone(), - deletion_file: anchor_deletion.clone(), + let anchor_deletion = + anchor.and_then(|anchor| deletion_by_name.get(&anchor.file_name).cloned().flatten()); + let (first_row_id, physical_count) = match (anchor, &anchor_deletion) { + (Some(anchor), Some(_)) => { + let (anchor_from, anchor_to) = anchor.row_id_range().unwrap(); + if anchor_from > from || anchor_to < to { + return Err(crate::Error::DataInvalid { + message: format!( + "data evolution anchor range [{anchor_from}, {anchor_to}] does not contain group [{from}, {to}]" + ), + source: None, + }); + } + (anchor_from, anchor.row_count) + } + _ => (from, to - from + 1), }; - let slicer = if anchor_deletion.is_some() { - live_row_slicer(table, &anchor_input).await? - } else { - LiveRowSlicer::new(physical_count, Vec::new())? + let slicer = match (anchor, anchor_deletion) { + (Some(anchor), Some(deletion_file)) => { + let anchor_input = InputFile { + file: anchor.clone(), + deletion_file: Some(deletion_file), + }; + live_row_slicer(table, &anchor_input).await? + } + _ => LiveRowSlicer::new(physical_count, Vec::new())?, }; let mut slicer = match slicer { Some(slicer) => slicer, diff --git a/crates/paimon/tests/incremental_batch_scan_test.rs b/crates/paimon/tests/incremental_batch_scan_test.rs index 3de0055c5..89d330ee1 100644 --- a/crates/paimon/tests/incremental_batch_scan_test.rs +++ b/crates/paimon/tests/incremental_batch_scan_test.rs @@ -3562,6 +3562,21 @@ async fn combined_delta_row_counts_match_the_rows_read() { ); rewrite(&table, 1, vec![new_name("BOB"), new_embedding([9.0, 9.0])]).await; assert_eq!(combined_delta_row_counts(&table, 0, 2).await, [(None, 8)]); + // A limit plans on the rows the split returns at least, its merged rows. + let mut builder = table.new_read_builder(); + builder.with_limit(1); + let scan = builder.new_incremental_scan(IncrementalScanMode::Delta, 0, 2); + let plan = scan.plan_combined_delta().await.unwrap(); + let batches: Vec = builder + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + let rows = batches.iter().map(RecordBatch::num_rows).sum::(); + assert_eq!((plan.stats().limit_early_stopped, rows), (true, 1)); // An update of the first insert's values merges into its rows; the // second insert is one file. @@ -3586,9 +3601,10 @@ async fn combined_delta_row_counts_match_the_rows_read() { } /// A limit stops planning once the splits planned return enough rows, also -/// for change events that may decode a row-id group file by file and so give -/// no merged row count: such a group returns at least its merged rows. Each -/// insert of the table writes a normal, a BLOB and a VECTOR file. +/// for change events that give no merged row count because their row-id +/// groups may be decoded file by file: a group returns at least its merged +/// rows either way. Each insert of the table writes a normal, a BLOB and a +/// VECTOR file, which merge. #[tokio::test] async fn combined_delta_limit_stops_at_the_rows_splits_return_at_least() { let path = "memory:/incremental_batch/limit_lower_bound"; @@ -3625,3 +3641,46 @@ async fn combined_delta_limit_stops_at_the_rows_splits_return_at_least() { assert_eq!((splits.len(), early, rows), expected, "{limit:?}"); } } + +/// Chunk shuffle slices a range of BLOB or VECTOR rewrites alone as it slices +/// other row ranges: change events load no deletion vectors, so their groups +/// need no normal file to anchor them. Together the chunks return the rows of +/// the range, each chunk the selected rows of every file of its group. +#[tokio::test] +async fn combined_delta_chunk_shuffle_slices_rewrites_of_dedicated_files_alone() { + let path = "memory:/incremental_batch/chunks_of_rewrites"; + let table = evolved_table(path, &[]).await; + // Snapshot 2 recomputes an embedding; snapshot 3 rewrites a picture and + // an embedding, a BLOB and a VECTOR file of the same rows. + rewrite(&table, 0, vec![new_embedding([9.0, 9.0])]).await; + let picture_and_embedding = vec![new_picture(b"zz"), new_embedding([7.0, 7.0])]; + rewrite(&table, 1, picture_and_embedding).await; + let builder = table.new_read_builder(); + let read = builder.new_read().unwrap(); + for (start, end) in [(1, 2), (2, 3), (1, 3)] { + let scan = || builder.new_incremental_scan(IncrementalScanMode::Delta, start, end); + let range = scan().plan_combined_delta().await.unwrap(); + let batches: Vec = read + .to_arrow(range.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + let expected = evolved_rows(&batches); + let chunks = scan().with_chunk_shuffle("7", 1).unwrap(); + let chunks = chunks.plan_combined_delta().await.unwrap(); + assert_eq!(chunks.splits().len(), 2, "({start},{end}]"); + let mut rows = Vec::new(); + for chunk in chunks.splits() { + let batches: Vec = read + .to_arrow(std::slice::from_ref(chunk)) + .unwrap() + .try_collect() + .await + .unwrap(); + rows.extend(evolved_rows(&batches)); + } + rows.sort(); + assert_eq!(rows, expected, "({start},{end}]"); + } +} diff --git a/docs/src/sql.md b/docs/src/sql.md index ba1792764..8ac987968 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -2426,9 +2426,9 @@ table. The table name resolves in the calling session: `my_table` and `my_db.my_table` use the current catalog (and database), as in a `FROM` clause, and -`my_catalog.my_db.my_table` names the catalog. As in the other Paimon table -functions, the name is split at dots, and the catalog, database and table names are -used as written, without SQL quoting or case folding. Dynamic options set with +`my_catalog.my_db.my_table` names the catalog. The name is split at dots, as in +the other Paimon table functions, and its parts are used as written, without SQL +quoting or case folding. Dynamic options set with `SET 'paimon.…'` apply as they do to a query on the table, except options that choose another range to read (time-travel selectors, `incremental-between*`, file creation-time filters and a `scan.mode` other than `default`), which are rejected From ff88c78f2477049f55a8f453b2526f23afb449a7 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sat, 31 Oct 2026 08:36:01 +0800 Subject: [PATCH 11/18] docs: match change events read file by file back to their rows A row of a data-evolution window read file by file carries only the columns its file wrote. The docs now say to join such rows by `_ROW_ID` with the table `VERSION AS OF` the range's end snapshot and to filter after the join, that a Python read has to project `_ROW_ID` for that, and that selected row ids, like positions and chunk sizes, count each row once. The chunk shuffle docstrings say chunks count row ids on data-evolution tables. --- bindings/python/python/pypaimon_rust/datafusion.pyi | 3 ++- bindings/python/src/read.rs | 3 ++- docs/src/python-binding.md | 10 ++++++---- docs/src/sql.md | 5 +++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/bindings/python/python/pypaimon_rust/datafusion.pyi b/bindings/python/python/pypaimon_rust/datafusion.pyi index 0ede5a1fb..5ecff424a 100644 --- a/bindings/python/python/pypaimon_rust/datafusion.pyi +++ b/bindings/python/python/pypaimon_rust/datafusion.pyi @@ -266,7 +266,8 @@ class TableScan: def with_row_position_slice(self, start: int, end: int) -> "TableScan": ... def with_row_position_shard(self, index: int, count: int) -> "TableScan": ... def with_chunk_shuffle(self, seed: str, chunk_size: int) -> "TableScan": - """Deterministically shuffle fixed-live-row chunks. ``seed`` is a decimal + """Deterministically shuffle fixed-live-row chunks; on Data Evolution + tables chunks count row ids. ``seed`` is a decimal integer string so arbitrarily large seeds keep Python ``random.Random`` semantics.""" ... diff --git a/bindings/python/src/read.rs b/bindings/python/src/read.rs index 4d9b9c324..f0fb8c4bf 100644 --- a/bindings/python/src/read.rs +++ b/bindings/python/src/read.rs @@ -767,7 +767,8 @@ impl PyTableScan { Ok(slf) } - /// Deterministically shuffle fixed-live-row chunks. `seed` is a decimal + /// Deterministically shuffle fixed-live-row chunks; on Data Evolution + /// tables chunks count row ids. `seed` is a decimal /// Python integer string so arbitrarily large seeds retain Python's /// `random.Random` semantics. fn with_chunk_shuffle( diff --git a/docs/src/python-binding.md b/docs/src/python-binding.md index 9f35774b9..0fe3f1bda 100644 --- a/docs/src/python-binding.md +++ b/docs/src/python-binding.md @@ -177,7 +177,9 @@ vectors and automatic global indexes are not applied to historical events. Builder filters, projections, and limits still apply. A Data Evolution range merges its files into rows as described for [`paimon_incremental_query`](sql.md#incremental-query); files that do not merge -are read as written, one row per row of each file. +are read as written, one row per row of each file. Project `_ROW_ID`, for +example `rb.with_projection(["_ROW_ID", "embedding"])`, to match those rows to +the table's rows. `split.is_streaming()` identifies this read contract. `split.serialize()` exports both batch and streaming splits to Java binary encoding, preserving the streaming @@ -202,9 +204,9 @@ intersect the positions after assignment, even when they exclude earlier files. Slices require `start < end`; shards require a positive count and `0 <= index < count`. Slice and shard selection are mutually exclusive and may also be applied to `new_incremental_scan` results, where positions count the -combined APPEND-delta batch. There positions and `with_chunk_shuffle` chunk -sizes count each Data Evolution row once, while a group read file by file -returns the selected rows from each of its files, so a selection can return +combined APPEND-delta batch. There positions, `with_chunk_shuffle` chunk sizes +and selected row ids count each Data Evolution row once, while a group read file +by file returns the selected rows of each of its files, so a selection can return more rows than it selects. The selection is encoded in the returned splits and survives serialization and direct native reads. diff --git a/docs/src/sql.md b/docs/src/sql.md index 8ac987968..62b0b5867 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -2421,8 +2421,9 @@ A `$audit_log` suffix puts `rowkind` first, followed by `_SEQUENCE_NUMBER` when query on the table, `_ROW_ID` included on a data-evolution table, so a column update returned without the row's other columns can be matched to its row. A row read file by file comes once per file that wrote it, and nothing in the output tells which is -newer; to read the rows' current values, join their `_ROW_ID`s with a query on the -table. +newer. To read the rows' values at the end of the range, join their `_ROW_ID`s with +the table `VERSION AS OF` the end snapshot, and filter after the join: a column a file +did not write is NULL in that file's rows. The table name resolves in the calling session: `my_table` and `my_db.my_table` use the current catalog (and database), as in a `FROM` clause, and From c141ccbeb5687311de2ed03d14e1f757f6837c07 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sat, 31 Oct 2026 11:44:50 +0800 Subject: [PATCH 12/18] fix(datafusion): refuse writes and vector_search under a session window UPDATE, DELETE and MERGE INTO find the rows they change by a query on the table through the session, and `vector_search` reads its hits that way. With `SET 'paimon.incremental-between-timestamp'`, the query returns the window's changes instead of the table's rows, so an UPDATE wrote values computed from older versions over newer ones, as it did on base. Since change events of a window holding a compaction are now read file by file, such a window also returns a row once per file that wrote it: a DELETE removed rows whose older version its filter matched, and `vector_search` returned the column update of each hit, its other columns NULL. The writes that refuse a time-travel selector, these three among them, now also refuse the window, and so does `vector_search`. --- .../datafusion/src/sql_context.rs | 19 ++++- .../datafusion/src/vector_search.rs | 13 +++ .../datafusion/tests/incremental_query.rs | 79 +++++++++++++++++++ 3 files changed, 107 insertions(+), 4 deletions(-) diff --git a/crates/integrations/datafusion/src/sql_context.rs b/crates/integrations/datafusion/src/sql_context.rs index acb000273..6f3144917 100644 --- a/crates/integrations/datafusion/src/sql_context.rs +++ b/crates/integrations/datafusion/src/sql_context.rs @@ -1609,12 +1609,13 @@ impl SQLContext { } /// Reject latest-table statement executors while a session-level - /// time-travel selector is active. + /// time-travel selector or incremental timestamp window is active. /// /// These executors load the latest catalog table, while session reads - /// resolve the selected snapshot schema. Silently ignoring the selector - /// here would be inconsistent. Ordinary INSERT uses the resolved provider - /// and its selected schema, so it does not need this guard. + /// resolve the selected snapshot schema, or return a window's changes + /// instead of the table's rows. Silently ignoring the selector here would + /// be inconsistent. Ordinary INSERT uses the resolved provider and its + /// selected schema, so it does not need this guard. fn ensure_no_time_travel_for_write(&self, operation: &str) -> DFResult<()> { use paimon::spec::{ SCAN_SNAPSHOT_ID_OPTION, SCAN_TAG_NAME_OPTION, SCAN_TIMESTAMP_MILLIS_OPTION, @@ -1637,6 +1638,16 @@ impl SQLContext { ))); } } + if !matches!( + CoreOptions::new(&options).incremental_timestamp_window(), + Ok(None) + ) { + let key = "incremental-between-timestamp"; + return Err(DataFusionError::Plan(format!( + "Cannot execute {operation} while scan option '{key}' is set; \ + RESET 'paimon.{key}' first" + ))); + } Ok(()) } diff --git a/crates/integrations/datafusion/src/vector_search.rs b/crates/integrations/datafusion/src/vector_search.rs index 715ce37ee..6bca78511 100644 --- a/crates/integrations/datafusion/src/vector_search.rs +++ b/crates/integrations/datafusion/src/vector_search.rs @@ -154,6 +154,19 @@ impl TableFunctionImpl for VectorSearchFunction { }, "vector_search: catalog access thread panicked", )?; + // The index is searched at one snapshot, while a read with a timestamp + // window returns that window's changes, which can hold a row once per + // file that wrote it, some of its columns NULL. + if !matches!( + CoreOptions::new(table.schema().options()).incremental_timestamp_window(), + Ok(None) + ) { + let key = "incremental-between-timestamp"; + return Err(DataFusionError::Plan(format!( + "{FUNCTION_NAME} searches one snapshot and cannot run while scan option \ + '{key}' is set; RESET 'paimon.{key}' first" + ))); + } let inner = PaimonTableProvider::try_new(table)?; let query_vector_json = diff --git a/crates/integrations/datafusion/tests/incremental_query.rs b/crates/integrations/datafusion/tests/incremental_query.rs index 4ad6d99e7..3d49548be 100644 --- a/crates/integrations/datafusion/tests/incremental_query.rs +++ b/crates/integrations/datafusion/tests/incremental_query.rs @@ -849,6 +849,85 @@ async fn test_a_read_of_the_range_as_its_window_counts_the_rows_it_returns() { assert_eq!(provider, (8, 8), "provider"); } +#[tokio::test] +async fn test_writes_refuse_a_session_window() { + use datafusion::arrow::util::pretty::pretty_format_batches; + // UPDATE, DELETE and MERGE INTO find the rows they change by a query on the + // table. A session window makes that query return the window's changes: a + // column update without the row's other columns, or a row once per file + // that wrote it. The write would then compute values from, and match, rows + // the table no longer holds. + let tmp = TempDir::new().unwrap(); + let ctx = context(&tmp).await; + run( + &ctx, + "CREATE TABLE paimon.default.de (id INT, v INT) WITH ( + 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true', + 'deletion-vectors.enabled' = 'true', + 'write-only' = 'true' + )", + ) + .await; + run( + &ctx, + "INSERT INTO paimon.default.de (id, v) VALUES (1, 1), (2, 2)", + ) + .await; + run(&ctx, "UPDATE paimon.default.de SET v = 20 WHERE id = 2").await; + let table = "SELECT id, v FROM paimon.default.de ORDER BY id"; + let format = |batches: Vec| pretty_format_batches(&batches).unwrap().to_string(); + let expected = format(run(&ctx, table).await); + + let key = "incremental-between-timestamp"; + run(&ctx, &format!("SET 'paimon.{key}' = '0,9999999999999'")).await; + for statement in [ + "UPDATE paimon.default.de SET v = v + 1 WHERE id = 2", + "DELETE FROM paimon.default.de WHERE id = 1", + "MERGE INTO paimon.default.de t USING (SELECT 2 AS id) s ON t.id = s.id + WHEN MATCHED THEN UPDATE SET v = t.v + 1", + ] { + let err = expect_error(&ctx, statement).await; + assert!( + err.contains(&format!( + "scan option '{key}' is set; RESET 'paimon.{key}' first" + )), + "{statement}: {err}" + ); + } + run(&ctx, &format!("RESET 'paimon.{key}'")).await; + assert_eq!(format(run(&ctx, table).await), expected); +} + +#[tokio::test] +async fn test_vector_search_refuses_a_session_window() { + // The index is searched at one snapshot, while the window's rows, read to + // return the hits, are its changes. + let tmp = TempDir::new().unwrap(); + let ctx = context(&tmp).await; + run( + &ctx, + "CREATE TABLE paimon.default.v (id INT, embedding ARRAY) WITH ( + 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true' + )", + ) + .await; + let key = "incremental-between-timestamp"; + run(&ctx, &format!("SET 'paimon.{key}' = '0,9999999999999'")).await; + let err = expect_error( + &ctx, + "SELECT * FROM vector_search('paimon.default.v', 'embedding', '[1.0, 0.0]', 1)", + ) + .await; + assert!( + err.contains(&format!( + "vector_search searches one snapshot and cannot run while scan option '{key}' is set" + )), + "{err}" + ); +} + /// Asserts that, for `name`, the function reads the rows a `FROM` clause in /// the same session reads. async fn assert_reads_as_from(ctx: &SQLContext, name: &str) { From c0d116bd7960ed2fb4ca185083e5c6edaa7be422 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sat, 31 Oct 2026 11:45:06 +0800 Subject: [PATCH 13/18] docs: say what a delta range returns of deleted rows and partitions A delta range returns the rows its APPEND commits wrote, rows a later DELETE removed included. Rows of a data-evolution update read on their own carry the partition columns as well as the updated ones. A row read file by file comes once per file, so the join back to the table's rows uses distinct `_ROW_ID`s. The function reads the range in one partition and applies `WHERE` filters after the read. --- docs/src/sql.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/src/sql.md b/docs/src/sql.md index 62b0b5867..a3e0864bf 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -2383,17 +2383,19 @@ SELECT * FROM paimon_incremental_query('my_db.my_table$audit_log', 1, 5, 'change The optional fourth argument selects the mode: -- `delta` returns the rows each APPEND commit wrote, every version of a key included. +- `delta` returns the rows each APPEND commit wrote, every version of a key included, + even rows a later `DELETE` removed. A data-evolution table's range is read as one batch, as Java's `incremental-between` scan reads it: the files it wrote for the same rows merge into rows when they are column slices of one row range, that is, its normal data files cover the same rows, its BLOB and VECTOR files lie within them, and each BLOB or VECTOR column has one layout, for BLOB the one the table now uses, and the merge would read each column where it is stored. For rows written before the range it returns every row of the - files the updates rewrote, unchanged rows included, with only the updated columns set - and the others NULL, NOT NULL columns included. (Java's reader instead fails when two - such updates of the same rows leave a NOT NULL column unwritten.) Other files are - returned as written, one row per row of each file. These include BLOB or VECTOR files + files the updates rewrote, unchanged rows included, with only the updated and the + partition columns set and the others NULL, NOT NULL columns included. (Java's reader + instead fails when two such updates of the same rows leave a NOT NULL column + unwritten.) Other files are returned as written, one row per row of each file. + These include BLOB or VECTOR files alone, such as recomputed embeddings, so two embedding updates of the same rows return both versions where Java returns the latest; every file of rows whose BLOB or VECTOR column the range wrote in two layouts (inline and in dedicated files, after a change @@ -2421,9 +2423,9 @@ A `$audit_log` suffix puts `rowkind` first, followed by `_SEQUENCE_NUMBER` when query on the table, `_ROW_ID` included on a data-evolution table, so a column update returned without the row's other columns can be matched to its row. A row read file by file comes once per file that wrote it, and nothing in the output tells which is -newer. To read the rows' values at the end of the range, join their `_ROW_ID`s with -the table `VERSION AS OF` the end snapshot, and filter after the join: a column a file -did not write is NULL in that file's rows. +newer. To read the rows' values at the end of the range, join their distinct `_ROW_ID`s +with the table `VERSION AS OF` the end snapshot, and filter after the join: a column a +file did not write, other than a partition column, is NULL in that file's rows. The table name resolves in the calling session: `my_table` and `my_db.my_table` use the current catalog (and database), as in a `FROM` clause, and @@ -2436,6 +2438,8 @@ creation-time filters and a `scan.mode` other than `default`), which are rejecte whether the session or the table read sets them. With `SET 'paimon.branch'`, the table read is that branch. +The range is read in one partition, and `WHERE` filters apply after the read. + ## Dynamic Options (SET / RESET) Use `SET` to configure session-scoped Paimon dynamic options that apply to subsequent table loads: From 87ebeaa6d8ef25d671f718459f0418ee9ff2146b Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sat, 31 Oct 2026 13:26:57 +0800 Subject: [PATCH 14/18] fix(datafusion): refuse a window the table stores, where rows are looked up c141ccb refused writes and `vector_search` only while the session set `incremental-between-timestamp`. A table can store the option too (`ALTER TABLE ... SET TBLPROPERTIES`), and then every read of it returns the window's changes, the target lookup of UPDATE, DELETE and MERGE INTO and the reads of `vector_search`, `full_text_search` and `hybrid_search` hits among them. A window holding a compaction is read file by file, so a DELETE removed a row its filter matched only in an older version, UPDATE and MERGE wrote values computed from older versions, `vector_search` returned the column-update row of each hit with its other columns NULL, and the other two returned each hit twice, where base failed or deleted nothing. These now check the table they read, as `paimon_incremental_query` checks its range options, and name the table or branch that stores the window. The writes and `vector_search` read with the session's options applied; the other two searches take no session options, so only a stored window stops them. TRUNCATE TABLE, DROP PARTITION and INSERT OVERWRITE ... PARTITION read no rows, so they no longer refuse a session window. --- .../datafusion/src/full_text_search.rs | 19 ++- .../datafusion/src/hybrid_search.rs | 16 ++- .../datafusion/src/incremental_query.rs | 10 +- .../datafusion/src/sql_context.rs | 51 +++++-- .../datafusion/src/table_loader.rs | 43 +++++- .../datafusion/src/vector_search.rs | 27 ++-- .../datafusion/tests/incremental_query.rs | 133 +++++++++++++----- 7 files changed, 216 insertions(+), 83 deletions(-) diff --git a/crates/integrations/datafusion/src/full_text_search.rs b/crates/integrations/datafusion/src/full_text_search.rs index a237d5998..9c5a905c2 100644 --- a/crates/integrations/datafusion/src/full_text_search.rs +++ b/crates/integrations/datafusion/src/full_text_search.rs @@ -24,6 +24,7 @@ //! //! Reference: [PaimonTableValuedFunctions.scala](https://github.com/apache/paimon/blob/master/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonTableValuedFunctions.scala) +use std::collections::HashMap; use std::fmt::Debug; use std::sync::Arc; @@ -33,7 +34,7 @@ use datafusion::catalog::Session; use datafusion::catalog::TableFunctionImpl; use datafusion::common::project_schema; use datafusion::datasource::{TableProvider, TableType}; -use datafusion::error::Result as DFResult; +use datafusion::error::{DataFusionError, Result as DFResult}; use datafusion::logical_expr::{Expr, TableProviderFilterPushDown}; use datafusion::physical_plan::empty::EmptyExec; use datafusion::physical_plan::ExecutionPlan; @@ -46,7 +47,7 @@ use crate::table::{PaimonScanBuilder, PaimonTableProvider}; use crate::table_function_args::{ extract_int_literal, extract_string_literal, parse_table_identifier, }; -use crate::table_loader::load_data_table_for_read; +use crate::table_loader::{ensure_no_incremental_window, load_data_table_for_read}; const FUNCTION_NAME: &str = "full_text_search"; @@ -111,7 +112,19 @@ impl TableFunctionImpl for FullTextSearchFunction { let catalog = Arc::clone(&self.catalog); let table = block_on_with_runtime( - async move { load_data_table_for_read(&catalog, &identifier, FUNCTION_NAME).await }, + async move { + let table = load_data_table_for_read(&catalog, &identifier, FUNCTION_NAME).await?; + // The index is searched at one snapshot, while a window stored + // on the table would make the read of its hits return that + // window's changes. Session options do not apply here. + ensure_no_incremental_window( + &format!("{FUNCTION_NAME} searches one snapshot and cannot run"), + &HashMap::new(), + &identifier, + &table, + )?; + Ok::<_, DataFusionError>(table) + }, "full_text_search: catalog access thread panicked", )?; diff --git a/crates/integrations/datafusion/src/hybrid_search.rs b/crates/integrations/datafusion/src/hybrid_search.rs index cc1898a27..191d915da 100644 --- a/crates/integrations/datafusion/src/hybrid_search.rs +++ b/crates/integrations/datafusion/src/hybrid_search.rs @@ -59,7 +59,7 @@ use crate::table::{ use crate::table_function_args::{ extract_int_literal, extract_string_literal, parse_table_identifier, }; -use crate::table_loader::load_data_table_for_read; +use crate::table_loader::{ensure_no_incremental_window, load_data_table_for_read}; const FUNCTION_NAME: &str = "hybrid_search"; const SEARCH_SCORE_COLUMN: &str = "__paimon_search_score"; @@ -133,7 +133,19 @@ impl TableFunctionImpl for HybridSearchFunction { parse_table_identifier(FUNCTION_NAME, &table_name, &self.default_database)?; let catalog = Arc::clone(&self.catalog); let table = block_on_with_runtime( - async move { load_data_table_for_read(&catalog, &identifier, FUNCTION_NAME).await }, + async move { + let table = load_data_table_for_read(&catalog, &identifier, FUNCTION_NAME).await?; + // The indexes are searched at one snapshot, while a window + // stored on the table would make the read of their hits return + // that window's changes. Session options do not apply here. + ensure_no_incremental_window( + &format!("{FUNCTION_NAME} searches one snapshot and cannot run"), + &HashMap::new(), + &identifier, + &table, + )?; + Ok::<_, DataFusionError>(table) + }, "hybrid_search: catalog access thread panicked", )?; diff --git a/crates/integrations/datafusion/src/incremental_query.rs b/crates/integrations/datafusion/src/incremental_query.rs index bf05fe7b8..93388dd76 100644 --- a/crates/integrations/datafusion/src/incremental_query.rs +++ b/crates/integrations/datafusion/src/incremental_query.rs @@ -58,7 +58,6 @@ use datafusion::physical_plan::{ }; use datafusion::prelude::SessionContext; use futures::{stream, StreamExt, TryStreamExt}; -use paimon::catalog::DEFAULT_MAIN_BRANCH; use paimon::resource::ResourceContext; use paimon::spec::{CoreOptions, DataField}; use paimon::table::{ @@ -76,7 +75,7 @@ use crate::table::{datafusion_arrow_schema, datafusion_read_fields}; use crate::table_function_args::{ extract_int_literal, extract_string_literal, resolve_table_argument, }; -use crate::table_loader::load_data_table_for_read; +use crate::table_loader::{load_data_table_for_read, table_read_name}; const FUNCTION_NAME: &str = "paimon_incremental_query"; const AUDIT_LOG_SUFFIX: &str = "$audit_log"; @@ -194,12 +193,7 @@ impl TableFunctionImpl for IncrementalQueryFunction { // Checked on the table read: a session `branch` reads that // branch, with that branch's options. ensure_no_range_options(table.schema().options(), |key| { - let read = match table.branch() { - DEFAULT_MAIN_BRANCH => format!("table '{}'", identifier.full_name()), - branch => { - format!("branch '{branch}' of table '{}'", identifier.full_name()) - } - }; + let read = table_read_name(&identifier, &table); format!( "{FUNCTION_NAME} reads the range its arguments give and cannot run on \ {read}, which sets scan option '{key}'" diff --git a/crates/integrations/datafusion/src/sql_context.rs b/crates/integrations/datafusion/src/sql_context.rs index 6f3144917..754c9e7f7 100644 --- a/crates/integrations/datafusion/src/sql_context.rs +++ b/crates/integrations/datafusion/src/sql_context.rs @@ -1609,13 +1609,12 @@ impl SQLContext { } /// Reject latest-table statement executors while a session-level - /// time-travel selector or incremental timestamp window is active. + /// time-travel selector is active. /// /// These executors load the latest catalog table, while session reads - /// resolve the selected snapshot schema, or return a window's changes - /// instead of the table's rows. Silently ignoring the selector here would - /// be inconsistent. Ordinary INSERT uses the resolved provider and its - /// selected schema, so it does not need this guard. + /// resolve the selected snapshot schema. Silently ignoring the selector + /// here would be inconsistent. Ordinary INSERT uses the resolved provider + /// and its selected schema, so it does not need this guard. fn ensure_no_time_travel_for_write(&self, operation: &str) -> DFResult<()> { use paimon::spec::{ SCAN_SNAPSHOT_ID_OPTION, SCAN_TAG_NAME_OPTION, SCAN_TIMESTAMP_MILLIS_OPTION, @@ -1638,19 +1637,35 @@ impl SQLContext { ))); } } - if !matches!( - CoreOptions::new(&options).incremental_timestamp_window(), - Ok(None) - ) { - let key = "incremental-between-timestamp"; - return Err(DataFusionError::Plan(format!( - "Cannot execute {operation} while scan option '{key}' is set; \ - RESET 'paimon.{key}' first" - ))); - } Ok(()) } + /// Reject UPDATE, DELETE and MERGE INTO while the query that finds their + /// target rows, a read of `table` in this session, would return the + /// changes of an incremental timestamp window: the write would change + /// rows by versions the table no longer holds. + async fn ensure_target_lookup_reads_rows( + &self, + identifier: &Identifier, + table: &paimon::table::Table, + operation: &str, + ) -> DFResult<()> { + let statement = format!("Cannot execute {operation}"); + let options = self.dynamic_options.read().unwrap().clone(); + if options.is_empty() { + return crate::table_loader::ensure_no_incremental_window( + &statement, &options, identifier, table, + ); + } + // The lookup's provider applies the session's options to the table, a + // session `branch` among them, so check the table it reads. + let read = table + .copy_with_time_travel(options.clone()) + .await + .map_err(to_datafusion_error)?; + crate::table_loader::ensure_no_incremental_window(&statement, &options, identifier, &read) + } + async fn handle_merge_into( &self, merge: &Merge, @@ -1672,6 +1687,8 @@ impl SQLContext { .get_table(&identifier) .await .map_err(to_datafusion_error)?; + self.ensure_target_lookup_reads_rows(&identifier, &table, "MERGE INTO") + .await?; crate::merge_into::execute_merge_into(self, merge, table, enable_ident_normalization).await } @@ -1697,6 +1714,8 @@ impl SQLContext { .get_table(&identifier) .await .map_err(to_datafusion_error)?; + self.ensure_target_lookup_reads_rows(&identifier, &table, "UPDATE") + .await?; crate::update::execute_update(self, update, table, enable_ident_normalization).await } @@ -1725,6 +1744,8 @@ impl SQLContext { .get_table(&identifier) .await .map_err(to_datafusion_error)?; + self.ensure_target_lookup_reads_rows(&identifier, &table, "DELETE") + .await?; let table_ref = table_name.to_string(); crate::delete::execute_delete(self, delete, table, &table_ref).await diff --git a/crates/integrations/datafusion/src/table_loader.rs b/crates/integrations/datafusion/src/table_loader.rs index 748853a8f..e99bb7dbf 100644 --- a/crates/integrations/datafusion/src/table_loader.rs +++ b/crates/integrations/datafusion/src/table_loader.rs @@ -15,14 +15,18 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; use std::sync::Arc; use datafusion::error::{DataFusionError, Result as DFResult}; -use paimon::catalog::{Catalog, Identifier, LoadedTable}; +use paimon::catalog::{Catalog, Identifier, LoadedTable, DEFAULT_MAIN_BRANCH}; +use paimon::spec::CoreOptions; use paimon::table::Table; use crate::error::to_datafusion_error; +const INCREMENTAL_WINDOW_OPTION: &str = "incremental-between-timestamp"; + /// [`Catalog::get_table`] for paths that need a `Table` rather than a /// [`paimon::catalog::LoadedTable`], rejecting an engine-served declared /// type: `get_table` is a trait method, so a catalog outside this repository @@ -137,3 +141,40 @@ pub(crate) async fn load_data_table_for_read( } Ok(table) } + +/// Names `read`, the table read for `identifier`, in a message: its branch +/// too when a session `branch` or the name chose one. +pub(crate) fn table_read_name(identifier: &Identifier, read: &Table) -> String { + match read.branch() { + DEFAULT_MAIN_BRANCH => format!("table '{}'", identifier.full_name()), + branch => format!("branch '{branch}' of table '{}'", identifier.full_name()), + } +} + +/// Refuses `statement`, which needs the rows of the table it reads, when that +/// read returns the changes of an incremental timestamp window instead: the +/// rows the window's commits wrote, older versions included, and, for rows +/// read file by file, a row once per file with the columns it did not write +/// NULL. `read` is the table read for `identifier`, with the session's +/// `options` applied. +pub(crate) fn ensure_no_incremental_window( + statement: &str, + options: &HashMap, + identifier: &Identifier, + read: &Table, +) -> DFResult<()> { + if matches!( + CoreOptions::new(read.schema().options()).incremental_timestamp_window(), + Ok(None) + ) { + return Ok(()); + } + let key = INCREMENTAL_WINDOW_OPTION; + let reason = if options.contains_key(key) { + format!("while scan option '{key}' is set; RESET 'paimon.{key}' first") + } else { + let read = table_read_name(identifier, read); + format!("on {read}, which sets scan option '{key}'") + }; + Err(DataFusionError::Plan(format!("{statement} {reason}"))) +} diff --git a/crates/integrations/datafusion/src/vector_search.rs b/crates/integrations/datafusion/src/vector_search.rs index 6bca78511..794c3871b 100644 --- a/crates/integrations/datafusion/src/vector_search.rs +++ b/crates/integrations/datafusion/src/vector_search.rs @@ -55,7 +55,7 @@ use crate::table::{datafusion_read_fields, PaimonTableProvider}; use crate::table_function_args::{ extract_int_literal, extract_string_literal, parse_table_identifier, }; -use crate::table_loader::load_data_table_for_read; +use crate::table_loader::{ensure_no_incremental_window, load_data_table_for_read}; use crate::DynamicOptions; const FUNCTION_NAME: &str = "vector_search"; @@ -146,27 +146,24 @@ impl TableFunctionImpl for VectorSearchFunction { table } else { table - .copy_with_time_travel(dynamic_options) + .copy_with_time_travel(dynamic_options.clone()) .await .map_err(to_datafusion_error)? }; + // The index is searched at one snapshot, while a read with a + // timestamp window returns that window's changes, which can + // hold a row once per file that wrote it, some of its columns + // NULL. + ensure_no_incremental_window( + &format!("{FUNCTION_NAME} searches one snapshot and cannot run"), + &dynamic_options, + &identifier, + &table, + )?; Ok::<_, DataFusionError>(table) }, "vector_search: catalog access thread panicked", )?; - // The index is searched at one snapshot, while a read with a timestamp - // window returns that window's changes, which can hold a row once per - // file that wrote it, some of its columns NULL. - if !matches!( - CoreOptions::new(table.schema().options()).incremental_timestamp_window(), - Ok(None) - ) { - let key = "incremental-between-timestamp"; - return Err(DataFusionError::Plan(format!( - "{FUNCTION_NAME} searches one snapshot and cannot run while scan option \ - '{key}' is set; RESET 'paimon.{key}' first" - ))); - } let inner = PaimonTableProvider::try_new(table)?; let query_vector_json = diff --git a/crates/integrations/datafusion/tests/incremental_query.rs b/crates/integrations/datafusion/tests/incremental_query.rs index 3d49548be..0a0412ae8 100644 --- a/crates/integrations/datafusion/tests/incremental_query.rs +++ b/crates/integrations/datafusion/tests/incremental_query.rs @@ -25,7 +25,7 @@ use datafusion::execution::runtime_env::RuntimeEnvBuilder; use futures::StreamExt; use paimon::catalog::{Catalog, Identifier}; use paimon::table::{IncrementalScanMode, IncrementalSplit}; -use paimon::{CatalogOptions, FileSystemCatalog, Options}; +use paimon::{CatalogOptions, FileSystemCatalog, Options, Table}; use paimon_datafusion::SQLContext; use tempfile::TempDir; @@ -83,6 +83,19 @@ async fn setup_three_snapshots() -> (TempDir, SQLContext) { run(&ctx, "INSERT INTO paimon.default.t VALUES (3, 'c')").await; (temp_dir, ctx) } + +/// Compacts the files of `table` in one COMPACT snapshot. +async fn compact(table: &Table) { + let compaction = table.new_compact_builder(); + let plan = compaction.new_planner().unwrap().plan().await.unwrap(); + let executor = compaction.new_executor().unwrap(); + let mut results = Vec::new(); + for task in plan.tasks() { + results.push(executor.execute(task).await.unwrap()); + } + let commit = compaction.new_commit().unwrap(); + commit.commit_compact(results).await.unwrap(); +} /// Collects the `id` column across batches into a sorted vector. fn sorted_ids(batches: &[RecordBatch]) -> Vec { use datafusion::arrow::array::{Array, Int32Array}; @@ -785,15 +798,7 @@ async fn test_a_read_of_the_range_as_its_window_counts_the_rows_it_returns() { let warehouse = catalog(&tmp); let identifier = Identifier::new("default", "de"); let table = warehouse.get_table(&identifier).await.unwrap(); - let compaction = table.new_compact_builder(); - let plan = compaction.new_planner().unwrap().plan().await.unwrap(); - let executor = compaction.new_executor().unwrap(); - let mut results = Vec::new(); - for task in plan.tasks() { - results.push(executor.execute(task).await.unwrap()); - } - let commit = compaction.new_commit().unwrap(); - commit.commit_compact(results).await.unwrap(); + compact(&table).await; run(&ctx, "UPDATE paimon.default.de SET v = 50").await; let latest = table.snapshot_manager().get_latest_snapshot().await; @@ -850,13 +855,14 @@ async fn test_a_read_of_the_range_as_its_window_counts_the_rows_it_returns() { } #[tokio::test] -async fn test_writes_refuse_a_session_window() { +async fn test_writes_refuse_an_incremental_window() { use datafusion::arrow::util::pretty::pretty_format_batches; // UPDATE, DELETE and MERGE INTO find the rows they change by a query on the - // table. A session window makes that query return the window's changes: a - // column update without the row's other columns, or a row once per file - // that wrote it. The write would then compute values from, and match, rows - // the table no longer holds. + // table, which a window, set by the session or stored on the table, makes + // return the window's changes. Two inserts, a compaction of their files + // and an update of every row are read file by file: `id` 2 comes back with + // its old `v` 2 next to the update's rows, whose `id` is NULL. The writes + // would change rows by versions the table no longer holds. let tmp = TempDir::new().unwrap(); let ctx = context(&tmp).await; run( @@ -865,67 +871,116 @@ async fn test_writes_refuse_a_session_window() { 'row-tracking.enabled' = 'true', 'data-evolution.enabled' = 'true', 'deletion-vectors.enabled' = 'true', + 'compaction.min.file-num' = '2', 'write-only' = 'true' )", ) .await; - run( - &ctx, - "INSERT INTO paimon.default.de (id, v) VALUES (1, 1), (2, 2)", - ) - .await; - run(&ctx, "UPDATE paimon.default.de SET v = 20 WHERE id = 2").await; + for values in ["(1, 1), (2, 2)", "(3, 3), (4, 4)"] { + run( + &ctx, + &format!("INSERT INTO paimon.default.de (id, v) VALUES {values}"), + ) + .await; + } + let identifier = Identifier::new("default", "de"); + compact(&catalog(&tmp).get_table(&identifier).await.unwrap()).await; + run(&ctx, "UPDATE paimon.default.de SET v = 50").await; let table = "SELECT id, v FROM paimon.default.de ORDER BY id"; let format = |batches: Vec| pretty_format_batches(&batches).unwrap().to_string(); let expected = format(run(&ctx, table).await); - - let key = "incremental-between-timestamp"; - run(&ctx, &format!("SET 'paimon.{key}' = '0,9999999999999'")).await; - for statement in [ + let statements = [ "UPDATE paimon.default.de SET v = v + 1 WHERE id = 2", - "DELETE FROM paimon.default.de WHERE id = 1", + "DELETE FROM paimon.default.de WHERE v = 2", "MERGE INTO paimon.default.de t USING (SELECT 2 AS id) s ON t.id = s.id WHEN MATCHED THEN UPDATE SET v = t.v + 1", - ] { + ]; + + let (key, window) = ("incremental-between-timestamp", "0,9999999999999"); + run(&ctx, &format!("SET 'paimon.{key}' = '{window}'")).await; + for statement in statements { let err = expect_error(&ctx, statement).await; assert!( err.contains(&format!( "scan option '{key}' is set; RESET 'paimon.{key}' first" )), - "{statement}: {err}" + "SET, {statement}: {err}" ); } run(&ctx, &format!("RESET 'paimon.{key}'")).await; assert_eq!(format(run(&ctx, table).await), expected); + + run( + &ctx, + &format!("ALTER TABLE paimon.default.de SET TBLPROPERTIES ('{key}' = '{window}')"), + ) + .await; + for statement in statements { + let err = expect_error(&ctx, statement).await; + assert!( + err.contains(&format!( + "on table 'default.de', which sets scan option '{key}'" + )), + "stored, {statement}: {err}" + ); + } } #[tokio::test] -async fn test_vector_search_refuses_a_session_window() { - // The index is searched at one snapshot, while the window's rows, read to - // return the hits, are its changes. +async fn test_search_functions_refuse_an_incremental_window() { + // An index is searched at one snapshot, while the read that returns the + // hits' rows returns the window's changes. `vector_search` takes the + // session's options; all three read the options the table stores. let tmp = TempDir::new().unwrap(); let ctx = context(&tmp).await; run( &ctx, - "CREATE TABLE paimon.default.v (id INT, embedding ARRAY) WITH ( + "CREATE TABLE paimon.default.v (id INT, embedding ARRAY, body STRING) WITH ( 'row-tracking.enabled' = 'true', 'data-evolution.enabled' = 'true' )", ) .await; - let key = "incremental-between-timestamp"; - run(&ctx, &format!("SET 'paimon.{key}' = '0,9999999999999'")).await; - let err = expect_error( - &ctx, - "SELECT * FROM vector_search('paimon.default.v', 'embedding', '[1.0, 0.0]', 1)", - ) - .await; + let (key, window) = ("incremental-between-timestamp", "0,9999999999999"); + let vector_search = + "SELECT * FROM vector_search('paimon.default.v', 'embedding', '[1.0, 0.0]', 1)"; + run(&ctx, &format!("SET 'paimon.{key}' = '{window}'")).await; + let err = expect_error(&ctx, vector_search).await; assert!( err.contains(&format!( "vector_search searches one snapshot and cannot run while scan option '{key}' is set" )), "{err}" ); + run(&ctx, &format!("RESET 'paimon.{key}'")).await; + + run( + &ctx, + &format!("ALTER TABLE paimon.default.v SET TBLPROPERTIES ('{key}' = '{window}')"), + ) + .await; + for (function, query) in [ + ("vector_search", vector_search), + ( + "full_text_search", + "SELECT * FROM full_text_search('paimon.default.v', 'body', 'paimon', 1)", + ), + ( + "hybrid_search", + "SELECT * FROM hybrid_search('paimon.default.v', + array(named_struct('field', 'embedding', 'query_vector', array(1.0, 0.0))), + array(), 1)", + ), + ] { + let err = expect_error(&ctx, query).await; + assert!( + err.contains(&format!( + "{function} searches one snapshot and cannot run on table 'default.v', \ + which sets scan option '{key}'" + )), + "{function}: {err}" + ); + } } /// Asserts that, for `name`, the function reads the rows a `FROM` clause in From a821eb84835744688b8fc32dd4a5cf84992074d7 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sat, 31 Oct 2026 13:27:09 +0800 Subject: [PATCH 15/18] docs: join Python rows of a range by distinct `_ROW_ID`s A row of a data-evolution range read file by file comes once per file that wrote it, so a Python read matches those rows to the table's rows by their distinct `_ROW_ID`s, as sql.md already says for SQL. --- docs/src/python-binding.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/python-binding.md b/docs/src/python-binding.md index 0fe3f1bda..7ec34db9f 100644 --- a/docs/src/python-binding.md +++ b/docs/src/python-binding.md @@ -178,8 +178,8 @@ Builder filters, projections, and limits still apply. A Data Evolution range merges its files into rows as described for [`paimon_incremental_query`](sql.md#incremental-query); files that do not merge are read as written, one row per row of each file. Project `_ROW_ID`, for -example `rb.with_projection(["_ROW_ID", "embedding"])`, to match those rows to -the table's rows. +example `rb.with_projection(["_ROW_ID", "embedding"])`, to match those rows by +their distinct `_ROW_ID`s to the table's rows. `split.is_streaming()` identifies this read contract. `split.serialize()` exports both batch and streaming splits to Java binary encoding, preserving the streaming From 510cff40a720f960f914b4c9f12f659ce630ec7b Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sat, 31 Oct 2026 15:14:19 +0800 Subject: [PATCH 16/18] fix(core): refuse blob views of an upstream that stores a window A REST read resolves blob views by reading the referenced upstream rows by `_ROW_ID` with the upstream table's own options. If the upstream stores `incremental-between-timestamp`, that read returns the window's changes, and since change events of a window holding a compaction are read file by file, a row comes once per file that wrote it with its BLOB NULL in the files that did not: every view resolved to NULL, where base failed. Resolution now fails with an error naming the upstream table, for every engine and binding that reads through core. --- .../paimon/src/table/data_evolution_reader.rs | 15 ++ crates/paimon/tests/rest_catalog_test.rs | 167 ++++++++++++++++++ 2 files changed, 182 insertions(+) diff --git a/crates/paimon/src/table/data_evolution_reader.rs b/crates/paimon/src/table/data_evolution_reader.rs index f802513ec..405ff4356 100644 --- a/crates/paimon/src/table/data_evolution_reader.rs +++ b/crates/paimon/src/table/data_evolution_reader.rs @@ -1878,6 +1878,21 @@ impl BlobViewLookup { }; for (identifier, refs) in by_table { let table = rest_env.get_table(&identifier).await?; + // A table that stores an incremental timestamp window reads that + // window's changes: a row once per file that wrote it, its BLOB + // NULL in the files that did not, so a view would resolve to + // whichever of them came last. + if !matches!( + CoreOptions::new(table.schema().options()).incremental_timestamp_window(), + Ok(None) + ) { + return Err(Error::Unsupported { + message: format!( + "Cannot resolve blob views of upstream table {}, which sets scan option 'incremental-between-timestamp'", + identifier.full_name() + ), + }); + } lookup .file_ios .insert(identifier.clone(), table.file_io().clone()); diff --git a/crates/paimon/tests/rest_catalog_test.rs b/crates/paimon/tests/rest_catalog_test.rs index fb7a2835e..8d2d35ce3 100644 --- a/crates/paimon/tests/rest_catalog_test.rs +++ b/crates/paimon/tests/rest_catalog_test.rs @@ -1304,6 +1304,173 @@ async fn test_blob_view_limit_only_resolves_selected_references() { } } +// An upstream table that stores `incremental-between-timestamp` reads that +// window's changes. Two inserts with pictures, a compaction of their files and +// an update of every name are read file by file: each row once per file that +// wrote it, its picture NULL in the files that did not, so every view resolved +// to NULL. The mock server also serves the same files without the window. +#[cfg(not(windows))] +#[tokio::test] +async fn test_blob_view_refuses_an_upstream_that_stores_a_window() { + let tmp = tempfile::tempdir().unwrap(); + let warehouse = format!("file://{}", tmp.path().display()); + let mut fs_options = Options::new(); + fs_options.set(CatalogOptions::WAREHOUSE, &warehouse); + let fs_catalog = FileSystemCatalog::new(fs_options).expect("create filesystem catalog"); + fs_catalog + .create_database("default", true, HashMap::new()) + .await + .unwrap(); + + let source_options = [("write-only", "true"), ("compaction.min.file-num", "2")]; + let source_id = Identifier::new("default", "source"); + fs_catalog + .create_table(&source_id, blob_schema(&source_options), false) + .await + .unwrap(); + let source = fs_catalog.get_table(&source_id).await.unwrap(); + write_batch( + &source, + blob_batch( + vec![1, 2], + vec!["a", "b"], + vec![b"p1".to_vec(), b"p2".to_vec()], + ), + "writer-1", + ) + .await; + write_batch( + &source, + blob_batch( + vec![3, 4], + vec!["c", "d"], + vec![b"p3".to_vec(), b"p4".to_vec()], + ), + "writer-2", + ) + .await; + let compaction = source.new_compact_builder(); + let plan = compaction.new_planner().unwrap().plan().await.unwrap(); + let executor = compaction.new_executor().unwrap(); + let mut results = Vec::new(); + for task in plan.tasks() { + results.push(executor.execute(task).await.unwrap()); + } + let commit = compaction.new_commit().unwrap(); + commit.commit_compact(results).await.unwrap(); + let source = fs_catalog.get_table(&source_id).await.unwrap(); + let update = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("_ROW_ID", ArrowDataType::Int64, false), + ArrowField::new("name", ArrowDataType::Utf8, true), + ])), + vec![ + Arc::new(Int64Array::from(vec![0, 1, 2, 3])), + Arc::new(StringArray::from(vec!["A", "B", "C", "D"])), + ], + ) + .unwrap(); + let write_builder = source.new_write_builder(); + let mut writer = write_builder + .new_data_evolution_writer(vec!["name".to_string()]) + .unwrap(); + writer.add_matched_batch(update).unwrap(); + let messages = writer.prepare_commit().await.unwrap(); + write_builder.new_commit().commit(messages).await.unwrap(); + + // One view table for each name the mock server serves the source under. + let picture_id = source + .schema() + .fields() + .iter() + .find(|field| field.name() == "picture") + .unwrap() + .id(); + let view_schema = blob_schema(&[("blob-view-field", "picture")]); + let mut views = Vec::new(); + for upstream in ["source", "source_window"] { + let view_id = Identifier::new("default", format!("view_of_{upstream}")); + fs_catalog + .create_table(&view_id, view_schema.clone(), false) + .await + .unwrap(); + let view = fs_catalog.get_table(&view_id).await.unwrap(); + let refs = (0..4) + .map(|row_id| { + BlobViewStruct::new(Identifier::new("default", upstream), picture_id, row_id) + .serialize() + .unwrap() + }) + .collect(); + write_batch( + &view, + blob_batch(vec![1, 2, 3, 4], vec!["v1", "v2", "v3", "v4"], refs), + "view-writer", + ) + .await; + views.push((view_id, view.location().to_string())); + } + + let mut defaults = HashMap::new(); + defaults.insert("prefix".to_string(), "mock-test".to_string()); + let server = start_mock_server( + "test_warehouse".to_string(), + warehouse.clone(), + ConfigResponse::new(defaults), + vec!["default".to_string()], + ) + .await; + server.add_table_with_schema( + "default", + "source", + blob_schema(&source_options), + source.location(), + ); + let window = ("incremental-between-timestamp", "0,9999999999999"); + server.add_table_with_schema( + "default", + "source_window", + blob_schema(&[source_options[0], source_options[1], window]), + source.location(), + ); + for (view_id, location) in &views { + server.add_table_with_schema("default", view_id.object(), view_schema.clone(), location); + } + let url = server.url().expect("Failed to get server URL"); + let mut rest_options = Options::new(); + rest_options.set("uri", &url); + rest_options.set("warehouse", "test_warehouse"); + rest_options.set("token.provider", "bear"); + rest_options.set("token", "test_token"); + let rest_catalog = RESTCatalog::new(rest_options, true) + .await + .expect("create rest catalog"); + + let source_view = rest_catalog + .get_table(&Identifier::new("default", "view_of_source")) + .await + .unwrap(); + let resolved = read_all(&source_view.new_read_builder()).await.unwrap(); + assert_eq!( + collect_blob_rows(&resolved), + (1..=4) + .map(|id| (id, format!("v{id}"), Some(format!("p{id}").into_bytes()))) + .collect::>() + ); + let window_view = rest_catalog + .get_table(&Identifier::new("default", "view_of_source_window")) + .await + .unwrap(); + let refused = read_all(&window_view.new_read_builder()).await; + assert!( + matches!(&refused, Err(paimon::Error::Unsupported { message }) + if message.contains("Cannot resolve blob views of upstream table default.source_window, \ + which sets scan option 'incremental-between-timestamp'")), + "{:?}", + refused.map(|batches| collect_blob_rows(&batches)) + ); +} + #[cfg(not(windows))] #[tokio::test] async fn test_rest_catalog_reads_format_table_with_bearer_and_local_file_dlf_auth() { From f847ccba14f025a811af286aeddf4e69a1c2d401 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sat, 31 Oct 2026 15:14:29 +0800 Subject: [PATCH 17/18] fix(datafusion): name a `$branch_` table once in window refusals A refusal named a branch read through a `t$branch_b` name as "branch 'b' of table 'db.t$branch_b'". It now names the table without the suffix, as for a branch the session selects. Tests now also pin that writes and `vector_search` check the branch the session or the name selects, and that TRUNCATE TABLE runs while a window is set. --- .../datafusion/src/table_loader.rs | 11 ++- .../datafusion/tests/incremental_query.rs | 71 +++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/crates/integrations/datafusion/src/table_loader.rs b/crates/integrations/datafusion/src/table_loader.rs index e99bb7dbf..a0b729830 100644 --- a/crates/integrations/datafusion/src/table_loader.rs +++ b/crates/integrations/datafusion/src/table_loader.rs @@ -147,7 +147,16 @@ pub(crate) async fn load_data_table_for_read( pub(crate) fn table_read_name(identifier: &Identifier, read: &Table) -> String { match read.branch() { DEFAULT_MAIN_BRANCH => format!("table '{}'", identifier.full_name()), - branch => format!("branch '{branch}' of table '{}'", identifier.full_name()), + branch => { + // `t$branch_b` already names the branch it reads. + let table = identifier + .table_name() + .unwrap_or_else(|_| identifier.object().to_string()); + format!( + "branch '{branch}' of table '{}.{table}'", + identifier.database() + ) + } } } diff --git a/crates/integrations/datafusion/tests/incremental_query.rs b/crates/integrations/datafusion/tests/incremental_query.rs index 0a0412ae8..e9caa3844 100644 --- a/crates/integrations/datafusion/tests/incremental_query.rs +++ b/crates/integrations/datafusion/tests/incremental_query.rs @@ -924,6 +924,77 @@ async fn test_writes_refuse_an_incremental_window() { "stored, {statement}: {err}" ); } + // TRUNCATE TABLE reads no rows, so a window does not keep it from running. + run(&ctx, &format!("SET 'paimon.{key}' = '{window}'")).await; + run(&ctx, "TRUNCATE TABLE paimon.default.de").await; +} + +#[tokio::test] +async fn test_writes_and_vector_search_check_the_branch_they_read() { + use paimon::spec::SchemaChange; + use paimon::table::{BranchManager, SnapshotManager, TagManager}; + // With a session `branch`, the target lookup of UPDATE and the reads of + // `vector_search` read that branch, so its options decide: here only b1 + // stores the window, and the main branch does not. + let tmp = TempDir::new().unwrap(); + let ctx = context(&tmp).await; + let key = "incremental-between-timestamp"; + run( + &ctx, + &format!( + "CREATE TABLE paimon.default.t (id INT, embedding ARRAY) WITH ( + 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true', + 'write-only' = 'true', + '{key}' = '0,9999999999999' + )" + ), + ) + .await; + run(&ctx, "INSERT INTO paimon.default.t (id) VALUES (1)").await; + let warehouse = catalog(&tmp); + let identifier = Identifier::new("default", "t"); + let loaded = warehouse.get_table(&identifier).await.unwrap(); + let (io, location) = (loaded.file_io().clone(), loaded.location().to_string()); + let snapshot = SnapshotManager::new(io.clone(), location.clone()) + .get_latest_snapshot() + .await + .unwrap() + .unwrap(); + TagManager::new(io.clone(), location.clone()) + .create("base", &snapshot) + .await + .unwrap(); + BranchManager::new(io, location) + .create_branch_from_tag("b1", "base") + .await + .unwrap(); + let unset = SchemaChange::remove_option(key.to_string()); + warehouse + .alter_table(&identifier, vec![unset], false) + .await + .unwrap(); + + let vector_search = + "SELECT * FROM vector_search('paimon.default.t', 'embedding', '[1.0, 0.0]', 1)"; + run(&ctx, vector_search).await; + // A name that selects the branch reads it as well. + let message = format!("branch 'b1' of table 'default.t', which sets scan option '{key}'"); + for query in [ + "SELECT * FROM vector_search('paimon.default.t$branch_b1', 'embedding', '[1.0, 0.0]', 1)", + "SELECT * FROM paimon_incremental_query('default.t$branch_b1', 0, 1)", + ] { + let err = expect_error(&ctx, query).await; + assert!(err.contains(&message), "{query}: {err}"); + } + run(&ctx, "SET 'paimon.branch' = 'b1'").await; + for statement in [ + "UPDATE paimon.default.t SET id = 2 WHERE id = 1", + vector_search, + ] { + let err = expect_error(&ctx, statement).await; + assert!(err.contains(&message), "{statement}: {err}"); + } } #[tokio::test] From c8f8b02c38a024054997ec59a18efb222a70e0f3 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sat, 31 Oct 2026 17:29:31 +0800 Subject: [PATCH 18/18] docs: say blob views of an upstream that stores a window fail to resolve 510cff4 refuses to resolve blob views whose upstream table stores `incremental-between-timestamp`. The Blob View section now says so and names both ways out: remove the option from the upstream table, or read the raw references with `blob-view.resolve.enabled=false`. --- docs/src/sql.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/src/sql.md b/docs/src/sql.md index e54ed8848..80a928fe1 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -378,6 +378,10 @@ SELECT id, picture FROM paimon.my_db.asset_refs; RESET 'paimon.blob-view.resolve.enabled'; ``` +Resolution fails if the upstream table stores `incremental-between-timestamp`, +since a read of it returns that window's changes instead of the table's rows. +Remove the option from the upstream table, or read the raw references as above. + Like ordinary BLOB reads, `paimon.blob-as-descriptor = true` makes resolved Blob View columns return serialized BLOB descriptors instead of loading the BLOB bytes.