From 134d152c6be7a5d3fb06f7df032a6a9ced20157f Mon Sep 17 00:00:00 2001 From: Dapeng Sun Date: Thu, 23 Jul 2026 04:23:39 +0800 Subject: [PATCH 1/7] datafusion: support REST format table partition commands --- Cargo.lock | 1 + crates/integrations/datafusion/Cargo.toml | 1 + .../datafusion/src/format_partition_repair.rs | 113 + crates/integrations/datafusion/src/lib.rs | 1 + .../datafusion/src/sql_context.rs | 3020 ++++++++++++++++- .../tests/rest_format_partition_sql.rs | 377 ++ crates/paimon/src/api/api_request.rs | 141 + crates/paimon/src/api/api_response.rs | 90 + crates/paimon/src/api/mod.rs | 12 +- crates/paimon/src/api/resource_paths.rs | 26 + crates/paimon/src/api/rest_api.rs | 113 +- crates/paimon/src/catalog/mod.rs | 41 + .../paimon/src/catalog/rest/rest_catalog.rs | 107 +- crates/paimon/src/spec/core_options.rs | 27 + crates/paimon/src/spec/mod.rs | 2 +- crates/paimon/src/spec/partition_utils.rs | 2 +- crates/paimon/src/table/format_partition.rs | 230 ++ crates/paimon/src/table/format_table_scan.rs | 327 +- crates/paimon/src/table/mod.rs | 45 +- crates/paimon/src/table/rest_env.rs | 72 +- crates/paimon/tests/format_partition_test.rs | 178 + crates/paimon/tests/mock_server.rs | 491 ++- crates/paimon/tests/rest_api_test.rs | 225 +- crates/paimon/tests/rest_catalog_test.rs | 788 ++++- .../paimon/tests/rest_object_models_test.rs | 30 +- ...22-rest-format-table-partition-commands.md | 292 ++ ...22-rest-format-table-partition-commands.md | 209 ++ docs/src/sql.md | 156 +- 28 files changed, 6943 insertions(+), 174 deletions(-) create mode 100644 crates/integrations/datafusion/src/format_partition_repair.rs create mode 100644 crates/integrations/datafusion/tests/rest_format_partition_sql.rs create mode 100644 crates/paimon/src/table/format_partition.rs create mode 100644 crates/paimon/tests/format_partition_test.rs create mode 100644 docs/designs/2026-07-22-rest-format-table-partition-commands.md create mode 100644 docs/plans/2026-07-22-rest-format-table-partition-commands.md diff --git a/Cargo.lock b/Cargo.lock index 43181baf..5f4425cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4492,6 +4492,7 @@ dependencies = [ "arrow-schema", "arrow-select", "async-trait", + "axum", "chrono", "constant_time_eq", "datafusion", diff --git a/crates/integrations/datafusion/Cargo.toml b/crates/integrations/datafusion/Cargo.toml index 37fb1947..ab0b362f 100644 --- a/crates/integrations/datafusion/Cargo.toml +++ b/crates/integrations/datafusion/Cargo.toml @@ -50,6 +50,7 @@ uuid = { version = "1", features = ["v4"] } [dev-dependencies] arrow-array = { workspace = true } arrow-schema = { workspace = true } +axum = { version = "0.7", features = ["macros", "tokio", "http1", "http2"] } flate2 = "1" parquet = { workspace = true } tar = "0.4" diff --git a/crates/integrations/datafusion/src/format_partition_repair.rs b/crates/integrations/datafusion/src/format_partition_repair.rs new file mode 100644 index 00000000..efa4848e --- /dev/null +++ b/crates/integrations/datafusion/src/format_partition_repair.rs @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::{BTreeMap, HashMap}; + +use paimon::catalog::{Catalog, Identifier}; +use paimon::spec::CoreOptions; +use paimon::table::{FormatTablePartitionPaths, Table}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RepairMode { + Add, + Drop, + Sync, +} + +pub(crate) async fn repair( + catalog: &dyn Catalog, + identifier: &Identifier, + table: &Table, + mode: RepairMode, +) -> paimon::Result { + let core_options = CoreOptions::new(table.schema().options()); + let paths = FormatTablePartitionPaths::new( + table.schema().partition_keys().iter().cloned(), + core_options.format_table_partition_only_value_in_path(), + ); + let table_path = core_options.path().unwrap_or_else(|| table.location()); + + // Complete both listings before mutating metadata. Filesystem discovery keeps + // raw directory values, so values such as month=01 round-trip unchanged. + let filesystem_specs = paths + .discover( + table.file_io(), + table_path, + core_options.partition_default_name(), + ) + .await?; + let registered_partitions = catalog.list_partitions(identifier).await?; + + let filesystem = index_specs(&paths, filesystem_specs)?; + let registered = index_specs( + &paths, + registered_partitions + .into_iter() + .map(|partition| partition.spec) + .collect(), + )?; + + let add = if matches!(mode, RepairMode::Add | RepairMode::Sync) { + filesystem + .iter() + .filter(|(name, _)| !registered.contains_key(*name)) + .map(|(_, spec)| spec.clone()) + .collect::>() + } else { + Vec::new() + }; + let drop = if matches!(mode, RepairMode::Drop | RepairMode::Sync) { + registered + .iter() + .filter(|(name, _)| !filesystem.contains_key(*name)) + .map(|(_, spec)| spec.clone()) + .collect::>() + } else { + Vec::new() + }; + + if !add.is_empty() { + catalog + .create_partitions(identifier, add.clone(), true) + .await?; + } + if !drop.is_empty() { + catalog.drop_partitions(identifier, drop.clone()).await?; + } + Ok(add.len() + drop.len()) +} + +fn index_specs( + paths: &FormatTablePartitionPaths, + specs: Vec>, +) -> paimon::Result>> { + let mut indexed = BTreeMap::new(); + for spec in specs { + let name = paths.partition_name(&spec)?; + if let Some(previous) = indexed.insert(name.clone(), spec.clone()) { + if previous != spec { + return Err(paimon::Error::DataInvalid { + message: format!( + "Conflicting partition specs map to the same canonical path {name}" + ), + source: None, + }); + } + } + } + Ok(indexed) +} diff --git a/crates/integrations/datafusion/src/lib.rs b/crates/integrations/datafusion/src/lib.rs index 05e4037a..ae8a42ca 100644 --- a/crates/integrations/datafusion/src/lib.rs +++ b/crates/integrations/datafusion/src/lib.rs @@ -43,6 +43,7 @@ mod catalog; mod delete; mod error; mod filter_pushdown; +mod format_partition_repair; #[cfg(feature = "fulltext")] mod full_text_search; mod hybrid_search; diff --git a/crates/integrations/datafusion/src/sql_context.rs b/crates/integrations/datafusion/src/sql_context.rs index 3190487b..d59ce83f 100644 --- a/crates/integrations/datafusion/src/sql_context.rs +++ b/crates/integrations/datafusion/src/sql_context.rs @@ -29,14 +29,17 @@ //! - `ALTER TABLE db.t DROP COLUMN col` //! - `ALTER TABLE db.t RENAME COLUMN old TO new` //! - `ALTER TABLE db.t RENAME TO new_name` +//! - `ALTER TABLE db.t ADD [IF NOT EXISTS] PARTITION (...) [PARTITION (...)]` //! - `ALTER TABLE db.t DROP PARTITION (col = val, ...)` +//! - `SHOW PARTITIONS db.t [PARTITION (...)]` +//! - `[MSCK] REPAIR TABLE db.t [{ADD|DROP|SYNC} PARTITIONS]` //! - `CREATE VIEW [IF NOT EXISTS] view [(col, ...)] AS query` //! - `DROP VIEW [IF EXISTS] view` //! - `CREATE FUNCTION name(args) RETURNS type [LANGUAGE SQL] RETURN expression` //! - `TRUNCATE TABLE db.t` //! - `TRUNCATE TABLE db.t PARTITION (col = val, ...)` -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; use datafusion::arrow::array::{ @@ -58,22 +61,23 @@ use datafusion::sql::sqlparser::ast::{ AlterTableOperation, BinaryLength, CharacterLength, ColumnDef, ColumnOption, CreateFunction, CreateFunctionBody, CreateTable, CreateTableOptions, CreateView, Delete, Expr as SqlExpr, FromTable, FunctionBehavior, FunctionReturnType, Insert, Merge, ObjectName, ObjectType, - RenameTableNameKind, Reset, ResetStatement, Set, ShowCreateObject, SqlOption, Statement, - TableFactor, TableObject, Truncate, Update, Value as SqlValue, + Partition as SqlPartition, RenameTableNameKind, Reset, ResetStatement, Set, ShowCreateObject, + SqlOption, Statement, TableFactor, TableObject, Truncate, Update, Value as SqlValue, }; use datafusion::sql::sqlparser::dialect::GenericDialect; use datafusion::sql::sqlparser::keywords::Keyword; use datafusion::sql::sqlparser::parser::Parser; -use datafusion::sql::sqlparser::tokenizer::{Token, Tokenizer}; +use datafusion::sql::sqlparser::tokenizer::{Token, TokenWithSpan, Tokenizer}; use futures::StreamExt; use paimon::catalog::{parse_object_name, Catalog, Identifier}; use paimon::spec::{ ArrayType as PaimonArrayType, BigIntType, BinaryType, BlobType, BooleanType, CharType, - DataField as PaimonDataField, DataType as PaimonDataType, DateType, Datum, DecimalType, - DoubleType, FloatType, IntType, LocalZonedTimestampType, MapType as PaimonMapType, + CoreOptions, DataField as PaimonDataField, DataType as PaimonDataType, DateType, Datum, + DecimalType, DoubleType, FloatType, IntType, LocalZonedTimestampType, MapType as PaimonMapType, RowType as PaimonRowType, SchemaChange, SmallIntType, TimestampType, TinyIntType, VarBinaryType, VarCharType, VariantType, }; +use paimon::table::FormatTablePartitionPaths; use crate::error::to_datafusion_error; use crate::table_loader::load_table_for_read; @@ -350,8 +354,8 @@ impl SQLContext { &self.dynamic_options } - /// Execute a SQL statement. ALTER TABLE is handled by Paimon directly; - /// everything else is delegated to DataFusion. + /// Execute SQL, intercepting Paimon-specific DDL/DML and delegating the + /// remaining statements to DataFusion. pub async fn sql(&self, sql: &str) -> DFResult { let is_create_table = looks_like_create_table(sql); let (rewritten_sql, partition_keys) = if is_create_table { @@ -363,6 +367,14 @@ impl SQLContext { // Time-travel queries are not DDL; skip our own parsing and handle directly. return self.handle_time_travel_query(&rewritten_sql).await; } + if let Some(show_partitions) = parse_show_partitions(&rewritten_sql)? { + return self.handle_show_partitions(&show_partitions).await; + } + if is_drop_partition_purge(&rewritten_sql)? { + return Err(DataFusionError::Plan( + "DROP PARTITION PURGE is not supported".to_string(), + )); + } let statements = parse_sql_statements(&rewritten_sql)?; @@ -388,6 +400,15 @@ impl SQLContext { obj_name, } => self.handle_show_create_table(sql, obj_name).await, Statement::AlterTable(alter_table) => { + if alter_table.location.is_some() + && alter_table.operations.iter().any(|operation| { + matches!(operation, AlterTableOperation::AddPartitions { .. }) + }) + { + return Err(DataFusionError::Plan( + "LOCATION is not supported for Format Table partitions".to_string(), + )); + } let (catalog, _catalog_name, _) = self.resolve_catalog_and_table(&alter_table.name)?; self.handle_alter_table( @@ -442,6 +463,7 @@ impl SQLContext { self.ctx.sql(sql).await } Statement::Truncate(truncate) => self.handle_truncate_table(truncate).await, + Statement::Msck(msck) => self.handle_msck(msck).await, Statement::CreateView(create_view) => { if create_view.temporary { // Temporary views are always handled by us (Paimon catalog temp storage) @@ -1030,6 +1052,37 @@ impl SQLContext { Self::ensure_main_branch_write_target(name, "ALTER TABLE")?; let identifier = self.resolve_table_name(name)?; + let drop_partitions = operations + .iter() + .filter_map(|operation| match operation { + AlterTableOperation::DropPartitions { + partitions, + if_exists, + } => Some((partitions.clone(), *if_exists)), + _ => None, + }) + .collect::>(); + if !drop_partitions.is_empty() { + if drop_partitions.len() != operations.len() { + return Err(DataFusionError::Plan( + "ALTER TABLE DROP PARTITION cannot be combined with other operations" + .to_string(), + )); + } + return self + .handle_drop_partitions(catalog, &identifier, &drop_partitions, if_exists) + .await; + } + if operations + .iter() + .any(|operation| matches!(operation, AlterTableOperation::AddPartitions { .. })) + && operations.len() != 1 + { + return Err(DataFusionError::Plan( + "ALTER TABLE ADD PARTITION cannot be combined with other operations".to_string(), + )); + } + let mut changes = Vec::new(); let mut rename_to: Option = None; @@ -1073,19 +1126,26 @@ impl SQLContext { } } } - AlterTableOperation::DropPartitions { - partitions, - if_exists: partition_if_exists, + AlterTableOperation::AddPartitions { + if_not_exists, + new_partitions, } => { return self - .handle_drop_partitions( + .handle_add_partitions( catalog, &identifier, - partitions, - if_exists || *partition_if_exists, + new_partitions, + *if_not_exists, + if_exists, ) .await; } + AlterTableOperation::DropPartitions { .. } => { + return Err(DataFusionError::Internal( + "DROP PARTITION should have been dispatched before ALTER processing" + .to_string(), + )); + } other => { return Err(DataFusionError::Plan(format!( "Unsupported ALTER TABLE operation: {other}" @@ -1646,27 +1706,159 @@ impl SQLContext { &self, catalog: &Arc, identifier: &Identifier, - partitions: &[SqlExpr], - if_exists: bool, + partition_specs: &[(Vec, bool)], + ignore_if_table_not_exists: bool, ) -> DFResult { - if partitions.is_empty() { + if partition_specs.is_empty() || partition_specs.iter().any(|(spec, _)| spec.is_empty()) { return Err(DataFusionError::Plan( "DROP PARTITIONS requires at least one partition specification".to_string(), )); } let table = match catalog.get_table(identifier).await { Ok(t) => t, - Err(e) if if_exists && is_table_not_exist(&e) => { + Err(e) if ignore_if_table_not_exists && is_table_not_exist(&e) => { return ok_result(&self.ctx); } Err(e) => return Err(to_datafusion_error(e)), }; + if table.has_catalog_managed_partitions() { + ensure_catalog_managed_format_table(&table, "ALTER TABLE DROP PARTITION")?; + let requested = partition_specs + .iter() + .map(|(expressions, ignore_if_not_exists)| { + Ok(( + parse_format_partition_spec(expressions, &table, false)?, + *ignore_if_not_exists, + )) + }) + .collect::>>()?; + let core_options = CoreOptions::new(table.schema().options()); + let partition_paths = FormatTablePartitionPaths::new( + table.schema().partition_keys().iter().cloned(), + core_options.format_table_partition_only_value_in_path(), + ); + let table_path = core_options.path().unwrap_or_else(|| table.location()); + let partition_key_count = table.schema().partition_keys().len(); + let complete_specs = requested + .iter() + .filter(|(spec, _)| spec.len() == partition_key_count) + .map(|(spec, _)| spec.clone()) + .collect::>(); + let exact_registered = if complete_specs.is_empty() { + Vec::new() + } else { + catalog + .list_partitions_by_names(identifier, complete_specs) + .await + .map_err(to_datafusion_error)? + }; + let exact_matches = requested + .iter() + .map(|(requested, _)| { + requested.len() == partition_key_count + && exact_registered + .iter() + .any(|partition| partition.spec == *requested) + }) + .collect::>(); + let needs_full_listing = requested.iter().enumerate().any(|(index, (spec, _))| { + spec.len() != partition_key_count || !exact_matches[index] + }); + let registered = if needs_full_listing { + catalog + .list_partitions(identifier) + .await + .map_err(to_datafusion_error)? + } else { + exact_registered + }; + + let normalized_registered = registered + .into_iter() + .map(|partition| { + let normalized = + normalize_format_partition_spec_values(&partition.spec, &table, true)?; + Ok((partition, normalized)) + }) + .collect::>>()?; + let mut matches = BTreeMap::new(); + for (index, (requested_spec, ignore_if_not_exists)) in requested.iter().enumerate() { + let requested_normalized = + normalize_format_partition_spec_values(requested_spec, &table, false)?; + let mut matched = false; + for (partition, normalized) in &normalized_registered { + let partition_matches = if exact_matches[index] { + partition.spec == *requested_spec + } else { + requested_normalized + .iter() + .all(|(key, value)| normalized.get(key) == Some(value)) + }; + if partition_matches { + matched = true; + let name = partition_paths + .partition_name(&partition.spec) + .map_err(to_datafusion_error)?; + let relative_path = partition_paths + .relative_path(&partition.spec) + .map_err(to_datafusion_error)?; + matches.entry(name).or_insert_with(|| { + ( + partition.spec.clone(), + format!("{}/{}", table_path.trim_end_matches('/'), relative_path), + ) + }); + } + } + if !matched + && requested_spec.len() == table.schema().partition_keys().len() + && !ignore_if_not_exists + { + return Err(DataFusionError::Plan(format!( + "Partition {requested_spec:?} does not exist in table {}", + identifier.full_name() + ))); + } + } + + if matches.is_empty() { + return ok_result(&self.ctx); + } + + catalog + .drop_partitions( + identifier, + matches.values().map(|(spec, _)| spec.clone()).collect(), + ) + .await + .map_err(to_datafusion_error)?; + for (_, path) in matches.into_values() { + table + .file_io() + .delete_dir(&path) + .await + .map_err(to_datafusion_error)?; + } + return ok_result(&self.ctx); + } + if CoreOptions::new(table.schema().options()).is_format_table() { + ensure_catalog_managed_format_table(&table, "ALTER TABLE DROP PARTITION")?; + } + let partition_values = parse_partition_values( - partitions, + &partition_specs[0].0, table.schema().fields(), table.schema().partition_keys(), )?; + let mut partition_values = partition_values; + for (expressions, _) in &partition_specs[1..] { + partition_values.extend(parse_partition_values( + expressions, + table.schema().fields(), + table.schema().partition_keys(), + )?); + } let wb = table.new_write_builder(); let commit = wb.try_new_commit().map_err(to_datafusion_error)?; @@ -1678,6 +1870,157 @@ impl SQLContext { ok_result(&self.ctx) } + async fn handle_add_partitions( + &self, + catalog: &Arc, + identifier: &Identifier, + partitions: &[SqlPartition], + ignore_if_exists: bool, + ignore_if_table_not_exists: bool, + ) -> DFResult { + let table = match catalog.get_table(identifier).await { + Ok(table) => table, + Err(error) if ignore_if_table_not_exists && is_table_not_exist(&error) => { + return ok_result(&self.ctx); + } + Err(error) => return Err(to_datafusion_error(error)), + }; + ensure_catalog_managed_format_table(&table, "ALTER TABLE ADD PARTITION")?; + if partitions.is_empty() { + return Err(DataFusionError::Plan( + "ADD PARTITION requires at least one partition specification".to_string(), + )); + } + + let core_options = CoreOptions::new(table.schema().options()); + let partition_paths = FormatTablePartitionPaths::new( + table.schema().partition_keys().iter().cloned(), + core_options.format_table_partition_only_value_in_path(), + ); + let table_path = core_options.path().unwrap_or_else(|| table.location()); + let mut specs = Vec::with_capacity(partitions.len()); + let mut directories = Vec::with_capacity(partitions.len()); + for partition in partitions { + let expressions = match partition { + SqlPartition::Partitions(expressions) => expressions, + other => { + return Err(DataFusionError::Plan(format!( + "Unsupported ADD PARTITION specification: {other}" + ))) + } + }; + let spec = parse_format_partition_spec(expressions, &table, true)?; + let relative_path = partition_paths + .relative_path(&spec) + .map_err(to_datafusion_error)?; + directories.push(format!( + "{}/{}", + table_path.trim_end_matches('/'), + relative_path + )); + specs.push(spec); + } + + catalog + .create_partitions(identifier, specs, ignore_if_exists) + .await + .map_err(to_datafusion_error)?; + for directory in directories { + table + .file_io() + .mkdirs(&directory) + .await + .map_err(to_datafusion_error)?; + } + ok_result(&self.ctx) + } + + async fn handle_msck( + &self, + msck: &datafusion::sql::sqlparser::ast::Msck, + ) -> DFResult { + let (catalog, _catalog_name, identifier) = + self.resolve_catalog_and_table(&msck.table_name)?; + let table = catalog + .get_table(&identifier) + .await + .map_err(to_datafusion_error)?; + ensure_catalog_managed_format_table(&table, "MSCK REPAIR TABLE")?; + let mode = match msck.partition_action { + None | Some(datafusion::sql::sqlparser::ast::AddDropSync::ADD) => { + crate::format_partition_repair::RepairMode::Add + } + Some(datafusion::sql::sqlparser::ast::AddDropSync::DROP) => { + crate::format_partition_repair::RepairMode::Drop + } + Some(datafusion::sql::sqlparser::ast::AddDropSync::SYNC) => { + crate::format_partition_repair::RepairMode::Sync + } + }; + crate::format_partition_repair::repair(catalog.as_ref(), &identifier, &table, mode) + .await + .map_err(to_datafusion_error)?; + ok_result(&self.ctx) + } + + async fn handle_show_partitions( + &self, + show_partitions: &ShowPartitionsStatement, + ) -> DFResult { + let (catalog, _catalog_name, identifier) = + self.resolve_catalog_and_table(&show_partitions.table_name)?; + let table = catalog + .get_table(&identifier) + .await + .map_err(to_datafusion_error)?; + ensure_catalog_managed_format_table(&table, "SHOW PARTITIONS")?; + let filter = if show_partitions.partition.is_empty() { + None + } else { + let spec = parse_format_partition_spec(&show_partitions.partition, &table, false)?; + Some(normalize_format_partition_spec_values( + &spec, &table, false, + )?) + }; + + let partition_paths = FormatTablePartitionPaths::new( + table.schema().partition_keys().iter().cloned(), + CoreOptions::new(table.schema().options()).format_table_partition_only_value_in_path(), + ); + let mut names = Vec::new(); + for partition in catalog + .list_partitions(&identifier) + .await + .map_err(to_datafusion_error)? + { + let normalized = normalize_format_partition_spec_values(&partition.spec, &table, true)?; + if !filter.as_ref().is_none_or(|filter| { + filter + .iter() + .all(|(key, value)| normalized.get(key) == Some(value)) + }) { + continue; + } + let display_spec = normalized + .into_iter() + .map(|(key, value)| (key, value.unwrap_or_else(|| "null".to_string()))) + .collect(); + let name = partition_paths + .partition_name(&display_spec) + .map_err(to_datafusion_error)?; + names.push(name); + } + names.sort(); + + let schema = Arc::new(Schema::new(vec![Field::new( + "partition", + ArrowDataType::Utf8, + false, + )])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(StringArray::from(names))])?; + self.ctx.read_batch(batch) + } + /// Returns the name of the current default catalog from DataFusion config. pub(crate) fn current_catalog_name(&self) -> String { self.ctx @@ -1958,11 +2301,73 @@ fn validate_persistent_create_function(create_function: &CreateFunction) -> DFRe Ok(()) } +#[derive(Debug)] +struct ShowPartitionsStatement { + table_name: ObjectName, + partition: Vec, +} + +fn parse_show_partitions(sql: &str) -> DFResult> { + let dialect = GenericDialect {}; + let tokens = Tokenizer::new(&dialect, sql) + .tokenize_with_location() + .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; + let significant = tokens + .iter() + .filter_map(|token| match &token.token { + Token::Whitespace(_) => None, + Token::Word(word) => Some(word.keyword), + _ => Some(Keyword::NoKeyword), + }) + .take(2) + .collect::>(); + if !matches!(significant.as_slice(), [Keyword::SHOW, Keyword::PARTITIONS]) { + return Ok(None); + } + + let mut parser = Parser::new(&dialect).with_tokens_with_locations(tokens); + parser + .expect_keyword_is(Keyword::SHOW) + .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; + parser + .expect_keyword_is(Keyword::PARTITIONS) + .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; + let table_name = parser + .parse_object_name(false) + .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; + let partition = if parser.parse_keyword(Keyword::PARTITION) { + parser + .expect_token(&Token::LParen) + .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; + let expressions = parser + .parse_comma_separated(Parser::parse_expr) + .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; + parser + .expect_token(&Token::RParen) + .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; + expressions + } else { + Vec::new() + }; + let _ = parser.consume_token(&Token::SemiColon); + if parser.peek_token().token != Token::EOF { + return Err(DataFusionError::Plan(format!( + "SQL parse error: unexpected token {} after SHOW PARTITIONS table name", + parser.peek_token().token + ))); + } + Ok(Some(ShowPartitionsStatement { + table_name, + partition, + })) +} + fn parse_sql_statements(sql: &str) -> DFResult> { let dialect = GenericDialect {}; let mut tokens = Tokenizer::new(&dialect, sql) .tokenize_with_location() .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; + tokens = rewrite_spark_multi_drop_partition_tokens(tokens); let significant = tokens .iter() .enumerate() @@ -1973,6 +2378,9 @@ fn parse_sql_statements(sql: &str) -> DFResult> { }) .take(5) .collect::>(); + if matches!(significant.first(), Some((_, Keyword::REPAIR))) { + return parse_sql_statements(&format!("MSCK {sql}")); + } let create_function_if_not_exists = matches!( significant.as_slice(), [ @@ -2028,6 +2436,138 @@ fn parse_sql_statements(sql: &str) -> DFResult> { Ok(statements) } +fn rewrite_spark_multi_drop_partition_tokens(tokens: Vec) -> Vec { + // sqlparser expects a repeated DROP keyword, while Spark accepts: + // `DROP IF EXISTS PARTITION (...), PARTITION (...)`. Insert DROP before + // each following top-level PARTITION and propagate the batch-level + // IF EXISTS flag so every generated AST operation keeps Spark semantics. + let first_keywords = tokens + .iter() + .filter_map(|token| match &token.token { + Token::Whitespace(_) => None, + Token::Word(word) => Some(word.keyword), + _ => Some(Keyword::NoKeyword), + }) + .take(2) + .collect::>(); + if !matches!(first_keywords.as_slice(), [Keyword::ALTER, Keyword::TABLE]) { + return tokens; + } + + let next_significant = |start: usize| { + (start..tokens.len()).find(|index| !matches!(tokens[*index].token, Token::Whitespace(_))) + }; + let mut insertions = HashMap::new(); + let mut depth = 0_usize; + let mut in_drop_partitions = false; + let mut propagate_if_exists = false; + let mut index = 0; + while index < tokens.len() { + match &tokens[index].token { + Token::LParen => depth += 1, + Token::RParen => depth = depth.saturating_sub(1), + Token::Word(word) if depth == 0 && word.keyword == Keyword::DROP => { + let Some(mut next) = next_significant(index + 1) else { + break; + }; + let mut if_exists = false; + if matches!( + &tokens[next].token, + Token::Word(word) if word.keyword == Keyword::IF + ) { + let Some(exists) = next_significant(next + 1) else { + break; + }; + if matches!( + &tokens[exists].token, + Token::Word(word) if word.keyword == Keyword::EXISTS + ) { + if_exists = true; + let Some(after_exists) = next_significant(exists + 1) else { + break; + }; + next = after_exists; + } + } + in_drop_partitions = matches!( + &tokens[next].token, + Token::Word(word) if word.keyword == Keyword::PARTITION + ); + propagate_if_exists = if_exists; + } + Token::Comma if depth == 0 && in_drop_partitions => { + let Some(next) = next_significant(index + 1) else { + break; + }; + match &tokens[next].token { + Token::Word(word) if word.keyword == Keyword::PARTITION => { + insertions.insert(next, propagate_if_exists); + } + Token::Word(word) if word.keyword == Keyword::DROP => {} + _ => in_drop_partitions = false, + } + } + _ => {} + } + index += 1; + } + + if insertions.is_empty() { + return tokens; + } + let mut rewritten = Vec::with_capacity(tokens.len() + insertions.len() * 3); + for (index, token) in tokens.into_iter().enumerate() { + if let Some(if_exists) = insertions.get(&index) { + rewritten.push(TokenWithSpan::wrap(Token::make_keyword("DROP"))); + if *if_exists { + rewritten.push(TokenWithSpan::wrap(Token::make_keyword("IF"))); + rewritten.push(TokenWithSpan::wrap(Token::make_keyword("EXISTS"))); + } + } + rewritten.push(token); + } + rewritten +} + +fn is_drop_partition_purge(sql: &str) -> DFResult { + let dialect = GenericDialect {}; + let mut tokens = Tokenizer::new(&dialect, sql) + .tokenize_with_location() + .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; + let Some(purge_index) = tokens.iter().enumerate().rev().find_map(|(index, token)| { + if matches!(token.token, Token::Whitespace(_) | Token::SemiColon) { + None + } else { + Some(index) + } + }) else { + return Ok(false); + }; + if !matches!( + &tokens[purge_index].token, + Token::Word(word) if word.keyword == Keyword::PURGE + ) { + return Ok(false); + } + + tokens.remove(purge_index); + tokens = rewrite_spark_multi_drop_partition_tokens(tokens); + let Ok(statements) = Parser::new(&dialect) + .with_tokens_with_locations(tokens) + .parse_statements() + else { + return Ok(false); + }; + Ok(matches!( + statements.as_slice(), + [Statement::AlterTable(alter_table)] + if alter_table + .operations + .iter() + .any(|operation| matches!(operation, AlterTableOperation::DropPartitions { .. })) + )) +} + fn validate_persistent_create_view(create_view: &CreateView) -> DFResult<()> { let unsupported = if create_view.or_alter { Some("CREATE OR ALTER VIEW is not supported") @@ -2651,36 +3191,55 @@ fn is_table_not_exist(e: &paimon::Error) -> bool { matches!(e, paimon::Error::TableNotExist { .. }) } -/// Parse partition expressions (`col = val, ...`) into partition value maps -/// suitable for `TableCommit::truncate_partitions`. -/// -/// All expressions are treated as belonging to a single partition specification. -/// For multiple partitions, callers should invoke this once per partition clause. -fn parse_partition_values( +fn ensure_catalog_managed_format_table(table: &paimon::Table, operation: &str) -> DFResult<()> { + if table.schema().partition_keys().is_empty() { + return Err(DataFusionError::Plan(format!( + "{operation} requires a partitioned table, but {} is not partitioned", + table.identifier().full_name() + ))); + } + if !table.has_catalog_managed_partitions() { + return Err(DataFusionError::Plan(format!( + "{operation} is supported only for a REST Format Table with catalog-managed partitions, but {} discovers partitions from the filesystem", + table.identifier().full_name() + ))); + } + Ok(()) +} + +fn parse_format_partition_spec( exprs: &[SqlExpr], - all_fields: &[PaimonDataField], - partition_keys: &[String], -) -> DFResult>>> { - let field_map: HashMap<&str, &PaimonDataField> = - all_fields.iter().map(|f| (f.name(), f)).collect(); + table: &paimon::Table, + require_complete: bool, +) -> DFResult> { + let field_map: HashMap<&str, &PaimonDataField> = table + .schema() + .fields() + .iter() + .map(|field| (field.name(), field)) + .collect(); + let partition_keys = table.schema().partition_keys(); + let core_options = CoreOptions::new(table.schema().options()); + let mut partition = HashMap::with_capacity(exprs.len()); - let mut partition = HashMap::new(); for expr in exprs { - let (col_name, val_expr) = match expr { + let (column_name, value_expr) = match expr { SqlExpr::BinaryOp { left, op: datafusion::sql::sqlparser::ast::BinaryOperator::Eq, right, } => { - let col = match left.as_ref() { - SqlExpr::Identifier(ident) => ident.value.clone(), + let column_name = match left.as_ref() { + SqlExpr::Identifier(identifier) => { + IdentNormalizer::default().normalize(identifier.clone()) + } other => { return Err(DataFusionError::Plan(format!( "Expected column name in partition spec, got: {other}" ))) } }; - (col, right.as_ref()) + (column_name, right.as_ref()) } other => { return Err(DataFusionError::Plan(format!( @@ -2689,17 +3248,474 @@ fn parse_partition_values( } }; - if !partition_keys.iter().any(|k| k == &col_name) { + if !partition_keys.contains(&column_name) { return Err(DataFusionError::Plan(format!( - "Column '{col_name}' is not a partition column" + "Column '{column_name}' is not a partition column" ))); } - - let field = field_map.get(col_name.as_str()).ok_or_else(|| { - DataFusionError::Plan(format!("Column '{col_name}' not found in table schema")) + if partition.contains_key(&column_name) { + return Err(DataFusionError::Plan(format!( + "Duplicate partition column '{column_name}'" + ))); + } + let field = field_map.get(column_name.as_str()).ok_or_else(|| { + DataFusionError::Plan(format!("Column '{column_name}' not found in table schema")) })?; - let datum = sql_expr_to_datum(val_expr, field.data_type())?; - partition.insert(col_name, Some(datum)); + let value = match format_partition_literal_to_datum(value_expr, field.data_type())? { + None => core_options.partition_default_name().to_string(), + Some(datum) => { + format_partition_datum_for_storage(&datum, field.data_type(), &core_options)? + } + }; + partition.insert(column_name, value); + } + + if require_complete { + let missing = partition_keys + .iter() + .filter(|key| !partition.contains_key(key.as_str())) + .cloned() + .collect::>(); + if !missing.is_empty() { + return Err(DataFusionError::Plan(format!( + "Incomplete partition spec: missing keys [{}]. All partition columns must be specified.", + missing.join(", ") + ))); + } + } + Ok(partition) +} + +fn format_partition_literal_to_datum( + expr: &SqlExpr, + data_type: &PaimonDataType, +) -> DFResult> { + if matches!( + expr, + SqlExpr::Value(value) if matches!(&value.value, SqlValue::Null) + ) { + return Ok(None); + } + let raw = format_partition_literal_as_string(expr)?; + let datum = match data_type { + PaimonDataType::Char(char_type) => Datum::String(normalize_format_partition_string( + &raw, + char_type.length(), + true, + "CHAR", + )?), + PaimonDataType::VarChar(varchar_type) => Datum::String(normalize_format_partition_string( + &raw, + varchar_type.length() as usize, + false, + "VARCHAR", + )?), + PaimonDataType::Boolean(_) => Datum::Bool(parse_format_partition_bool(&raw)?), + PaimonDataType::TinyInt(_) => { + Datum::TinyInt(raw.trim().parse::().map_err(|error| { + DataFusionError::Plan(format!("Invalid TINYINT partition value '{raw}': {error}")) + })?) + } + PaimonDataType::SmallInt(_) => { + Datum::SmallInt(raw.trim().parse::().map_err(|error| { + DataFusionError::Plan(format!("Invalid SMALLINT partition value '{raw}': {error}")) + })?) + } + PaimonDataType::Int(_) => Datum::Int(raw.trim().parse::().map_err(|error| { + DataFusionError::Plan(format!("Invalid INT partition value '{raw}': {error}")) + })?), + PaimonDataType::BigInt(_) => Datum::Long(raw.trim().parse::().map_err(|error| { + DataFusionError::Plan(format!("Invalid BIGINT partition value '{raw}': {error}")) + })?), + PaimonDataType::Date(_) => Datum::Date(parse_spark_partition_date(&raw)?), + other => { + return Err(DataFusionError::NotImplemented(format!( + "Partition literals of type {other:?} are not implemented yet" + ))) + } + }; + Ok(Some(datum)) +} + +fn normalize_format_partition_string( + value: &str, + length: usize, + pad_to_length: bool, + type_name: &str, +) -> DFResult { + let mut normalized = value.to_string(); + let mut character_count = normalized.chars().count(); + if character_count > length { + for _ in 0..(character_count - length) { + if !normalized.ends_with(' ') { + break; + } + normalized.pop(); + character_count -= 1; + } + if character_count > length { + return Err(DataFusionError::Plan(format!( + "{type_name} partition value '{value}' exceeds maximum length {length}" + ))); + } + } + if pad_to_length && character_count < length { + normalized.push_str(&" ".repeat(length - character_count)); + Ok(normalized) + } else { + Ok(normalized) + } +} + +fn format_partition_literal_as_string(expr: &SqlExpr) -> DFResult { + match expr { + SqlExpr::TypedString(typed) => { + let value = typed.value.value.clone().into_string().ok_or_else(|| { + DataFusionError::Plan(format!("Unsupported typed partition literal: {expr}")) + })?; + match &typed.data_type { + datafusion::sql::sqlparser::ast::DataType::Date => { + format_partition_date(parse_spark_partition_date(&value)?) + } + _ => Err(DataFusionError::Plan(format!( + "Unsupported typed partition literal: {expr}" + ))), + } + } + SqlExpr::Value(value) => match &value.value { + SqlValue::Number(value, _) => Ok(canonicalize_integer_literal_if_possible(value)), + SqlValue::Boolean(value) => Ok(value.to_string()), + SqlValue::Null => Err(DataFusionError::Plan( + "NULL partition values must be handled before string conversion".to_string(), + )), + _ => value.value.clone().into_string().ok_or_else(|| { + DataFusionError::Plan(format!("Unsupported partition literal: {expr}")) + }), + }, + SqlExpr::UnaryOp { + op: + datafusion::sql::sqlparser::ast::UnaryOperator::Plus + | datafusion::sql::sqlparser::ast::UnaryOperator::Minus, + expr: inner, + } => { + let value = format_partition_literal_as_string(inner)?; + let sign = if matches!( + expr, + SqlExpr::UnaryOp { + op: datafusion::sql::sqlparser::ast::UnaryOperator::Minus, + .. + } + ) { + "-" + } else { + "" + }; + Ok(format!("{sign}{value}")) + } + other => Err(DataFusionError::Plan(format!( + "Unsupported partition value expression: {other}" + ))), + } +} + +fn canonicalize_integer_literal_if_possible(value: &str) -> String { + if let Ok(value) = value.parse::() { + value.to_string() + } else if let Ok(value) = value.parse::() { + value.to_string() + } else { + value.to_string() + } +} + +fn parse_format_partition_bool(value: &str) -> DFResult { + match value.trim().to_ascii_lowercase().as_str() { + "t" | "true" | "y" | "yes" | "1" => Ok(true), + "f" | "false" | "n" | "no" | "0" => Ok(false), + _ => Err(DataFusionError::Plan(format!( + "Invalid BOOLEAN partition value '{value}'" + ))), + } +} + +fn parse_spark_partition_date(value: &str) -> DFResult { + let value = value.trim(); + let bytes = value.as_bytes(); + if bytes.is_empty() { + return Err(DataFusionError::Plan("Invalid DATE ''".to_string())); + } + + let mut sign = 1_i32; + let mut offset = 0; + if matches!(bytes[0], b'+' | b'-') { + sign = if bytes[0] == b'-' { -1 } else { 1 }; + offset = 1; + } + + let mut segments = [1_i32, 1_i32, 1_i32]; + let mut segment_index = 0; + let mut segment_value = 0_i32; + let mut segment_digits = 0_usize; + while offset < bytes.len() && segment_index < 3 && !matches!(bytes[offset], b' ' | b'T') { + let byte = bytes[offset]; + if segment_index < 2 && byte == b'-' { + if !valid_spark_date_segment(segment_index, segment_digits) { + return Err(DataFusionError::Plan(format!("Invalid DATE '{value}'"))); + } + segments[segment_index] = segment_value; + segment_index += 1; + segment_value = 0; + segment_digits = 0; + } else if byte.is_ascii_digit() { + segment_value = segment_value + .checked_mul(10) + .and_then(|current| current.checked_add(i32::from(byte - b'0'))) + .ok_or_else(|| DataFusionError::Plan(format!("Invalid DATE '{value}'")))?; + segment_digits += 1; + } else { + return Err(DataFusionError::Plan(format!("Invalid DATE '{value}'"))); + } + offset += 1; + } + + if !valid_spark_date_segment(segment_index, segment_digits) + || (segment_index < 2 && offset < bytes.len()) + { + return Err(DataFusionError::Plan(format!("Invalid DATE '{value}'"))); + } + segments[segment_index] = segment_value; + + let year = sign * segments[0]; + if !(0..=9999).contains(&year) { + return Err(DataFusionError::Plan(format!( + "DATE '{value}' is outside Paimon DATE range 0000-01-01..9999-12-31" + ))); + } + let date = chrono::NaiveDate::from_ymd_opt(year, segments[1] as u32, segments[2] as u32) + .ok_or_else(|| DataFusionError::Plan(format!("Invalid DATE '{value}'")))?; + i32::try_from((date - chrono::NaiveDate::default()).num_days()).map_err(|error| { + DataFusionError::Plan(format!( + "DATE '{value}' is outside the supported partition range: {error}" + )) + }) +} + +fn valid_spark_date_segment(segment_index: usize, digits: usize) -> bool { + if segment_index == 0 { + (4..=7).contains(&digits) + } else { + (1..=2).contains(&digits) + } +} + +fn format_partition_date(epoch_days: i32) -> DFResult { + let ce_days = epoch_days.checked_add(719_163).ok_or_else(|| { + DataFusionError::Plan(format!( + "DATE partition value {epoch_days} is outside the supported range" + )) + })?; + Ok(chrono::NaiveDate::from_num_days_from_ce_opt(ce_days) + .ok_or_else(|| { + DataFusionError::Plan(format!( + "DATE partition value {epoch_days} is outside the supported range" + )) + })? + .format("%Y-%m-%d") + .to_string()) +} + +fn format_partition_datum_for_storage( + datum: &Datum, + data_type: &PaimonDataType, + core_options: &CoreOptions<'_>, +) -> DFResult { + match (datum, data_type) { + (Datum::String(value), PaimonDataType::Char(_) | PaimonDataType::VarChar(_)) => { + if value.trim().is_empty() { + Ok(core_options.partition_default_name().to_string()) + } else { + Ok(value.clone()) + } + } + (Datum::Bool(value), PaimonDataType::Boolean(_)) => Ok(value.to_string()), + (Datum::TinyInt(value), PaimonDataType::TinyInt(_)) => Ok(value.to_string()), + (Datum::SmallInt(value), PaimonDataType::SmallInt(_)) => Ok(value.to_string()), + (Datum::Int(value), PaimonDataType::Int(_)) => Ok(value.to_string()), + (Datum::Long(value), PaimonDataType::BigInt(_)) => Ok(value.to_string()), + (Datum::Date(value), PaimonDataType::Date(_)) => { + if core_options.legacy_partition_name() { + Ok(value.to_string()) + } else { + format_partition_date(*value) + } + } + _ => Err(DataFusionError::NotImplemented(format!( + "Partition literals of type {data_type:?} are not implemented yet" + ))), + } +} + +fn normalize_format_partition_spec_values( + spec: &HashMap, + table: &paimon::Table, + require_complete: bool, +) -> DFResult>> { + let partition_keys = table.schema().partition_keys(); + if spec.keys().any(|key| !partition_keys.contains(key)) + || (require_complete + && (spec.len() != partition_keys.len() + || partition_keys.iter().any(|key| !spec.contains_key(key)))) + { + return Err(DataFusionError::Plan(format!( + "Invalid partition spec {spec:?} for table {}: expected {}keys {partition_keys:?}", + table.identifier().full_name(), + if require_complete { + "exactly " + } else { + "only " + } + ))); + } + let fields = table + .schema() + .partition_fields() + .into_iter() + .map(|field| (field.name().to_string(), field)) + .collect::>(); + let core_options = CoreOptions::new(table.schema().options()); + let default_partition_name = core_options.partition_default_name(); + let mut normalized = HashMap::with_capacity(spec.len()); + for key in partition_keys.iter().filter(|key| spec.contains_key(*key)) { + let raw = &spec[key]; + let value = if raw == default_partition_name { + None + } else { + let field = fields.get(key).ok_or_else(|| { + DataFusionError::Plan(format!( + "Partition column '{key}' is missing from table {} schema", + table.identifier().full_name() + )) + })?; + Some(match field.data_type() { + PaimonDataType::Char(char_type) => { + normalize_format_partition_string(raw, char_type.length(), true, "CHAR")? + } + PaimonDataType::VarChar(varchar_type) => normalize_format_partition_string( + raw, + varchar_type.length() as usize, + false, + "VARCHAR", + )?, + PaimonDataType::Boolean(_) => parse_format_partition_bool(raw)?.to_string(), + PaimonDataType::TinyInt(_) => raw + .trim() + .parse::() + .map_err(|error| { + DataFusionError::Plan(format!( + "Invalid TINYINT partition value '{raw}' for column '{key}': {error}" + )) + })? + .to_string(), + PaimonDataType::SmallInt(_) => raw + .trim() + .parse::() + .map_err(|error| { + DataFusionError::Plan(format!( + "Invalid SMALLINT partition value '{raw}' for column '{key}': {error}" + )) + })? + .to_string(), + PaimonDataType::Int(_) => raw + .trim() + .parse::() + .map_err(|error| { + DataFusionError::Plan(format!( + "Invalid INT partition value '{raw}' for column '{key}': {error}" + )) + })? + .to_string(), + PaimonDataType::BigInt(_) => raw + .trim() + .parse::() + .map_err(|error| { + DataFusionError::Plan(format!( + "Invalid BIGINT partition value '{raw}' for column '{key}': {error}" + )) + })? + .to_string(), + PaimonDataType::Date(_) => { + let epoch_days = if core_options.legacy_partition_name() { + raw.trim().parse::().map_err(|error| { + DataFusionError::Plan(format!( + "Invalid legacy DATE partition value '{raw}' for column '{key}': {error}" + )) + })? + } else { + parse_spark_partition_date(raw)? + }; + format_partition_date(epoch_days)? + } + other => { + return Err(DataFusionError::NotImplemented(format!( + "SHOW/DROP PARTITION values of type {other:?} are not implemented yet" + ))) + } + }) + }; + normalized.insert(key.clone(), value); + } + Ok(normalized) +} + +/// Parse partition expressions (`col = val, ...`) into partition value maps +/// suitable for `TableCommit::truncate_partitions`. +/// +/// All expressions are treated as belonging to a single partition specification. +/// For multiple partitions, callers should invoke this once per partition clause. +fn parse_partition_values( + exprs: &[SqlExpr], + all_fields: &[PaimonDataField], + partition_keys: &[String], +) -> DFResult>>> { + let field_map: HashMap<&str, &PaimonDataField> = + all_fields.iter().map(|f| (f.name(), f)).collect(); + + let mut partition = HashMap::new(); + for expr in exprs { + let (col_name, val_expr) = match expr { + SqlExpr::BinaryOp { + left, + op: datafusion::sql::sqlparser::ast::BinaryOperator::Eq, + right, + } => { + let col = match left.as_ref() { + SqlExpr::Identifier(ident) => ident.value.clone(), + other => { + return Err(DataFusionError::Plan(format!( + "Expected column name in partition spec, got: {other}" + ))) + } + }; + (col, right.as_ref()) + } + other => { + return Err(DataFusionError::Plan(format!( + "Expected 'column = value' in partition spec, got: {other}" + ))) + } + }; + + if !partition_keys.iter().any(|k| k == &col_name) { + return Err(DataFusionError::Plan(format!( + "Column '{col_name}' is not a partition column" + ))); + } + + let field = field_map.get(col_name.as_str()).ok_or_else(|| { + DataFusionError::Plan(format!("Column '{col_name}' not found in table schema")) + })?; + let datum = sql_expr_to_datum(val_expr, field.data_type())?; + partition.insert(col_name, Some(datum)); } let missing: Vec<&str> = partition_keys @@ -2783,6 +3799,21 @@ fn parse_static_partitions( /// Convert a SQL literal expression to a Paimon Datum. fn sql_expr_to_datum(expr: &SqlExpr, data_type: &PaimonDataType) -> DFResult { + if let SqlExpr::TypedString(typed) = expr { + if matches!( + typed.data_type, + datafusion::sql::sqlparser::ast::DataType::Date + ) && matches!(data_type, PaimonDataType::Date(_)) + { + if let SqlValue::SingleQuotedString(value) = &typed.value.value { + return parse_date_datum(value); + } + } + return Err(DataFusionError::Plan(format!( + "Cannot convert {expr} to {data_type:?}" + ))); + } + let (value, negate) = match expr { SqlExpr::Value(v) => (&v.value, false), SqlExpr::UnaryOp { @@ -2806,14 +3837,13 @@ fn sql_expr_to_datum(expr: &SqlExpr, data_type: &PaimonDataType) -> DFResult parse_number_datum(n, data_type, negate), - (SqlValue::SingleQuotedString(s), PaimonDataType::VarChar(_)) if !negate => { + (SqlValue::SingleQuotedString(s), PaimonDataType::Char(_) | PaimonDataType::VarChar(_)) + if !negate => + { Ok(Datum::String(s.clone())) } (SqlValue::SingleQuotedString(s), PaimonDataType::Date(_)) if !negate => { - let date = chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d") - .map_err(|e| DataFusionError::Plan(format!("Invalid DATE '{s}': {e}")))?; - let epoch = chrono::NaiveDate::from_ymd_opt(1970, 1, 1).unwrap(); - Ok(Datum::Date((date - epoch).num_days() as i32)) + parse_date_datum(s) } (SqlValue::Boolean(b), PaimonDataType::Boolean(_)) if !negate => Ok(Datum::Bool(*b)), _ if negate => Err(DataFusionError::Plan(format!( @@ -2825,6 +3855,18 @@ fn sql_expr_to_datum(expr: &SqlExpr, data_type: &PaimonDataType) -> DFResult DFResult { + let date = chrono::NaiveDate::parse_from_str(value, "%Y-%m-%d") + .map_err(|e| DataFusionError::Plan(format!("Invalid DATE '{value}': {e}")))?; + let epoch_days = + i32::try_from((date - chrono::NaiveDate::default()).num_days()).map_err(|e| { + DataFusionError::Plan(format!( + "DATE '{value}' is outside the supported partition range: {e}" + )) + })?; + Ok(Datum::Date(epoch_days)) +} + fn parse_number_datum(n: &str, data_type: &PaimonDataType, negate: bool) -> DFResult { let s: String = if negate { format!("-{n}") @@ -3274,10 +4316,14 @@ mod tests { use async_trait::async_trait; use datafusion::arrow::array::StringViewArray; use paimon::catalog::Database; + use paimon::common::Options; + use paimon::io::FileIO; use paimon::spec::{ - DataField as PaimonDataField, DataType as PaimonDataType, IntType, Schema as PaimonSchema, + DataField as PaimonDataField, DataType as PaimonDataType, IntType, + Partition as PaimonPartition, Schema as PaimonSchema, TableSchema, }; - use paimon::table::Table; + use paimon::table::{RESTEnv, Table}; + use paimon::{CatalogOptions, RESTApi}; // ==================== Mock Catalog ==================== @@ -3306,6 +4352,12 @@ mod tests { existing_table: Mutex>, functions: Mutex>, views: Mutex>, + partition_specs: Mutex>>, + get_table_calls: Mutex, + list_partitions_calls: Mutex, + list_partitions_by_names_calls: Mutex>>>, + partition_mutation_calls: Mutex>, + list_partitions_error: Mutex, drop_view_supported: bool, } @@ -3316,6 +4368,12 @@ mod tests { existing_table: Mutex::new(None), functions: Mutex::new(HashMap::new()), views: Mutex::new(HashMap::new()), + partition_specs: Mutex::new(Vec::new()), + get_table_calls: Mutex::new(0), + list_partitions_calls: Mutex::new(0), + list_partitions_by_names_calls: Mutex::new(Vec::new()), + partition_mutation_calls: Mutex::new(Vec::new()), + list_partitions_error: Mutex::new(false), drop_view_supported: true, } } @@ -3344,6 +4402,55 @@ mod tests { .unwrap() .insert(view.identifier().clone(), view); } + + fn set_table(&self, table: Table) { + *self.existing_table.lock().unwrap() = Some(table); + } + + fn set_partition_specs(&self, specs: Vec>) { + *self.partition_specs.lock().unwrap() = specs; + } + + fn partition_specs(&self) -> Vec> { + self.partition_specs.lock().unwrap().clone() + } + + fn get_table_calls(&self) -> usize { + *self.get_table_calls.lock().unwrap() + } + + fn list_partitions_calls(&self) -> usize { + *self.list_partitions_calls.lock().unwrap() + } + + fn list_partitions_by_names_calls(&self) -> Vec>> { + self.list_partitions_by_names_calls.lock().unwrap().clone() + } + + fn partition_mutation_calls(&self) -> Vec<&'static str> { + self.partition_mutation_calls.lock().unwrap().clone() + } + + fn set_list_partitions_error(&self, enabled: bool) { + *self.list_partitions_error.lock().unwrap() = enabled; + } + } + + fn test_partition(spec: HashMap) -> PaimonPartition { + PaimonPartition { + spec, + record_count: 0, + file_size_in_bytes: 0, + file_count: 0, + last_file_creation_time: 0, + total_buckets: 0, + done: false, + created_at: None, + created_by: None, + updated_at: None, + updated_by: None, + options: None, + } } #[async_trait] @@ -3371,6 +4478,7 @@ mod tests { Ok(()) } async fn get_table(&self, _identifier: &Identifier) -> paimon::Result { + *self.get_table_calls.lock().unwrap() += 1; if let Some(table) = self.existing_table.lock().unwrap().clone() { return Ok(table); } @@ -3428,6 +4536,82 @@ mod tests { Ok(()) } + async fn create_partitions( + &self, + _identifier: &Identifier, + partitions: Vec>, + ignore_if_exists: bool, + ) -> paimon::Result<()> { + self.partition_mutation_calls.lock().unwrap().push("create"); + let mut current = self.partition_specs.lock().unwrap(); + if !ignore_if_exists && partitions.iter().any(|spec| current.contains(spec)) { + return Err(paimon::Error::DataInvalid { + message: "Some partitions already exist".to_string(), + source: None, + }); + } + for spec in partitions { + if !current.contains(&spec) { + current.push(spec); + } + } + Ok(()) + } + + async fn drop_partitions( + &self, + _identifier: &Identifier, + partitions: Vec>, + ) -> paimon::Result<()> { + self.partition_mutation_calls.lock().unwrap().push("drop"); + self.partition_specs + .lock() + .unwrap() + .retain(|spec| !partitions.contains(spec)); + Ok(()) + } + + async fn list_partitions_by_names( + &self, + _identifier: &Identifier, + partitions: Vec>, + ) -> paimon::Result> { + self.list_partitions_by_names_calls + .lock() + .unwrap() + .push(partitions.clone()); + Ok(self + .partition_specs + .lock() + .unwrap() + .iter() + .filter(|spec| partitions.contains(spec)) + .cloned() + .map(test_partition) + .collect()) + } + + async fn list_partitions( + &self, + _identifier: &Identifier, + ) -> paimon::Result> { + *self.list_partitions_calls.lock().unwrap() += 1; + if *self.list_partitions_error.lock().unwrap() { + return Err(paimon::Error::UnexpectedError { + message: "Injected partition listing failure".to_string(), + source: None, + }); + } + Ok(self + .partition_specs + .lock() + .unwrap() + .iter() + .cloned() + .map(test_partition) + .collect()) + } + async fn list_functions(&self, database_name: &str) -> paimon::Result> { Ok(self .functions @@ -3541,13 +4725,97 @@ mod tests { ctx } - fn add_unary_sql_function( - catalog: &MockCatalog, - name: &str, - definition: &str, - deterministic: bool, - ) { - add_unary_sql_function_in_database(catalog, "default", name, definition, deterministic); + async fn managed_format_table( + identifier: Identifier, + location: &str, + partition_keys: &[&str], + ) -> Table { + managed_format_table_with_partition_fields( + identifier, + location, + partition_keys + .iter() + .map(|key| { + ( + *key, + PaimonDataType::VarChar(VarCharType::new(255).unwrap()), + ) + }) + .collect(), + ) + .await + } + + async fn managed_format_table_with_partition_fields( + identifier: Identifier, + location: &str, + partition_fields: Vec<(&str, PaimonDataType)>, + ) -> Table { + managed_format_table_with_partition_fields_and_options( + identifier, + location, + partition_fields, + &[], + ) + .await + } + + async fn managed_format_table_with_partition_fields_and_options( + identifier: Identifier, + location: &str, + partition_fields: Vec<(&str, PaimonDataType)>, + extra_options: &[(&str, &str)], + ) -> Table { + let partition_keys = partition_fields + .iter() + .map(|(key, _)| *key) + .collect::>(); + let mut builder = PaimonSchema::builder() + .column("id", PaimonDataType::Int(IntType::new())) + .option("type", "format-table") + .option("metastore.partitioned-table", "true") + .option("path", location); + for (key, data_type) in partition_fields { + builder = builder.column(key, data_type); + } + for (key, value) in extra_options { + builder = builder.option(*key, *value); + } + let schema = builder.partition_keys(partition_keys).build().unwrap(); + let table_schema = TableSchema::new(0, &schema); + let file_io = FileIO::from_path(location).unwrap().build().unwrap(); + + let mut options = Options::new(); + options.set(CatalogOptions::URI, "http://127.0.0.1:1"); + options.set("token.provider", "bear"); + options.set("token", "test-token"); + let api = Arc::new(RESTApi::new(options.clone(), false).await.unwrap()); + let rest_env = RESTEnv::new_for_table( + identifier.clone(), + "test-uuid".to_string(), + api, + options, + false, + &table_schema, + false, + ) + .unwrap(); + Table::new( + file_io, + identifier, + location.to_string(), + table_schema, + Some(rest_env), + ) + } + + fn add_unary_sql_function( + catalog: &MockCatalog, + name: &str, + definition: &str, + deterministic: bool, + ) { + add_unary_sql_function_in_database(catalog, "default", name, definition, deterministic); } fn add_unary_sql_function_in_database( @@ -6316,6 +7584,1646 @@ mod tests { .contains("exactly one SQL statement")); } + #[tokio::test] + async fn test_msck_repair_table_dispatches_to_paimon_catalog() { + let catalog = Arc::new(MockCatalog::new()); + let sql_context = make_sql_context(catalog).await; + + let error = sql_context + .sql("MSCK REPAIR TABLE mydb.missing") + .await + .unwrap_err(); + + assert!( + error + .to_string() + .contains("Table mydb.missing does not exist"), + "expected catalog table error, got: {error}" + ); + } + + #[tokio::test] + async fn test_bare_repair_table_dispatches_to_paimon_catalog() { + let catalog = Arc::new(MockCatalog::new()); + let sql_context = make_sql_context(catalog).await; + + let error = sql_context + .sql("REPAIR TABLE mydb.missing") + .await + .unwrap_err(); + + assert!( + error + .to_string() + .contains("Table mydb.missing does not exist"), + "expected catalog table error, got: {error}" + ); + } + + #[tokio::test] + async fn test_show_partitions_dispatches_to_paimon_catalog() { + let catalog = Arc::new(MockCatalog::new()); + let sql_context = make_sql_context(catalog).await; + + let error = sql_context + .sql("SHOW PARTITIONS mydb.missing") + .await + .unwrap_err(); + + assert!( + error + .to_string() + .contains("Table mydb.missing does not exist"), + "expected catalog table error, got: {error}" + ); + } + + #[test] + fn test_parse_show_partitions_with_partition_filter() { + let statement = parse_show_partitions( + "SHOW PARTITIONS mydb.events PARTITION (dt = '2026-07-22', region = 3)", + ) + .unwrap() + .unwrap(); + + assert_eq!(statement.table_name.to_string(), "mydb.events"); + assert_eq!( + statement + .partition + .iter() + .map(ToString::to_string) + .collect::>(), + vec!["dt = '2026-07-22'", "region = 3"] + ); + } + + #[test] + fn test_parse_spark_multi_drop_partitions() { + let statements = parse_sql_statements( + "ALTER TABLE mydb.events \ + DROP IF EXISTS PARTITION (dt = '2026-07-21'), \ + PARTITION (dt = '2026-07-22')", + ) + .unwrap(); + let [Statement::AlterTable(alter_table)] = statements.as_slice() else { + panic!("expected ALTER TABLE"); + }; + + assert_eq!(alter_table.operations.len(), 2); + assert!(alter_table.operations.iter().all(|operation| matches!( + operation, + AlterTableOperation::DropPartitions { + if_exists: true, + .. + } + ))); + } + + #[tokio::test] + async fn test_show_partitions_returns_sorted_catalog_registrations() { + let catalog = Arc::new(MockCatalog::new()); + let identifier = Identifier::new("mydb", "events"); + catalog.set_table( + managed_format_table(identifier, "memory:/show_partitions", &["dt", "hour"]).await, + ); + catalog.set_partition_specs(vec![ + HashMap::from([ + ("dt".to_string(), "2026-07-22".to_string()), + ("hour".to_string(), "10".to_string()), + ]), + HashMap::from([ + ("dt".to_string(), "2026-07-21".to_string()), + ("hour".to_string(), "09".to_string()), + ]), + ]); + let sql_context = make_sql_context(catalog).await; + + let batches = sql_context + .sql("SHOW PARTITIONS mydb.events") + .await + .unwrap() + .collect() + .await + .unwrap(); + + assert_eq!(batches[0].schema().field(0).name(), "partition"); + let partitions = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + (0..batches[0].num_rows()) + .map(|index| partitions.value(index)) + .collect::>(), + vec!["dt=2026-07-21/hour=09", "dt=2026-07-22/hour=10",] + ); + } + + #[tokio::test] + async fn test_show_partitions_filters_by_non_prefix_partition_key() { + let catalog = Arc::new(MockCatalog::new()); + let identifier = Identifier::new("mydb", "events"); + catalog.set_table( + managed_format_table( + identifier, + "memory:/show_partitions_filter", + &["dt", "hour"], + ) + .await, + ); + catalog.set_partition_specs(vec![ + HashMap::from([ + ("dt".to_string(), "2026-07-22".to_string()), + ("hour".to_string(), "10".to_string()), + ]), + HashMap::from([ + ("dt".to_string(), "2026-07-21".to_string()), + ("hour".to_string(), "09".to_string()), + ]), + ]); + let sql_context = make_sql_context(catalog).await; + + let batches = sql_context + .sql("SHOW PARTITIONS mydb.events PARTITION (hour = '10')") + .await + .unwrap() + .collect() + .await + .unwrap(); + + let partitions = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(batches[0].num_rows(), 1); + assert_eq!(partitions.value(0), "dt=2026-07-22/hour=10"); + } + + #[tokio::test] + async fn test_show_partitions_formats_default_partition_as_null() { + let catalog = Arc::new(MockCatalog::new()); + let identifier = Identifier::new("mydb", "events"); + catalog.set_table( + managed_format_table(identifier, "memory:/show_default_partition", &["dt"]).await, + ); + catalog.set_partition_specs(vec![HashMap::from([( + "dt".to_string(), + "__DEFAULT_PARTITION__".to_string(), + )])]); + let sql_context = make_sql_context(catalog).await; + + let batches = sql_context + .sql("SHOW PARTITIONS mydb.events") + .await + .unwrap() + .collect() + .await + .unwrap(); + let partitions = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + + assert_eq!(partitions.value(0), "dt=null"); + } + + #[tokio::test] + async fn test_show_partitions_formats_date_and_integer_values_by_type() { + let catalog = Arc::new(MockCatalog::new()); + let identifier = Identifier::new("mydb", "events"); + catalog.set_table( + managed_format_table_with_partition_fields( + identifier, + "memory:/show_typed_partitions", + vec![ + ("dt", PaimonDataType::Date(DateType::new())), + ("month", PaimonDataType::Int(IntType::new())), + ], + ) + .await, + ); + catalog.set_partition_specs(vec![HashMap::from([ + ("dt".to_string(), "20656".to_string()), + ("month".to_string(), "01".to_string()), + ])]); + let sql_context = make_sql_context(catalog).await; + + let batches = sql_context + .sql("SHOW PARTITIONS mydb.events PARTITION (month = '1')") + .await + .unwrap() + .collect() + .await + .unwrap(); + let partitions = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + + assert_eq!(partitions.value(0), "dt=2026-07-22/month=1"); + } + + #[tokio::test] + async fn test_show_partitions_preserves_duplicate_normalized_names() { + let catalog = Arc::new(MockCatalog::new()); + let identifier = Identifier::new("mydb", "events"); + catalog.set_table( + managed_format_table_with_partition_fields( + identifier, + "memory:/show_duplicate_typed_partitions", + vec![("month", PaimonDataType::Int(IntType::new()))], + ) + .await, + ); + catalog.set_partition_specs(vec![ + HashMap::from([("month".to_string(), "01".to_string())]), + HashMap::from([("month".to_string(), "1".to_string())]), + ]); + let sql_context = make_sql_context(catalog).await; + + let batches = sql_context + .sql("SHOW PARTITIONS mydb.events") + .await + .unwrap() + .collect() + .await + .unwrap(); + let partitions = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + + assert_eq!( + (0..batches[0].num_rows()) + .map(|index| partitions.value(index)) + .collect::>(), + vec!["month=1", "month=1"] + ); + } + + #[tokio::test] + async fn test_add_partition_dispatches_to_paimon_catalog() { + let catalog = Arc::new(MockCatalog::new()); + let sql_context = make_sql_context(catalog).await; + + let error = sql_context + .sql("ALTER TABLE mydb.missing ADD PARTITION (dt = '2026-07-22')") + .await + .unwrap_err(); + + assert!( + error + .to_string() + .contains("Table mydb.missing does not exist"), + "expected catalog table error, got: {error}" + ); + } + + #[tokio::test] + async fn test_add_format_partitions_registers_batch_and_creates_directories() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql( + "ALTER TABLE mydb.events ADD \ + PARTITION (dt = '2026-07-22') \ + PARTITION (dt = '2026-07-21')", + ) + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![ + HashMap::from([("dt".to_string(), "2026-07-22".to_string())]), + HashMap::from([("dt".to_string(), "2026-07-21".to_string())]), + ] + ); + assert!(temp_dir.path().join("dt=2026-07-22").is_dir()); + assert!(temp_dir.path().join("dt=2026-07-21").is_dir()); + } + + #[tokio::test] + async fn test_add_format_partition_strict_conflict_and_if_not_exists_retry() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, + ); + let spec = HashMap::from([("dt".to_string(), "2026-07-22".to_string())]); + catalog.set_partition_specs(vec![spec.clone()]); + let sql_context = make_sql_context(catalog.clone()).await; + + let error = sql_context + .sql("ALTER TABLE mydb.events ADD PARTITION (dt = '2026-07-22')") + .await + .unwrap_err(); + assert!( + error.to_string().contains("already exist"), + "expected strict duplicate error, got: {error}" + ); + assert!(!temp_dir.path().join("dt=2026-07-22").exists()); + + sql_context + .sql( + "ALTER TABLE mydb.events \ + ADD IF NOT EXISTS PARTITION (dt = '2026-07-22')", + ) + .await + .unwrap(); + + assert_eq!(catalog.partition_specs(), vec![spec]); + assert!(temp_dir.path().join("dt=2026-07-22").is_dir()); + assert_eq!(catalog.partition_mutation_calls(), vec!["create", "create"]); + } + + #[tokio::test] + async fn test_add_format_partition_normalizes_integer_literal() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![("month", PaimonDataType::Int(IntType::new()))], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("ALTER TABLE mydb.events ADD PARTITION (month = 01)") + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([("month".to_string(), "1".to_string())])] + ); + assert!(temp_dir.path().join("month=1").is_dir()); + assert!(!temp_dir.path().join("month=01").exists()); + } + + #[tokio::test] + async fn test_add_format_partition_normalizes_numeric_literal_for_string_column() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table(Identifier::new("mydb", "events"), &location, &["label"]).await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("ALTER TABLE mydb.events ADD PARTITION (label = 01)") + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([("label".to_string(), "1".to_string())])] + ); + assert!(temp_dir.path().join("label=1").is_dir()); + assert!(!temp_dir.path().join("label=01").exists()); + } + + #[tokio::test] + async fn test_add_format_partition_accepts_char_literal() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![("region", PaimonDataType::Char(CharType::new(2).unwrap()))], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("ALTER TABLE mydb.events ADD PARTITION (region = 'us')") + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([("region".to_string(), "us".to_string())])] + ); + assert!(temp_dir.path().join("region=us").is_dir()); + } + + #[tokio::test] + async fn test_add_format_partition_pads_short_char_literal() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![("region", PaimonDataType::Char(CharType::new(4).unwrap()))], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("ALTER TABLE mydb.events ADD PARTITION (region = 'us')") + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([("region".to_string(), "us ".to_string())])] + ); + assert!(temp_dir.path().join("region=us ").is_dir()); + } + + #[tokio::test] + async fn test_add_format_partition_rejects_oversize_char_and_varchar_literals() { + for data_type in [ + PaimonDataType::Char(CharType::new(5).unwrap()), + PaimonDataType::VarChar(VarCharType::new(5).unwrap()), + ] { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![("region", data_type)], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + let error = sql_context + .sql("ALTER TABLE mydb.events ADD PARTITION (region = 'abcdef')") + .await + .unwrap_err(); + + assert!( + error.to_string().contains("exceeds maximum length 5"), + "unexpected error: {error}" + ); + assert!(catalog.partition_specs().is_empty()); + } + } + + #[tokio::test] + async fn test_add_format_partition_trims_excess_trailing_spaces_for_char_and_varchar() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![ + ("fixed", PaimonDataType::Char(CharType::new(5).unwrap())), + ( + "variable", + PaimonDataType::VarChar(VarCharType::new(5).unwrap()), + ), + ], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql( + "ALTER TABLE mydb.events ADD PARTITION (\ + fixed = 'a ', variable = 'a ')", + ) + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([ + ("fixed".to_string(), "a ".to_string()), + ("variable".to_string(), "a ".to_string()), + ])] + ); + } + + #[tokio::test] + async fn test_add_format_partition_maps_null_to_default_partition_name() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("ALTER TABLE mydb.events ADD PARTITION (dt = NULL)") + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([( + "dt".to_string(), + "__DEFAULT_PARTITION__".to_string(), + )])] + ); + assert!(temp_dir.path().join("dt=__DEFAULT_PARTITION__").is_dir()); + } + + #[tokio::test] + async fn test_add_format_partition_maps_blank_string_to_default_partition_name() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("ALTER TABLE mydb.events ADD PARTITION (dt = ' ')") + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([( + "dt".to_string(), + "__DEFAULT_PARTITION__".to_string(), + )])] + ); + assert!(temp_dir.path().join("dt=__DEFAULT_PARTITION__").is_dir()); + } + + #[tokio::test] + async fn test_add_format_partition_normalizes_unquoted_partition_column_name() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("ALTER TABLE mydb.events ADD PARTITION (DT = '2026-07-22')") + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([( + "dt".to_string(), + "2026-07-22".to_string(), + )])] + ); + } + + #[tokio::test] + async fn test_add_format_partition_normalizes_common_typed_literals() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![ + ("shard", PaimonDataType::BigInt(BigIntType::new())), + ("active", PaimonDataType::Boolean(BooleanType::new())), + ("dt", PaimonDataType::Date(DateType::new())), + ], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql( + "ALTER TABLE mydb.events ADD PARTITION (\ + shard = 9223372036854775807, active = true, dt = '2026-07-22')", + ) + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([ + ("shard".to_string(), "9223372036854775807".to_string()), + ("active".to_string(), "true".to_string()), + ("dt".to_string(), "20656".to_string()), + ])] + ); + assert!(temp_dir + .path() + .join("shard=9223372036854775807/active=true/dt=20656") + .is_dir()); + } + + #[tokio::test] + async fn test_add_format_partition_casts_literals_through_partition_column_types() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![ + ("month", PaimonDataType::Int(IntType::new())), + ( + "label", + PaimonDataType::VarChar(VarCharType::new(255).unwrap()), + ), + ("active", PaimonDataType::Boolean(BooleanType::new())), + ], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql( + "ALTER TABLE mydb.events ADD PARTITION (\ + month = '01', label = 20260722, active = 'TRUE')", + ) + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([ + ("month".to_string(), "1".to_string()), + ("label".to_string(), "20260722".to_string()), + ("active".to_string(), "true".to_string()), + ])] + ); + } + + #[tokio::test] + async fn test_add_format_partition_trims_string_before_integer_cast() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![("month", PaimonDataType::Int(IntType::new()))], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("ALTER TABLE mydb.events ADD PARTITION (month = ' 01 ')") + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([("month".to_string(), "1".to_string())])] + ); + assert!(temp_dir.path().join("month=1").is_dir()); + } + + #[tokio::test] + async fn test_add_format_partition_accepts_spark_boolean_spellings() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![("active", PaimonDataType::Boolean(BooleanType::new()))], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql( + "ALTER TABLE mydb.events ADD \ + PARTITION (active = 'yes') \ + PARTITION (active = 0)", + ) + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![ + HashMap::from([("active".to_string(), "true".to_string())]), + HashMap::from([("active".to_string(), "false".to_string())]), + ] + ); + assert!(temp_dir.path().join("active=true").is_dir()); + assert!(temp_dir.path().join("active=false").is_dir()); + } + + #[tokio::test] + async fn test_add_format_partition_trims_string_before_boolean_cast() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![("active", PaimonDataType::Boolean(BooleanType::new()))], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("ALTER TABLE mydb.events ADD PARTITION (active = ' yes ')") + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([("active".to_string(), "true".to_string())])] + ); + assert!(temp_dir.path().join("active=true").is_dir()); + } + + #[tokio::test] + async fn test_add_format_partition_accepts_tinyint_and_smallint_literals() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![ + ("tiny", PaimonDataType::TinyInt(TinyIntType::new())), + ("small", PaimonDataType::SmallInt(SmallIntType::new())), + ], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql( + "ALTER TABLE mydb.events \ + ADD PARTITION (tiny = -128, small = 32767)", + ) + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([ + ("tiny".to_string(), "-128".to_string()), + ("small".to_string(), "32767".to_string()), + ])] + ); + assert!(temp_dir.path().join("tiny=-128/small=32767").is_dir()); + } + + #[tokio::test] + async fn test_add_format_partition_accepts_typed_date_literal() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![("dt", PaimonDataType::Date(DateType::new()))], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("ALTER TABLE mydb.events ADD PARTITION (dt = DATE '2026-07-22')") + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([("dt".to_string(), "20656".to_string())])] + ); + assert!(temp_dir.path().join("dt=20656").is_dir()); + } + + #[tokio::test] + async fn test_add_format_partition_normalizes_typed_date_before_varchar_cast() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![( + "label", + PaimonDataType::VarChar(VarCharType::new(10).unwrap()), + )], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("ALTER TABLE mydb.events ADD PARTITION (label = DATE '2026')") + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([( + "label".to_string(), + "2026-01-01".to_string() + )])] + ); + assert!(temp_dir.path().join("label=2026-01-01").is_dir()); + } + + #[tokio::test] + async fn test_add_format_partition_rejects_unsupported_typed_literal() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![( + "label", + PaimonDataType::VarChar(VarCharType::new(32).unwrap()), + )], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + let error = sql_context + .sql( + "ALTER TABLE mydb.events \ + ADD PARTITION (label = TIMESTAMP '2026-07-22')", + ) + .await + .unwrap_err(); + + assert!(error + .to_string() + .contains("Unsupported typed partition literal")); + assert!(catalog.partition_specs().is_empty()); + } + + #[tokio::test] + async fn test_add_format_partition_parses_spark_year_only_date_string() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![("dt", PaimonDataType::Date(DateType::new()))], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("ALTER TABLE mydb.events ADD PARTITION (dt = '2026')") + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([("dt".to_string(), "20454".to_string())])] + ); + assert!(temp_dir.path().join("dt=20454").is_dir()); + } + + #[tokio::test] + async fn test_add_format_partition_rejects_date_outside_paimon_range() { + for value in ["-0001-01-01", "10000-01-01"] { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![("dt", PaimonDataType::Date(DateType::new()))], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + let error = sql_context + .sql(&format!( + "ALTER TABLE mydb.events ADD PARTITION (dt = '{value}')" + )) + .await + .unwrap_err(); + + assert!( + error + .to_string() + .contains("outside Paimon DATE range 0000-01-01..9999-12-31"), + "unexpected error for {value}: {error}" + ); + assert!(catalog.partition_specs().is_empty()); + } + } + + #[tokio::test] + async fn test_add_format_partition_rejects_compact_numeric_date_without_mutation() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![("dt", PaimonDataType::Date(DateType::new()))], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + let error = sql_context + .sql("ALTER TABLE mydb.events ADD PARTITION (dt = 20260722)") + .await + .unwrap_err(); + + assert!( + error.to_string().contains("Invalid DATE '20260722'"), + "expected invalid date error, got: {error}" + ); + assert!(catalog.partition_specs().is_empty()); + assert!(catalog.partition_mutation_calls().is_empty()); + assert!(!temp_dir.path().join("dt=20260722").exists()); + } + + #[tokio::test] + async fn test_add_format_partition_rejects_location_explicitly() { + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table( + Identifier::new("mydb", "events"), + "memory:/add_partition_location", + &["dt"], + ) + .await, + ); + let sql_context = make_sql_context(catalog).await; + + let error = sql_context + .sql( + "ALTER TABLE mydb.events ADD PARTITION (dt = '2026-07-22') \ + LOCATION 'memory:/other'", + ) + .await + .unwrap_err(); + + assert!( + error + .to_string() + .contains("LOCATION is not supported for Format Table partitions"), + "expected explicit LOCATION rejection, got: {error}" + ); + } + + #[tokio::test] + async fn test_drop_managed_format_partition_unregisters_then_deletes_directory() { + let temp_dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(temp_dir.path().join("dt=2026-07-22")).unwrap(); + std::fs::create_dir_all(temp_dir.path().join("dt=2026-07-21")).unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, + ); + catalog.set_partition_specs(vec![ + HashMap::from([("dt".to_string(), "2026-07-22".to_string())]), + HashMap::from([("dt".to_string(), "2026-07-21".to_string())]), + ]); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("ALTER TABLE mydb.events DROP PARTITION (dt = '2026-07-22')") + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([( + "dt".to_string(), + "2026-07-21".to_string() + )])] + ); + assert!(!temp_dir.path().join("dt=2026-07-22").exists()); + assert!(temp_dir.path().join("dt=2026-07-21").is_dir()); + } + + #[tokio::test] + async fn test_drop_complete_format_partition_uses_list_by_names() { + let temp_dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(temp_dir.path().join("dt=2026-07-22/hour=10")).unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table( + Identifier::new("mydb", "events"), + &location, + &["dt", "hour"], + ) + .await, + ); + let spec = HashMap::from([ + ("dt".to_string(), "2026-07-22".to_string()), + ("hour".to_string(), "10".to_string()), + ]); + catalog.set_partition_specs(vec![spec.clone()]); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql( + "ALTER TABLE mydb.events \ + DROP PARTITION (dt = '2026-07-22', hour = '10')", + ) + .await + .unwrap(); + + assert_eq!(catalog.list_partitions_calls(), 0); + assert_eq!(catalog.list_partitions_by_names_calls(), vec![vec![spec]]); + } + + #[tokio::test] + async fn test_drop_multiple_managed_format_partitions_uses_spark_syntax() { + let temp_dir = tempfile::tempdir().unwrap(); + for dt in ["2026-07-21", "2026-07-22"] { + std::fs::create_dir_all(temp_dir.path().join(format!("dt={dt}"))).unwrap(); + } + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, + ); + catalog.set_partition_specs(vec![ + HashMap::from([("dt".to_string(), "2026-07-21".to_string())]), + HashMap::from([("dt".to_string(), "2026-07-22".to_string())]), + ]); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql( + "ALTER TABLE mydb.events \ + DROP PARTITION (dt = '2026-07-21'), \ + PARTITION (dt = '2026-07-22')", + ) + .await + .unwrap(); + + assert!(catalog.partition_specs().is_empty()); + assert!(!temp_dir.path().join("dt=2026-07-21").exists()); + assert!(!temp_dir.path().join("dt=2026-07-22").exists()); + } + + #[tokio::test] + async fn test_drop_partial_managed_format_partition_expands_registered_matches() { + let temp_dir = tempfile::tempdir().unwrap(); + for (dt, region) in [ + ("2026-07-21", "us"), + ("2026-07-22", "us"), + ("2026-07-22", "eu"), + ] { + std::fs::create_dir_all(temp_dir.path().join(format!("dt={dt}/region={region}"))) + .unwrap(); + } + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table( + Identifier::new("mydb", "events"), + &location, + &["dt", "region"], + ) + .await, + ); + catalog.set_partition_specs(vec![ + HashMap::from([ + ("dt".to_string(), "2026-07-21".to_string()), + ("region".to_string(), "us".to_string()), + ]), + HashMap::from([ + ("dt".to_string(), "2026-07-22".to_string()), + ("region".to_string(), "us".to_string()), + ]), + HashMap::from([ + ("dt".to_string(), "2026-07-22".to_string()), + ("region".to_string(), "eu".to_string()), + ]), + ]); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("ALTER TABLE mydb.events DROP PARTITION (region = 'us')") + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([ + ("dt".to_string(), "2026-07-22".to_string()), + ("region".to_string(), "eu".to_string()), + ])] + ); + assert_eq!(catalog.list_partitions_calls(), 1); + assert!(!temp_dir.path().join("dt=2026-07-21/region=us").exists()); + assert!(!temp_dir.path().join("dt=2026-07-22/region=us").exists()); + assert!(temp_dir.path().join("dt=2026-07-22/region=eu").is_dir()); + } + + #[tokio::test] + async fn test_drop_complete_partition_falls_back_to_typed_match_for_msck_raw_value() { + let temp_dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(temp_dir.path().join("month=01")).unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![("month", PaimonDataType::Int(IntType::new()))], + ) + .await, + ); + catalog.set_partition_specs(vec![HashMap::from([( + "month".to_string(), + "01".to_string(), + )])]); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("ALTER TABLE mydb.events DROP PARTITION (month = 1)") + .await + .unwrap(); + + assert!(catalog.partition_specs().is_empty()); + assert_eq!(catalog.list_partitions_calls(), 1); + assert!(!temp_dir.path().join("month=01").exists()); + } + + #[tokio::test] + async fn test_drop_batch_fallback_does_not_expand_exact_partition_to_typed_aliases() { + let temp_dir = tempfile::tempdir().unwrap(); + for month in ["1", "01", "02"] { + std::fs::create_dir_all(temp_dir.path().join(format!("month={month}"))).unwrap(); + } + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![("month", PaimonDataType::Int(IntType::new()))], + ) + .await, + ); + catalog.set_partition_specs(vec![ + HashMap::from([("month".to_string(), "1".to_string())]), + HashMap::from([("month".to_string(), "01".to_string())]), + HashMap::from([("month".to_string(), "02".to_string())]), + ]); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql( + "ALTER TABLE mydb.events \ + DROP PARTITION (month = 1), \ + PARTITION (month = 2)", + ) + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([("month".to_string(), "01".to_string())])] + ); + assert!(!temp_dir.path().join("month=1").exists()); + assert!(temp_dir.path().join("month=01").is_dir()); + assert!(!temp_dir.path().join("month=02").exists()); + } + + #[tokio::test] + async fn test_drop_mixed_complete_and_partial_specs_keep_complete_match_exact() { + let temp_dir = tempfile::tempdir().unwrap(); + for (region, month) in [("us", "1"), ("us", "01"), ("eu", "02")] { + std::fs::create_dir_all( + temp_dir + .path() + .join(format!("region={region}/month={month}")), + ) + .unwrap(); + } + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![ + ( + "region", + PaimonDataType::VarChar(VarCharType::new(32).unwrap()), + ), + ("month", PaimonDataType::Int(IntType::new())), + ], + ) + .await, + ); + catalog.set_partition_specs(vec![ + HashMap::from([ + ("region".to_string(), "us".to_string()), + ("month".to_string(), "1".to_string()), + ]), + HashMap::from([ + ("region".to_string(), "us".to_string()), + ("month".to_string(), "01".to_string()), + ]), + HashMap::from([ + ("region".to_string(), "eu".to_string()), + ("month".to_string(), "02".to_string()), + ]), + ]); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql( + "ALTER TABLE mydb.events \ + DROP PARTITION (region = 'us', month = 1), \ + PARTITION (region = 'eu')", + ) + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([ + ("region".to_string(), "us".to_string()), + ("month".to_string(), "01".to_string()), + ])] + ); + assert!(!temp_dir.path().join("region=us/month=1").exists()); + assert!(temp_dir.path().join("region=us/month=01").is_dir()); + assert!(!temp_dir.path().join("region=eu/month=02").exists()); + } + + #[tokio::test] + async fn test_drop_integer_partition_matches_whitespace_msck_raw_value() { + let temp_dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(temp_dir.path().join("month= 01")).unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![("month", PaimonDataType::Int(IntType::new()))], + ) + .await, + ); + catalog.set_partition_specs(vec![HashMap::from([( + "month".to_string(), + " 01".to_string(), + )])]); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("ALTER TABLE mydb.events DROP PARTITION (month = 1)") + .await + .unwrap(); + + assert!(catalog.partition_specs().is_empty()); + assert_eq!(catalog.list_partitions_calls(), 1); + assert!(!temp_dir.path().join("month= 01").exists()); + } + + #[tokio::test] + async fn test_drop_char_partition_matches_unpadded_msck_raw_value() { + let temp_dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(temp_dir.path().join("region=us")).unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![("region", PaimonDataType::Char(CharType::new(4).unwrap()))], + ) + .await, + ); + catalog.set_partition_specs(vec![HashMap::from([( + "region".to_string(), + "us".to_string(), + )])]); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("ALTER TABLE mydb.events DROP PARTITION (region = 'us')") + .await + .unwrap(); + + assert!(catalog.partition_specs().is_empty()); + assert_eq!(catalog.list_partitions_calls(), 1); + assert!(!temp_dir.path().join("region=us").exists()); + } + + #[tokio::test] + async fn test_drop_missing_complete_partition_requires_partition_if_exists() { + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table( + Identifier::new("mydb", "events"), + "memory:/drop_missing_partition", + &["dt"], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + let error = sql_context + .sql( + "ALTER TABLE IF EXISTS mydb.events \ + DROP PARTITION (dt = '2026-07-22')", + ) + .await + .unwrap_err(); + assert!( + error.to_string().contains("does not exist"), + "expected missing partition error, got: {error}" + ); + assert!(catalog.partition_mutation_calls().is_empty()); + + sql_context + .sql( + "ALTER TABLE mydb.events \ + DROP IF EXISTS PARTITION (dt = '2026-07-22')", + ) + .await + .unwrap(); + assert!(catalog.partition_mutation_calls().is_empty()); + } + + #[tokio::test] + async fn test_drop_format_partition_rejects_purge_before_catalog_access() { + let catalog = Arc::new(MockCatalog::new()); + let sql_context = make_sql_context(catalog.clone()).await; + + let error = sql_context + .sql( + "ALTER TABLE mydb.events \ + DROP PARTITION (dt = '2026-07-22') PURGE", + ) + .await + .unwrap_err(); + + assert!( + error + .to_string() + .contains("DROP PARTITION PURGE is not supported"), + "expected explicit PURGE rejection, got: {error}" + ); + assert_eq!(catalog.get_table_calls(), 0); + assert_eq!(catalog.list_partitions_calls(), 0); + assert!(catalog.list_partitions_by_names_calls().is_empty()); + } + + #[tokio::test] + async fn test_msck_add_preserves_raw_filesystem_partition_values() { + let temp_dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(temp_dir.path().join("month=01")).unwrap(); + std::fs::create_dir_all(temp_dir.path().join("month=02")).unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![("month", PaimonDataType::Int(IntType::new()))], + ) + .await, + ); + catalog.set_partition_specs(vec![HashMap::from([( + "month".to_string(), + "02".to_string(), + )])]); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("MSCK REPAIR TABLE mydb.events") + .await + .unwrap(); + + let mut specs = catalog.partition_specs(); + specs.sort_by_key(|spec| spec["month"].clone()); + assert_eq!( + specs, + vec![ + HashMap::from([("month".to_string(), "01".to_string())]), + HashMap::from([("month".to_string(), "02".to_string())]), + ] + ); + assert!(temp_dir.path().join("month=01").is_dir()); + assert!(temp_dir.path().join("month=02").is_dir()); + } + + #[tokio::test] + async fn test_msck_rejects_non_round_tripping_partition_path_without_mutation() { + let temp_dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(temp_dir.path().join("dt=value%ZZ")).unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields( + Identifier::new("mydb", "events"), + &location, + vec![("dt", PaimonDataType::VarChar(VarCharType::new(32).unwrap()))], + ) + .await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + let error = sql_context + .sql("MSCK REPAIR TABLE mydb.events") + .await + .unwrap_err(); + + assert!( + error.to_string().contains("cannot round-trip"), + "expected a non-round-tripping path error, got: {error}" + ); + assert!(catalog.partition_specs().is_empty()); + assert!(catalog.partition_mutation_calls().is_empty()); + assert!(temp_dir.path().join("dt=value%ZZ").is_dir()); + } + + #[tokio::test] + async fn test_msck_listing_failure_performs_no_partition_mutation() { + let temp_dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(temp_dir.path().join("dt=2026-07-22")).unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, + ); + catalog.set_list_partitions_error(true); + let sql_context = make_sql_context(catalog.clone()).await; + + let error = sql_context + .sql("MSCK REPAIR TABLE mydb.events SYNC PARTITIONS") + .await + .unwrap_err(); + + assert!( + error + .to_string() + .contains("Injected partition listing failure"), + "expected listing failure, got: {error}" + ); + assert!(catalog.partition_mutation_calls().is_empty()); + assert!(catalog.partition_specs().is_empty()); + } + + #[tokio::test] + async fn test_msck_drop_removes_only_missing_filesystem_partitions() { + let temp_dir = tempfile::tempdir().unwrap(); + for dt in ["2026-07-21", "2026-07-22"] { + std::fs::create_dir_all(temp_dir.path().join(format!("dt={dt}"))).unwrap(); + } + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, + ); + catalog.set_partition_specs(vec![ + HashMap::from([("dt".to_string(), "2026-07-20".to_string())]), + HashMap::from([("dt".to_string(), "2026-07-21".to_string())]), + ]); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("MSCK REPAIR TABLE mydb.events DROP PARTITIONS") + .await + .unwrap(); + + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([( + "dt".to_string(), + "2026-07-21".to_string(), + )])] + ); + assert_eq!(catalog.partition_mutation_calls(), vec!["drop"]); + assert!(temp_dir.path().join("dt=2026-07-21").is_dir()); + assert!(temp_dir.path().join("dt=2026-07-22").is_dir()); + } + + #[tokio::test] + async fn test_msck_sync_adds_before_drop_and_rerun_is_idempotent() { + let temp_dir = tempfile::tempdir().unwrap(); + for dt in ["2026-07-21", "2026-07-22"] { + std::fs::create_dir_all(temp_dir.path().join(format!("dt={dt}"))).unwrap(); + } + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, + ); + catalog.set_partition_specs(vec![ + HashMap::from([("dt".to_string(), "2026-07-21".to_string())]), + HashMap::from([("dt".to_string(), "2026-07-20".to_string())]), + ]); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("MSCK REPAIR TABLE mydb.events SYNC PARTITIONS") + .await + .unwrap(); + let mut specs = catalog.partition_specs(); + specs.sort_by_key(|spec| spec["dt"].clone()); + assert_eq!( + specs, + vec![ + HashMap::from([("dt".to_string(), "2026-07-21".to_string())]), + HashMap::from([("dt".to_string(), "2026-07-22".to_string())]), + ] + ); + assert_eq!(catalog.partition_mutation_calls(), vec!["create", "drop"]); + assert!(temp_dir.path().join("dt=2026-07-21").is_dir()); + assert!(temp_dir.path().join("dt=2026-07-22").is_dir()); + + sql_context + .sql("MSCK REPAIR TABLE mydb.events SYNC PARTITIONS") + .await + .unwrap(); + assert_eq!(catalog.partition_mutation_calls(), vec!["create", "drop"]); + } + + #[tokio::test] + async fn test_msck_sync_keeps_value_only_default_partition_directory() { + let temp_dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(temp_dir.path().join("__DEFAULT_PARTITION__")).unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table_with_partition_fields_and_options( + Identifier::new("mydb", "events"), + &location, + vec![( + "dt", + PaimonDataType::VarChar(VarCharType::new(255).unwrap()), + )], + &[("format-table.partition-path-only-value", "true")], + ) + .await, + ); + let default_partition = + HashMap::from([("dt".to_string(), "__DEFAULT_PARTITION__".to_string())]); + catalog.set_partition_specs(vec![default_partition.clone()]); + let sql_context = make_sql_context(catalog.clone()).await; + + sql_context + .sql("MSCK REPAIR TABLE mydb.events SYNC PARTITIONS") + .await + .unwrap(); + + assert_eq!(catalog.partition_specs(), vec![default_partition]); + assert!(catalog.partition_mutation_calls().is_empty()); + assert!(temp_dir.path().join("__DEFAULT_PARTITION__").is_dir()); + } + #[tokio::test] async fn test_create_external_table_rejected() { let catalog = Arc::new(MockCatalog::new()); diff --git a/crates/integrations/datafusion/tests/rest_format_partition_sql.rs b/crates/integrations/datafusion/tests/rest_format_partition_sql.rs new file mode 100644 index 00000000..1e03f200 --- /dev/null +++ b/crates/integrations/datafusion/tests/rest_format_partition_sql.rs @@ -0,0 +1,377 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_array::StringArray; +use paimon::api::ConfigResponse; +use paimon::catalog::RESTCatalog; +use paimon::common::Options; +use paimon::spec::{BigIntType, BooleanType, DataType, DateType, IntType, Schema, VarCharType}; +use paimon_datafusion::SQLContext; + +#[path = "../../../paimon/tests/mock_server.rs"] +mod mock_server; +use mock_server::start_mock_server; + +#[cfg(not(windows))] +#[tokio::test] +async fn rest_managed_format_partition_sql_end_to_end() { + let temp_dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(temp_dir.path().join("dt=2026-07-21")).unwrap(); + let table_path = format!("file://{}", temp_dir.path().display()); + + let prefix = "mock-test"; + let server = start_mock_server( + "test_warehouse".to_string(), + temp_dir.path().to_string_lossy().into_owned(), + ConfigResponse::new(HashMap::from([("prefix".to_string(), prefix.to_string())])), + vec!["default".to_string()], + ) + .await; + let schema = Schema::builder() + .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) + .column("id", DataType::BigInt(BigIntType::new())) + .partition_keys(["dt"]) + .option("type", "format-table") + .option("file.format", "parquet") + .option("metastore.partitioned-table", "true") + .build() + .unwrap(); + server.add_table_with_schema("default", "events", schema, &table_path); + server.set_table_external("default", "events", false); + + let mut options = Options::new(); + options.set("uri", server.url().unwrap()); + options.set("warehouse", "test_warehouse"); + options.set("token.provider", "bear"); + options.set("token", "test-token"); + let catalog = Arc::new(RESTCatalog::new(options, true).await.unwrap()); + let mut context = SQLContext::new(); + context.register_catalog("paimon", catalog).await.unwrap(); + + context + .sql("ALTER TABLE paimon.default.events ADD PARTITION (dt = '2026-07-22')") + .await + .unwrap(); + assert_eq!( + show_partitions(&context).await, + vec!["dt=2026-07-22".to_string()] + ); + assert!(temp_dir.path().join("dt=2026-07-22").is_dir()); + + context + .sql("REPAIR TABLE paimon.default.events ADD PARTITIONS") + .await + .unwrap(); + assert_eq!( + show_partitions(&context).await, + vec!["dt=2026-07-21".to_string(), "dt=2026-07-22".to_string()] + ); + + std::fs::remove_dir_all(temp_dir.path().join("dt=2026-07-22")).unwrap(); + context + .sql("MSCK REPAIR TABLE paimon.default.events SYNC PARTITIONS") + .await + .unwrap(); + assert_eq!( + show_partitions(&context).await, + vec!["dt=2026-07-21".to_string()] + ); + + context + .sql("ALTER TABLE paimon.default.events ADD PARTITION (dt = '2026-07-22')") + .await + .unwrap(); + context + .sql( + "ALTER TABLE paimon.default.events \ + DROP PARTITION (dt = '2026-07-21'), \ + PARTITION (dt = '2026-07-22')", + ) + .await + .unwrap(); + assert!(show_partitions(&context).await.is_empty()); + assert!(!temp_dir.path().join("dt=2026-07-21").exists()); + assert!(!temp_dir.path().join("dt=2026-07-22").exists()); +} + +#[cfg(not(windows))] +#[tokio::test] +async fn rest_managed_format_partition_sql_normalizes_typed_and_default_values() { + let temp_dir = tempfile::tempdir().unwrap(); + let table_path = format!("file://{}", temp_dir.path().display()); + + let prefix = "mock-test"; + let server = start_mock_server( + "test_warehouse".to_string(), + temp_dir.path().to_string_lossy().into_owned(), + ConfigResponse::new(HashMap::from([("prefix".to_string(), prefix.to_string())])), + vec!["default".to_string()], + ) + .await; + let schema = Schema::builder() + .column("dt", DataType::Date(DateType::new())) + .column("month", DataType::Int(IntType::new())) + .column("active", DataType::Boolean(BooleanType::new())) + .column("label", DataType::VarChar(VarCharType::new(255).unwrap())) + .column("id", DataType::BigInt(BigIntType::new())) + .partition_keys(["dt", "month", "active", "label"]) + .option("type", "format-table") + .option("file.format", "parquet") + .option("metastore.partitioned-table", "true") + .build() + .unwrap(); + server.add_table_with_schema("default", "events", schema, &table_path); + server.set_table_external("default", "events", false); + + let mut options = Options::new(); + options.set("uri", server.url().unwrap()); + options.set("warehouse", "test_warehouse"); + options.set("token.provider", "bear"); + options.set("token", "test-token"); + let catalog = Arc::new(RESTCatalog::new(options, true).await.unwrap()); + let mut context = SQLContext::new(); + context.register_catalog("paimon", catalog).await.unwrap(); + + context + .sql( + "ALTER TABLE paimon.default.events ADD PARTITION (\ + dt = '2026', month = '01', active = 'yes', label = 01)", + ) + .await + .unwrap(); + assert_eq!( + show_partitions(&context).await, + vec!["dt=2026-01-01/month=1/active=true/label=1".to_string()] + ); + assert!(temp_dir + .path() + .join("dt=20454/month=1/active=true/label=1") + .is_dir()); + + context + .sql("ALTER TABLE paimon.default.events DROP PARTITION (active = 'yes')") + .await + .unwrap(); + assert!(show_partitions(&context).await.is_empty()); + + context + .sql( + "ALTER TABLE paimon.default.events ADD PARTITION (\ + dt = NULL, month = NULL, active = NULL, label = NULL)", + ) + .await + .unwrap(); + assert_eq!( + show_partitions(&context).await, + vec!["dt=null/month=null/active=null/label=null".to_string()] + ); + assert!(temp_dir + .path() + .join( + "dt=__DEFAULT_PARTITION__/month=__DEFAULT_PARTITION__/\ + active=__DEFAULT_PARTITION__/label=__DEFAULT_PARTITION__", + ) + .is_dir()); +} + +#[cfg(not(windows))] +#[tokio::test] +async fn rest_drop_batch_fallback_preserves_exact_partition_aliases() { + let temp_dir = tempfile::tempdir().unwrap(); + for month in ["1", "01", "02"] { + std::fs::create_dir_all(temp_dir.path().join(format!("month={month}"))).unwrap(); + } + let table_path = format!("file://{}", temp_dir.path().display()); + + let prefix = "mock-test"; + let server = start_mock_server( + "test_warehouse".to_string(), + temp_dir.path().to_string_lossy().into_owned(), + ConfigResponse::new(HashMap::from([("prefix".to_string(), prefix.to_string())])), + vec!["default".to_string()], + ) + .await; + let schema = Schema::builder() + .column("month", DataType::Int(IntType::new())) + .column("id", DataType::BigInt(BigIntType::new())) + .partition_keys(["month"]) + .option("type", "format-table") + .option("file.format", "parquet") + .option("metastore.partitioned-table", "true") + .build() + .unwrap(); + server.add_table_with_schema("default", "events", schema, &table_path); + server.set_table_external("default", "events", false); + server.set_table_partitions( + "default", + "events", + vec![ + HashMap::from([("month".to_string(), "1".to_string())]), + HashMap::from([("month".to_string(), "01".to_string())]), + HashMap::from([("month".to_string(), "02".to_string())]), + ], + ); + + let mut options = Options::new(); + options.set("uri", server.url().unwrap()); + options.set("warehouse", "test_warehouse"); + options.set("token.provider", "bear"); + options.set("token", "test-token"); + let catalog = Arc::new(RESTCatalog::new(options, true).await.unwrap()); + let mut context = SQLContext::new(); + context.register_catalog("paimon", catalog).await.unwrap(); + + context + .sql( + "ALTER TABLE paimon.default.events \ + DROP PARTITION (month = 1), \ + PARTITION (month = 2)", + ) + .await + .unwrap(); + + assert_eq!(show_partitions(&context).await, vec!["month=1".to_string()]); + assert!(!temp_dir.path().join("month=1").exists()); + assert!(temp_dir.path().join("month=01").is_dir()); + assert!(!temp_dir.path().join("month=02").exists()); + + let (_, _, request) = server.last_drop_partitions_call().unwrap(); + assert_eq!(request.partition_specs.len(), 2); + assert!(request + .partition_specs + .contains(&HashMap::from([("month".to_string(), "1".to_string())]))); + assert!(request + .partition_specs + .contains(&HashMap::from([("month".to_string(), "02".to_string())]))); +} + +#[cfg(not(windows))] +#[tokio::test] +async fn rest_drop_mixed_complete_and_partial_specs_keep_complete_match_exact() { + let temp_dir = tempfile::tempdir().unwrap(); + for (region, month) in [("us", "1"), ("us", "01"), ("eu", "02")] { + std::fs::create_dir_all( + temp_dir + .path() + .join(format!("region={region}/month={month}")), + ) + .unwrap(); + } + let table_path = format!("file://{}", temp_dir.path().display()); + + let prefix = "mock-test"; + let server = start_mock_server( + "test_warehouse".to_string(), + temp_dir.path().to_string_lossy().into_owned(), + ConfigResponse::new(HashMap::from([("prefix".to_string(), prefix.to_string())])), + vec!["default".to_string()], + ) + .await; + let schema = Schema::builder() + .column("region", DataType::VarChar(VarCharType::new(32).unwrap())) + .column("month", DataType::Int(IntType::new())) + .column("id", DataType::BigInt(BigIntType::new())) + .partition_keys(["region", "month"]) + .option("type", "format-table") + .option("file.format", "parquet") + .option("metastore.partitioned-table", "true") + .build() + .unwrap(); + server.add_table_with_schema("default", "events", schema, &table_path); + server.set_table_external("default", "events", false); + server.set_table_partitions( + "default", + "events", + vec![ + HashMap::from([ + ("region".to_string(), "us".to_string()), + ("month".to_string(), "1".to_string()), + ]), + HashMap::from([ + ("region".to_string(), "us".to_string()), + ("month".to_string(), "01".to_string()), + ]), + HashMap::from([ + ("region".to_string(), "eu".to_string()), + ("month".to_string(), "02".to_string()), + ]), + ], + ); + + let mut options = Options::new(); + options.set("uri", server.url().unwrap()); + options.set("warehouse", "test_warehouse"); + options.set("token.provider", "bear"); + options.set("token", "test-token"); + let catalog = Arc::new(RESTCatalog::new(options, true).await.unwrap()); + let mut context = SQLContext::new(); + context.register_catalog("paimon", catalog).await.unwrap(); + + context + .sql( + "ALTER TABLE paimon.default.events \ + DROP PARTITION (region = 'us', month = 1), \ + PARTITION (region = 'eu')", + ) + .await + .unwrap(); + + assert_eq!( + show_partitions(&context).await, + vec!["region=us/month=1".to_string()] + ); + assert!(!temp_dir.path().join("region=us/month=1").exists()); + assert!(temp_dir.path().join("region=us/month=01").is_dir()); + assert!(!temp_dir.path().join("region=eu/month=02").exists()); + + let (_, _, request) = server.last_drop_partitions_call().unwrap(); + assert_eq!(request.partition_specs.len(), 2); + assert!(request.partition_specs.contains(&HashMap::from([ + ("region".to_string(), "us".to_string()), + ("month".to_string(), "1".to_string()), + ]))); + assert!(request.partition_specs.contains(&HashMap::from([ + ("region".to_string(), "eu".to_string()), + ("month".to_string(), "02".to_string()), + ]))); +} + +async fn show_partitions(context: &SQLContext) -> Vec { + let batches = context + .sql("SHOW PARTITIONS paimon.default.events") + .await + .unwrap() + .collect() + .await + .unwrap(); + batches + .iter() + .flat_map(|batch| { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + (0..batch.num_rows()) + .map(|index| values.value(index).to_string()) + .collect::>() + }) + .collect() +} diff --git a/crates/paimon/src/api/api_request.rs b/crates/paimon/src/api/api_request.rs index 84ee7c5c..2d2f6bf9 100644 --- a/crates/paimon/src/api/api_request.rs +++ b/crates/paimon/src/api/api_request.rs @@ -167,6 +167,68 @@ impl AlterTableRequest { } } +/// Request to create table partitions. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CreatePartitionsRequest { + /// Partition specs to register. + pub partition_specs: Vec>, + /// Whether already registered partitions should be ignored. + #[serde(default = "default_true")] + pub ignore_if_exists: bool, +} + +impl CreatePartitionsRequest { + /// Create a new CreatePartitionsRequest. + pub fn new(partition_specs: Vec>, ignore_if_exists: bool) -> Self { + Self { + partition_specs, + ignore_if_exists, + } + } +} + +/// Request to drop table partitions. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DropPartitionsRequest { + /// Partition specs to unregister. + pub partition_specs: Vec>, + /// Whether missing partitions should be ignored. + #[serde(default = "default_true")] + pub ignore_if_not_exists: bool, +} + +impl DropPartitionsRequest { + /// Create a new DropPartitionsRequest. + pub fn new(partition_specs: Vec>, ignore_if_not_exists: bool) -> Self { + Self { + partition_specs, + ignore_if_not_exists, + } + } +} + +/// Request to list table partitions by exact specs. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ListPartitionsByNamesRequest { + /// Partition specs to retrieve. + #[serde(rename = "specs")] + pub partition_specs: Vec>, +} + +impl ListPartitionsByNamesRequest { + /// Create a new ListPartitionsByNamesRequest. + pub fn new(partition_specs: Vec>) -> Self { + Self { partition_specs } + } +} + +fn default_true() -> bool { + true +} + /// Request for auth table query: the projected columns of the query. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct AuthTableQueryRequest { @@ -220,6 +282,85 @@ mod tests { assert_eq!(serde_json::to_string(&req).unwrap(), "{}"); } + #[test] + fn test_create_partitions_request_serialization() { + let req = CreatePartitionsRequest::new( + vec![HashMap::from([ + ("dt".to_string(), "2026-07-22".to_string()), + ("hour".to_string(), "10".to_string()), + ])], + false, + ); + + assert_eq!( + serde_json::to_value(req).unwrap(), + serde_json::json!({ + "partitionSpecs": [{ + "dt": "2026-07-22", + "hour": "10" + }], + "ignoreIfExists": false + }) + ); + } + + #[test] + fn test_create_partitions_request_defaults_ignore_if_exists() { + let req: CreatePartitionsRequest = serde_json::from_value(serde_json::json!({ + "partitionSpecs": [{"dt": "2026-07-22"}] + })) + .unwrap(); + + assert!(req.ignore_if_exists); + } + + #[test] + fn test_drop_partitions_request_serialization() { + let req = DropPartitionsRequest::new( + vec![HashMap::from([( + "dt".to_string(), + "2026-07-22".to_string(), + )])], + false, + ); + + assert_eq!( + serde_json::to_value(req).unwrap(), + serde_json::json!({ + "partitionSpecs": [{"dt": "2026-07-22"}], + "ignoreIfNotExists": false + }) + ); + } + + #[test] + fn test_drop_partitions_request_defaults_ignore_if_not_exists() { + let req: DropPartitionsRequest = serde_json::from_value(serde_json::json!({ + "partitionSpecs": [{"dt": "2026-07-22"}] + })) + .unwrap(); + + assert!(req.ignore_if_not_exists); + } + + #[test] + fn test_list_partitions_by_names_request_serialization() { + let req = ListPartitionsByNamesRequest::new(vec![HashMap::from([ + ("dt".to_string(), "2026-07-22".to_string()), + ("hour".to_string(), "10".to_string()), + ])]); + + assert_eq!( + serde_json::to_value(req).unwrap(), + serde_json::json!({ + "specs": [{ + "dt": "2026-07-22", + "hour": "10" + }] + }) + ); + } + #[test] fn test_rename_table_request_serialization() { let source = Identifier::new("db1".to_string(), "table1".to_string()); diff --git a/crates/paimon/src/api/api_response.rs b/crates/paimon/src/api/api_response.rs index 9d68cf3d..0f866178 100644 --- a/crates/paimon/src/api/api_response.rs +++ b/crates/paimon/src/api/api_response.rs @@ -421,6 +421,30 @@ impl ListTablesResponse { } } +/// Response for creating table partitions. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CreatePartitionsResponse { + /// Partition specs created by the request. + #[serde(default, deserialize_with = "deserialize_null_to_default")] + pub created: Vec>, + /// Partition specs which already existed. + #[serde(default, deserialize_with = "deserialize_null_to_default")] + pub existed: Vec>, +} + +/// Response for dropping table partitions. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DropPartitionsResponse { + /// Partition specs dropped by the request. + #[serde(default, deserialize_with = "deserialize_null_to_default")] + pub dropped: Vec>, + /// Partition specs which did not exist. + #[serde(default, deserialize_with = "deserialize_null_to_default")] + pub missing: Vec>, +} + /// Response for listing partitions. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -661,6 +685,72 @@ mod tests { assert!(json.contains("\"nextPageToken\":\"token123\"")); } + #[test] + fn test_create_partitions_response_deserialization() { + let response: CreatePartitionsResponse = serde_json::from_value(serde_json::json!({ + "created": [{"dt": "2026-07-22"}], + "existed": [{"dt": "2026-07-21"}] + })) + .unwrap(); + + assert_eq!( + response.created, + vec![HashMap::from([( + "dt".to_string(), + "2026-07-22".to_string() + )])] + ); + assert_eq!( + response.existed, + vec![HashMap::from([( + "dt".to_string(), + "2026-07-21".to_string() + )])] + ); + } + + #[test] + fn test_create_partitions_response_defaults_null_and_missing_lists() { + let response: CreatePartitionsResponse = + serde_json::from_value(serde_json::json!({"created": null})).unwrap(); + + assert!(response.created.is_empty()); + assert!(response.existed.is_empty()); + } + + #[test] + fn test_drop_partitions_response_deserialization() { + let response: DropPartitionsResponse = serde_json::from_value(serde_json::json!({ + "dropped": [{"dt": "2026-07-22"}], + "missing": [{"dt": "2026-07-21"}] + })) + .unwrap(); + + assert_eq!( + response.dropped, + vec![HashMap::from([( + "dt".to_string(), + "2026-07-22".to_string() + )])] + ); + assert_eq!( + response.missing, + vec![HashMap::from([( + "dt".to_string(), + "2026-07-21".to_string() + )])] + ); + } + + #[test] + fn test_drop_partitions_response_defaults_null_and_missing_lists() { + let response: DropPartitionsResponse = + serde_json::from_value(serde_json::json!({"dropped": null})).unwrap(); + + assert!(response.dropped.is_empty()); + assert!(response.missing.is_empty()); + } + #[test] fn test_audit_response_options() { let audit = AuditRESTResponse::new( diff --git a/crates/paimon/src/api/mod.rs b/crates/paimon/src/api/mod.rs index 809b9021..e5b5689e 100644 --- a/crates/paimon/src/api/mod.rs +++ b/crates/paimon/src/api/mod.rs @@ -32,15 +32,17 @@ mod api_response; // Re-export request types pub use api_request::{ AlterDatabaseRequest, AlterTableRequest, AuthTableQueryRequest, CreateDatabaseRequest, - CreateFunctionRequest, CreateTableRequest, CreateViewRequest, RenameTableRequest, + CreateFunctionRequest, CreatePartitionsRequest, CreateTableRequest, CreateViewRequest, + DropPartitionsRequest, ListPartitionsByNamesRequest, RenameTableRequest, }; // Re-export response types pub use api_response::{ - AuditRESTResponse, AuthTableQueryResponse, ConfigResponse, ErrorResponse, GetDatabaseResponse, - GetFunctionResponse, GetTableResponse, GetTableTokenResponse, GetViewResponse, - ListDatabasesResponse, ListFunctionsResponse, ListPartitionsResponse, ListTablesResponse, - ListViewsResponse, PagedList, + AuditRESTResponse, AuthTableQueryResponse, ConfigResponse, CreatePartitionsResponse, + DropPartitionsResponse, ErrorResponse, GetDatabaseResponse, GetFunctionResponse, + GetTableResponse, GetTableTokenResponse, GetViewResponse, ListDatabasesResponse, + ListFunctionsResponse, ListPartitionsResponse, ListTablesResponse, ListViewsResponse, + PagedList, }; // Re-export error types diff --git a/crates/paimon/src/api/resource_paths.rs b/crates/paimon/src/api/resource_paths.rs index 38c39c9a..7ad46325 100644 --- a/crates/paimon/src/api/resource_paths.rs +++ b/crates/paimon/src/api/resource_paths.rs @@ -216,6 +216,19 @@ impl ResourcePaths { Self::PARTITIONS ) } + + /// Get the drop partitions endpoint path for a table. + pub fn drop_partitions(&self, database_name: &str, table_name: &str) -> String { + format!("{}/drop", self.partitions(database_name, table_name)) + } + + /// Get the list partitions by names endpoint path for a table. + pub fn list_partitions_by_names(&self, database_name: &str, table_name: &str) -> String { + format!( + "{}/list-by-names", + self.partitions(database_name, table_name) + ) + } } #[cfg(test)] @@ -296,4 +309,17 @@ mod tests { "/v1/catalog/databases/analytics/functions/rectangle+area" ); } + + #[test] + fn test_partition_operation_paths_encode_names() { + let paths = ResourcePaths::new("catalog"); + assert_eq!( + paths.drop_partitions("analytics db", "user events"), + "/v1/catalog/databases/analytics+db/tables/user+events/partitions/drop" + ); + assert_eq!( + paths.list_partitions_by_names("analytics db", "user events"), + "/v1/catalog/databases/analytics+db/tables/user+events/partitions/list-by-names" + ); + } } diff --git a/crates/paimon/src/api/rest_api.rs b/crates/paimon/src/api/rest_api.rs index cdde2b23..8c77bbf7 100644 --- a/crates/paimon/src/api/rest_api.rs +++ b/crates/paimon/src/api/rest_api.rs @@ -20,7 +20,7 @@ //! This module provides a REST API client for interacting with //! Paimon rest catalog services, supporting database operations. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use crate::api::rest_client::HttpClient; use crate::catalog::{Function, Identifier, ViewSchema}; @@ -30,12 +30,14 @@ use crate::Result; use super::api_request::{ AlterDatabaseRequest, AlterTableRequest, AuthTableQueryRequest, CreateDatabaseRequest, - CreateFunctionRequest, CreateTableRequest, CreateViewRequest, RenameTableRequest, + CreateFunctionRequest, CreatePartitionsRequest, CreateTableRequest, CreateViewRequest, + DropPartitionsRequest, ListPartitionsByNamesRequest, RenameTableRequest, }; use super::api_response::{ - AuthTableQueryResponse, ConfigResponse, GetDatabaseResponse, GetFunctionResponse, - GetTableResponse, GetViewResponse, ListDatabasesResponse, ListFunctionsResponse, - ListPartitionsResponse, ListTablesResponse, ListViewsResponse, PagedList, + AuthTableQueryResponse, ConfigResponse, CreatePartitionsResponse, DropPartitionsResponse, + GetDatabaseResponse, GetFunctionResponse, GetTableResponse, GetViewResponse, + ListDatabasesResponse, ListFunctionsResponse, ListPartitionsResponse, ListTablesResponse, + ListViewsResponse, PagedList, }; use super::auth::{AuthProviderFactory, RESTAuthFunction}; use super::resource_paths::ResourcePaths; @@ -72,6 +74,24 @@ fn validate_non_empty_multi(values: &[(&str, &str)]) -> Result<()> { Ok(()) } +fn validate_partition_request_size( + identifier: &Identifier, + partition_spec_count: usize, +) -> Result<()> { + if partition_spec_count > RESTApi::MAX_PARTITION_SPECS_PER_REQUEST { + return Err(crate::Error::DataInvalid { + message: format!( + "REST partition requests for table {} accept at most {} specs, got {}", + identifier.full_name(), + RESTApi::MAX_PARTITION_SPECS_PER_REQUEST, + partition_spec_count + ), + source: None, + }); + } + Ok(()) +} + /// REST API wrapper for Paimon catalog operations. /// /// This struct provides methods for database and table CRUD operations @@ -86,6 +106,8 @@ impl RESTApi { // Constants for query parameters and headers pub const HEADER_PREFIX: &'static str = "header."; pub const MAX_RESULTS: &'static str = "maxResults"; + /// Maximum partition specs accepted by one REST request. + pub const MAX_PARTITION_SPECS_PER_REQUEST: usize = 1000; pub const PAGE_TOKEN: &'static str = "pageToken"; pub const DATABASE_NAME_PATTERN: &'static str = "databaseNamePattern"; pub const TABLE_NAME_PATTERN: &'static str = "tableNamePattern"; @@ -556,6 +578,68 @@ impl RESTApi { // ==================== Partition Operations ==================== + /// Create table partitions with one REST request. + /// + /// Requests larger than [`Self::MAX_PARTITION_SPECS_PER_REQUEST`] are + /// rejected locally. Use [`crate::catalog::RESTCatalog`] when idempotent + /// batches should be split automatically. + pub async fn create_partitions( + &self, + identifier: &Identifier, + partition_specs: Vec>, + ignore_if_exists: bool, + ) -> Result { + let database = identifier.database(); + let table = identifier.object(); + validate_non_empty_multi(&[(database, "database name"), (table, "table name")])?; + validate_partition_request_size(identifier, partition_specs.len())?; + let path = self.resource_paths.partitions(database, table); + let request = CreatePartitionsRequest::new(partition_specs, ignore_if_exists); + self.client.post(&path, &request).await + } + + /// Drop table partitions with one REST request. + /// + /// Requests larger than [`Self::MAX_PARTITION_SPECS_PER_REQUEST`] are + /// rejected locally. Use [`crate::catalog::RESTCatalog`] to split larger + /// idempotent batches. + pub async fn drop_partitions( + &self, + identifier: &Identifier, + partition_specs: Vec>, + ignore_if_not_exists: bool, + ) -> Result { + let database = identifier.database(); + let table = identifier.object(); + validate_non_empty_multi(&[(database, "database name"), (table, "table name")])?; + validate_partition_request_size(identifier, partition_specs.len())?; + let path = self.resource_paths.drop_partitions(database, table); + let request = DropPartitionsRequest::new(partition_specs, ignore_if_not_exists); + self.client.post(&path, &request).await + } + + /// List table partitions by exact specs with one REST request. + /// + /// Requests larger than [`Self::MAX_PARTITION_SPECS_PER_REQUEST`] are + /// rejected locally. Use [`crate::catalog::RESTCatalog`] to split larger + /// batches. + pub async fn list_partitions_by_names( + &self, + identifier: &Identifier, + partition_specs: Vec>, + ) -> Result> { + let database = identifier.database(); + let table = identifier.object(); + validate_non_empty_multi(&[(database, "database name"), (table, "table name")])?; + validate_partition_request_size(identifier, partition_specs.len())?; + let path = self + .resource_paths + .list_partitions_by_names(database, table); + let request = ListPartitionsByNamesRequest::new(partition_specs); + let response: ListPartitionsResponse = self.client.post(&path, &request).await?; + Ok(response.partitions.unwrap_or_default()) + } + /// List all partitions of a table, paging internally. pub async fn list_partitions(&self, identifier: &Identifier) -> Result> { let database = identifier.database(); @@ -564,16 +648,27 @@ impl RESTApi { let mut results = Vec::new(); let mut page_token: Option = None; + let mut seen_page_tokens = HashSet::new(); loop { let paged = self .list_partitions_paged(identifier, None, page_token.as_deref()) .await?; - let is_empty = paged.elements.is_empty(); results.extend(paged.elements); - page_token = paged.next_page_token; - if page_token.is_none() || is_empty { - break; + page_token = paged.next_page_token.filter(|token| !token.is_empty()); + match page_token.as_ref() { + Some(token) if !seen_page_tokens.insert(token.clone()) => { + return Err(crate::Error::UnexpectedError { + message: format!( + "REST catalog returned repeated partition page token '{token}' \ + for table {}", + identifier.full_name() + ), + source: None, + }); + } + Some(_) => {} + None => break, } } diff --git a/crates/paimon/src/catalog/mod.rs b/crates/paimon/src/catalog/mod.rs index 38a414b9..95fe7701 100644 --- a/crates/paimon/src/catalog/mod.rs +++ b/crates/paimon/src/catalog/mod.rs @@ -445,6 +445,47 @@ pub trait Catalog: Send + Sync { }) } + /// Register table partition specs in the catalog. + /// + /// When `ignore_if_exists` is false, an existing spec must reject the + /// entire supplied batch without registering any of it. When true, + /// existing specs are ignored and retries must converge. + async fn create_partitions( + &self, + _identifier: &Identifier, + _partition_specs: Vec>, + _ignore_if_exists: bool, + ) -> Result<()> { + Err(Error::Unsupported { + message: "Catalog does not support creating partitions".to_string(), + }) + } + + /// Unregister table partition specs from the catalog. + /// + /// Missing specs are ignored so callers can safely retry a partially + /// applied batch. + async fn drop_partitions( + &self, + _identifier: &Identifier, + _partition_specs: Vec>, + ) -> Result<()> { + Err(Error::Unsupported { + message: "Catalog does not support dropping partitions".to_string(), + }) + } + + /// List table partitions matching exact partition specs. + async fn list_partitions_by_names( + &self, + _identifier: &Identifier, + _partition_specs: Vec>, + ) -> Result> { + Err(Error::Unsupported { + message: "Catalog does not support listing partitions by names".to_string(), + }) + } + /// List partitions for a table. /// /// Default impl scans the table's manifest entries via diff --git a/crates/paimon/src/catalog/rest/rest_catalog.rs b/crates/paimon/src/catalog/rest/rest_catalog.rs index 2057d3f5..cad07039 100644 --- a/crates/paimon/src/catalog/rest/rest_catalog.rs +++ b/crates/paimon/src/catalog/rest/rest_catalog.rs @@ -363,13 +363,73 @@ impl Catalog for RESTCatalog { )) } + async fn create_partitions( + &self, + identifier: &Identifier, + partition_specs: Vec>, + ignore_if_exists: bool, + ) -> Result<()> { + if !ignore_if_exists { + return self + .api + .create_partitions(identifier, partition_specs, false) + .await + .map(|_| ()) + .map_err(|error| map_rest_error_for_create_partitions(error, identifier)); + } + + for batch in partition_specs.chunks(RESTApi::MAX_PARTITION_SPECS_PER_REQUEST) { + self.api + .create_partitions(identifier, batch.to_vec(), true) + .await + .map_err(|error| map_rest_error_for_create_partitions(error, identifier))?; + } + Ok(()) + } + + async fn drop_partitions( + &self, + identifier: &Identifier, + partition_specs: Vec>, + ) -> Result<()> { + for batch in partition_specs.chunks(RESTApi::MAX_PARTITION_SPECS_PER_REQUEST) { + self.api + .drop_partitions(identifier, batch.to_vec(), true) + .await + .map_err(|error| map_rest_error_for_partition_request(error, identifier))?; + } + Ok(()) + } + + async fn list_partitions_by_names( + &self, + identifier: &Identifier, + partition_specs: Vec>, + ) -> Result> { + let mut found = Vec::new(); + for batch in partition_specs.chunks(RESTApi::MAX_PARTITION_SPECS_PER_REQUEST) { + found.extend( + self.api + .list_partitions_by_names(identifier, batch.to_vec()) + .await + .map_err(|error| map_rest_error_for_partition_request(error, identifier))?, + ); + } + Ok(found) + } + async fn list_partitions(&self, identifier: &Identifier) -> Result> { match self.api.list_partitions(identifier).await { Ok(parts) => Ok(parts), - Err(Error::RestApi { - source: RestError::NotImplemented { .. }, - }) => { + Err( + error @ Error::RestApi { + source: RestError::NotImplemented { .. }, + }, + ) => { let table = self.get_table(identifier).await?; + if table.has_catalog_managed_partitions() { + return Err(error); + } list_partitions_from_file_system(&table).await } Err(e) => Err(map_rest_error_for_table(e, identifier)), @@ -388,10 +448,15 @@ impl Catalog for RESTCatalog { .await { Ok(page) => Ok(page), - Err(Error::RestApi { - source: RestError::NotImplemented { .. }, - }) => { + Err( + error @ Error::RestApi { + source: RestError::NotImplemented { .. }, + }, + ) => { let table = self.get_table(identifier).await?; + if table.has_catalog_managed_partitions() { + return Err(error); + } let parts = list_partitions_from_file_system(&table).await?; Ok(PagedList::new(parts, None)) } @@ -445,6 +510,36 @@ fn map_rest_error_for_table(err: Error, identifier: &Identifier) -> Error { } } +fn map_rest_error_for_create_partitions(err: Error, identifier: &Identifier) -> Error { + match err { + Error::RestApi { + source: RestError::AlreadyExists { message, .. }, + } => Error::DataInvalid { + message: format!( + "Some partitions of table {} already exist: {message}", + identifier.full_name() + ), + source: None, + }, + other => map_rest_error_for_partition_request(other, identifier), + } +} + +fn map_rest_error_for_partition_request(err: Error, identifier: &Identifier) -> Error { + match err { + Error::RestApi { + source: RestError::BadRequest { message }, + } => Error::DataInvalid { + message: format!( + "Invalid partition request for table {}: {message}", + identifier.full_name() + ), + source: None, + }, + other => map_rest_error_for_table(other, identifier), + } +} + /// Map a REST API error from creating a persistent view. fn map_rest_error_for_create_view(err: Error, identifier: &Identifier) -> Error { match err { diff --git a/crates/paimon/src/spec/core_options.rs b/crates/paimon/src/spec/core_options.rs index fd4809bc..efa0db82 100644 --- a/crates/paimon/src/spec/core_options.rs +++ b/crates/paimon/src/spec/core_options.rs @@ -32,6 +32,8 @@ const SOURCE_SPLIT_TARGET_SIZE_OPTION: &str = "source.split.target-size"; const SOURCE_SPLIT_OPEN_FILE_COST_OPTION: &str = "source.split.open-file-cost"; const PARTITION_DEFAULT_NAME_OPTION: &str = "partition.default-name"; const PARTITION_LEGACY_NAME_OPTION: &str = "partition.legacy-name"; +pub(crate) const METASTORE_PARTITIONED_TABLE_OPTION: &str = "metastore.partitioned-table"; +const FORMAT_TABLE_IMPLEMENTATION_OPTION: &str = "format-table.implementation"; const FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE_OPTION: &str = "format-table.partition-path-only-value"; pub(crate) const BUCKET_KEY_OPTION: &str = "bucket-key"; @@ -534,6 +536,19 @@ impl<'a> CoreOptions<'a> { .unwrap_or(false) } + pub fn partitioned_table_in_metastore(&self) -> bool { + self.options + .get(METASTORE_PARTITIONED_TABLE_OPTION) + .map(|value| value.eq_ignore_ascii_case("true")) + .unwrap_or(false) + } + + pub fn format_table_uses_engine_implementation(&self) -> bool { + self.options + .get(FORMAT_TABLE_IMPLEMENTATION_OPTION) + .is_some_and(|value| value.eq_ignore_ascii_case("engine")) + } + pub fn global_index_enabled(&self) -> bool { self.options .get(GLOBAL_INDEX_ENABLED_OPTION) @@ -1469,6 +1484,18 @@ mod tests { assert!(core.format_table_partition_only_value_in_path()); } + #[test] + fn test_partitioned_table_in_metastore_option() { + let empty = HashMap::new(); + assert!(!CoreOptions::new(&empty).partitioned_table_in_metastore()); + + let options = HashMap::from([( + "metastore.partitioned-table".to_string(), + "TrUe".to_string(), + )]); + assert!(CoreOptions::new(&options).partitioned_table_in_metastore()); + } + #[test] fn test_try_time_travel_selector_rejects_conflicting_selectors() { let options = HashMap::from([ diff --git a/crates/paimon/src/spec/mod.rs b/crates/paimon/src/spec/mod.rs index 13f9c3d6..71d39244 100644 --- a/crates/paimon/src/spec/mod.rs +++ b/crates/paimon/src/spec/mod.rs @@ -94,7 +94,7 @@ pub use types::*; mod partition; pub use partition::Partition; mod partition_utils; -pub(crate) use partition_utils::PartitionComputer; +pub(crate) use partition_utils::{escape_path_name, PartitionComputer}; mod predicate; pub(crate) use predicate::datum_cmp; pub(crate) use predicate::eval_row; diff --git a/crates/paimon/src/spec/partition_utils.rs b/crates/paimon/src/spec/partition_utils.rs index 3db1884c..f81bc941 100644 --- a/crates/paimon/src/spec/partition_utils.rs +++ b/crates/paimon/src/spec/partition_utils.rs @@ -464,7 +464,7 @@ fn format_timestamp_non_legacy(dt: NaiveDateTime, precision: u32) -> String { /// Escape a path component following Java `PartitionPathUtils.escapePathName`. /// /// Characters that need escaping are encoded as `%XX` (uppercase hex). -fn escape_path_name(path: &str) -> String { +pub(crate) fn escape_path_name(path: &str) -> String { if !path.chars().any(needs_escaping) { return path.to_string(); } diff --git a/crates/paimon/src/table/format_partition.rs b/crates/paimon/src/table/format_partition.rs new file mode 100644 index 00000000..3d2c2528 --- /dev/null +++ b/crates/paimon/src/table/format_partition.rs @@ -0,0 +1,230 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Format Table partition path helpers shared by readers and SQL administration. + +use std::collections::HashMap; + +use crate::io::FileIO; +use crate::spec::escape_path_name; + +/// Generates canonical names and physical paths for Format Table partitions. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FormatTablePartitionPaths { + partition_keys: Vec, + only_value_in_path: bool, +} + +impl FormatTablePartitionPaths { + /// Create helpers for the declared partition-key order and physical layout. + pub fn new(partition_keys: I, only_value_in_path: bool) -> Self + where + I: IntoIterator, + S: Into, + { + Self { + partition_keys: partition_keys.into_iter().map(Into::into).collect(), + only_value_in_path, + } + } + + /// Return the canonical logical partition name (`key=value/...`). + pub fn partition_name(&self, spec: &HashMap) -> crate::Result { + let values = self.ordered_values(spec)?; + Ok(self + .partition_keys + .iter() + .zip(values) + .map(|(key, value)| format!("{}={}", escape_path_name(key), escape_path_name(value))) + .collect::>() + .join("/")) + } + + /// Return the physical partition path relative to the table location. + pub fn relative_path(&self, spec: &HashMap) -> crate::Result { + if !self.only_value_in_path { + return self.partition_name(spec); + } + Ok(self + .ordered_values(spec)? + .into_iter() + .map(escape_path_name) + .collect::>() + .join("/")) + } + + /// Discover complete raw partition specs from the table directory. + /// + /// Hidden directories whose names begin with `.` or `_`, segments that do + /// not match the configured layout, and paths shallower than the declared + /// partition depth are ignored. In value-only layouts, the configured + /// default-partition directory is the only underscore-prefixed exception. + /// A matching segment that is malformed or not canonically escaped returns + /// an error rather than being skipped, because its catalog spec would not + /// resolve back to the same physical path. + /// Results are sorted and deduplicated by canonical `key=value/...` name. + pub async fn discover( + &self, + file_io: &FileIO, + table_path: &str, + default_partition_name: &str, + ) -> crate::Result>> { + let mut frontier = vec![(table_path.trim_end_matches('/').to_string(), HashMap::new())]; + for key in &self.partition_keys { + let mut next = Vec::new(); + for (path, spec) in frontier { + for status in file_io.list_status(&path).await? { + if !status.is_dir { + continue; + } + let Some(segment) = last_path_segment(&status.path) else { + continue; + }; + let is_value_only_default = + self.only_value_in_path && segment == default_partition_name; + if segment.starts_with('.') + || (segment.starts_with('_') && !is_value_only_default) + { + continue; + } + let Some(value) = self.partition_value_from_segment(key, segment)? else { + continue; + }; + let mut child_spec = spec.clone(); + child_spec.insert(key.clone(), value); + next.push((status.path.trim_end_matches('/').to_string(), child_spec)); + } + } + frontier = next; + } + + let mut partitions = frontier + .into_iter() + .map(|(_, spec)| Ok((self.partition_name(&spec)?, spec))) + .collect::>>()?; + partitions.sort_by(|left, right| left.0.cmp(&right.0)); + partitions.dedup_by(|left, right| left.0 == right.0); + Ok(partitions.into_iter().map(|(_, spec)| spec).collect()) + } + + fn ordered_values<'a>(&self, spec: &'a HashMap) -> crate::Result> { + if spec.len() != self.partition_keys.len() + || self + .partition_keys + .iter() + .any(|key| !spec.contains_key(key)) + { + return Err(crate::Error::DataInvalid { + message: format!( + "Partition spec {spec:?} must contain exactly keys {:?}", + self.partition_keys + ), + source: None, + }); + } + + let mut values = Vec::with_capacity(self.partition_keys.len()); + for key in &self.partition_keys { + let value = spec[key].as_str(); + if value.is_empty() || (self.only_value_in_path && matches!(value, "." | "..")) { + return Err(crate::Error::DataInvalid { + message: format!( + "Partition value {value:?} cannot be used as a partition path component" + ), + source: None, + }); + } + values.push(value); + } + Ok(values) + } + + fn partition_value_from_segment( + &self, + key: &str, + segment: &str, + ) -> crate::Result> { + let value = if self.only_value_in_path { + unescape_canonical_path_name(segment)? + } else { + let Some((segment_key, value)) = segment.split_once('=') else { + return Ok(None); + }; + let decoded_key = + unescape_path_name(segment_key).ok_or_else(|| crate::Error::DataInvalid { + message: format!( + "Partition path segment {segment_key:?} is not valid percent-encoded text" + ), + source: None, + })?; + if decoded_key != key { + return Ok(None); + } + ensure_canonical_path_name(segment_key, &decoded_key)?; + unescape_canonical_path_name(value)? + }; + Ok(Some(value)) + } +} + +fn unescape_canonical_path_name(value: &str) -> crate::Result { + let decoded = unescape_path_name(value).ok_or_else(|| crate::Error::DataInvalid { + message: format!("Partition path segment {value:?} is not valid percent-encoded text"), + source: None, + })?; + ensure_canonical_path_name(value, &decoded)?; + Ok(decoded) +} + +fn ensure_canonical_path_name(value: &str, decoded: &str) -> crate::Result<()> { + let canonical = escape_path_name(decoded); + if canonical != value { + return Err(crate::Error::DataInvalid { + message: format!( + "Partition path segment {value:?} cannot round-trip through catalog metadata; \ + its canonical escaped form is {canonical:?}" + ), + source: None, + }); + } + Ok(()) +} + +pub(crate) fn unescape_path_name(value: &str) -> Option { + let mut out = String::with_capacity(value.len()); + let mut chars = value.chars().peekable(); + while let Some(character) = chars.next() { + if character == '%' { + let mut lookahead = chars.clone(); + if let (Some(high), Some(low)) = (lookahead.next(), lookahead.next()) { + if let (Some(high), Some(low)) = (high.to_digit(16), low.to_digit(16)) { + let decoded = char::from_u32((high << 4) | low)?; + chars.next(); + chars.next(); + out.push(decoded); + continue; + } + } + } + out.push(character); + } + Some(out) +} + +fn last_path_segment(path: &str) -> Option<&str> { + path.trim_end_matches('/').rsplit('/').next() +} diff --git a/crates/paimon/src/table/format_table_scan.rs b/crates/paimon/src/table/format_table_scan.rs index 28907664..ea735bc6 100644 --- a/crates/paimon/src/table/format_table_scan.rs +++ b/crates/paimon/src/table/format_table_scan.rs @@ -17,15 +17,17 @@ //! Scan implementation for Java-compatible `type=format-table` metadata. -use super::{Plan, ScanTrace, Table}; +use super::format_partition::{unescape_path_name, FormatTablePartitionPaths}; +use super::{CatalogManagedPartitionOptions, Plan, ScanTrace, Table}; use crate::spec::stats::BinaryTableStats; use crate::spec::{ - extract_datum, BinaryRow, BinaryRowBuilder, CoreOptions, DataField, DataFileMeta, DataType, - Datum, PartitionComputer, Predicate, PredicateOperator, + escape_path_name, extract_datum, BinaryRow, BinaryRowBuilder, CoreOptions, DataField, + DataFileMeta, DataType, Datum, PartitionComputer, Predicate, PredicateOperator, }; use crate::table::partition_filter::PartitionFilter; use crate::table::source::DataSplitBuilder; use chrono::NaiveDate; +use std::collections::HashSet; #[derive(Debug, Clone)] pub(crate) struct FormatTableScan<'a> { @@ -65,17 +67,22 @@ impl<'a> FormatTableScan<'a> { async fn plan_inner(&self, trace: Option<&mut ScanTrace>) -> crate::Result { let core_options = CoreOptions::new(self.table.schema().options()); - let format_extension = supported_format_table_extension(core_options.file_format())?; + let managed_options = self.table.catalog_managed_partition_options(); + let file_format = managed_options + .map(|options| options.file_format.as_str()) + .unwrap_or_else(|| core_options.file_format()); + let format_extension = supported_format_table_extension(file_format)?; let schema_id = self.table.schema().id(); - let table_path = core_options - .path() + let table_path = managed_options + .map(|options| options.table_path.as_str()) + .or_else(|| core_options.path()) .unwrap_or_else(|| self.table.location()) .trim_end_matches('/') .to_string(); let partition_fields = self.table.schema().partition_fields(); let mut splits = Vec::new(); - for scan_root in self.scan_roots(&core_options, &table_path)? { + for scan_root in self.scan_roots(&core_options, &table_path).await? { let statuses = self .list_status_recursive_if_exists(&scan_root.path) .await?; @@ -111,7 +118,7 @@ impl<'a> FormatTableScan<'a> { Ok(Plan::new(splits)) } - fn scan_roots( + async fn scan_roots( &self, core_options: &CoreOptions<'_>, table_path: &str, @@ -124,6 +131,26 @@ impl<'a> FormatTableScan<'a> { partition: BinaryRow::new(0), }]); } + if self.table.has_catalog_managed_partitions() { + let managed_options = + self.table + .catalog_managed_partition_options() + .ok_or_else(|| crate::Error::DataInvalid { + message: format!( + "Format table {} is missing catalog-managed partition options", + self.table.identifier().full_name() + ), + source: None, + })?; + return self + .managed_scan_roots( + table_path, + partition_keys, + &partition_fields, + managed_options, + ) + .await; + } let Some(PartitionFilter::PartitionSet { partitions, .. }) = &self.partition_filter else { if let Some(PartitionFilter::Predicate(predicate)) = &self.partition_filter { @@ -178,19 +205,78 @@ impl<'a> FormatTableScan<'a> { Ok(roots) } + async fn managed_scan_roots( + &self, + table_path: &str, + partition_keys: &[String], + partition_fields: &[DataField], + managed_options: &CatalogManagedPartitionOptions, + ) -> crate::Result> { + let rest_env = self + .table + .rest_env() + .ok_or_else(|| crate::Error::DataInvalid { + message: format!( + "Format table {} has catalog-managed partitions but no REST environment", + self.table.identifier().full_name() + ), + source: None, + })?; + let partitions = rest_env + .api() + .list_partitions(rest_env.identifier()) + .await?; + let only_value_in_path = managed_options.only_value_in_path; + let partition_paths = + FormatTablePartitionPaths::new(partition_keys.iter().cloned(), only_value_in_path); + let mut seen_paths = HashSet::with_capacity(partitions.len()); + let mut roots = Vec::with_capacity(partitions.len()); + for partition in partitions { + let partition_path = + partition_paths + .relative_path(&partition.spec) + .map_err(|error| crate::Error::DataInvalid { + message: format!( + "Catalog returned invalid partition metadata for format table {}", + self.table.identifier().full_name() + ), + source: Some(Box::new(error)), + })?; + if !seen_paths.insert(partition_path.clone()) { + continue; + } + let path = join_path(table_path, &partition_path); + let partition = partition_row_from_path( + table_path, + &path, + partition_fields, + partition_keys, + &managed_options.default_partition_name, + only_value_in_path, + )? + .ok_or_else(|| crate::Error::DataInvalid { + message: format!( + "Catalog returned corrupt partition metadata for format table {}", + self.table.identifier().full_name() + ), + source: None, + })?; + if self.partition_matches(&partition)? { + roots.push(ScanRoot { path, partition }); + } + } + roots.sort_by(|left, right| left.path.cmp(&right.path)); + Ok(roots) + } + async fn list_status_recursive_if_exists( &self, path: &str, ) -> crate::Result> { match self.table.file_io().list_status_recursive(path).await { Ok(statuses) => Ok(statuses), - Err(err) => { - if !self.table.file_io().exists(path).await.unwrap_or(true) { - Ok(Vec::new()) - } else { - Err(err) - } - } + Err(err) if is_missing_partition_directory_error(&err) => Ok(Vec::new()), + Err(err) => Err(err), } } @@ -310,6 +396,14 @@ fn join_path(parent: &str, child: &str) -> String { } } +fn is_missing_partition_directory_error(error: &crate::Error) -> bool { + matches!( + error, + crate::Error::IoUnexpected { source, .. } + if source.kind() == opendal::ErrorKind::NotFound + ) +} + fn leading_equality_partition_path( table_path: &str, partition_keys: &[String], @@ -439,41 +533,6 @@ fn format_partition_date(epoch_days: i32) -> String { date.format("%Y-%m-%d").to_string() } -fn escape_path_name(path: &str) -> String { - let mut result = String::with_capacity(path.len()); - for byte in path.bytes() { - if should_escape(byte) { - result.push('%'); - result.push_str(&format!("{byte:02X}")); - } else { - result.push(byte as char); - } - } - result -} - -fn should_escape(byte: u8) -> bool { - byte <= 0x1F - || byte >= 0x7F - || matches!( - byte, - b'"' | b'#' - | b'%' - | b'\'' - | b'*' - | b'/' - | b':' - | b'=' - | b'?' - | b'\\' - | b'\x7F' - | b'{' - | b'[' - | b']' - | b'^' - ) -} - fn partition_row_from_path( table_path: &str, file_parent: &str, @@ -543,44 +602,26 @@ fn partition_segment_value(segment: &str, key: &str) -> Option { } } -fn unescape_path_name(value: &str) -> Option { - let bytes = value.as_bytes(); - let mut out = Vec::with_capacity(bytes.len()); - let mut i = 0; - while i < bytes.len() { - if bytes[i] == b'%' { - if i + 2 >= bytes.len() { - return None; - } - let hi = hex_value(bytes[i + 1])?; - let lo = hex_value(bytes[i + 2])?; - out.push((hi << 4) | lo); - i += 3; - } else { - out.push(bytes[i]); - i += 1; - } - } - String::from_utf8(out).ok() -} - -fn hex_value(byte: u8) -> Option { - match byte { - b'0'..=b'9' => Some(byte - b'0'), - b'a'..=b'f' => Some(byte - b'a' + 10), - b'A'..=b'F' => Some(byte - b'A' + 10), - _ => None, - } -} - fn parse_partition_datum(value: &str, data_type: &DataType) -> Option { match data_type { - DataType::Boolean(_) => value.parse::().ok().map(Datum::Bool), + DataType::Boolean(_) => match value.to_ascii_lowercase().as_str() { + "t" | "true" | "y" | "yes" | "1" => Some(Datum::Bool(true)), + "f" | "false" | "n" | "no" | "0" => Some(Datum::Bool(false)), + _ => None, + }, DataType::TinyInt(_) => value.parse::().ok().map(Datum::TinyInt), DataType::SmallInt(_) => value.parse::().ok().map(Datum::SmallInt), DataType::Int(_) => value.parse::().ok().map(Datum::Int), DataType::BigInt(_) => value.parse::().ok().map(Datum::Long), - DataType::Char(_) | DataType::VarChar(_) => Some(Datum::String(value.to_string())), + DataType::Char(char_type) if value.chars().count() <= char_type.length() => { + Some(Datum::String(value.to_string())) + } + DataType::VarChar(varchar_type) + if value.chars().count() <= varchar_type.length() as usize => + { + Some(Datum::String(value.to_string())) + } + DataType::Char(_) | DataType::VarChar(_) => None, DataType::Date(_) => parse_partition_date(value).map(Datum::Date), DataType::Time(_) => value.parse::().ok().map(Datum::Time), _ => None, @@ -588,16 +629,52 @@ fn parse_partition_datum(value: &str, data_type: &DataType) -> Option { } fn parse_partition_date(value: &str) -> Option { - if let Ok(epoch_days) = value.parse::() { - return Some(epoch_days); + let is_numeric = value + .strip_prefix('-') + .unwrap_or(value) + .chars() + .all(|character| character.is_ascii_digit()) + && value != "-"; + if is_numeric { + return value.parse::().ok(); + } + + let date_value = value + .find(' ') + .filter(|index| *index > 0) + .map(|index| &value[..index]) + .unwrap_or(value); + let mut segments = date_value.split('-'); + let year = parse_date_segment(segments.next()?)?; + if !(0..=9999).contains(&year) { + return None; } - let date = NaiveDate::parse_from_str(value, "%Y-%m-%d").ok()?; + let month = match segments.next() { + Some(value) => parse_date_segment(value)?, + None => 1, + }; + let day = match segments.next() { + Some(value) => parse_date_segment(value)?, + None => 1, + }; + if segments.next().is_some() { + return None; + } + let date = NaiveDate::from_ymd_opt(year, month as u32, day as u32)?; date.signed_duration_since(NaiveDate::from_ymd_opt(1970, 1, 1).unwrap()) .num_days() .try_into() .ok() } +fn parse_date_segment(value: &str) -> Option { + let value = value.trim(); + if value.is_empty() || !value.chars().all(|character| character.is_ascii_digit()) { + return None; + } + value.parse().ok() +} + fn supported_format_table_extension(format: &str) -> crate::Result<&'static str> { match format.to_ascii_lowercase().as_str() { "parquet" => Ok(".parquet"), @@ -639,3 +716,83 @@ fn data_file_meta(file_name: String, file_size: i64, schema_id: i64) -> DataFile write_cols: None, } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::spec::{BooleanType, CharType, DateType, VarCharType}; + + #[test] + fn test_parse_partition_boolean_uses_java_spellings() { + let data_type = DataType::Boolean(BooleanType::new()); + + assert_eq!( + parse_partition_datum("1", &data_type), + Some(Datum::Bool(true)) + ); + assert_eq!( + parse_partition_datum("yes", &data_type), + Some(Datum::Bool(true)) + ); + assert_eq!( + parse_partition_datum("0", &data_type), + Some(Datum::Bool(false)) + ); + assert_eq!( + parse_partition_datum("no", &data_type), + Some(Datum::Bool(false)) + ); + } + + #[test] + fn test_parse_partition_date_accepts_java_partial_month() { + let data_type = DataType::Date(DateType::new()); + + assert_eq!( + parse_partition_datum("2026-07", &data_type), + parse_partition_datum("2026-07-01", &data_type) + ); + } + + #[test] + fn test_parse_partition_date_rejects_outside_paimon_range() { + let data_type = DataType::Date(DateType::new()); + + assert_eq!(parse_partition_datum("10000-01-01", &data_type), None); + } + + #[test] + fn test_parse_partition_string_enforces_declared_character_length() { + let char_type = DataType::Char(CharType::new(3).unwrap()); + let varchar_type = DataType::VarChar(VarCharType::new(3).unwrap()); + + assert_eq!( + parse_partition_datum("中文a", &char_type), + Some(Datum::String("中文a".to_string())) + ); + assert_eq!(parse_partition_datum("中文ab", &char_type), None); + assert_eq!( + parse_partition_datum("中文a", &varchar_type), + Some(Datum::String("中文a".to_string())) + ); + assert_eq!(parse_partition_datum("中文ab", &varchar_type), None); + } + + #[test] + fn test_only_not_found_listing_errors_mean_missing_partition_directory() { + let not_found = crate::Error::IoUnexpected { + message: "missing".to_string(), + source: Box::new(opendal::Error::new(opendal::ErrorKind::NotFound, "missing")), + }; + let permission_denied = crate::Error::IoUnexpected { + message: "denied".to_string(), + source: Box::new(opendal::Error::new( + opendal::ErrorKind::PermissionDenied, + "denied", + )), + }; + + assert!(is_missing_partition_directory_error(¬_found)); + assert!(!is_missing_partition_directory_error(&permission_denied)); + } +} diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index 5c34918d..536e014b 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -38,6 +38,7 @@ pub mod data_evolution_writer; mod data_file_reader; mod data_file_writer; mod dedicated_format_file_writer; +mod format_partition; mod format_read_builder; mod format_table_read; mod format_table_scan; @@ -94,6 +95,7 @@ pub use btree_global_index_build_builder::BTreeGlobalIndexBuildBuilder; pub use commit_message::CommitMessage; pub use cow_writer::{CopyOnWriteMergeWriter, FileInfo}; pub use data_evolution_writer::{DataEvolutionDeleteWriter, DataEvolutionWriter}; +pub use format_partition::FormatTablePartitionPaths; #[cfg(feature = "fulltext")] pub use full_text_search_builder::FullTextSearchBuilder; use futures::stream::BoxStream; @@ -133,6 +135,14 @@ use crate::io::FileIO; use crate::spec::{CoreOptions, DataField, Snapshot, TableSchema}; use std::collections::HashMap; +#[derive(Debug, Clone)] +pub(crate) struct CatalogManagedPartitionOptions { + table_path: String, + file_format: String, + default_partition_name: String, + only_value_in_path: bool, +} + /// Table represents a table in the catalog. #[derive(Debug, Clone)] pub struct Table { @@ -144,6 +154,7 @@ pub struct Table { branch: String, branch_reference: bool, rest_env: Option, + catalog_managed_partition_options: Option, /// True when this table copy was switched to a historical schema by /// [`Table::copy_with_time_travel`]. Such a copy is read-only. time_traveled: bool, @@ -164,6 +175,17 @@ impl Table { ) -> Self { let schema_manager = SchemaManager::new(file_io.clone(), location.clone()); let branch = DEFAULT_MAIN_BRANCH.to_string(); + let options = CoreOptions::new(schema.options()); + let catalog_managed_partition_options = rest_env + .as_ref() + .filter(|rest_env| rest_env.has_catalog_managed_partitions()) + .filter(|_| options.is_format_table() && options.partitioned_table_in_metastore()) + .map(|_| CatalogManagedPartitionOptions { + table_path: options.path().unwrap_or(&location).to_string(), + file_format: options.file_format().to_string(), + default_partition_name: options.partition_default_name().to_string(), + only_value_in_path: options.format_table_partition_only_value_in_path(), + }); Self { file_io, identifier, @@ -173,6 +195,7 @@ impl Table { branch, branch_reference: false, rest_env, + catalog_managed_partition_options, time_traveled: false, travel_snapshot: None, } @@ -252,7 +275,20 @@ impl Table { } pub(crate) fn is_format_table(&self) -> bool { - CoreOptions::new(self.schema.options()).is_format_table() + self.has_catalog_managed_partitions() + || CoreOptions::new(self.schema.options()).is_format_table() + } + + /// Whether metadata accepted at the REST catalog trust boundary marked + /// this as an internal Format Table with catalog-managed partitions. + pub fn has_catalog_managed_partitions(&self) -> bool { + self.catalog_managed_partition_options.is_some() + } + + pub(crate) fn catalog_managed_partition_options( + &self, + ) -> Option<&CatalogManagedPartitionOptions> { + self.catalog_managed_partition_options.as_ref() } /// Create a read builder for scan/read. @@ -314,6 +350,11 @@ impl Table { /// `FileStoreTable.copyWithoutTimeTravel`. Use /// [`Table::copy_with_time_travel`] when the options may select a /// historical snapshot whose schema should be used for reading. + /// + /// Catalog-managed Format Tables retain the validated REST provenance and + /// physical partition layout captured when they were loaded. Dynamic + /// options therefore cannot redirect those scans to a different partition + /// source, table path, file format, or value-only layout. pub fn copy_with_options(&self, extra: HashMap) -> Self { // Changing the time-travel selector invalidates the resolved snapshot // (a time-travelled schema then has no matching snapshot anymore, and @@ -334,6 +375,7 @@ impl Table { branch: self.branch.clone(), branch_reference: self.branch_reference, rest_env: self.rest_env.clone(), + catalog_managed_partition_options: self.catalog_managed_partition_options.clone(), time_traveled: self.time_traveled, travel_snapshot: if selector_changed { None @@ -414,6 +456,7 @@ impl Table { branch, branch_reference: true, rest_env: self.rest_env.clone(), + catalog_managed_partition_options: self.catalog_managed_partition_options.clone(), time_traveled: false, travel_snapshot: None, }) diff --git a/crates/paimon/src/table/rest_env.rs b/crates/paimon/src/table/rest_env.rs index b7771001..076d1f38 100644 --- a/crates/paimon/src/table/rest_env.rs +++ b/crates/paimon/src/table/rest_env.rs @@ -38,6 +38,7 @@ pub struct RESTEnv { api: Arc, options: Options, data_token_enabled: bool, + catalog_managed_partitions: bool, } impl std::fmt::Debug for RESTEnv { @@ -64,9 +65,69 @@ impl RESTEnv { api, options, data_token_enabled, + catalog_managed_partitions: false, } } + /// Create a REST environment from table metadata returned by a REST catalog. + /// + /// Catalog-managed Format Table partitions are enabled only when the + /// supplied schema requests them and the REST table is internal. Callers + /// must pass the server-provided schema and `is_external` value unchanged; + /// [`RESTEnv::new`] never enables this capability. + /// + /// # Trust boundary + /// + /// This constructor is public so custom REST catalog implementations can + /// attach their loaded table metadata. It does not fetch or authenticate + /// that metadata itself. Supplying fabricated schema or ownership metadata + /// can incorrectly grant catalog-managed partition capability. + pub fn new_for_table( + identifier: Identifier, + uuid: String, + api: Arc, + options: Options, + data_token_enabled: bool, + table_schema: &TableSchema, + is_external: bool, + ) -> Result { + let core_options = CoreOptions::new(table_schema.options()); + let catalog_managed_partitions = + core_options.is_format_table() && core_options.partitioned_table_in_metastore(); + if catalog_managed_partitions && core_options.format_table_uses_engine_implementation() { + return Err(Error::DataInvalid { + message: format!( + "Cannot combine catalog-managed partitions \ + (metastore.partitioned-table=true) with \ + format-table.implementation=engine for format table {}", + identifier.full_name() + ), + source: None, + }); + } + if catalog_managed_partitions && is_external { + return Err(Error::DataInvalid { + message: format!( + "Catalog-managed partitions are only supported for internal tables, but format table {} is external", + identifier.full_name() + ), + source: None, + }); + } + Ok(Self { + identifier, + uuid, + api, + options, + data_token_enabled, + catalog_managed_partitions, + }) + } + + pub(crate) fn has_catalog_managed_partitions(&self) -> bool { + self.catalog_managed_partitions + } + /// Get the REST API client. pub fn api(&self) -> &Arc { &self.api @@ -132,7 +193,6 @@ impl RESTEnv { ), source: None, })?; - let uuid = response.id.ok_or_else(|| Error::DataInvalid { message: format!( "Table {} response missing id (uuid)", @@ -151,7 +211,15 @@ impl RESTEnv { builder.build()? }; - let rest_env = RESTEnv::new(identifier.clone(), uuid, api, options, data_token_enabled); + let rest_env = RESTEnv::new_for_table( + identifier.clone(), + uuid, + api, + options, + data_token_enabled, + &table_schema, + is_external, + )?; Ok(Table::new( file_io, diff --git a/crates/paimon/tests/format_partition_test.rs b/crates/paimon/tests/format_partition_test.rs new file mode 100644 index 00000000..32ea9236 --- /dev/null +++ b/crates/paimon/tests/format_partition_test.rs @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; + +use paimon::io::FileIO; +use paimon::table::FormatTablePartitionPaths; + +#[test] +fn format_partition_paths_escape_values_and_honor_layout() { + let spec = HashMap::from([ + ("dt".to_string(), "2026/07=22".to_string()), + ("hour".to_string(), "10".to_string()), + ]); + + let keyed = FormatTablePartitionPaths::new(["dt", "hour"], false); + assert_eq!( + keyed.partition_name(&spec).unwrap(), + "dt=2026%2F07%3D22/hour=10" + ); + assert_eq!( + keyed.relative_path(&spec).unwrap(), + "dt=2026%2F07%3D22/hour=10" + ); + + let value_only = FormatTablePartitionPaths::new(["dt", "hour"], true); + assert_eq!( + value_only.partition_name(&spec).unwrap(), + "dt=2026%2F07%3D22/hour=10" + ); + assert_eq!( + value_only.relative_path(&spec).unwrap(), + "2026%2F07%3D22/10" + ); +} + +#[test] +fn format_partition_paths_match_java_escaping_for_unicode_and_closing_brace() { + let spec = HashMap::from([("region".to_string(), "杭州}".to_string())]); + let paths = FormatTablePartitionPaths::new(["region"], false); + + assert_eq!(paths.partition_name(&spec).unwrap(), "region=杭州%7D"); + assert_eq!(paths.relative_path(&spec).unwrap(), "region=杭州%7D"); +} + +#[cfg(not(windows))] +#[tokio::test] +async fn discover_format_partitions_returns_complete_sorted_specs() { + let tmp = tempfile::tempdir().unwrap(); + for relative in [ + "dt=2026-07-22/hour=10", + "dt=2026-07-21/hour=09", + "dt=2026-07-20", + ".staging/dt=2026-07-19/hour=08", + ] { + std::fs::create_dir_all(tmp.path().join(relative)).unwrap(); + } + let table_path = format!("file://{}", tmp.path().display()); + let file_io = FileIO::from_path(&table_path).unwrap().build().unwrap(); + let paths = FormatTablePartitionPaths::new(["dt", "hour"], false); + + let partitions = paths + .discover(&file_io, &table_path, "__DEFAULT_PARTITION__") + .await + .unwrap(); + + assert_eq!( + partitions, + vec![ + HashMap::from([ + ("dt".to_string(), "2026-07-21".to_string()), + ("hour".to_string(), "09".to_string()), + ]), + HashMap::from([ + ("dt".to_string(), "2026-07-22".to_string()), + ("hour".to_string(), "10".to_string()), + ]), + ] + ); +} + +#[cfg(not(windows))] +#[tokio::test] +async fn discover_format_partitions_rejects_non_round_tripping_percent_escape() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(tmp.path().join("dt=value%ZZ")).unwrap(); + let table_path = format!("file://{}", tmp.path().display()); + let file_io = FileIO::from_path(&table_path).unwrap().build().unwrap(); + let paths = FormatTablePartitionPaths::new(["dt"], false); + + let error = paths + .discover(&file_io, &table_path, "__DEFAULT_PARTITION__") + .await + .unwrap_err(); + + assert!( + error.to_string().contains("cannot round-trip"), + "expected a non-round-tripping path error, got: {error}" + ); +} + +#[cfg(not(windows))] +#[tokio::test] +async fn discover_format_partitions_ignores_nonmatching_malformed_key_segment() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(tmp.path().join("wrong%ZZ=value")).unwrap(); + std::fs::create_dir_all(tmp.path().join("dt=valid")).unwrap(); + let table_path = format!("file://{}", tmp.path().display()); + let file_io = FileIO::from_path(&table_path).unwrap().build().unwrap(); + let paths = FormatTablePartitionPaths::new(["dt"], false); + + let partitions = paths + .discover(&file_io, &table_path, "__DEFAULT_PARTITION__") + .await + .unwrap(); + + assert_eq!( + partitions, + vec![HashMap::from([("dt".to_string(), "valid".to_string(),)])] + ); +} + +#[cfg(not(windows))] +#[tokio::test] +async fn discover_value_only_partitions_rejects_parent_traversal_value() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(tmp.path().join("%2E%2E")).unwrap(); + let table_path = format!("file://{}", tmp.path().display()); + let file_io = FileIO::from_path(&table_path).unwrap().build().unwrap(); + let paths = FormatTablePartitionPaths::new(["dt"], true); + + let error = paths + .discover(&file_io, &table_path, "__DEFAULT_PARTITION__") + .await + .unwrap_err(); + + assert!( + error.to_string().contains(".."), + "expected traversal value in error, got: {error}" + ); +} + +#[cfg(not(windows))] +#[tokio::test] +async fn discover_value_only_partitions_keeps_default_partition_directory() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(tmp.path().join("__DEFAULT_PARTITION__")).unwrap(); + let table_path = format!("file://{}", tmp.path().display()); + let file_io = FileIO::from_path(&table_path).unwrap().build().unwrap(); + let paths = FormatTablePartitionPaths::new(["dt"], true); + + let partitions = paths + .discover(&file_io, &table_path, "__DEFAULT_PARTITION__") + .await + .unwrap(); + + assert_eq!( + partitions, + vec![HashMap::from([( + "dt".to_string(), + "__DEFAULT_PARTITION__".to_string(), + )])] + ); +} diff --git a/crates/paimon/tests/mock_server.rs b/crates/paimon/tests/mock_server.rs index 6c8a0048..9f767237 100644 --- a/crates/paimon/tests/mock_server.rs +++ b/crates/paimon/tests/mock_server.rs @@ -35,12 +35,16 @@ use tokio::task::JoinHandle; use paimon::api::{ AlterDatabaseRequest, AlterTableRequest, AuditRESTResponse, ConfigResponse, - CreateFunctionRequest, CreateViewRequest, ErrorResponse, GetDatabaseResponse, + CreateFunctionRequest, CreatePartitionsRequest, CreatePartitionsResponse, CreateViewRequest, + DropPartitionsRequest, DropPartitionsResponse, ErrorResponse, GetDatabaseResponse, GetFunctionResponse, GetTableResponse, GetViewResponse, ListDatabasesResponse, - ListFunctionsResponse, ListTablesResponse, ListViewsResponse, RenameTableRequest, - ResourcePaths, + ListFunctionsResponse, ListPartitionsByNamesRequest, ListPartitionsResponse, + ListTablesResponse, ListViewsResponse, RenameTableRequest, ResourcePaths, }; use paimon::catalog::{Function, Identifier}; +use paimon::spec::Partition; + +type PartitionPageResponse = (Vec>, Option); #[derive(Clone, Debug, Default)] struct MockState { @@ -48,17 +52,48 @@ struct MockState { tables: HashMap, views: HashMap, functions: HashMap, + partitions: HashMap>, + partition_pages: HashMap>>, + partition_page_tokens: HashMap>>, + partition_list_calls: HashMap, view_function_endpoints_unsupported: bool, drop_view_error_status: Option, list_page_size: Option, no_permission_databases: HashSet, no_permission_tables: HashSet, + last_create_partitions_call: Option<(String, String, CreatePartitionsRequest)>, + create_partitions_call_sizes: Vec, + last_drop_partitions_call: Option<(String, String, DropPartitionsRequest)>, + drop_partitions_call_sizes: Vec, + last_list_partitions_by_names_call: Option<(String, String, ListPartitionsByNamesRequest)>, + list_partitions_by_names_call_sizes: Vec, + create_partitions_error_status: Option, + drop_partitions_error_status: Option, + list_partitions_error_status: Option, + list_partitions_by_names_error_status: Option, /// ECS metadata role name (for token loader testing) ecs_role_name: Option, /// ECS metadata token (for token loader testing) ecs_token: Option, } +fn partition_from_spec(spec: HashMap) -> Partition { + Partition { + spec, + record_count: 0, + file_size_in_bytes: 0, + file_count: 0, + last_file_creation_time: 0, + total_buckets: 0, + done: false, + created_at: None, + created_by: None, + updated_at: None, + updated_by: None, + options: None, + } +} + fn paginate_names( names: Vec, params: &HashMap, @@ -771,6 +806,261 @@ impl RESTServer { } } + /// Handle POST /databases/:db/tables/:table/partitions - create partitions. + pub async fn create_partitions( + Path((db, table)): Path<(String, String)>, + Extension(state): Extension>, + Json(request): Json, + ) -> impl IntoResponse { + let (error_status, table_exists) = { + let mut inner = state.inner.lock().unwrap(); + inner + .create_partitions_call_sizes + .push(request.partition_specs.len()); + inner.last_create_partitions_call = Some((db.clone(), table.clone(), request.clone())); + ( + inner.create_partitions_error_status, + inner.tables.contains_key(&format!("{db}.{table}")), + ) + }; + if let Some(status) = error_status { + let error = ErrorResponse::new( + Some("partition".to_string()), + Some(table.clone()), + Some("Some partitions already exist".to_string()), + Some(status.as_u16() as i32), + ); + return (status, Json(error)).into_response(); + } + if !table_exists { + let error = ErrorResponse::new( + Some("table".to_string()), + Some(table), + Some("Not Found".to_string()), + Some(404), + ); + return (StatusCode::NOT_FOUND, Json(error)).into_response(); + } + + let mut inner = state.inner.lock().unwrap(); + let key = format!("{db}.{table}"); + let registered = inner.partitions.entry(key.clone()).or_default(); + let mut seen = Vec::new(); + let conflicts = request.partition_specs.iter().any(|spec| { + registered.iter().any(|partition| partition.spec == *spec) || seen.contains(spec) || { + seen.push(spec.clone()); + false + } + }); + if conflicts && !request.ignore_if_exists { + let error = ErrorResponse::new( + Some("partition".to_string()), + Some(table), + Some("Some partitions already exist".to_string()), + Some(StatusCode::CONFLICT.as_u16() as i32), + ); + return (StatusCode::CONFLICT, Json(error)).into_response(); + } + + let mut created = Vec::new(); + let mut existed = Vec::new(); + for spec in request.partition_specs { + if registered.iter().any(|partition| partition.spec == spec) { + existed.push(spec); + } else { + registered.push(partition_from_spec(spec.clone())); + created.push(spec); + } + } + inner.partition_pages.remove(&key); + let response = CreatePartitionsResponse { created, existed }; + (StatusCode::OK, Json(response)).into_response() + } + + /// Handle GET /databases/:db/tables/:table/partitions - list partitions. + pub async fn list_partitions( + Path((db, table)): Path<(String, String)>, + Query(params): Query>, + Extension(state): Extension>, + ) -> impl IntoResponse { + let mut inner = state.inner.lock().unwrap(); + let key = format!("{db}.{table}"); + if !inner.tables.contains_key(&key) { + let error = ErrorResponse::new( + Some("table".to_string()), + Some(table), + Some("Not Found".to_string()), + Some(404), + ); + return (StatusCode::NOT_FOUND, Json(error)).into_response(); + } + if let Some(status) = inner.list_partitions_error_status { + let error = ErrorResponse::new( + Some("partition".to_string()), + Some(table), + Some("Partition listing is not implemented".to_string()), + Some(status.as_u16() as i32), + ); + return (status, Json(error)).into_response(); + } + if let Some(pages) = inner.partition_pages.get(&key) { + let pages = pages.clone(); + let scripted_tokens = inner.partition_page_tokens.get(&key).cloned(); + let request_index = { + let calls = inner.partition_list_calls.entry(key.clone()).or_default(); + let request_index = *calls; + *calls += 1; + request_index + }; + let page_index = if scripted_tokens.is_some() { + request_index + } else { + params + .get("pageToken") + .and_then(|token| token.parse::().ok()) + .unwrap_or(0) + }; + let partitions = pages.get(page_index).cloned().unwrap_or_default(); + let next_page_token = scripted_tokens.map_or_else( + || (page_index + 1 < pages.len()).then(|| (page_index + 1).to_string()), + |tokens| tokens.get(page_index).cloned().flatten(), + ); + return ( + StatusCode::OK, + Json(ListPartitionsResponse::new( + Some(partitions), + next_page_token, + )), + ) + .into_response(); + } + let partitions = inner.partitions.get(&key).cloned().unwrap_or_default(); + ( + StatusCode::OK, + Json(ListPartitionsResponse::new(Some(partitions), None)), + ) + .into_response() + } + + /// Handle POST /databases/:db/tables/:table/partitions/drop - drop partitions. + pub async fn drop_partitions( + Path((db, table)): Path<(String, String)>, + Extension(state): Extension>, + Json(request): Json, + ) -> impl IntoResponse { + let (error_status, table_exists) = { + let mut inner = state.inner.lock().unwrap(); + inner + .drop_partitions_call_sizes + .push(request.partition_specs.len()); + inner.last_drop_partitions_call = Some((db.clone(), table.clone(), request.clone())); + ( + inner.drop_partitions_error_status, + inner.tables.contains_key(&format!("{db}.{table}")), + ) + }; + if let Some(status) = error_status { + let error = ErrorResponse::new( + Some("partition".to_string()), + Some(table.clone()), + Some("Invalid partition request".to_string()), + Some(status.as_u16() as i32), + ); + return (status, Json(error)).into_response(); + } + if !table_exists { + let error = ErrorResponse::new( + Some("table".to_string()), + Some(table), + Some("Not Found".to_string()), + Some(404), + ); + return (StatusCode::NOT_FOUND, Json(error)).into_response(); + } + + let mut inner = state.inner.lock().unwrap(); + let key = format!("{db}.{table}"); + let registered = inner.partitions.entry(key.clone()).or_default(); + let missing = request + .partition_specs + .iter() + .filter(|spec| !registered.iter().any(|partition| partition.spec == **spec)) + .cloned() + .collect::>(); + if !missing.is_empty() && !request.ignore_if_not_exists { + let error = ErrorResponse::new( + Some("partition".to_string()), + Some(table), + Some("Some partitions do not exist".to_string()), + Some(StatusCode::NOT_FOUND.as_u16() as i32), + ); + return (StatusCode::NOT_FOUND, Json(error)).into_response(); + } + + let mut dropped = Vec::new(); + registered.retain(|partition| { + if request.partition_specs.contains(&partition.spec) { + dropped.push(partition.spec.clone()); + false + } else { + true + } + }); + inner.partition_pages.remove(&key); + let response = DropPartitionsResponse { dropped, missing }; + (StatusCode::OK, Json(response)).into_response() + } + + /// Handle POST /databases/:db/tables/:table/partitions/list-by-names. + pub async fn list_partitions_by_names( + Path((db, table)): Path<(String, String)>, + Extension(state): Extension>, + Json(request): Json, + ) -> impl IntoResponse { + let (error_status, table_exists) = { + let mut inner = state.inner.lock().unwrap(); + inner + .list_partitions_by_names_call_sizes + .push(request.partition_specs.len()); + inner.last_list_partitions_by_names_call = + Some((db.clone(), table.clone(), request.clone())); + ( + inner.list_partitions_by_names_error_status, + inner.tables.contains_key(&format!("{db}.{table}")), + ) + }; + if let Some(status) = error_status { + let error = ErrorResponse::new( + Some("partition".to_string()), + Some(table.clone()), + Some("Invalid partition request".to_string()), + Some(status.as_u16() as i32), + ); + return (status, Json(error)).into_response(); + } + if !table_exists { + let error = ErrorResponse::new( + Some("table".to_string()), + Some(table), + Some("Not Found".to_string()), + Some(404), + ); + return (StatusCode::NOT_FOUND, Json(error)).into_response(); + } + + let inner = state.inner.lock().unwrap(); + let partitions = inner + .partitions + .get(&format!("{db}.{table}")) + .into_iter() + .flatten() + .filter(|partition| request.partition_specs.contains(&partition.spec)) + .cloned() + .collect(); + let response = ListPartitionsResponse::new(Some(partitions), None); + (StatusCode::OK, Json(response)).into_response() + } + /// Handle POST /rename-table - rename a table. pub async fn rename_table( Extension(state): Extension>, @@ -930,6 +1220,29 @@ impl RESTServer { self.inner.lock().unwrap().drop_view_error_status = status; } + /// Make the create-partitions endpoint return the given status. + pub fn set_create_partitions_error_status(&self, status: Option) { + self.inner.lock().unwrap().create_partitions_error_status = status; + } + + /// Make the drop-partitions endpoint return the given status. + pub fn set_drop_partitions_error_status(&self, status: Option) { + self.inner.lock().unwrap().drop_partitions_error_status = status; + } + + /// Make the list-partitions endpoint return the given status. + pub fn set_list_partitions_error_status(&self, status: Option) { + self.inner.lock().unwrap().list_partitions_error_status = status; + } + + /// Make the list-partitions-by-names endpoint return the given status. + pub fn set_list_partitions_by_names_error_status(&self, status: Option) { + self.inner + .lock() + .unwrap() + .list_partitions_by_names_error_status = status; + } + /// Add a table with schema and path to the server state. /// /// This is needed for `RESTCatalog::get_table` which requires @@ -972,6 +1285,166 @@ impl RESTServer { let mut s = self.inner.lock().unwrap(); s.no_permission_tables.insert(format!("{database}.{table}")); } + + /// Set whether a stored table is external. + pub fn set_table_external(&self, database: &str, table: &str, is_external: bool) { + let key = format!("{database}.{table}"); + let mut state = self.inner.lock().unwrap(); + state + .tables + .get_mut(&key) + .unwrap_or_else(|| panic!("table {key} does not exist")) + .is_external = Some(is_external); + } + + /// Set the catalog-registered partitions for a table. + pub fn set_table_partitions( + &self, + database: &str, + table: &str, + partition_specs: Vec>, + ) { + let partitions = partition_specs + .into_iter() + .map(|spec| Partition { + spec, + record_count: 0, + file_size_in_bytes: 0, + file_count: 0, + last_file_creation_time: 0, + total_buckets: 0, + done: false, + created_at: None, + created_by: None, + updated_at: None, + updated_by: None, + options: None, + }) + .collect(); + self.inner + .lock() + .unwrap() + .partitions + .insert(format!("{database}.{table}"), partitions); + } + + /// Set explicit pages returned by the list-partitions endpoint. + pub fn set_table_partition_pages( + &self, + database: &str, + table: &str, + pages: Vec>>, + ) { + let pages = pages + .into_iter() + .map(|specs| { + specs + .into_iter() + .map(|spec| Partition { + spec, + record_count: 0, + file_size_in_bytes: 0, + file_count: 0, + last_file_creation_time: 0, + total_buckets: 0, + done: false, + created_at: None, + created_by: None, + updated_at: None, + updated_by: None, + options: None, + }) + .collect() + }) + .collect(); + self.inner + .lock() + .unwrap() + .partition_pages + .insert(format!("{database}.{table}"), pages); + } + + /// Set explicit list-partitions pages and response tokens in request order. + pub fn set_table_partition_page_responses( + &self, + database: &str, + table: &str, + responses: Vec, + ) { + let (pages, tokens): (Vec<_>, Vec<_>) = responses + .into_iter() + .map(|(specs, token)| (specs.into_iter().map(partition_from_spec).collect(), token)) + .unzip(); + let key = format!("{database}.{table}"); + let mut inner = self.inner.lock().unwrap(); + inner.partition_pages.insert(key.clone(), pages); + inner.partition_page_tokens.insert(key.clone(), tokens); + inner.partition_list_calls.remove(&key); + } + + /// Return how many paged partition-list requests the table received. + pub fn table_partition_list_calls(&self, database: &str, table: &str) -> usize { + self.inner + .lock() + .unwrap() + .partition_list_calls + .get(&format!("{database}.{table}")) + .copied() + .unwrap_or_default() + } + + /// Return the last create-partitions call received by the server. + pub fn last_create_partitions_call(&self) -> Option<(String, String, CreatePartitionsRequest)> { + self.inner + .lock() + .unwrap() + .last_create_partitions_call + .clone() + } + + /// Return the partition counts of all create-partitions calls. + pub fn create_partitions_call_sizes(&self) -> Vec { + self.inner + .lock() + .unwrap() + .create_partitions_call_sizes + .clone() + } + + /// Return the last drop-partitions call received by the server. + pub fn last_drop_partitions_call(&self) -> Option<(String, String, DropPartitionsRequest)> { + self.inner.lock().unwrap().last_drop_partitions_call.clone() + } + + /// Return the partition counts of all drop-partitions calls. + pub fn drop_partitions_call_sizes(&self) -> Vec { + self.inner + .lock() + .unwrap() + .drop_partitions_call_sizes + .clone() + } + + /// Return the last list-partitions-by-names call received by the server. + pub fn last_list_partitions_by_names_call( + &self, + ) -> Option<(String, String, ListPartitionsByNamesRequest)> { + self.inner + .lock() + .unwrap() + .last_list_partitions_by_names_call + .clone() + } + + /// Return the partition counts of all list-partitions-by-names calls. + pub fn list_partitions_by_names_call_sizes(&self) -> Vec { + self.inner + .lock() + .unwrap() + .list_partitions_by_names_call_sizes + .clone() + } + /// Get the server URL. pub fn url(&self) -> Option { self.addr.map(|a| format!("http://{a}")) @@ -1089,6 +1562,18 @@ pub async fn start_mock_server( .post(RESTServer::alter_table) .delete(RESTServer::drop_table), ) + .route( + &format!("{prefix}/databases/:db/tables/:table/partitions"), + get(RESTServer::list_partitions).post(RESTServer::create_partitions), + ) + .route( + &format!("{prefix}/databases/:db/tables/:table/partitions/drop"), + post(RESTServer::drop_partitions), + ) + .route( + &format!("{prefix}/databases/:db/tables/:table/partitions/list-by-names"), + post(RESTServer::list_partitions_by_names), + ) .route( &format!("{prefix}/databases/:db/views"), get(RESTServer::list_views).post(RESTServer::create_view), diff --git a/crates/paimon/tests/rest_api_test.rs b/crates/paimon/tests/rest_api_test.rs index 99b952ce..fc876974 100644 --- a/crates/paimon/tests/rest_api_test.rs +++ b/crates/paimon/tests/rest_api_test.rs @@ -24,7 +24,9 @@ use std::collections::HashMap; use paimon::api::auth::{DLFECSTokenLoader, DLFToken, DLFTokenLoader}; use paimon::api::rest_api::RESTApi; -use paimon::api::ConfigResponse; +use paimon::api::{ + ConfigResponse, CreatePartitionsRequest, DropPartitionsRequest, ListPartitionsByNamesRequest, +}; use paimon::catalog::{Function, FunctionDefinition, Identifier, ViewSchema}; use paimon::common::Options; use paimon::spec::DataField; @@ -594,6 +596,227 @@ async fn test_drop_table_no_permission() { assert!(result.is_err(), "dropping no-permission table should fail"); } +// ==================== Partition Tests ==================== + +#[tokio::test] +async fn test_list_partitions_by_names_rejects_more_than_1000_specs() { + let ctx = setup_test_server(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + let partition_specs = (0..=1000) + .map(|day| HashMap::from([("day".to_string(), day.to_string())])) + .collect::>(); + + let error = ctx + .api + .list_partitions_by_names(&identifier, partition_specs) + .await + .unwrap_err(); + + assert!( + matches!(error, paimon::Error::DataInvalid { .. }) + && error.to_string().contains("at most 1000"), + "expected a request-size validation error, got: {error}" + ); + assert!(ctx.server.last_list_partitions_by_names_call().is_none()); +} + +#[tokio::test] +async fn test_create_partitions_posts_java_compatible_request() { + let ctx = setup_test_server(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + let partition_specs = vec![HashMap::from([ + ("dt".to_string(), "2026-07-22".to_string()), + ("hour".to_string(), "10".to_string()), + ])]; + + let response = ctx + .api + .create_partitions(&identifier, partition_specs.clone(), false) + .await + .unwrap(); + + assert_eq!(response.created, partition_specs.clone()); + assert!(response.existed.is_empty()); + assert_eq!( + ctx.server.last_create_partitions_call(), + Some(( + "default".to_string(), + "managed_table".to_string(), + CreatePartitionsRequest::new(partition_specs, false), + )) + ); +} + +#[tokio::test] +async fn test_drop_partitions_posts_java_compatible_request() { + let ctx = setup_test_server(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + let partition_specs = vec![HashMap::from([( + "dt".to_string(), + "2026-07-22".to_string(), + )])]; + ctx.server + .set_table_partitions("default", "managed_table", partition_specs.clone()); + + let response = ctx + .api + .drop_partitions(&identifier, partition_specs.clone(), false) + .await + .unwrap(); + + assert_eq!(response.dropped, partition_specs.clone()); + assert!(response.missing.is_empty()); + assert_eq!( + ctx.server.last_drop_partitions_call(), + Some(( + "default".to_string(), + "managed_table".to_string(), + DropPartitionsRequest::new(partition_specs, false), + )) + ); +} + +#[tokio::test] +async fn test_list_partitions_by_names_posts_java_compatible_request() { + let ctx = setup_test_server(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + let partition_specs = vec![ + HashMap::from([("dt".to_string(), "2026-07-22".to_string())]), + HashMap::from([("dt".to_string(), "2026-07-21".to_string())]), + ]; + ctx.server + .set_table_partitions("default", "managed_table", partition_specs.clone()); + + let partitions = ctx + .api + .list_partitions_by_names(&identifier, partition_specs.clone()) + .await + .unwrap(); + + assert_eq!( + partitions + .into_iter() + .map(|partition| partition.spec) + .collect::>(), + partition_specs.clone() + ); + assert_eq!( + ctx.server.last_list_partitions_by_names_call(), + Some(( + "default".to_string(), + "managed_table".to_string(), + ListPartitionsByNamesRequest::new(partition_specs), + )) + ); +} + +#[tokio::test] +async fn test_list_partitions_follows_token_after_empty_page() { + let ctx = setup_test_server(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + let expected = HashMap::from([("dt".to_string(), "2026-07-22".to_string())]); + ctx.server.set_table_partition_pages( + "default", + "managed_table", + vec![Vec::new(), vec![expected.clone()]], + ); + + let partitions = ctx.api.list_partitions(&identifier).await.unwrap(); + + assert_eq!(partitions.len(), 1); + assert_eq!(partitions[0].spec, expected); +} + +#[tokio::test] +async fn test_list_partitions_stops_on_empty_next_page_token() { + let ctx = setup_test_server(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + let expected = HashMap::from([("dt".to_string(), "2026-07-22".to_string())]); + let unexpected = HashMap::from([("dt".to_string(), "2026-07-23".to_string())]); + ctx.server.set_table_partition_page_responses( + "default", + "managed_table", + vec![ + (vec![expected.clone()], Some(String::new())), + (vec![unexpected], None), + ], + ); + + let partitions = ctx.api.list_partitions(&identifier).await.unwrap(); + + assert_eq!( + partitions + .into_iter() + .map(|partition| partition.spec) + .collect::>(), + vec![expected] + ); + assert_eq!( + ctx.server + .table_partition_list_calls("default", "managed_table"), + 1 + ); +} + +#[tokio::test] +async fn test_list_partitions_rejects_repeated_page_token() { + let ctx = setup_test_server(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + ctx.server.set_table_partition_page_responses( + "default", + "managed_table", + vec![ + ( + vec![HashMap::from([( + "dt".to_string(), + "2026-07-20".to_string(), + )])], + Some("a".to_string()), + ), + ( + vec![HashMap::from([( + "dt".to_string(), + "2026-07-21".to_string(), + )])], + Some("b".to_string()), + ), + ( + vec![HashMap::from([( + "dt".to_string(), + "2026-07-22".to_string(), + )])], + Some("a".to_string()), + ), + (Vec::new(), None), + ], + ); + + let error = ctx.api.list_partitions(&identifier).await.unwrap_err(); + + assert!( + error + .to_string() + .contains("repeated partition page token 'a'"), + "expected repeated-token error, got: {error}" + ); + assert!( + error.to_string().contains("default.managed_table"), + "expected table context, got: {error}" + ); + assert_eq!( + ctx.server + .table_partition_list_calls("default", "managed_table"), + 3 + ); +} + // ==================== Rename Table Tests ==================== #[tokio::test] diff --git a/crates/paimon/tests/rest_catalog_test.rs b/crates/paimon/tests/rest_catalog_test.rs index 2000f9e4..ccfa908e 100644 --- a/crates/paimon/tests/rest_catalog_test.rs +++ b/crates/paimon/tests/rest_catalog_test.rs @@ -27,14 +27,16 @@ use arrow_array::{Array, BinaryArray, Int32Array, Int64Array, RecordBatch, Strin use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema}; use axum::http::StatusCode; use futures::TryStreamExt; -use paimon::api::ConfigResponse; +use paimon::api::{ + ConfigResponse, CreatePartitionsRequest, DropPartitionsRequest, ListPartitionsByNamesRequest, +}; use paimon::catalog::{Catalog, Function, FunctionDefinition, Identifier, RESTCatalog, ViewSchema}; use paimon::common::Options; use paimon::spec::{ BigIntType, BlobType, BlobViewStruct, DataField, DataType, Datum, IntType, PredicateBuilder, Schema, SchemaChange, VarCharType, }; -use paimon::{CatalogOptions, FileSystemCatalog, Table}; +use paimon::{CatalogOptions, FileSystemCatalog, RESTEnv, Table}; mod mock_server; use mock_server::{start_mock_server, RESTServer}; @@ -167,6 +169,390 @@ fn collect_blob_rows(batches: &[RecordBatch]) -> Vec<(i32, String, Option>(); + + ctx.catalog + .create_partitions(&identifier, partition_specs, true) + .await + .unwrap(); + + assert_eq!(ctx.server.create_partitions_call_sizes(), vec![1000, 1]); +} + +#[tokio::test] +async fn test_rest_catalog_rejects_large_strict_create_before_request() { + let ctx = setup_catalog(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + let partition_specs = (0..1001) + .map(|value| HashMap::from([("dt".to_string(), value.to_string())])) + .collect::>(); + + let error = ctx + .catalog + .create_partitions(&identifier, partition_specs, false) + .await + .unwrap_err(); + + assert!( + matches!(error, paimon::Error::DataInvalid { .. }) + && error.to_string().contains("at most 1000"), + "expected a request-size validation error, got: {error}" + ); + assert!(ctx.server.create_partitions_call_sizes().is_empty()); +} + +#[tokio::test] +async fn test_rest_catalog_created_partitions_are_visible_to_list() { + let ctx = setup_catalog(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + let partition_specs = vec![HashMap::from([( + "dt".to_string(), + "2026-07-22".to_string(), + )])]; + + ctx.catalog + .create_partitions(&identifier, partition_specs.clone(), false) + .await + .unwrap(); + + let listed = ctx.catalog.list_partitions(&identifier).await.unwrap(); + assert_eq!( + listed + .into_iter() + .map(|partition| partition.spec) + .collect::>(), + partition_specs + ); +} + +#[tokio::test] +async fn test_rest_catalog_maps_strict_create_partition_conflict() { + let ctx = setup_catalog(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server + .set_create_partitions_error_status(Some(StatusCode::CONFLICT)); + + let error = ctx + .catalog + .create_partitions( + &identifier, + vec![HashMap::from([( + "dt".to_string(), + "2026-07-22".to_string(), + )])], + false, + ) + .await + .unwrap_err(); + + assert!(matches!( + error, + paimon::Error::DataInvalid { message, .. } + if message.contains("default.managed_table") + && message.contains("already exist") + )); +} + +#[tokio::test] +async fn test_rest_catalog_maps_invalid_partition_request() { + let ctx = setup_catalog(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server + .set_create_partitions_error_status(Some(StatusCode::BAD_REQUEST)); + + let error = ctx + .catalog + .create_partitions( + &identifier, + vec![HashMap::from([( + "unknown".to_string(), + "value".to_string(), + )])], + false, + ) + .await + .unwrap_err(); + + assert!(matches!( + error, + paimon::Error::DataInvalid { message, .. } + if message.contains("default.managed_table") + )); +} + +#[tokio::test] +async fn test_rest_catalog_drop_partitions_delegates_to_rest_metadata_only() { + let ctx = setup_catalog(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + let partition_specs = vec![HashMap::from([( + "dt".to_string(), + "2026-07-22".to_string(), + )])]; + + ctx.catalog + .drop_partitions(&identifier, partition_specs.clone()) + .await + .unwrap(); + + assert_eq!( + ctx.server.last_drop_partitions_call(), + Some(( + "default".to_string(), + "managed_table".to_string(), + DropPartitionsRequest::new(partition_specs, true), + )) + ); +} + +#[tokio::test] +async fn test_rest_catalog_maps_invalid_drop_partition_request() { + let ctx = setup_catalog(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + ctx.server + .set_drop_partitions_error_status(Some(StatusCode::BAD_REQUEST)); + + let error = ctx + .catalog + .drop_partitions( + &identifier, + vec![HashMap::from([( + "unknown".to_string(), + "value".to_string(), + )])], + ) + .await + .unwrap_err(); + + assert!(matches!( + error, + paimon::Error::DataInvalid { message, .. } + if message.contains("default.managed_table") + )); +} + +#[tokio::test] +async fn test_rest_catalog_batches_drop_partitions() { + let ctx = setup_catalog(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + let partition_specs = (0..1001) + .map(|value| HashMap::from([("dt".to_string(), value.to_string())])) + .collect::>(); + ctx.server + .set_table_partitions("default", "managed_table", partition_specs.clone()); + + ctx.catalog + .drop_partitions(&identifier, partition_specs) + .await + .unwrap(); + + assert_eq!(ctx.server.drop_partitions_call_sizes(), vec![1000, 1]); +} + +#[tokio::test] +async fn test_rest_catalog_dropped_partitions_are_removed_from_list() { + let ctx = setup_catalog(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + let partition_specs = vec![HashMap::from([( + "dt".to_string(), + "2026-07-22".to_string(), + )])]; + ctx.server + .set_table_partitions("default", "managed_table", partition_specs.clone()); + + ctx.catalog + .drop_partitions(&identifier, partition_specs) + .await + .unwrap(); + + assert!(ctx + .catalog + .list_partitions(&identifier) + .await + .unwrap() + .is_empty()); +} + +#[tokio::test] +async fn test_rest_catalog_lists_partitions_by_names_through_rest() { + let ctx = setup_catalog(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + let existing = HashMap::from([("dt".to_string(), "2026-07-22".to_string())]); + let missing = HashMap::from([("dt".to_string(), "2026-07-23".to_string())]); + ctx.server + .set_table_partitions("default", "managed_table", vec![existing.clone()]); + let partition_specs = vec![existing.clone(), missing]; + + let partitions = ctx + .catalog + .list_partitions_by_names(&identifier, partition_specs.clone()) + .await + .unwrap(); + + assert_eq!(partitions.len(), 1); + assert_eq!(partitions[0].spec, existing); + assert_eq!( + ctx.server.last_list_partitions_by_names_call(), + Some(( + "default".to_string(), + "managed_table".to_string(), + ListPartitionsByNamesRequest::new(partition_specs), + )) + ); +} + +#[tokio::test] +async fn test_rest_catalog_maps_invalid_list_partitions_by_names_request() { + let ctx = setup_catalog(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + ctx.server + .set_list_partitions_by_names_error_status(Some(StatusCode::BAD_REQUEST)); + + let error = ctx + .catalog + .list_partitions_by_names( + &identifier, + vec![HashMap::from([( + "unknown".to_string(), + "value".to_string(), + )])], + ) + .await + .unwrap_err(); + + assert!(matches!( + error, + paimon::Error::DataInvalid { message, .. } + if message.contains("default.managed_table") + )); +} + +#[tokio::test] +async fn test_rest_catalog_batches_list_partitions_by_names() { + let ctx = setup_catalog(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + let partition_specs = (0..1001) + .map(|value| HashMap::from([("dt".to_string(), value.to_string())])) + .collect::>(); + + ctx.catalog + .list_partitions_by_names(&identifier, partition_specs) + .await + .unwrap(); + + assert_eq!( + ctx.server.list_partitions_by_names_call_sizes(), + vec![1000, 1] + ); +} + +#[tokio::test] +async fn test_rest_catalog_partition_operations_map_missing_table() { + let ctx = setup_catalog(vec!["default"]).await; + let identifier = Identifier::new("default", "missing"); + let partition_specs = vec![HashMap::from([( + "dt".to_string(), + "2026-07-22".to_string(), + )])]; + + let create_error = ctx + .catalog + .create_partitions(&identifier, partition_specs.clone(), false) + .await + .unwrap_err(); + let drop_error = ctx + .catalog + .drop_partitions(&identifier, partition_specs.clone()) + .await + .unwrap_err(); + let list_error = ctx + .catalog + .list_partitions_by_names(&identifier, partition_specs) + .await + .unwrap_err(); + + for error in [create_error, drop_error, list_error] { + assert!(matches!( + error, + paimon::Error::TableNotExist { full_name } if full_name == "default.missing" + )); + } +} + // ==================== Database Tests ==================== #[tokio::test] @@ -579,6 +965,404 @@ async fn test_rest_catalog_reads_format_table() { ); } +#[tokio::test] +async fn test_rest_catalog_marks_internal_format_table_as_catalog_managed() { + let ctx = setup_catalog(vec!["default"]).await; + let schema = Schema::builder() + .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) + .column("id", DataType::BigInt(BigIntType::new())) + .partition_keys(["dt"]) + .option("type", "format-table") + .option("file.format", "parquet") + .option("metastore.partitioned-table", "true") + .build() + .unwrap(); + let identifier = Identifier::new("default", "managed_format"); + ctx.server.add_table_with_schema( + "default", + "managed_format", + schema, + "memory:/managed_format", + ); + ctx.server + .set_table_external("default", "managed_format", false); + + let table = ctx.catalog.get_table(&identifier).await.unwrap(); + + assert!(table.has_catalog_managed_partitions()); + assert!(table + .copy_with_options(HashMap::from([( + "metastore.partitioned-table".to_string(), + "false".to_string(), + )])) + .has_catalog_managed_partitions()); +} + +#[tokio::test] +async fn test_plain_rest_env_does_not_assert_internal_catalog_managed_table_provenance() { + let ctx = setup_catalog(vec!["default"]).await; + let schema = Schema::builder() + .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) + .column("id", DataType::BigInt(BigIntType::new())) + .partition_keys(["dt"]) + .option("type", "format-table") + .option("file.format", "parquet") + .option("metastore.partitioned-table", "true") + .build() + .unwrap(); + let identifier = Identifier::new("default", "managed_provenance"); + ctx.server.add_table_with_schema( + "default", + "managed_provenance", + schema, + "memory:/managed_provenance", + ); + ctx.server + .set_table_external("default", "managed_provenance", false); + + let loaded = ctx.catalog.get_table(&identifier).await.unwrap(); + let plain_rest_env = RESTEnv::new( + identifier.clone(), + "manual-uuid".to_string(), + loaded.rest_env().unwrap().api().clone(), + Options::new(), + false, + ); + let manually_constructed = Table::new( + loaded.file_io().clone(), + identifier, + loaded.location().to_string(), + loaded.schema().clone(), + Some(plain_rest_env), + ); + + assert!(!manually_constructed.has_catalog_managed_partitions()); +} + +#[cfg(not(windows))] +#[tokio::test] +async fn test_managed_format_partition_listing_does_not_fallback_to_filesystem() { + let temp_dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(temp_dir.path().join("dt=2026-07-22")).unwrap(); + let table_path = format!("file://{}", temp_dir.path().display()); + let ctx = setup_catalog(vec!["default"]).await; + let schema = Schema::builder() + .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) + .column("id", DataType::BigInt(BigIntType::new())) + .partition_keys(["dt"]) + .option("type", "format-table") + .option("file.format", "parquet") + .option("metastore.partitioned-table", "true") + .build() + .unwrap(); + let identifier = Identifier::new("default", "managed_no_fallback"); + ctx.server + .add_table_with_schema("default", "managed_no_fallback", schema, &table_path); + ctx.server + .set_table_external("default", "managed_no_fallback", false); + ctx.server + .set_list_partitions_error_status(Some(StatusCode::NOT_IMPLEMENTED)); + + let error = ctx.catalog.list_partitions(&identifier).await.unwrap_err(); + + assert!(matches!( + error, + paimon::Error::RestApi { + source: paimon::api::RestError::NotImplemented { .. } + } + )); + + let paged_error = ctx + .catalog + .list_partitions_paged(&identifier, Some(10), None) + .await + .unwrap_err(); + assert!(matches!( + paged_error, + paimon::Error::RestApi { + source: paimon::api::RestError::NotImplemented { .. } + } + )); +} + +#[tokio::test] +async fn test_rest_catalog_rejects_external_catalog_managed_format_table() { + let ctx = setup_catalog(vec!["default"]).await; + let schema = Schema::builder() + .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) + .column("id", DataType::BigInt(BigIntType::new())) + .partition_keys(["dt"]) + .option("type", "format-table") + .option("file.format", "parquet") + .option("metastore.partitioned-table", "true") + .build() + .unwrap(); + let identifier = Identifier::new("default", "external_managed_format"); + ctx.server.add_table_with_schema( + "default", + "external_managed_format", + schema, + "memory:/external_managed_format", + ); + + let error = ctx.catalog.get_table(&identifier).await.unwrap_err(); + + assert!(matches!( + error, + paimon::Error::DataInvalid { message, .. } + if message.contains("default.external_managed_format") + && message.contains("internal") + )); +} + +#[tokio::test] +async fn test_rest_catalog_rejects_engine_managed_format_table() { + let ctx = setup_catalog(vec!["default"]).await; + let schema = Schema::builder() + .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) + .column("id", DataType::BigInt(BigIntType::new())) + .partition_keys(["dt"]) + .option("type", "format-table") + .option("file.format", "parquet") + .option("metastore.partitioned-table", "true") + .option("format-table.implementation", "engine") + .build() + .unwrap(); + let identifier = Identifier::new("default", "engine_managed_format"); + ctx.server.add_table_with_schema( + "default", + "engine_managed_format", + schema, + "memory:/engine_managed_format", + ); + ctx.server + .set_table_external("default", "engine_managed_format", false); + + let error = ctx.catalog.get_table(&identifier).await.unwrap_err(); + + assert!(matches!( + error, + paimon::Error::DataInvalid { message, .. } + if message.contains("metastore.partitioned-table") + && message.contains("format-table.implementation=engine") + )); +} + +#[cfg(not(windows))] +#[tokio::test] +async fn test_managed_format_scan_hides_unregistered_partition() { + let tmp = tempfile::tempdir().unwrap(); + for dt in ["2026-07-21", "2026-07-22"] { + let partition_dir = tmp.path().join(format!("dt={dt}")); + std::fs::create_dir_all(&partition_dir).unwrap(); + std::fs::write(partition_dir.join("part-0.parquet"), b"data").unwrap(); + } + let table_path = format!("file://{}", tmp.path().display()); + + let ctx = setup_catalog(vec!["default"]).await; + let schema = Schema::builder() + .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) + .column("id", DataType::BigInt(BigIntType::new())) + .partition_keys(["dt"]) + .option("type", "format-table") + .option("file.format", "parquet") + .option("metastore.partitioned-table", "true") + .build() + .unwrap(); + let identifier = Identifier::new("default", "managed_visibility"); + ctx.server + .add_table_with_schema("default", "managed_visibility", schema, &table_path); + ctx.server + .set_table_external("default", "managed_visibility", false); + ctx.server.set_table_partitions( + "default", + "managed_visibility", + vec![HashMap::from([( + "dt".to_string(), + "2026-07-22".to_string(), + )])], + ); + + let table = ctx.catalog.get_table(&identifier).await.unwrap(); + let plan = table.new_read_builder().new_scan().plan().await.unwrap(); + + assert_eq!(plan.splits().len(), 1); + assert!(plan.splits()[0].bucket_path().ends_with("/dt=2026-07-22")); +} + +#[cfg(not(windows))] +#[tokio::test] +async fn test_managed_format_scan_preserves_loaded_partition_layout_in_dynamic_copy() { + let tmp = tempfile::tempdir().unwrap(); + let partition_dir = tmp.path().join("dt=2026-07-22"); + std::fs::create_dir_all(&partition_dir).unwrap(); + std::fs::write(partition_dir.join("part-0.parquet"), b"data").unwrap(); + let table_path = format!("file://{}", tmp.path().display()); + + let ctx = setup_catalog(vec!["default"]).await; + let schema = Schema::builder() + .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) + .column("id", DataType::BigInt(BigIntType::new())) + .partition_keys(["dt"]) + .option("type", "format-table") + .option("file.format", "parquet") + .option("metastore.partitioned-table", "true") + .build() + .unwrap(); + let identifier = Identifier::new("default", "managed_dynamic_layout"); + ctx.server + .add_table_with_schema("default", "managed_dynamic_layout", schema, &table_path); + ctx.server + .set_table_external("default", "managed_dynamic_layout", false); + ctx.server.set_table_partitions( + "default", + "managed_dynamic_layout", + vec![HashMap::from([( + "dt".to_string(), + "2026-07-22".to_string(), + )])], + ); + + let table = ctx.catalog.get_table(&identifier).await.unwrap(); + let copied = table.copy_with_options(HashMap::from([ + ( + "format-table.partition-path-only-value".to_string(), + "true".to_string(), + ), + ("path".to_string(), "memory:/wrong-path".to_string()), + ("type".to_string(), "table".to_string()), + ])); + let plan = copied.new_read_builder().new_scan().plan().await.unwrap(); + + assert_eq!(plan.splits().len(), 1); + assert!(plan.splits()[0].bucket_path().ends_with("/dt=2026-07-22")); +} + +#[cfg(not(windows))] +#[tokio::test] +async fn test_managed_format_scan_treats_registered_missing_directory_as_empty() { + let tmp = tempfile::tempdir().unwrap(); + let table_path = format!("file://{}", tmp.path().display()); + + let ctx = setup_catalog(vec!["default"]).await; + let schema = Schema::builder() + .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) + .column("id", DataType::BigInt(BigIntType::new())) + .partition_keys(["dt"]) + .option("type", "format-table") + .option("file.format", "parquet") + .option("metastore.partitioned-table", "true") + .build() + .unwrap(); + let identifier = Identifier::new("default", "managed_missing_directory"); + ctx.server + .add_table_with_schema("default", "managed_missing_directory", schema, &table_path); + ctx.server + .set_table_external("default", "managed_missing_directory", false); + ctx.server.set_table_partitions( + "default", + "managed_missing_directory", + vec![HashMap::from([( + "dt".to_string(), + "2026-07-22".to_string(), + )])], + ); + + let table = ctx.catalog.get_table(&identifier).await.unwrap(); + let plan = table.new_read_builder().new_scan().plan().await.unwrap(); + + assert!(plan.splits().is_empty()); +} + +#[cfg(not(windows))] +#[tokio::test] +async fn test_managed_format_scan_propagates_partition_listing_error() { + let tmp = tempfile::tempdir().unwrap(); + let partition_dir = tmp.path().join("dt=2026-07-22"); + std::fs::create_dir_all(&partition_dir).unwrap(); + std::fs::write(partition_dir.join("part-0.parquet"), b"data").unwrap(); + let table_path = format!("file://{}", tmp.path().display()); + + let ctx = setup_catalog(vec!["default"]).await; + let schema = Schema::builder() + .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) + .column("id", DataType::BigInt(BigIntType::new())) + .partition_keys(["dt"]) + .option("type", "format-table") + .option("file.format", "parquet") + .option("metastore.partitioned-table", "true") + .build() + .unwrap(); + let identifier = Identifier::new("default", "managed_scan_error"); + ctx.server + .add_table_with_schema("default", "managed_scan_error", schema, &table_path); + ctx.server + .set_table_external("default", "managed_scan_error", false); + ctx.server + .set_list_partitions_error_status(Some(StatusCode::NOT_IMPLEMENTED)); + + let table = ctx.catalog.get_table(&identifier).await.unwrap(); + let error = table + .new_read_builder() + .new_scan() + .plan() + .await + .unwrap_err(); + + assert!(matches!( + error, + paimon::Error::RestApi { + source: paimon::api::RestError::NotImplemented { .. } + } + )); +} + +#[cfg(not(windows))] +#[tokio::test] +async fn test_managed_format_scan_reports_table_for_malformed_partition_metadata() { + let tmp = tempfile::tempdir().unwrap(); + let table_path = format!("file://{}", tmp.path().display()); + + let ctx = setup_catalog(vec!["default"]).await; + let schema = Schema::builder() + .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) + .column("id", DataType::BigInt(BigIntType::new())) + .partition_keys(["dt"]) + .option("type", "format-table") + .option("file.format", "parquet") + .option("metastore.partitioned-table", "true") + .build() + .unwrap(); + let identifier = Identifier::new("default", "managed_corrupt_metadata"); + ctx.server + .add_table_with_schema("default", "managed_corrupt_metadata", schema, &table_path); + ctx.server + .set_table_external("default", "managed_corrupt_metadata", false); + ctx.server.set_table_partitions( + "default", + "managed_corrupt_metadata", + vec![HashMap::from([ + ("dt".to_string(), "2026-07-22".to_string()), + ("unexpected".to_string(), "value".to_string()), + ])], + ); + + let table = ctx.catalog.get_table(&identifier).await.unwrap(); + let error = table + .new_read_builder() + .new_scan() + .plan() + .await + .unwrap_err(); + + assert!( + error + .to_string() + .contains("default.managed_corrupt_metadata"), + "expected table context, got: {error}" + ); +} + #[cfg(not(windows))] #[tokio::test] async fn test_rest_catalog_prunes_format_table_partition_filter() { diff --git a/crates/paimon/tests/rest_object_models_test.rs b/crates/paimon/tests/rest_object_models_test.rs index f4e3af1e..522d10c1 100644 --- a/crates/paimon/tests/rest_object_models_test.rs +++ b/crates/paimon/tests/rest_object_models_test.rs @@ -17,11 +17,39 @@ use std::collections::HashMap; -use paimon::api::{CreateFunctionRequest, CreateViewRequest}; +use paimon::api::{ + CreateFunctionRequest, CreatePartitionsRequest, CreatePartitionsResponse, CreateViewRequest, + DropPartitionsRequest, DropPartitionsResponse, ListPartitionsByNamesRequest, +}; use paimon::catalog::{Function, FunctionDefinition, Identifier, View, ViewSchema}; use paimon::spec::DataField; use serde_json::json; +#[test] +fn partition_rest_models_are_public_api() { + let spec = HashMap::from([("dt".to_string(), "2026-07-22".to_string())]); + + let create = CreatePartitionsRequest::new(vec![spec.clone()], false); + let drop = DropPartitionsRequest::new(vec![spec.clone()], false); + let list = ListPartitionsByNamesRequest::new(vec![spec.clone()]); + let created: CreatePartitionsResponse = serde_json::from_value(json!({ + "created": [spec.clone()], + "existed": [] + })) + .unwrap(); + let dropped: DropPartitionsResponse = serde_json::from_value(json!({ + "dropped": [spec], + "missing": [] + })) + .unwrap(); + + assert!(!create.ignore_if_exists); + assert!(!drop.ignore_if_not_exists); + assert_eq!(list.partition_specs.len(), 1); + assert_eq!(created.created.len(), 1); + assert_eq!(dropped.dropped.len(), 1); +} + #[test] fn construct_view_schema_contract() { let fields: Vec = serde_json::from_value(json!([ diff --git a/docs/designs/2026-07-22-rest-format-table-partition-commands.md b/docs/designs/2026-07-22-rest-format-table-partition-commands.md new file mode 100644 index 00000000..2896490b --- /dev/null +++ b/docs/designs/2026-07-22-rest-format-table-partition-commands.md @@ -0,0 +1,292 @@ +# REST Format Table Partition Commands Design + +## Problem Statement + +`paimon-rust` can read a REST-backed `type=format-table`, and the REST client can +list table partitions. It does not yet support the catalog-managed partition +contract used by internal REST Format Tables: + +- `metastore.partitioned-table=true` makes the REST catalog the source of truth + for partition visibility. +- The client needs partition registration and unregistration APIs. +- DataFusion needs Spark-compatible partition administration commands, + including `SHOW PARTITIONS`, `ALTER TABLE ... ADD/DROP PARTITION`, and + `MSCK REPAIR TABLE`. + +Adding SQL commands without changing scan visibility would be inconsistent: +`SHOW PARTITIONS` would display catalog registrations while `SELECT` would +still discover every directory from the filesystem. + +## Chosen Approach + +Implement a REST-first vertical slice: + +1. Add the minimal partition operations to the REST API and `Catalog` contract. +2. Recognize catalog-managed internal REST Format Tables at load time. +3. Make their scans use REST partition registrations without filesystem + fallback. +4. Add Spark-compatible partition commands to `paimon-datafusion`. + +Do not introduce Java Paimon's serializable `FormatTablePartitionManager` +abstraction. Rust Format Table writes are currently unsupported, and this +feature targets REST-backed administration and reads only. + +## Design Details + +### Managed Table Capability + +A table has catalog-managed partitions only when all of the following are true: + +- `type=format-table` +- `metastore.partitioned-table=true` +- `format-table.implementation` is not `engine` +- the table is internal (`isExternal=false`) +- the table was loaded from a REST catalog + +Loading fails if an external REST Format Table requests catalog-managed +partitions, or if it combines catalog-managed partitions with +`format-table.implementation=engine`, whose contract is filesystem discovery +by the engine. A non-REST table cannot acquire the capability because it has +no `RESTEnv`. A plain public `RESTEnv::new` also does not assert this capability. +Catalog implementations that use `RESTEnv::new_for_table` must pass the +server-provided schema and `isExternal` value unchanged. This public +constructor is an explicit catalog trust boundary rather than an independent +metadata verifier; the built-in `RESTCatalog` invokes it only with the table +response it just loaded. + +`Table` exposes a predicate describing the effective capability. Runtime code +uses the effective capability rather than checking the option alone. Copies +retain the loaded table path, file format, and partition layout for managed +scans, so dynamic options cannot silently redirect them to a different +directory or value-only layout. + +### REST Partition Protocol + +Add wire-compatible paths and models: + +- `POST .../partitions` + - request: `partitionSpecs`, `ignoreIfExists` + - response: `created`, `existed` +- `POST .../partitions/drop` + - request: `partitionSpecs`, `ignoreIfNotExists` + - response: `dropped`, `missing` +- `POST .../partitions/list-by-names` + - request: `specs` + - response: existing `ListPartitionsResponse` + +The existing paged `GET .../partitions` remains the listing primitive for the +first version. An empty next-page token terminates pagination, and a repeated +non-empty token fails fast instead of looping forever. Predicate transport and +`list-by-filter` are deferred. + +`Catalog` gains minimal async methods for create, drop, and list-by-names. +Default implementations return `Unsupported`; `RESTCatalog` maps REST errors to +catalog-level errors. + +Catalog services cap a request at 1000 partition specs, and the low-level +`RESTApi` rejects larger single requests locally. Strict create remains one +request so a conflict rejects the whole SQL batch and therefore cannot exceed +1000 specs. Idempotent create, drop, and list-by-names are split into batches of +1000; a failure may therefore occur after earlier idempotent batches have +succeeded, and retrying converges. + +### Managed Format Table Scan + +For a partitioned managed Format Table, planning: + +1. Lists every registered partition from `RESTEnv`. +2. Validates each returned spec is complete and contains only declared + partition keys. +3. Converts raw registered values into partition rows. +4. Applies the existing local partition predicate. +5. Builds one scan root per surviving registration. + +An unregistered directory is invisible. A registered partition whose directory +does not exist reads as empty only when storage reports `NotFound`. Other +partition-directory listing failures propagate. REST listing errors also +propagate; managed scans never fall back to filesystem discovery. + +Unmanaged Format Tables retain the current filesystem behavior. + +### SQL Surface + +`SQLContext` supports: + +```sql +SHOW PARTITIONS table [PARTITION (...)] + +ALTER TABLE table +ADD [IF NOT EXISTS] PARTITION (...) [PARTITION (...) ...] + +ALTER TABLE table +DROP [IF EXISTS] PARTITION (...) +[, PARTITION (...) ...] + +[MSCK] REPAIR TABLE table +[{ADD|DROP|SYNC} PARTITIONS] +``` + +`sqlparser` already represents `MSCK`, ADD PARTITION, and DROP PARTITION. +Compatibility handling rewrites bare Spark `REPAIR TABLE` into the parser's +`MSCK REPAIR TABLE` form. Spark's comma-separated DROP syntax is rewritten into +multiple `DropPartitions` AST operations and then executed as one resolved +batch. `SHOW PARTITIONS` receives a small dedicated parser because it is not +represented by the current parser AST. + +Every command is accepted only for a partitioned Format Table with the +effective catalog-managed REST capability. + +### SHOW PARTITIONS + +`SHOW PARTITIONS` reads REST registrations and returns one non-null UTF-8 column +named `partition`. Each name uses declared partition-key order: + +```text +dt=2026-07-22/hour=10 +``` + +Rows are sorted lexicographically by the complete partition name, with one row +per REST registration. Type-equivalent raw registrations may therefore produce +duplicate displayed names. Registered values are cast through their +partition-column types before display and filter comparison: default-partition +values display as `null`, dates display as `yyyy-MM-dd`, and numeric text such +as `01` displays as `1`. Non-character casts ignore surrounding whitespace. +`CHAR` values use Spark write-side padding and both `CHAR` and `VARCHAR` +enforce their declared character limits. + +An optional spec filters registrations locally. It may be partial and does not +need to be a leading prefix. + +### ADD PARTITION + +ADD requires complete specs. + +1. Convert SQL constants to strings and cast them through the partition column + types, matching Spark partition-spec normalization. +2. Validate the resulting path remains under the table location. +3. Send the batch to REST with the SQL `IF NOT EXISTS` flag. +4. Create each partition directory after successful registration. + +Strict ADD is never split because the REST contract rejects the whole batch +atomically if any partition already exists, so one statement may contain at +most 1000 specs. `IF NOT EXISTS` ADD is split into 1000-spec requests. SQL +`NULL` and blank character values map to `partition.default-name`. Directory +creation failure leaves the registration visible but its missing directory +reads as empty; retrying with `IF NOT EXISTS` continues directory creation. + +For the supported scalar types, non-character string casts trim surrounding +whitespace. `CHAR(n)` pads short values; `CHAR(n)` and `VARCHAR(n)` trim only +excess ASCII trailing spaces and reject remaining overlong content. Typed +`DATE` literals are converted to their canonical date string before the target +partition-column cast. Other typed literals are rejected until their Spark +source-type formatting is implemented. Paimon `DATE` values remain within the +declared `0000-01-01..9999-12-31` range. + +Partition path components use Java-compatible escaping: Unicode is preserved, +while reserved characters such as `/`, `=`, `%`, and `}` are percent-encoded. +`LOCATION` is rejected for Format Tables. + +### DROP PARTITION + +One or more complete or partial specs are supported. + +1. Resolve complete specs with `list-by-names`. If an exact lookup misses a + type-equivalent raw value registered by MSCK (for example integer `01`, + surrounding whitespace on a non-character value, or an unpadded `CHAR` + value), fall back to one full traversal. Any partial spec also uses one full + traversal. Exact-vs-fallback resolution is tracked independently for each + requested spec, so an exact complete match never expands to typed aliases + merely because another complete miss or partial spec requires the full + traversal. +2. Expand each partial spec to exact registered leaf partitions. +3. For complete missing specs, fail unless `IF EXISTS` was specified. +4. Unregister the exact resolved set. +5. Delete those partition directories. + +Only registered partitions are deleted. Data in an unregistered directory is +never removed by DROP. Unregistering precedes deletion so a partial deletion +failure cannot expose the remaining files through managed scans. +REST unregister requests are split into batches of 1000. `PURGE` is rejected +before any catalog access. + +### MSCK Repair + +The repair engine lives in `paimon-datafusion`; it is SQL/engine workflow rather +than a general core catalog abstraction. + +It discovers raw filesystem partition specs without casting directory values +through their logical data types. This preserves values such as `month=01`. +Each segment that matches the declared partition layout must also round-trip +exactly through permissive Java-compatible unescaping and canonical escaping. +Malformed, lowercase, or otherwise over-escaped percent spellings fail repair +before mutation: the REST partition model stores the raw logical spec but has +no separate physical-path spelling to make a noncanonical directory readable +later. Unrelated directory names that do not decode to the expected partition +key remain ignored. + +It fully lists both filesystem and REST state before mutation, then computes: + +```text +ADD = filesystem - registered +DROP = registered - filesystem +``` + +Plain MSCK defaults to ADD. SYNC applies ADD before DROP. Operations are sorted +by canonical partition path for deterministic behavior. ADD is idempotent; +DROP ignores missing registrations, so a rerun converges after a partial +failure. Their REST calls use the same 1000-spec batching rules. + +MSCK changes metadata only and never creates or deletes partition directories. + +## Error Handling + +- Missing table: preserve `TableNotExist`. +- Non-REST or unmanaged Format Table: explicit `Unsupported`. +- External table requesting managed partitions: `DataInvalid`. +- REST 404/403/409/400: map to the existing catalog error taxonomy. +- Malformed or incomplete REST partition metadata: fail closed with table + context. +- Filesystem or REST listing failure during MSCK: perform no mutation. +- Noncanonical filesystem partition paths that cannot round-trip through REST + metadata: fail before mutation. + +## Testing + +Core/REST tests cover: + +- request and response JSON shapes; +- resource paths; +- REST catalog create/drop/list-by-names behavior and error mapping; +- empty and repeated partition-list page tokens; +- managed-table load validation; +- plain `RESTEnv` construction not asserting managed internal-table provenance; +- managed scans hiding unregistered directories; +- registered missing directories reading as empty; +- non-`NotFound` directory listing failures propagating; +- listing failures not falling back to the filesystem; +- malformed registration context and dynamic-layout copies. + +DataFusion tests cover: + +- parser compatibility, including Spark comma-separated DROP; +- typed SHOW output and partial filtering; +- strict/idempotent ADD, Spark whitespace and `CHAR`/`VARCHAR` normalization, + typed-literal conversion, and Paimon date-range validation; +- complete, partial, missing, multi-spec, mixed complete/partial, and raw-value + DROP; +- path safety; +- bare/ADD/DROP/SYNC MSCK; +- raw directory values and rejection of non-round-tripping paths; +- fail-closed listing and idempotent repair reruns. + +## Open Questions + +None for the first version. + +## Out of Scope + +- Format Table writes and automatic post-write registration. +- Non-REST catalogs. +- A general `FormatTablePartitionManager`. +- REST predicate/list-by-filter pushdown. +- Procedures, dry-run, or per-partition repair reports. diff --git a/docs/plans/2026-07-22-rest-format-table-partition-commands.md b/docs/plans/2026-07-22-rest-format-table-partition-commands.md new file mode 100644 index 00000000..c2e71636 --- /dev/null +++ b/docs/plans/2026-07-22-rest-format-table-partition-commands.md @@ -0,0 +1,209 @@ +# Implementation Plan: REST Format Table Partition Commands + +## Prerequisites + +- [x] Design approved in `docs/designs/2026-07-22-rest-format-table-partition-commands.md`. +- [x] Worktree is clean on `main`. +- [x] Make the Rust toolchain available on `PATH`. + +## Tasks + +### Task 1: Add REST partition wire models [x] + +- **Files**: + - `crates/paimon/src/api/api_request.rs` + - `crates/paimon/src/api/api_response.rs` + - `crates/paimon/src/api/mod.rs` +- **Changes**: + - Add create, drop, and list-by-names request types. + - Add create and drop response types. + - Add serde tests for Java-compatible field names and defaults. +- **Verify**: + - `cargo test --locked -p paimon api:: --lib` +- **Dependencies**: None + +### Task 2: Add REST partition paths and client methods [x] + +- **Files**: + - `crates/paimon/src/api/resource_paths.rs` + - `crates/paimon/src/api/rest_api.rs` + - `crates/paimon/tests/rest_api_test.rs` +- **Changes**: + - Add `/partitions/drop` and `/partitions/list-by-names`. + - Add create, drop, and list-by-names client methods. + - Assert HTTP methods, paths, and JSON bodies with the mock server. +- **Verify**: + - `cargo test --locked -p paimon --test rest_api_test partition` +- **Dependencies**: Task 1 + +### Task 3: Extend the catalog partition contract [x] + +- **Files**: + - `crates/paimon/src/catalog/mod.rs` + - `crates/paimon/src/catalog/rest/rest_catalog.rs` + - `crates/paimon/tests/rest_catalog_test.rs` +- **Changes**: + - Add default unsupported create/drop/list-by-names methods. + - Implement them in `RESTCatalog`. + - Preserve strict create and metadata-only drop semantics. + - Add REST error-mapping tests. +- **Verify**: + - `cargo test --locked -p paimon --test rest_catalog_test partition` +- **Dependencies**: Task 2 + +### Task 4: Recognize managed internal REST Format Tables [x] + +- **Files**: + - `crates/paimon/src/spec/core_options.rs` + - `crates/paimon/src/table/mod.rs` + - `crates/paimon/src/table/rest_env.rs` + - `crates/paimon/tests/rest_catalog_test.rs` +- **Changes**: + - Parse `metastore.partitioned-table`. + - Record validated internal-table provenance in `RESTEnv` and add the + effective managed-format-table predicate. + - Reject external managed Format Tables and + `format-table.implementation=engine` while loading. + - Keep plain manual `RESTEnv` construction fail-closed. + - Add capability and validation tests. +- **Verify**: + - `cargo test --locked -p paimon --test rest_catalog_test managed_format` +- **Dependencies**: Task 3 + +### Task 5: Make managed Format Table scans catalog-authoritative [x] + +- **Files**: + - `crates/paimon/src/table/format_table_scan.rs` + - `crates/paimon/tests/rest_catalog_test.rs` +- **Changes**: + - List registered partitions through `RESTEnv`. + - Validate and convert raw specs to scan roots. + - Apply local partition filtering. + - Treat missing registered directories as empty. + - Prohibit REST-to-filesystem fallback. + - Preserve the loaded path, format, and partition layout across dynamic + table copies. +- **Verify**: + - `cargo test --locked -p paimon --test rest_catalog_test managed_format` +- **Dependencies**: Task 4 + +### Task 6: Add reusable Format Table partition path helpers [x] + +- **Files**: + - `crates/paimon/src/table/format_table_scan.rs` + - `crates/paimon/src/table/mod.rs` + - `crates/paimon/src/table/format_partition.rs` (new, if extraction is clearer) +- **Changes**: + - Centralize raw path discovery, spec validation, path construction, and + canonical ordering for scan and SQL administration. + - Reject physical path spellings that cannot round-trip through REST + partition metadata and canonical escaping. + - Add focused integration tests for escaped paths, value-only layouts, + hidden directories, and default-partition discovery. +- **Verify**: + - `cargo test --locked -p paimon --test format_partition_test` +- **Dependencies**: Task 5 + +### Task 7: Add SQL parsing for SHOW and REPAIR compatibility [x] + +- **Files**: + - `crates/integrations/datafusion/src/sql_context.rs` +- **Changes**: + - Dispatch `Statement::Msck`. + - Rewrite bare `REPAIR TABLE`. + - Rewrite Spark comma-separated DROP partition clauses into sqlparser AST + operations. + - Parse Spark-compatible `SHOW PARTITIONS ... [PARTITION (...)]`. + - Add parser-only tests before execution handlers. +- **Verify**: + - `cargo test --locked -p paimon-datafusion --lib sql_context::tests::test_parse_` +- **Dependencies**: Task 4 + +### Task 8: Implement SHOW PARTITIONS [x] + +- **Files**: + - `crates/integrations/datafusion/src/sql_context.rs` +- **Changes**: + - Validate the target capability. + - Cast REST values through partition-column types, apply optional partial + filtering, sort, and return the `partition` DataFrame. + - Add success and rejection tests. +- **Verify**: + - `cargo test --locked -p paimon-datafusion show_partitions` +- **Dependencies**: Tasks 3, 7 + +### Task 9: Implement ADD PARTITION [x] + +- **Files**: + - `crates/integrations/datafusion/src/sql_context.rs` +- **Changes**: + - Handle `AlterTableOperation::AddPartitions`. + - Normalize Spark-style literals, NULL/default values, and complete typed + specs, including non-character whitespace trimming, `CHAR`/`VARCHAR` + length semantics, typed `DATE` conversion, and the Paimon `DATE` range; + reject LOCATION. + - Keep strict registration atomic, batch idempotent REST requests by 1000, + and then create Java-compatible escaped directories. + - Add strict, IF NOT EXISTS, and path-safety tests. +- **Verify**: + - `cargo test --locked -p paimon-datafusion add_format_partition` +- **Dependencies**: Tasks 3, 6, 7 + +### Task 10: Implement managed DROP PARTITION [x] + +- **Files**: + - `crates/integrations/datafusion/src/sql_context.rs` +- **Changes**: + - Route managed Format Tables away from snapshot truncation. + - Resolve multiple complete/partial specs against REST registrations, + including typed fallback for raw values registered by MSCK, such as + leading-zero or whitespace-padded numerics and unpadded `CHAR` values. + - Keep exact complete matches exact when another requested spec triggers a + full-list fallback. + - Enforce IF EXISTS, unregister first, then delete directories. + - Add complete, partial, missing, and unregistered-data tests. +- **Verify**: + - `cargo test --locked -p paimon-datafusion drop_format_partition` +- **Dependencies**: Tasks 3, 6, 7 + +### Task 11: Implement MSCK repair [x] + +- **Files**: + - `crates/integrations/datafusion/src/format_partition_repair.rs` (new) + - `crates/integrations/datafusion/src/lib.rs` + - `crates/integrations/datafusion/src/sql_context.rs` +- **Changes**: + - Discover raw filesystem specs. + - Fail before mutation when a physical partition path cannot round-trip + through canonical REST partition metadata. + - Fully list REST registrations before mutation. + - Compute deterministic ADD/DROP/SYNC diffs. + - Add metadata-only, raw-value, failure, and rerun tests. +- **Verify**: + - `cargo test --locked -p paimon-datafusion --lib sql_context::tests::test_msck_` +- **Dependencies**: Tasks 3, 6, 7 + +### Task 12: Document the SQL surface [x] + +- **Files**: + - `docs/src/sql.md` +- **Changes**: + - Document capability requirements, syntax, examples, and metadata/data + semantics. +- **Verify**: + - `rg "MSCK REPAIR TABLE|SHOW PARTITIONS|metastore.partitioned-table" docs/src/sql.md` +- **Dependencies**: Tasks 8-11 + +## Post-Implementation + +- [x] `cargo fmt --all -- --check` +- [x] `cargo test --locked -p paimon --all-targets` +- [x] `cargo test --locked -p paimon-datafusion --lib` +- [ ] `cargo test --locked -p paimon-datafusion --all-targets` + - Executed with `--no-fail-fast`; every target completed. Only + `vector_search_tests::test_vector_search_top3` and + `vector_search_tests::test_vector_search_top6_returns_all` failed because + `liblumina_py.so` is unavailable in the test environment. +- [x] `cargo clippy --locked -p paimon -p paimon-datafusion --all-targets -- -D warnings` +- [x] Review the complete diff and run `dlf-review-quick` and + `dlf-review-zen`. diff --git a/docs/src/sql.md b/docs/src/sql.md index ac810727..eeea63a0 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -40,7 +40,7 @@ Mosaic support is always available and currently read-only. SQL queries can read SQL support has two layers: - DataFusion provides the parser, query planner, optimizer, execution engine, expressions, scalar functions, aggregate functions, and window functions. SQL statements that `SQLContext` does not intercept are delegated to DataFusion. This includes the DataFusion SQL surface for `SELECT` queries, CTEs (including recursive CTEs), subqueries, joins including `LATERAL` joins, SQL lambda functions, grouping, `HAVING`, window clauses, `QUALIFY`, set operations, `ORDER BY`, `LIMIT`/`OFFSET`, `EXPLAIN`, information-schema commands such as `SHOW TABLES`, `DESCRIBE`, `COPY`, and ordinary `INSERT`. -- Paimon-specific table management and row-level writes are implemented by `SQLContext`. This includes Paimon `CREATE TABLE`, `ALTER TABLE`, `DROP TABLE`, `CREATE TEMPORARY TABLE`, `CREATE TEMPORARY VIEW`, REST Catalog persistent `CREATE VIEW`, `DROP VIEW`, and `CREATE FUNCTION`, `DROP TEMPORARY TABLE` / `VIEW`, `INSERT OVERWRITE ... PARTITION`, `UPDATE`, `DELETE`, `MERGE INTO`, `TRUNCATE TABLE`, `ALTER TABLE ... DROP PARTITION`, `CALL sys.*`, Paimon time travel, and `SET` / `RESET 'paimon.*'`. +- Paimon-specific table management and row-level writes are implemented by `SQLContext`. This includes Paimon `CREATE TABLE`, `ALTER TABLE`, `DROP TABLE`, `CREATE TEMPORARY TABLE`, `CREATE TEMPORARY VIEW`, REST Catalog persistent `CREATE VIEW`, `DROP VIEW`, and `CREATE FUNCTION`, REST-managed Format Table partition commands (`SHOW PARTITIONS`, `ADD/DROP PARTITION`, and `MSCK REPAIR TABLE`), `DROP TEMPORARY TABLE` / `VIEW`, `INSERT OVERWRITE ... PARTITION`, `UPDATE`, `DELETE`, `MERGE INTO`, `TRUNCATE TABLE`, `CALL sys.*`, Paimon time travel, and `SET` / `RESET 'paimon.*'`. Not every DataFusion DDL/DML statement maps to a Paimon table operation. For Paimon catalogs, `CREATE EXTERNAL TABLE`, `LOCATION`, `CREATE MATERIALIZED VIEW`, and persistent `CREATE TABLE AS SELECT` are rejected or not implemented. Persistent `CREATE FUNCTION` is supported only for the REST Catalog SQL scalar form documented below. DataFusion `COPY` can export query results to files; it does not create or commit Paimon table files. @@ -703,6 +703,153 @@ ALTER TABLE paimon.my_db.users SET TBLPROPERTIES('data-evolution.enabled' = 'tru ALTER TABLE IF EXISTS paimon.my_db.users ADD COLUMN age INT; ``` +### REST-managed Format Table partitions + +`SQLContext` supports Spark-compatible partition administration for a +catalog-managed Format Table. The target must satisfy all of these conditions: + +- it was loaded from a REST Catalog; +- it is an internal table (`isExternal=false`); +- it has `type=format-table`; +- it is partitioned; +- it has `metastore.partitioned-table=true`; +- it does not use `format-table.implementation=engine`. + +For such a table, REST partition registrations are authoritative for both +`SHOW PARTITIONS` and scans. A directory that is not registered is invisible, +while a registered partition whose directory is absent reads as empty when +storage reports `NotFound`. Other partition-directory listing failures +propagate instead of being treated as empty. Filesystem-discovered Format +Tables and non-REST catalogs do not support these commands. Dynamic query +options cannot change the validated partition source, table path, file format, +or value-only partition layout of a loaded managed table. + +#### SHOW PARTITIONS + +```sql +SHOW PARTITIONS paimon.my_db.events; +SHOW PARTITIONS paimon.my_db.events +PARTITION (dt = '2026-07-22'); +SHOW PARTITIONS paimon.my_db.events +PARTITION (region = 'us'); +``` + +The optional filter may be partial and does not need to be a leading partition +prefix. Results contain one non-null string column named `partition`. Each name +uses declared partition-key order, and rows are sorted lexicographically by the +complete name, for example `dt=2026-07-22/region=us`. The result contains one +row per REST registration, so type-equivalent raw registrations can produce +duplicate displayed names. + +REST values are cast through their partition-column types before display and +filtering. Default-partition values display as `null`, dates as `yyyy-MM-dd`, +and an integer directory value such as `01` displays as `1`. Type-equivalent +raw values registered by MSCK are normalized for display and filtering; for +example, surrounding whitespace is ignored for numeric, boolean, and date +casts, while character values preserve their whitespace. + +#### ADD PARTITION + +```sql +ALTER TABLE paimon.my_db.events +ADD PARTITION (dt = '2026-07-22', region = 'us'); + +ALTER TABLE paimon.my_db.events +ADD IF NOT EXISTS +PARTITION (dt = '2026-07-22', region = 'us') +PARTITION (dt = '2026-07-23', region = 'eu'); +``` + +Every ADD specification must contain all partition columns. The complete batch +is registered through REST before its partition directories are created. +Strict ADD leaves duplicate detection to the REST catalog; `IF NOT EXISTS` +makes retries converge. SQL constants are converted to strings and cast through +the partition-column types, so quoted numeric/boolean values and numeric values +for string columns are accepted. Integer types from `TINYINT` through `BIGINT`, +`BOOLEAN`, `CHAR`/`VARCHAR`, and `DATE` are supported. SQL `NULL` and +blank character values map to `partition.default-name`. + +Normalization follows Spark partition-literal behavior for the supported +types. Integer SQL literals lose redundant leading zeros even when the target +column is character data (`VARCHAR label = 01` becomes `label=1`; a `CHAR` +target also applies its fixed-width padding). String inputs to integer, +boolean, and date columns ignore surrounding whitespace, while `CHAR` and +`VARCHAR` values preserve it. `CHAR(n)` right-pads short values with ASCII +spaces. Both `CHAR(n)` and `VARCHAR(n)` count Unicode characters, reject +overlong non-space content, and trim only the excess ASCII trailing spaces +needed to fit `n`. Boolean values accept `t`/`true`/`y`/`yes`/`1` and +`f`/`false`/`n`/`no`/`0`, case-insensitively. Date strings may omit the month +or day. Typed literals are normalized before the partition-column cast, so +`DATE '2026'` becomes `2026-01-01` even for a `CHAR`/`VARCHAR` partition +column. Compact numeric dates such as `20260722` are rejected instead of being +interpreted as `yyyyMMdd`. Paimon `DATE` partition values are limited to +`0000-01-01` through `9999-12-31`. Typed literals other than `DATE` are +rejected until their Spark source-type formatting is implemented. + +Strict ADD is sent as one REST request to preserve whole-batch conflict +semantics and therefore accepts at most 1000 specs. `IF NOT EXISTS` ADD is +split into requests of at most 1000 specs. +Partition values use Java-compatible path escaping: Unicode is preserved and +reserved characters such as `/`, `=`, `%`, and `}` are percent-encoded. +`LOCATION` is not supported. + +#### DROP PARTITION + +```sql +-- Complete specification +ALTER TABLE paimon.my_db.events +DROP PARTITION (dt = '2026-07-22', region = 'us'); + +-- Partial, including a non-leading key +ALTER TABLE paimon.my_db.events +DROP PARTITION (region = 'us'); + +-- Ignore a missing complete registration +ALTER TABLE paimon.my_db.events +DROP IF EXISTS PARTITION (dt = '2026-07-22', region = 'us'); + +-- Spark-compatible multi-partition DROP +ALTER TABLE paimon.my_db.events +DROP PARTITION (dt = '2026-07-22', region = 'us'), + PARTITION (dt = '2026-07-23', region = 'eu'); +``` + +Complete specifications first use the REST list-by-names lookup. If MSCK +registered a type-equivalent raw value such as integer `01`, whitespace-padded +numeric text, or an unpadded `CHAR` value, DROP falls back to a full +registration traversal so the normalized SQL specification still matches it. +Fallback is tracked per requested specification: an exact complete match stays +exact even when another complete miss or partial specification requires a full +traversal. Partial specifications are expanded against that full registration +traversal. Matching registrations are removed before their directories are +deleted. Therefore DROP never deletes an unregistered directory. A missing +complete specification fails unless partition-level `IF EXISTS` is present. +REST unregister calls are split into batches of at most 1000 specs. `PURGE` is +rejected. + +#### MSCK REPAIR TABLE + +```sql +MSCK REPAIR TABLE paimon.my_db.events; +REPAIR TABLE paimon.my_db.events ADD PARTITIONS; +MSCK REPAIR TABLE paimon.my_db.events DROP PARTITIONS; +MSCK REPAIR TABLE paimon.my_db.events SYNC PARTITIONS; +``` + +Plain `MSCK REPAIR TABLE` and `ADD PARTITIONS` register partitions that exist +on the filesystem but not in REST. `DROP PARTITIONS` unregisters REST +partitions whose directories are absent. `SYNC PARTITIONS` performs both, +adding before dropping. Repair is metadata-only: it never creates or deletes +partition directories. Filesystem values are preserved as written, so a path +such as `month=01` is registered with the raw value `01`; `SHOW PARTITIONS` +still displays its typed value as `month=1`. REST create/drop calls use batches +of at most 1000 specs. Every segment that matches the declared partition +layout must round-trip exactly through Java-compatible unescaping and canonical +escaping. Noncanonical spellings such as malformed `%ZZ`, lowercase percent +escapes, or over-escaped safe characters fail the repair before any REST +mutation because the REST partition model has no separate field in which to +retain the physical spelling. + ## DML The table type determines which row-level DML operations are supported: @@ -870,6 +1017,11 @@ Multiple partition key-value pairs can be specified: ALTER TABLE paimon.my_db.events DROP PARTITION (dt = '2024-01-01', region = 'us'); ``` +For a REST-managed Format Table, use the dedicated semantics documented in +[REST-managed Format Table partitions](#rest-managed-format-table-partitions): +REST metadata is unregistered before physical directories are deleted, and +multiple `PARTITION (...)` clauses are supported. + ## Procedures Use `CALL` to invoke built-in procedures. All procedures are under the `sys` namespace. @@ -1962,6 +2114,8 @@ the normal physical format without wrapping the writer. | `'deletion-vectors.enabled' = 'true'` | Enable deletion vectors | | `'cross-partition-update.enabled' = 'true'` | Allow cross-partition updates | | `'changelog-producer' = 'input'` | Changelog producer (PK tables with input mode reject writes) | +| `'metastore.partitioned-table' = 'true'` | Use REST registrations as the authoritative partition set for an internal Format Table | +| `'format-table.partition-path-only-value' = 'true'` | Store Format Table partition directories as value-only path components instead of `key=value` components | ## Full Example From 552170f74e25d28f49a8a1691e5183ec4fcf4d21 Mon Sep 17 00:00:00 2001 From: Dapeng Sun Date: Thu, 23 Jul 2026 05:47:21 +0800 Subject: [PATCH 2/7] docs: add ASF headers to partition design files --- ...22-rest-format-table-partition-commands.md | 19 +++++++++++++++++++ ...22-rest-format-table-partition-commands.md | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/docs/designs/2026-07-22-rest-format-table-partition-commands.md b/docs/designs/2026-07-22-rest-format-table-partition-commands.md index 2896490b..1a4fb2b2 100644 --- a/docs/designs/2026-07-22-rest-format-table-partition-commands.md +++ b/docs/designs/2026-07-22-rest-format-table-partition-commands.md @@ -1,3 +1,22 @@ + + # REST Format Table Partition Commands Design ## Problem Statement diff --git a/docs/plans/2026-07-22-rest-format-table-partition-commands.md b/docs/plans/2026-07-22-rest-format-table-partition-commands.md index c2e71636..ab13bc44 100644 --- a/docs/plans/2026-07-22-rest-format-table-partition-commands.md +++ b/docs/plans/2026-07-22-rest-format-table-partition-commands.md @@ -1,3 +1,22 @@ + + # Implementation Plan: REST Format Table Partition Commands ## Prerequisites From 5307eef11b770e5a750abfac0cea897616a25baa Mon Sep 17 00:00:00 2001 From: Dapeng Sun Date: Thu, 23 Jul 2026 05:49:31 +0800 Subject: [PATCH 3/7] docs: remove temporary design artifacts --- ...22-rest-format-table-partition-commands.md | 311 ------------------ ...22-rest-format-table-partition-commands.md | 228 ------------- 2 files changed, 539 deletions(-) delete mode 100644 docs/designs/2026-07-22-rest-format-table-partition-commands.md delete mode 100644 docs/plans/2026-07-22-rest-format-table-partition-commands.md diff --git a/docs/designs/2026-07-22-rest-format-table-partition-commands.md b/docs/designs/2026-07-22-rest-format-table-partition-commands.md deleted file mode 100644 index 1a4fb2b2..00000000 --- a/docs/designs/2026-07-22-rest-format-table-partition-commands.md +++ /dev/null @@ -1,311 +0,0 @@ - - -# REST Format Table Partition Commands Design - -## Problem Statement - -`paimon-rust` can read a REST-backed `type=format-table`, and the REST client can -list table partitions. It does not yet support the catalog-managed partition -contract used by internal REST Format Tables: - -- `metastore.partitioned-table=true` makes the REST catalog the source of truth - for partition visibility. -- The client needs partition registration and unregistration APIs. -- DataFusion needs Spark-compatible partition administration commands, - including `SHOW PARTITIONS`, `ALTER TABLE ... ADD/DROP PARTITION`, and - `MSCK REPAIR TABLE`. - -Adding SQL commands without changing scan visibility would be inconsistent: -`SHOW PARTITIONS` would display catalog registrations while `SELECT` would -still discover every directory from the filesystem. - -## Chosen Approach - -Implement a REST-first vertical slice: - -1. Add the minimal partition operations to the REST API and `Catalog` contract. -2. Recognize catalog-managed internal REST Format Tables at load time. -3. Make their scans use REST partition registrations without filesystem - fallback. -4. Add Spark-compatible partition commands to `paimon-datafusion`. - -Do not introduce Java Paimon's serializable `FormatTablePartitionManager` -abstraction. Rust Format Table writes are currently unsupported, and this -feature targets REST-backed administration and reads only. - -## Design Details - -### Managed Table Capability - -A table has catalog-managed partitions only when all of the following are true: - -- `type=format-table` -- `metastore.partitioned-table=true` -- `format-table.implementation` is not `engine` -- the table is internal (`isExternal=false`) -- the table was loaded from a REST catalog - -Loading fails if an external REST Format Table requests catalog-managed -partitions, or if it combines catalog-managed partitions with -`format-table.implementation=engine`, whose contract is filesystem discovery -by the engine. A non-REST table cannot acquire the capability because it has -no `RESTEnv`. A plain public `RESTEnv::new` also does not assert this capability. -Catalog implementations that use `RESTEnv::new_for_table` must pass the -server-provided schema and `isExternal` value unchanged. This public -constructor is an explicit catalog trust boundary rather than an independent -metadata verifier; the built-in `RESTCatalog` invokes it only with the table -response it just loaded. - -`Table` exposes a predicate describing the effective capability. Runtime code -uses the effective capability rather than checking the option alone. Copies -retain the loaded table path, file format, and partition layout for managed -scans, so dynamic options cannot silently redirect them to a different -directory or value-only layout. - -### REST Partition Protocol - -Add wire-compatible paths and models: - -- `POST .../partitions` - - request: `partitionSpecs`, `ignoreIfExists` - - response: `created`, `existed` -- `POST .../partitions/drop` - - request: `partitionSpecs`, `ignoreIfNotExists` - - response: `dropped`, `missing` -- `POST .../partitions/list-by-names` - - request: `specs` - - response: existing `ListPartitionsResponse` - -The existing paged `GET .../partitions` remains the listing primitive for the -first version. An empty next-page token terminates pagination, and a repeated -non-empty token fails fast instead of looping forever. Predicate transport and -`list-by-filter` are deferred. - -`Catalog` gains minimal async methods for create, drop, and list-by-names. -Default implementations return `Unsupported`; `RESTCatalog` maps REST errors to -catalog-level errors. - -Catalog services cap a request at 1000 partition specs, and the low-level -`RESTApi` rejects larger single requests locally. Strict create remains one -request so a conflict rejects the whole SQL batch and therefore cannot exceed -1000 specs. Idempotent create, drop, and list-by-names are split into batches of -1000; a failure may therefore occur after earlier idempotent batches have -succeeded, and retrying converges. - -### Managed Format Table Scan - -For a partitioned managed Format Table, planning: - -1. Lists every registered partition from `RESTEnv`. -2. Validates each returned spec is complete and contains only declared - partition keys. -3. Converts raw registered values into partition rows. -4. Applies the existing local partition predicate. -5. Builds one scan root per surviving registration. - -An unregistered directory is invisible. A registered partition whose directory -does not exist reads as empty only when storage reports `NotFound`. Other -partition-directory listing failures propagate. REST listing errors also -propagate; managed scans never fall back to filesystem discovery. - -Unmanaged Format Tables retain the current filesystem behavior. - -### SQL Surface - -`SQLContext` supports: - -```sql -SHOW PARTITIONS table [PARTITION (...)] - -ALTER TABLE table -ADD [IF NOT EXISTS] PARTITION (...) [PARTITION (...) ...] - -ALTER TABLE table -DROP [IF EXISTS] PARTITION (...) -[, PARTITION (...) ...] - -[MSCK] REPAIR TABLE table -[{ADD|DROP|SYNC} PARTITIONS] -``` - -`sqlparser` already represents `MSCK`, ADD PARTITION, and DROP PARTITION. -Compatibility handling rewrites bare Spark `REPAIR TABLE` into the parser's -`MSCK REPAIR TABLE` form. Spark's comma-separated DROP syntax is rewritten into -multiple `DropPartitions` AST operations and then executed as one resolved -batch. `SHOW PARTITIONS` receives a small dedicated parser because it is not -represented by the current parser AST. - -Every command is accepted only for a partitioned Format Table with the -effective catalog-managed REST capability. - -### SHOW PARTITIONS - -`SHOW PARTITIONS` reads REST registrations and returns one non-null UTF-8 column -named `partition`. Each name uses declared partition-key order: - -```text -dt=2026-07-22/hour=10 -``` - -Rows are sorted lexicographically by the complete partition name, with one row -per REST registration. Type-equivalent raw registrations may therefore produce -duplicate displayed names. Registered values are cast through their -partition-column types before display and filter comparison: default-partition -values display as `null`, dates display as `yyyy-MM-dd`, and numeric text such -as `01` displays as `1`. Non-character casts ignore surrounding whitespace. -`CHAR` values use Spark write-side padding and both `CHAR` and `VARCHAR` -enforce their declared character limits. - -An optional spec filters registrations locally. It may be partial and does not -need to be a leading prefix. - -### ADD PARTITION - -ADD requires complete specs. - -1. Convert SQL constants to strings and cast them through the partition column - types, matching Spark partition-spec normalization. -2. Validate the resulting path remains under the table location. -3. Send the batch to REST with the SQL `IF NOT EXISTS` flag. -4. Create each partition directory after successful registration. - -Strict ADD is never split because the REST contract rejects the whole batch -atomically if any partition already exists, so one statement may contain at -most 1000 specs. `IF NOT EXISTS` ADD is split into 1000-spec requests. SQL -`NULL` and blank character values map to `partition.default-name`. Directory -creation failure leaves the registration visible but its missing directory -reads as empty; retrying with `IF NOT EXISTS` continues directory creation. - -For the supported scalar types, non-character string casts trim surrounding -whitespace. `CHAR(n)` pads short values; `CHAR(n)` and `VARCHAR(n)` trim only -excess ASCII trailing spaces and reject remaining overlong content. Typed -`DATE` literals are converted to their canonical date string before the target -partition-column cast. Other typed literals are rejected until their Spark -source-type formatting is implemented. Paimon `DATE` values remain within the -declared `0000-01-01..9999-12-31` range. - -Partition path components use Java-compatible escaping: Unicode is preserved, -while reserved characters such as `/`, `=`, `%`, and `}` are percent-encoded. -`LOCATION` is rejected for Format Tables. - -### DROP PARTITION - -One or more complete or partial specs are supported. - -1. Resolve complete specs with `list-by-names`. If an exact lookup misses a - type-equivalent raw value registered by MSCK (for example integer `01`, - surrounding whitespace on a non-character value, or an unpadded `CHAR` - value), fall back to one full traversal. Any partial spec also uses one full - traversal. Exact-vs-fallback resolution is tracked independently for each - requested spec, so an exact complete match never expands to typed aliases - merely because another complete miss or partial spec requires the full - traversal. -2. Expand each partial spec to exact registered leaf partitions. -3. For complete missing specs, fail unless `IF EXISTS` was specified. -4. Unregister the exact resolved set. -5. Delete those partition directories. - -Only registered partitions are deleted. Data in an unregistered directory is -never removed by DROP. Unregistering precedes deletion so a partial deletion -failure cannot expose the remaining files through managed scans. -REST unregister requests are split into batches of 1000. `PURGE` is rejected -before any catalog access. - -### MSCK Repair - -The repair engine lives in `paimon-datafusion`; it is SQL/engine workflow rather -than a general core catalog abstraction. - -It discovers raw filesystem partition specs without casting directory values -through their logical data types. This preserves values such as `month=01`. -Each segment that matches the declared partition layout must also round-trip -exactly through permissive Java-compatible unescaping and canonical escaping. -Malformed, lowercase, or otherwise over-escaped percent spellings fail repair -before mutation: the REST partition model stores the raw logical spec but has -no separate physical-path spelling to make a noncanonical directory readable -later. Unrelated directory names that do not decode to the expected partition -key remain ignored. - -It fully lists both filesystem and REST state before mutation, then computes: - -```text -ADD = filesystem - registered -DROP = registered - filesystem -``` - -Plain MSCK defaults to ADD. SYNC applies ADD before DROP. Operations are sorted -by canonical partition path for deterministic behavior. ADD is idempotent; -DROP ignores missing registrations, so a rerun converges after a partial -failure. Their REST calls use the same 1000-spec batching rules. - -MSCK changes metadata only and never creates or deletes partition directories. - -## Error Handling - -- Missing table: preserve `TableNotExist`. -- Non-REST or unmanaged Format Table: explicit `Unsupported`. -- External table requesting managed partitions: `DataInvalid`. -- REST 404/403/409/400: map to the existing catalog error taxonomy. -- Malformed or incomplete REST partition metadata: fail closed with table - context. -- Filesystem or REST listing failure during MSCK: perform no mutation. -- Noncanonical filesystem partition paths that cannot round-trip through REST - metadata: fail before mutation. - -## Testing - -Core/REST tests cover: - -- request and response JSON shapes; -- resource paths; -- REST catalog create/drop/list-by-names behavior and error mapping; -- empty and repeated partition-list page tokens; -- managed-table load validation; -- plain `RESTEnv` construction not asserting managed internal-table provenance; -- managed scans hiding unregistered directories; -- registered missing directories reading as empty; -- non-`NotFound` directory listing failures propagating; -- listing failures not falling back to the filesystem; -- malformed registration context and dynamic-layout copies. - -DataFusion tests cover: - -- parser compatibility, including Spark comma-separated DROP; -- typed SHOW output and partial filtering; -- strict/idempotent ADD, Spark whitespace and `CHAR`/`VARCHAR` normalization, - typed-literal conversion, and Paimon date-range validation; -- complete, partial, missing, multi-spec, mixed complete/partial, and raw-value - DROP; -- path safety; -- bare/ADD/DROP/SYNC MSCK; -- raw directory values and rejection of non-round-tripping paths; -- fail-closed listing and idempotent repair reruns. - -## Open Questions - -None for the first version. - -## Out of Scope - -- Format Table writes and automatic post-write registration. -- Non-REST catalogs. -- A general `FormatTablePartitionManager`. -- REST predicate/list-by-filter pushdown. -- Procedures, dry-run, or per-partition repair reports. diff --git a/docs/plans/2026-07-22-rest-format-table-partition-commands.md b/docs/plans/2026-07-22-rest-format-table-partition-commands.md deleted file mode 100644 index ab13bc44..00000000 --- a/docs/plans/2026-07-22-rest-format-table-partition-commands.md +++ /dev/null @@ -1,228 +0,0 @@ - - -# Implementation Plan: REST Format Table Partition Commands - -## Prerequisites - -- [x] Design approved in `docs/designs/2026-07-22-rest-format-table-partition-commands.md`. -- [x] Worktree is clean on `main`. -- [x] Make the Rust toolchain available on `PATH`. - -## Tasks - -### Task 1: Add REST partition wire models [x] - -- **Files**: - - `crates/paimon/src/api/api_request.rs` - - `crates/paimon/src/api/api_response.rs` - - `crates/paimon/src/api/mod.rs` -- **Changes**: - - Add create, drop, and list-by-names request types. - - Add create and drop response types. - - Add serde tests for Java-compatible field names and defaults. -- **Verify**: - - `cargo test --locked -p paimon api:: --lib` -- **Dependencies**: None - -### Task 2: Add REST partition paths and client methods [x] - -- **Files**: - - `crates/paimon/src/api/resource_paths.rs` - - `crates/paimon/src/api/rest_api.rs` - - `crates/paimon/tests/rest_api_test.rs` -- **Changes**: - - Add `/partitions/drop` and `/partitions/list-by-names`. - - Add create, drop, and list-by-names client methods. - - Assert HTTP methods, paths, and JSON bodies with the mock server. -- **Verify**: - - `cargo test --locked -p paimon --test rest_api_test partition` -- **Dependencies**: Task 1 - -### Task 3: Extend the catalog partition contract [x] - -- **Files**: - - `crates/paimon/src/catalog/mod.rs` - - `crates/paimon/src/catalog/rest/rest_catalog.rs` - - `crates/paimon/tests/rest_catalog_test.rs` -- **Changes**: - - Add default unsupported create/drop/list-by-names methods. - - Implement them in `RESTCatalog`. - - Preserve strict create and metadata-only drop semantics. - - Add REST error-mapping tests. -- **Verify**: - - `cargo test --locked -p paimon --test rest_catalog_test partition` -- **Dependencies**: Task 2 - -### Task 4: Recognize managed internal REST Format Tables [x] - -- **Files**: - - `crates/paimon/src/spec/core_options.rs` - - `crates/paimon/src/table/mod.rs` - - `crates/paimon/src/table/rest_env.rs` - - `crates/paimon/tests/rest_catalog_test.rs` -- **Changes**: - - Parse `metastore.partitioned-table`. - - Record validated internal-table provenance in `RESTEnv` and add the - effective managed-format-table predicate. - - Reject external managed Format Tables and - `format-table.implementation=engine` while loading. - - Keep plain manual `RESTEnv` construction fail-closed. - - Add capability and validation tests. -- **Verify**: - - `cargo test --locked -p paimon --test rest_catalog_test managed_format` -- **Dependencies**: Task 3 - -### Task 5: Make managed Format Table scans catalog-authoritative [x] - -- **Files**: - - `crates/paimon/src/table/format_table_scan.rs` - - `crates/paimon/tests/rest_catalog_test.rs` -- **Changes**: - - List registered partitions through `RESTEnv`. - - Validate and convert raw specs to scan roots. - - Apply local partition filtering. - - Treat missing registered directories as empty. - - Prohibit REST-to-filesystem fallback. - - Preserve the loaded path, format, and partition layout across dynamic - table copies. -- **Verify**: - - `cargo test --locked -p paimon --test rest_catalog_test managed_format` -- **Dependencies**: Task 4 - -### Task 6: Add reusable Format Table partition path helpers [x] - -- **Files**: - - `crates/paimon/src/table/format_table_scan.rs` - - `crates/paimon/src/table/mod.rs` - - `crates/paimon/src/table/format_partition.rs` (new, if extraction is clearer) -- **Changes**: - - Centralize raw path discovery, spec validation, path construction, and - canonical ordering for scan and SQL administration. - - Reject physical path spellings that cannot round-trip through REST - partition metadata and canonical escaping. - - Add focused integration tests for escaped paths, value-only layouts, - hidden directories, and default-partition discovery. -- **Verify**: - - `cargo test --locked -p paimon --test format_partition_test` -- **Dependencies**: Task 5 - -### Task 7: Add SQL parsing for SHOW and REPAIR compatibility [x] - -- **Files**: - - `crates/integrations/datafusion/src/sql_context.rs` -- **Changes**: - - Dispatch `Statement::Msck`. - - Rewrite bare `REPAIR TABLE`. - - Rewrite Spark comma-separated DROP partition clauses into sqlparser AST - operations. - - Parse Spark-compatible `SHOW PARTITIONS ... [PARTITION (...)]`. - - Add parser-only tests before execution handlers. -- **Verify**: - - `cargo test --locked -p paimon-datafusion --lib sql_context::tests::test_parse_` -- **Dependencies**: Task 4 - -### Task 8: Implement SHOW PARTITIONS [x] - -- **Files**: - - `crates/integrations/datafusion/src/sql_context.rs` -- **Changes**: - - Validate the target capability. - - Cast REST values through partition-column types, apply optional partial - filtering, sort, and return the `partition` DataFrame. - - Add success and rejection tests. -- **Verify**: - - `cargo test --locked -p paimon-datafusion show_partitions` -- **Dependencies**: Tasks 3, 7 - -### Task 9: Implement ADD PARTITION [x] - -- **Files**: - - `crates/integrations/datafusion/src/sql_context.rs` -- **Changes**: - - Handle `AlterTableOperation::AddPartitions`. - - Normalize Spark-style literals, NULL/default values, and complete typed - specs, including non-character whitespace trimming, `CHAR`/`VARCHAR` - length semantics, typed `DATE` conversion, and the Paimon `DATE` range; - reject LOCATION. - - Keep strict registration atomic, batch idempotent REST requests by 1000, - and then create Java-compatible escaped directories. - - Add strict, IF NOT EXISTS, and path-safety tests. -- **Verify**: - - `cargo test --locked -p paimon-datafusion add_format_partition` -- **Dependencies**: Tasks 3, 6, 7 - -### Task 10: Implement managed DROP PARTITION [x] - -- **Files**: - - `crates/integrations/datafusion/src/sql_context.rs` -- **Changes**: - - Route managed Format Tables away from snapshot truncation. - - Resolve multiple complete/partial specs against REST registrations, - including typed fallback for raw values registered by MSCK, such as - leading-zero or whitespace-padded numerics and unpadded `CHAR` values. - - Keep exact complete matches exact when another requested spec triggers a - full-list fallback. - - Enforce IF EXISTS, unregister first, then delete directories. - - Add complete, partial, missing, and unregistered-data tests. -- **Verify**: - - `cargo test --locked -p paimon-datafusion drop_format_partition` -- **Dependencies**: Tasks 3, 6, 7 - -### Task 11: Implement MSCK repair [x] - -- **Files**: - - `crates/integrations/datafusion/src/format_partition_repair.rs` (new) - - `crates/integrations/datafusion/src/lib.rs` - - `crates/integrations/datafusion/src/sql_context.rs` -- **Changes**: - - Discover raw filesystem specs. - - Fail before mutation when a physical partition path cannot round-trip - through canonical REST partition metadata. - - Fully list REST registrations before mutation. - - Compute deterministic ADD/DROP/SYNC diffs. - - Add metadata-only, raw-value, failure, and rerun tests. -- **Verify**: - - `cargo test --locked -p paimon-datafusion --lib sql_context::tests::test_msck_` -- **Dependencies**: Tasks 3, 6, 7 - -### Task 12: Document the SQL surface [x] - -- **Files**: - - `docs/src/sql.md` -- **Changes**: - - Document capability requirements, syntax, examples, and metadata/data - semantics. -- **Verify**: - - `rg "MSCK REPAIR TABLE|SHOW PARTITIONS|metastore.partitioned-table" docs/src/sql.md` -- **Dependencies**: Tasks 8-11 - -## Post-Implementation - -- [x] `cargo fmt --all -- --check` -- [x] `cargo test --locked -p paimon --all-targets` -- [x] `cargo test --locked -p paimon-datafusion --lib` -- [ ] `cargo test --locked -p paimon-datafusion --all-targets` - - Executed with `--no-fail-fast`; every target completed. Only - `vector_search_tests::test_vector_search_top3` and - `vector_search_tests::test_vector_search_top6_returns_all` failed because - `liblumina_py.so` is unavailable in the test environment. -- [x] `cargo clippy --locked -p paimon -p paimon-datafusion --all-targets -- -D warnings` -- [x] Review the complete diff and run `dlf-review-quick` and - `dlf-review-zen`. From 7b7c315610144e3df795de9cd6aed160eea4f585 Mon Sep 17 00:00:00 2001 From: Dapeng Sun Date: Thu, 23 Jul 2026 08:26:38 +0800 Subject: [PATCH 4/7] datafusion: refine REST format table partition commands --- .../datafusion/src/format_partition_repair.rs | 65 +-- .../datafusion/src/sql_context.rs | 548 ++++++++++++------ .../tests/rest_format_partition_sql.rs | 228 +++----- crates/paimon/src/api/api_request.rs | 65 ++- crates/paimon/src/api/api_response.rs | 50 +- crates/paimon/src/api/resource_paths.rs | 4 +- crates/paimon/src/api/rest_api.rs | 67 +-- crates/paimon/src/catalog/mod.rs | 15 +- .../paimon/src/catalog/rest/rest_catalog.rs | 56 +- crates/paimon/src/spec/core_options.rs | 60 +- crates/paimon/src/spec/mod.rs | 2 +- crates/paimon/src/spec/partition_utils.rs | 31 + crates/paimon/src/table/format_partition.rs | 288 +++++++-- crates/paimon/src/table/format_table_scan.rs | 249 ++------ crates/paimon/src/table/mod.rs | 15 +- crates/paimon/src/table/rest_env.rs | 156 +++-- crates/paimon/tests/format_partition_test.rs | 60 +- crates/paimon/tests/mock_server.rs | 327 ++++++----- crates/paimon/tests/rest_api_test.rs | 73 ++- crates/paimon/tests/rest_catalog_test.rs | 525 ++++++++++++----- .../paimon/tests/rest_object_models_test.rs | 14 +- docs/src/sql.md | 108 ++-- scripts/release_licenses.py | 83 ++- third-party-licenses/README.md | 6 + third-party-licenses/jieba-rs-0.10.3.LICENSE | 22 + 25 files changed, 1986 insertions(+), 1131 deletions(-) create mode 100644 third-party-licenses/jieba-rs-0.10.3.LICENSE diff --git a/crates/integrations/datafusion/src/format_partition_repair.rs b/crates/integrations/datafusion/src/format_partition_repair.rs index efa4848e..7271bf1d 100644 --- a/crates/integrations/datafusion/src/format_partition_repair.rs +++ b/crates/integrations/datafusion/src/format_partition_repair.rs @@ -33,17 +33,17 @@ pub(crate) async fn repair( identifier: &Identifier, table: &Table, mode: RepairMode, -) -> paimon::Result { +) -> paimon::Result<()> { let core_options = CoreOptions::new(table.schema().options()); - let paths = FormatTablePartitionPaths::new( + let partition_paths = FormatTablePartitionPaths::new( table.schema().partition_keys().iter().cloned(), core_options.format_table_partition_only_value_in_path(), ); - let table_path = core_options.path().unwrap_or_else(|| table.location()); + let table_path = table.location(); - // Complete both listings before mutating metadata. Filesystem discovery keeps - // raw directory values, so values such as month=01 round-trip unchanged. - let filesystem_specs = paths + // Load both views before changing catalog metadata so a listing failure leaves + // metadata unchanged. Discovery preserves raw directory values such as month=01. + let discovered_specs = partition_paths .discover( table.file_io(), table_path, @@ -52,62 +52,53 @@ pub(crate) async fn repair( .await?; let registered_partitions = catalog.list_partitions(identifier).await?; - let filesystem = index_specs(&paths, filesystem_specs)?; - let registered = index_specs( - &paths, + let discovered_by_name = index_specs_by_name(&partition_paths, discovered_specs)?; + let registered_by_name = index_specs_by_name( + &partition_paths, registered_partitions .into_iter() .map(|partition| partition.spec) .collect(), )?; - let add = if matches!(mode, RepairMode::Add | RepairMode::Sync) { - filesystem + let to_register = if matches!(mode, RepairMode::Add | RepairMode::Sync) { + discovered_by_name .iter() - .filter(|(name, _)| !registered.contains_key(*name)) + .filter(|(name, _)| !registered_by_name.contains_key(*name)) .map(|(_, spec)| spec.clone()) .collect::>() } else { Vec::new() }; - let drop = if matches!(mode, RepairMode::Drop | RepairMode::Sync) { - registered + let to_unregister = if matches!(mode, RepairMode::Drop | RepairMode::Sync) { + registered_by_name .iter() - .filter(|(name, _)| !filesystem.contains_key(*name)) + .filter(|(name, _)| !discovered_by_name.contains_key(*name)) .map(|(_, spec)| spec.clone()) .collect::>() } else { Vec::new() }; - if !add.is_empty() { + if !to_register.is_empty() { catalog - .create_partitions(identifier, add.clone(), true) + .create_partitions(identifier, to_register, true) .await?; } - if !drop.is_empty() { - catalog.drop_partitions(identifier, drop.clone()).await?; + if !to_unregister.is_empty() { + catalog + .unregister_partitions(identifier, to_unregister) + .await?; } - Ok(add.len() + drop.len()) + Ok(()) } -fn index_specs( - paths: &FormatTablePartitionPaths, +fn index_specs_by_name( + partition_paths: &FormatTablePartitionPaths, specs: Vec>, ) -> paimon::Result>> { - let mut indexed = BTreeMap::new(); - for spec in specs { - let name = paths.partition_name(&spec)?; - if let Some(previous) = indexed.insert(name.clone(), spec.clone()) { - if previous != spec { - return Err(paimon::Error::DataInvalid { - message: format!( - "Conflicting partition specs map to the same canonical path {name}" - ), - source: None, - }); - } - } - } - Ok(indexed) + specs + .into_iter() + .map(|spec| Ok((partition_paths.partition_name(&spec)?, spec))) + .collect() } diff --git a/crates/integrations/datafusion/src/sql_context.rs b/crates/integrations/datafusion/src/sql_context.rs index d59ce83f..e87420d6 100644 --- a/crates/integrations/datafusion/src/sql_context.rs +++ b/crates/integrations/datafusion/src/sql_context.rs @@ -30,7 +30,7 @@ //! - `ALTER TABLE db.t RENAME COLUMN old TO new` //! - `ALTER TABLE db.t RENAME TO new_name` //! - `ALTER TABLE db.t ADD [IF NOT EXISTS] PARTITION (...) [PARTITION (...)]` -//! - `ALTER TABLE db.t DROP PARTITION (col = val, ...)` +//! - `ALTER TABLE db.t DROP [IF EXISTS] PARTITION (...) [, PARTITION (...)]` //! - `SHOW PARTITIONS db.t [PARTITION (...)]` //! - `[MSCK] REPAIR TABLE db.t [{ADD|DROP|SYNC} PARTITIONS]` //! - `CREATE VIEW [IF NOT EXISTS] view [(col, ...)] AS query` @@ -58,11 +58,12 @@ use datafusion::logical_expr::{Expr as LogicalExpr, LogicalPlan, Volatility}; use datafusion::prelude::{DataFrame, SessionContext}; use datafusion::sql::planner::IdentNormalizer; use datafusion::sql::sqlparser::ast::{ - AlterTableOperation, BinaryLength, CharacterLength, ColumnDef, ColumnOption, CreateFunction, - CreateFunctionBody, CreateTable, CreateTableOptions, CreateView, Delete, Expr as SqlExpr, - FromTable, FunctionBehavior, FunctionReturnType, Insert, Merge, ObjectName, ObjectType, - Partition as SqlPartition, RenameTableNameKind, Reset, ResetStatement, Set, ShowCreateObject, - SqlOption, Statement, TableFactor, TableObject, Truncate, Update, Value as SqlValue, + AddDropSync, AlterTableOperation, BinaryLength, CharacterLength, ColumnDef, ColumnOption, + CreateFunction, CreateFunctionBody, CreateTable, CreateTableOptions, CreateView, Delete, + Expr as SqlExpr, FromTable, FunctionBehavior, FunctionReturnType, Insert, Merge, Msck, + ObjectName, ObjectType, Partition as SqlPartition, RenameTableNameKind, Reset, ResetStatement, + Set, ShowCreateObject, SqlOption, Statement, TableFactor, TableObject, Truncate, Update, + Value as SqlValue, }; use datafusion::sql::sqlparser::dialect::GenericDialect; use datafusion::sql::sqlparser::keywords::Keyword; @@ -77,7 +78,7 @@ use paimon::spec::{ RowType as PaimonRowType, SchemaChange, SmallIntType, TimestampType, TinyIntType, VarBinaryType, VarCharType, VariantType, }; -use paimon::table::FormatTablePartitionPaths; +use paimon::table::{parse_format_partition_value, FormatTablePartitionPaths}; use crate::error::to_datafusion_error; use crate::table_loader::load_table_for_read; @@ -1058,7 +1059,7 @@ impl SQLContext { AlterTableOperation::DropPartitions { partitions, if_exists, - } => Some((partitions.clone(), *if_exists)), + } => Some((partitions.as_slice(), *if_exists)), _ => None, }) .collect::>(); @@ -1706,7 +1707,7 @@ impl SQLContext { &self, catalog: &Arc, identifier: &Identifier, - partition_specs: &[(Vec, bool)], + partition_specs: &[(&[SqlExpr], bool)], ignore_if_table_not_exists: bool, ) -> DFResult { if partition_specs.is_empty() || partition_specs.iter().any(|(spec, _)| spec.is_empty()) { @@ -1738,7 +1739,7 @@ impl SQLContext { table.schema().partition_keys().iter().cloned(), core_options.format_table_partition_only_value_in_path(), ); - let table_path = core_options.path().unwrap_or_else(|| table.location()); + let table_path = table.location(); let partition_key_count = table.schema().partition_keys().len(); let complete_specs = requested .iter() @@ -1765,7 +1766,7 @@ impl SQLContext { let needs_full_listing = requested.iter().enumerate().any(|(index, (spec, _))| { spec.len() != partition_key_count || !exact_matches[index] }); - let registered = if needs_full_listing { + let registered_partitions = if needs_full_listing { catalog .list_partitions(identifier) .await @@ -1774,20 +1775,28 @@ impl SQLContext { exact_registered }; - let normalized_registered = registered + let normalized_partitions = registered_partitions .into_iter() .map(|partition| { - let normalized = - normalize_format_partition_spec_values(&partition.spec, &table, true)?; + let normalized = normalize_format_partition_spec_values( + &partition.spec, + &table, + FormatPartitionValueSource::CatalogMetadata, + true, + )?; Ok((partition, normalized)) }) .collect::>>()?; - let mut matches = BTreeMap::new(); + let mut selected_partitions = BTreeMap::new(); for (index, (requested_spec, ignore_if_not_exists)) in requested.iter().enumerate() { - let requested_normalized = - normalize_format_partition_spec_values(requested_spec, &table, false)?; + let requested_normalized = normalize_format_partition_spec_values( + requested_spec, + &table, + FormatPartitionValueSource::SqlLiteral, + false, + )?; let mut matched = false; - for (partition, normalized) in &normalized_registered { + for (partition, normalized) in &normalized_partitions { let partition_matches = if exact_matches[index] { partition.spec == *requested_spec } else { @@ -1803,7 +1812,7 @@ impl SQLContext { let relative_path = partition_paths .relative_path(&partition.spec) .map_err(to_datafusion_error)?; - matches.entry(name).or_insert_with(|| { + selected_partitions.entry(name).or_insert_with(|| { ( partition.spec.clone(), format!("{}/{}", table_path.trim_end_matches('/'), relative_path), @@ -1822,18 +1831,21 @@ impl SQLContext { } } - if matches.is_empty() { + if selected_partitions.is_empty() { return ok_result(&self.ctx); } catalog - .drop_partitions( + .unregister_partitions( identifier, - matches.values().map(|(spec, _)| spec.clone()).collect(), + selected_partitions + .values() + .map(|(spec, _)| spec.clone()) + .collect(), ) .await .map_err(to_datafusion_error)?; - for (_, path) in matches.into_values() { + for (_, path) in selected_partitions.into_values() { table .file_io() .delete_dir(&path) @@ -1847,7 +1859,7 @@ impl SQLContext { } let partition_values = parse_partition_values( - &partition_specs[0].0, + partition_specs[0].0, table.schema().fields(), table.schema().partition_keys(), )?; @@ -1897,7 +1909,7 @@ impl SQLContext { table.schema().partition_keys().iter().cloned(), core_options.format_table_partition_only_value_in_path(), ); - let table_path = core_options.path().unwrap_or_else(|| table.location()); + let table_path = table.location(); let mut specs = Vec::with_capacity(partitions.len()); let mut directories = Vec::with_capacity(partitions.len()); for partition in partitions { @@ -1935,10 +1947,13 @@ impl SQLContext { ok_result(&self.ctx) } - async fn handle_msck( - &self, - msck: &datafusion::sql::sqlparser::ast::Msck, - ) -> DFResult { + async fn handle_msck(&self, msck: &Msck) -> DFResult { + if !msck.repair { + return Err(DataFusionError::Plan( + "MSCK requires the REPAIR keyword".to_string(), + )); + } + Self::ensure_main_branch_write_target(&msck.table_name, "MSCK REPAIR TABLE")?; let (catalog, _catalog_name, identifier) = self.resolve_catalog_and_table(&msck.table_name)?; let table = catalog @@ -1947,15 +1962,9 @@ impl SQLContext { .map_err(to_datafusion_error)?; ensure_catalog_managed_format_table(&table, "MSCK REPAIR TABLE")?; let mode = match msck.partition_action { - None | Some(datafusion::sql::sqlparser::ast::AddDropSync::ADD) => { - crate::format_partition_repair::RepairMode::Add - } - Some(datafusion::sql::sqlparser::ast::AddDropSync::DROP) => { - crate::format_partition_repair::RepairMode::Drop - } - Some(datafusion::sql::sqlparser::ast::AddDropSync::SYNC) => { - crate::format_partition_repair::RepairMode::Sync - } + None | Some(AddDropSync::ADD) => crate::format_partition_repair::RepairMode::Add, + Some(AddDropSync::DROP) => crate::format_partition_repair::RepairMode::Drop, + Some(AddDropSync::SYNC) => crate::format_partition_repair::RepairMode::Sync, }; crate::format_partition_repair::repair(catalog.as_ref(), &identifier, &table, mode) .await @@ -1974,12 +1983,16 @@ impl SQLContext { .await .map_err(to_datafusion_error)?; ensure_catalog_managed_format_table(&table, "SHOW PARTITIONS")?; - let filter = if show_partitions.partition.is_empty() { + let filter = if show_partitions.partition_filter.is_empty() { None } else { - let spec = parse_format_partition_spec(&show_partitions.partition, &table, false)?; + let spec = + parse_format_partition_spec(&show_partitions.partition_filter, &table, false)?; Some(normalize_format_partition_spec_values( - &spec, &table, false, + &spec, + &table, + FormatPartitionValueSource::SqlLiteral, + false, )?) }; @@ -1993,7 +2006,12 @@ impl SQLContext { .await .map_err(to_datafusion_error)? { - let normalized = normalize_format_partition_spec_values(&partition.spec, &table, true)?; + let normalized = normalize_format_partition_spec_values( + &partition.spec, + &table, + FormatPartitionValueSource::CatalogMetadata, + true, + )?; if !filter.as_ref().is_none_or(|filter| { filter .iter() @@ -2304,7 +2322,7 @@ fn validate_persistent_create_function(create_function: &CreateFunction) -> DFRe #[derive(Debug)] struct ShowPartitionsStatement { table_name: ObjectName, - partition: Vec, + partition_filter: Vec, } fn parse_show_partitions(sql: &str) -> DFResult> { @@ -2335,7 +2353,7 @@ fn parse_show_partitions(sql: &str) -> DFResult> let table_name = parser .parse_object_name(false) .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; - let partition = if parser.parse_keyword(Keyword::PARTITION) { + let partition_filter = if parser.parse_keyword(Keyword::PARTITION) { parser .expect_token(&Token::LParen) .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; @@ -2352,13 +2370,13 @@ fn parse_show_partitions(sql: &str) -> DFResult> let _ = parser.consume_token(&Token::SemiColon); if parser.peek_token().token != Token::EOF { return Err(DataFusionError::Plan(format!( - "SQL parse error: unexpected token {} after SHOW PARTITIONS table name", + "SQL parse error: unexpected token {} after SHOW PARTITIONS statement", parser.peek_token().token ))); } Ok(Some(ShowPartitionsStatement { table_name, - partition, + partition_filter, })) } @@ -2437,10 +2455,8 @@ fn parse_sql_statements(sql: &str) -> DFResult> { } fn rewrite_spark_multi_drop_partition_tokens(tokens: Vec) -> Vec { - // sqlparser expects a repeated DROP keyword, while Spark accepts: - // `DROP IF EXISTS PARTITION (...), PARTITION (...)`. Insert DROP before - // each following top-level PARTITION and propagate the batch-level - // IF EXISTS flag so every generated AST operation keeps Spark semantics. + // sqlparser requires DROP before every partition clause. Spark allows + // `DROP IF EXISTS PARTITION (...), PARTITION (...)`. let first_keywords = tokens .iter() .filter_map(|token| match &token.token { @@ -3200,7 +3216,7 @@ fn ensure_catalog_managed_format_table(table: &paimon::Table, operation: &str) - } if !table.has_catalog_managed_partitions() { return Err(DataFusionError::Plan(format!( - "{operation} is supported only for a REST Format Table with catalog-managed partitions, but {} discovers partitions from the filesystem", + "{operation} is supported only for catalog-managed partitions on an internal Format Table loaded from REST Catalog; table {} does not have this configuration", table.identifier().full_name() ))); } @@ -3385,39 +3401,44 @@ fn format_partition_literal_as_string(expr: &SqlExpr) -> DFResult { SqlExpr::Value(value) => match &value.value { SqlValue::Number(value, _) => Ok(canonicalize_integer_literal_if_possible(value)), SqlValue::Boolean(value) => Ok(value.to_string()), - SqlValue::Null => Err(DataFusionError::Plan( - "NULL partition values must be handled before string conversion".to_string(), + SqlValue::Null => Err(DataFusionError::Internal( + "NULL partition literal reached string conversion".to_string(), )), _ => value.value.clone().into_string().ok_or_else(|| { DataFusionError::Plan(format!("Unsupported partition literal: {expr}")) }), }, SqlExpr::UnaryOp { - op: - datafusion::sql::sqlparser::ast::UnaryOperator::Plus - | datafusion::sql::sqlparser::ast::UnaryOperator::Minus, + op: datafusion::sql::sqlparser::ast::UnaryOperator::Plus, expr: inner, - } => { - let value = format_partition_literal_as_string(inner)?; - let sign = if matches!( - expr, - SqlExpr::UnaryOp { - op: datafusion::sql::sqlparser::ast::UnaryOperator::Minus, - .. - } - ) { - "-" - } else { - "" - }; - Ok(format!("{sign}{value}")) - } + } => format_signed_partition_number(inner, ""), + SqlExpr::UnaryOp { + op: datafusion::sql::sqlparser::ast::UnaryOperator::Minus, + expr: inner, + } => format_signed_partition_number(inner, "-"), other => Err(DataFusionError::Plan(format!( "Unsupported partition value expression: {other}" ))), } } +fn format_signed_partition_number(expr: &SqlExpr, sign: &str) -> DFResult { + let SqlExpr::Value(value) = expr else { + return Err(DataFusionError::Plan(format!( + "Unary signs are supported only for numeric partition literals, got: {expr}" + ))); + }; + let SqlValue::Number(value, _) = &value.value else { + return Err(DataFusionError::Plan(format!( + "Unary signs are supported only for numeric partition literals, got: {expr}" + ))); + }; + Ok(format!( + "{sign}{}", + canonicalize_integer_literal_if_possible(value) + )) +} + fn canonicalize_integer_literal_if_possible(value: &str) -> String { if let Ok(value) = value.parse::() { value.to_string() @@ -3555,25 +3576,32 @@ fn format_partition_datum_for_storage( } } +#[derive(Debug, Clone, Copy)] +enum FormatPartitionValueSource { + SqlLiteral, + CatalogMetadata, +} + fn normalize_format_partition_spec_values( spec: &HashMap, table: &paimon::Table, + source: FormatPartitionValueSource, require_complete: bool, ) -> DFResult>> { let partition_keys = table.schema().partition_keys(); - if spec.keys().any(|key| !partition_keys.contains(key)) - || (require_complete - && (spec.len() != partition_keys.len() - || partition_keys.iter().any(|key| !spec.contains_key(key)))) + if spec.keys().any(|key| !partition_keys.contains(key)) { + return Err(DataFusionError::Plan(format!( + "Invalid partition spec {spec:?} for table {}: partition keys are {partition_keys:?}", + table.identifier().full_name() + ))); + } + if require_complete + && (spec.len() != partition_keys.len() + || partition_keys.iter().any(|key| !spec.contains_key(key))) { return Err(DataFusionError::Plan(format!( - "Invalid partition spec {spec:?} for table {}: expected {}keys {partition_keys:?}", - table.identifier().full_name(), - if require_complete { - "exactly " - } else { - "only " - } + "Invalid partition spec {spec:?} for table {}: expected exactly {partition_keys:?}", + table.identifier().full_name() ))); } let fields = table @@ -3596,70 +3624,16 @@ fn normalize_format_partition_spec_values( table.identifier().full_name() )) })?; - Some(match field.data_type() { - PaimonDataType::Char(char_type) => { - normalize_format_partition_string(raw, char_type.length(), true, "CHAR")? + Some(match source { + FormatPartitionValueSource::CatalogMetadata => { + normalize_catalog_format_partition_value(raw, key, field.data_type())? } - PaimonDataType::VarChar(varchar_type) => normalize_format_partition_string( + FormatPartitionValueSource::SqlLiteral => normalize_sql_format_partition_value( raw, - varchar_type.length() as usize, - false, - "VARCHAR", + key, + field.data_type(), + &core_options, )?, - PaimonDataType::Boolean(_) => parse_format_partition_bool(raw)?.to_string(), - PaimonDataType::TinyInt(_) => raw - .trim() - .parse::() - .map_err(|error| { - DataFusionError::Plan(format!( - "Invalid TINYINT partition value '{raw}' for column '{key}': {error}" - )) - })? - .to_string(), - PaimonDataType::SmallInt(_) => raw - .trim() - .parse::() - .map_err(|error| { - DataFusionError::Plan(format!( - "Invalid SMALLINT partition value '{raw}' for column '{key}': {error}" - )) - })? - .to_string(), - PaimonDataType::Int(_) => raw - .trim() - .parse::() - .map_err(|error| { - DataFusionError::Plan(format!( - "Invalid INT partition value '{raw}' for column '{key}': {error}" - )) - })? - .to_string(), - PaimonDataType::BigInt(_) => raw - .trim() - .parse::() - .map_err(|error| { - DataFusionError::Plan(format!( - "Invalid BIGINT partition value '{raw}' for column '{key}': {error}" - )) - })? - .to_string(), - PaimonDataType::Date(_) => { - let epoch_days = if core_options.legacy_partition_name() { - raw.trim().parse::().map_err(|error| { - DataFusionError::Plan(format!( - "Invalid legacy DATE partition value '{raw}' for column '{key}': {error}" - )) - })? - } else { - parse_spark_partition_date(raw)? - }; - format_partition_date(epoch_days)? - } - other => { - return Err(DataFusionError::NotImplemented(format!( - "SHOW/DROP PARTITION values of type {other:?} are not implemented yet" - ))) - } }) }; normalized.insert(key.clone(), value); @@ -3667,6 +3641,114 @@ fn normalize_format_partition_spec_values( Ok(normalized) } +fn normalize_sql_format_partition_value( + raw: &str, + key: &str, + data_type: &PaimonDataType, + core_options: &CoreOptions<'_>, +) -> DFResult { + match data_type { + PaimonDataType::Char(char_type) => { + normalize_format_partition_string(raw, char_type.length(), true, "CHAR") + } + PaimonDataType::VarChar(varchar_type) => { + normalize_format_partition_string(raw, varchar_type.length() as usize, false, "VARCHAR") + } + PaimonDataType::Boolean(_) => Ok(parse_format_partition_bool(raw)?.to_string()), + PaimonDataType::TinyInt(_) => raw + .trim() + .parse::() + .map(|value| value.to_string()) + .map_err(|error| { + DataFusionError::Plan(format!( + "Invalid TINYINT partition value '{raw}' for column '{key}': {error}" + )) + }), + PaimonDataType::SmallInt(_) => raw + .trim() + .parse::() + .map(|value| value.to_string()) + .map_err(|error| { + DataFusionError::Plan(format!( + "Invalid SMALLINT partition value '{raw}' for column '{key}': {error}" + )) + }), + PaimonDataType::Int(_) => raw + .trim() + .parse::() + .map(|value| value.to_string()) + .map_err(|error| { + DataFusionError::Plan(format!( + "Invalid INT partition value '{raw}' for column '{key}': {error}" + )) + }), + PaimonDataType::BigInt(_) => raw + .trim() + .parse::() + .map(|value| value.to_string()) + .map_err(|error| { + DataFusionError::Plan(format!( + "Invalid BIGINT partition value '{raw}' for column '{key}': {error}" + )) + }), + PaimonDataType::Date(_) => { + let epoch_days = if core_options.legacy_partition_name() { + raw.trim().parse::().map_err(|error| { + DataFusionError::Plan(format!( + "Invalid legacy DATE partition value '{raw}' for column '{key}': {error}" + )) + })? + } else { + parse_spark_partition_date(raw)? + }; + format_partition_date(epoch_days) + } + other => Err(DataFusionError::NotImplemented(format!( + "SHOW/DROP PARTITION values of type {other:?} are not implemented yet" + ))), + } +} + +fn normalize_catalog_format_partition_value( + raw: &str, + key: &str, + data_type: &PaimonDataType, +) -> DFResult { + match data_type { + PaimonDataType::Char(_) + | PaimonDataType::VarChar(_) + | PaimonDataType::Boolean(_) + | PaimonDataType::TinyInt(_) + | PaimonDataType::SmallInt(_) + | PaimonDataType::Int(_) + | PaimonDataType::BigInt(_) + | PaimonDataType::Date(_) => {} + other => { + return Err(DataFusionError::NotImplemented(format!( + "SHOW/DROP PARTITION values of type {other:?} are not implemented yet" + ))) + } + } + + match (data_type, parse_format_partition_value(raw, data_type)) { + (PaimonDataType::Char(_) | PaimonDataType::VarChar(_), Some(Datum::String(value))) => { + Ok(value) + } + (PaimonDataType::Boolean(_), Some(Datum::Bool(value))) => Ok(value.to_string()), + (PaimonDataType::TinyInt(_), Some(Datum::TinyInt(value))) => Ok(value.to_string()), + (PaimonDataType::SmallInt(_), Some(Datum::SmallInt(value))) => Ok(value.to_string()), + (PaimonDataType::Int(_), Some(Datum::Int(value))) => Ok(value.to_string()), + (PaimonDataType::BigInt(_), Some(Datum::Long(value))) => Ok(value.to_string()), + (PaimonDataType::Date(_), Some(Datum::Date(value))) => format_partition_date(value), + (_, None) => Err(DataFusionError::Plan(format!( + "Invalid catalog partition value {raw:?} for column '{key}' with type {data_type:?}" + ))), + (_, Some(datum)) => Err(DataFusionError::Internal(format!( + "Format partition parser returned {datum:?} for column '{key}' with type {data_type:?}" + ))), + } +} + /// Parse partition expressions (`col = val, ...`) into partition value maps /// suitable for `TableCommit::truncate_partitions`. /// @@ -4320,7 +4402,7 @@ mod tests { use paimon::io::FileIO; use paimon::spec::{ DataField as PaimonDataField, DataType as PaimonDataType, IntType, - Partition as PaimonPartition, Schema as PaimonSchema, TableSchema, + Partition as PaimonPartition, Schema as PaimonSchema, TableSchema, TimeType, }; use paimon::table::{RESTEnv, Table}; use paimon::{CatalogOptions, RESTApi}; @@ -4357,7 +4439,7 @@ mod tests { list_partitions_calls: Mutex, list_partitions_by_names_calls: Mutex>>>, partition_mutation_calls: Mutex>, - list_partitions_error: Mutex, + fail_list_partitions: Mutex, drop_view_supported: bool, } @@ -4373,7 +4455,7 @@ mod tests { list_partitions_calls: Mutex::new(0), list_partitions_by_names_calls: Mutex::new(Vec::new()), partition_mutation_calls: Mutex::new(Vec::new()), - list_partitions_error: Mutex::new(false), + fail_list_partitions: Mutex::new(false), drop_view_supported: true, } } @@ -4431,12 +4513,12 @@ mod tests { self.partition_mutation_calls.lock().unwrap().clone() } - fn set_list_partitions_error(&self, enabled: bool) { - *self.list_partitions_error.lock().unwrap() = enabled; + fn fail_list_partitions(&self) { + *self.fail_list_partitions.lock().unwrap() = true; } } - fn test_partition(spec: HashMap) -> PaimonPartition { + fn partition_from_spec(spec: HashMap) -> PaimonPartition { PaimonPartition { spec, record_count: 0, @@ -4477,13 +4559,13 @@ mod tests { ) -> paimon::Result<()> { Ok(()) } - async fn get_table(&self, _identifier: &Identifier) -> paimon::Result
{ + async fn get_table(&self, identifier: &Identifier) -> paimon::Result
{ *self.get_table_calls.lock().unwrap() += 1; if let Some(table) = self.existing_table.lock().unwrap().clone() { return Ok(table); } Err(paimon::Error::TableNotExist { - full_name: _identifier.to_string(), + full_name: identifier.to_string(), }) } async fn list_tables(&self, _database_name: &str) -> paimon::Result> { @@ -4558,7 +4640,7 @@ mod tests { Ok(()) } - async fn drop_partitions( + async fn unregister_partitions( &self, _identifier: &Identifier, partitions: Vec>, @@ -4587,7 +4669,7 @@ mod tests { .iter() .filter(|spec| partitions.contains(spec)) .cloned() - .map(test_partition) + .map(partition_from_spec) .collect()) } @@ -4596,7 +4678,7 @@ mod tests { _identifier: &Identifier, ) -> paimon::Result> { *self.list_partitions_calls.lock().unwrap() += 1; - if *self.list_partitions_error.lock().unwrap() { + if *self.fail_list_partitions.lock().unwrap() { return Err(paimon::Error::UnexpectedError { message: "Injected partition listing failure".to_string(), source: None, @@ -4608,7 +4690,7 @@ mod tests { .unwrap() .iter() .cloned() - .map(test_partition) + .map(partition_from_spec) .collect()) } @@ -7585,7 +7667,7 @@ mod tests { } #[tokio::test] - async fn test_msck_repair_table_dispatches_to_paimon_catalog() { + async fn test_msck_repair_table_reports_missing_catalog_table() { let catalog = Arc::new(MockCatalog::new()); let sql_context = make_sql_context(catalog).await; @@ -7603,7 +7685,7 @@ mod tests { } #[tokio::test] - async fn test_bare_repair_table_dispatches_to_paimon_catalog() { + async fn test_repair_table_reports_missing_catalog_table() { let catalog = Arc::new(MockCatalog::new()); let sql_context = make_sql_context(catalog).await; @@ -7621,7 +7703,61 @@ mod tests { } #[tokio::test] - async fn test_show_partitions_dispatches_to_paimon_catalog() { + async fn test_msck_without_repair_is_rejected_before_catalog_access() { + let catalog = Arc::new(MockCatalog::new()); + let sql_context = make_sql_context(catalog.clone()).await; + + let error = sql_context.sql("MSCK TABLE mydb.events").await.unwrap_err(); + + assert!( + error + .to_string() + .contains("MSCK requires the REPAIR keyword"), + "expected missing REPAIR error, got: {error}" + ); + assert_eq!(catalog.get_table_calls(), 0); + } + + #[tokio::test] + async fn test_msck_repair_table_rejects_branch_target_before_catalog_access() { + let catalog = Arc::new(MockCatalog::new()); + let sql_context = make_sql_context(catalog.clone()).await; + + let error = sql_context + .sql("MSCK REPAIR TABLE mydb.events$branch_dev") + .await + .unwrap_err(); + + assert!( + error + .to_string() + .contains("MSCK REPAIR TABLE on Paimon branch 'dev' is not supported"), + "expected branch write rejection, got: {error}" + ); + assert_eq!(catalog.get_table_calls(), 0); + } + + #[tokio::test] + async fn test_repair_table_rejects_branch_target_before_catalog_access() { + let catalog = Arc::new(MockCatalog::new()); + let sql_context = make_sql_context(catalog.clone()).await; + + let error = sql_context + .sql("REPAIR TABLE mydb.events$branch_dev") + .await + .unwrap_err(); + + assert!( + error + .to_string() + .contains("MSCK REPAIR TABLE on Paimon branch 'dev' is not supported"), + "expected branch write rejection, got: {error}" + ); + assert_eq!(catalog.get_table_calls(), 0); + } + + #[tokio::test] + async fn test_show_partitions_reports_missing_catalog_table() { let catalog = Arc::new(MockCatalog::new()); let sql_context = make_sql_context(catalog).await; @@ -7649,7 +7785,7 @@ mod tests { assert_eq!(statement.table_name.to_string(), "mydb.events"); assert_eq!( statement - .partition + .partition_filter .iter() .map(ToString::to_string) .collect::>(), @@ -7679,6 +7815,18 @@ mod tests { ))); } + #[test] + fn test_catalog_partition_normalization_rejects_unsupported_type() { + let error = normalize_catalog_format_partition_value( + "1234", + "event_time", + &PaimonDataType::Time(TimeType::new(0).unwrap()), + ) + .unwrap_err(); + + assert!(matches!(error, DataFusionError::NotImplemented(_))); + } + #[tokio::test] async fn test_show_partitions_returns_sorted_catalog_registrations() { let catalog = Arc::new(MockCatalog::new()); @@ -7795,13 +7943,14 @@ mod tests { let catalog = Arc::new(MockCatalog::new()); let identifier = Identifier::new("mydb", "events"); catalog.set_table( - managed_format_table_with_partition_fields( + managed_format_table_with_partition_fields_and_options( identifier, "memory:/show_typed_partitions", vec![ ("dt", PaimonDataType::Date(DateType::new())), ("month", PaimonDataType::Int(IntType::new())), ], + &[("partition.legacy-name", "false")], ) .await, ); @@ -7867,7 +8016,7 @@ mod tests { } #[tokio::test] - async fn test_add_partition_dispatches_to_paimon_catalog() { + async fn test_add_partition_reports_missing_catalog_table() { let catalog = Arc::new(MockCatalog::new()); let sql_context = make_sql_context(catalog).await; @@ -7915,7 +8064,7 @@ mod tests { } #[tokio::test] - async fn test_add_format_partition_strict_conflict_and_if_not_exists_retry() { + async fn test_add_format_partition_respects_if_not_exists() { let temp_dir = tempfile::tempdir().unwrap(); let location = format!("file://{}", temp_dir.path().display()); let catalog = Arc::new(MockCatalog::new()); @@ -7932,7 +8081,7 @@ mod tests { .unwrap_err(); assert!( error.to_string().contains("already exist"), - "expected strict duplicate error, got: {error}" + "expected duplicate partition error, got: {error}" ); assert!(!temp_dir.path().join("dt=2026-07-22").exists()); @@ -8000,6 +8149,35 @@ mod tests { assert!(!temp_dir.path().join("label=01").exists()); } + #[tokio::test] + async fn test_add_format_partition_rejects_sign_on_string_literal() { + let temp_dir = tempfile::tempdir().unwrap(); + let location = format!("file://{}", temp_dir.path().display()); + let catalog = Arc::new(MockCatalog::new()); + catalog.set_table( + managed_format_table(Identifier::new("mydb", "events"), &location, &["label"]).await, + ); + let sql_context = make_sql_context(catalog.clone()).await; + + for literal in ["+'us'", "-'us'"] { + let error = sql_context + .sql(&format!( + "ALTER TABLE mydb.events ADD PARTITION (label = {literal})" + )) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("Unary signs are supported only for numeric partition literals"), + "expected signed string rejection for {literal}, got: {error}" + ); + } + + assert!(catalog.partition_specs().is_empty()); + assert!(catalog.partition_mutation_calls().is_empty()); + } + #[tokio::test] async fn test_add_format_partition_accepts_char_literal() { let temp_dir = tempfile::tempdir().unwrap(); @@ -8766,7 +8944,7 @@ mod tests { } #[tokio::test] - async fn test_drop_complete_partition_falls_back_to_typed_match_for_msck_raw_value() { + async fn test_drop_complete_partition_matches_msck_value_by_type() { let temp_dir = tempfile::tempdir().unwrap(); std::fs::create_dir_all(temp_dir.path().join("month=01")).unwrap(); let location = format!("file://{}", temp_dir.path().display()); @@ -8796,7 +8974,7 @@ mod tests { } #[tokio::test] - async fn test_drop_batch_fallback_does_not_expand_exact_partition_to_typed_aliases() { + async fn test_drop_batch_preserves_exact_partition_aliases() { let temp_dir = tempfile::tempdir().unwrap(); for month in ["1", "01", "02"] { std::fs::create_dir_all(temp_dir.path().join(format!("month={month}"))).unwrap(); @@ -8901,7 +9079,7 @@ mod tests { } #[tokio::test] - async fn test_drop_integer_partition_matches_whitespace_msck_raw_value() { + async fn test_drop_rejects_whitespace_padded_catalog_integer() { let temp_dir = tempfile::tempdir().unwrap(); std::fs::create_dir_all(temp_dir.path().join("month= 01")).unwrap(); let location = format!("file://{}", temp_dir.path().display()); @@ -8920,18 +9098,27 @@ mod tests { )])]); let sql_context = make_sql_context(catalog.clone()).await; - sql_context + let error = sql_context .sql("ALTER TABLE mydb.events DROP PARTITION (month = 1)") .await - .unwrap(); + .unwrap_err(); - assert!(catalog.partition_specs().is_empty()); + assert!( + error + .to_string() + .contains("Invalid catalog partition value \" 01\" for column 'month'"), + "expected malformed catalog metadata error, got: {error}" + ); + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([("month".to_string(), " 01".to_string())])] + ); assert_eq!(catalog.list_partitions_calls(), 1); - assert!(!temp_dir.path().join("month= 01").exists()); + assert!(temp_dir.path().join("month= 01").is_dir()); } #[tokio::test] - async fn test_drop_char_partition_matches_unpadded_msck_raw_value() { + async fn test_drop_char_partition_requires_exact_catalog_value() { let temp_dir = tempfile::tempdir().unwrap(); std::fs::create_dir_all(temp_dir.path().join("region=us")).unwrap(); let location = format!("file://{}", temp_dir.path().display()); @@ -8950,14 +9137,21 @@ mod tests { )])]); let sql_context = make_sql_context(catalog.clone()).await; - sql_context + let error = sql_context .sql("ALTER TABLE mydb.events DROP PARTITION (region = 'us')") .await - .unwrap(); + .unwrap_err(); - assert!(catalog.partition_specs().is_empty()); + assert!( + error.to_string().contains("does not exist"), + "expected catalog CHAR value to remain distinct, got: {error}" + ); + assert_eq!( + catalog.partition_specs(), + vec![HashMap::from([("region".to_string(), "us".to_string())])] + ); assert_eq!(catalog.list_partitions_calls(), 1); - assert!(!temp_dir.path().join("region=us").exists()); + assert!(temp_dir.path().join("region=us").is_dir()); } #[tokio::test] @@ -9021,7 +9215,7 @@ mod tests { } #[tokio::test] - async fn test_msck_add_preserves_raw_filesystem_partition_values() { + async fn test_msck_add_preserves_filesystem_partition_values() { let temp_dir = tempfile::tempdir().unwrap(); std::fs::create_dir_all(temp_dir.path().join("month=01")).unwrap(); std::fs::create_dir_all(temp_dir.path().join("month=02")).unwrap(); @@ -9098,7 +9292,7 @@ mod tests { catalog.set_table( managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, ); - catalog.set_list_partitions_error(true); + catalog.fail_list_partitions(); let sql_context = make_sql_context(catalog.clone()).await; let error = sql_context diff --git a/crates/integrations/datafusion/tests/rest_format_partition_sql.rs b/crates/integrations/datafusion/tests/rest_format_partition_sql.rs index 1e03f200..1e9765aa 100644 --- a/crates/integrations/datafusion/tests/rest_format_partition_sql.rs +++ b/crates/integrations/datafusion/tests/rest_format_partition_sql.rs @@ -15,55 +15,82 @@ // specific language governing permissions and limitations // under the License. +#[path = "../../../paimon/tests/mock_server.rs"] +mod mock_server; + use std::collections::HashMap; use std::sync::Arc; use arrow_array::StringArray; use paimon::api::ConfigResponse; use paimon::catalog::RESTCatalog; -use paimon::common::Options; use paimon::spec::{BigIntType, BooleanType, DataType, DateType, IntType, Schema, VarCharType}; +use paimon::{CatalogOptions, Options}; use paimon_datafusion::SQLContext; +use tempfile::TempDir; -#[path = "../../../paimon/tests/mock_server.rs"] -mod mock_server; -use mock_server::start_mock_server; +use mock_server::{start_mock_server, RESTServer}; -#[cfg(not(windows))] -#[tokio::test] -async fn rest_managed_format_partition_sql_end_to_end() { - let temp_dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(temp_dir.path().join("dt=2026-07-21")).unwrap(); - let table_path = format!("file://{}", temp_dir.path().display()); +const DATABASE: &str = "default"; +const TABLE: &str = "events"; +const WAREHOUSE: &str = "test_warehouse"; - let prefix = "mock-test"; +async fn setup_rest_table(temp_dir: &TempDir, schema: Schema) -> (RESTServer, SQLContext) { let server = start_mock_server( - "test_warehouse".to_string(), + WAREHOUSE.to_string(), temp_dir.path().to_string_lossy().into_owned(), - ConfigResponse::new(HashMap::from([("prefix".to_string(), prefix.to_string())])), - vec!["default".to_string()], + ConfigResponse::new(HashMap::from([( + CatalogOptions::PREFIX.to_string(), + "mock-test".to_string(), + )])), + vec![DATABASE.to_string()], ) .await; - let schema = Schema::builder() - .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) + server.add_table_with_schema( + DATABASE, + TABLE, + schema, + &format!("file://{}", temp_dir.path().display()), + ); + server.set_table_external(DATABASE, TABLE, false); + + let mut options = Options::new(); + options.set(CatalogOptions::URI, server.url().unwrap()); + options.set(CatalogOptions::WAREHOUSE, WAREHOUSE); + options.set(CatalogOptions::TOKEN_PROVIDER, "bear"); + options.set(CatalogOptions::TOKEN, "test-token"); + let catalog = Arc::new(RESTCatalog::new(options, true).await.unwrap()); + let mut context = SQLContext::new(); + context.register_catalog("paimon", catalog).await.unwrap(); + (server, context) +} + +fn format_table_schema(partition_columns: &[(&str, DataType)]) -> Schema { + let partition_keys = partition_columns + .iter() + .map(|(name, _)| (*name).to_string()) + .collect::>(); + partition_columns + .iter() + .fold(Schema::builder(), |builder, (name, data_type)| { + builder.column(*name, data_type.clone()) + }) .column("id", DataType::BigInt(BigIntType::new())) - .partition_keys(["dt"]) + .partition_keys(partition_keys) .option("type", "format-table") .option("file.format", "parquet") .option("metastore.partitioned-table", "true") .build() - .unwrap(); - server.add_table_with_schema("default", "events", schema, &table_path); - server.set_table_external("default", "events", false); + .unwrap() +} - let mut options = Options::new(); - options.set("uri", server.url().unwrap()); - options.set("warehouse", "test_warehouse"); - options.set("token.provider", "bear"); - options.set("token", "test-token"); - let catalog = Arc::new(RESTCatalog::new(options, true).await.unwrap()); - let mut context = SQLContext::new(); - context.register_catalog("paimon", catalog).await.unwrap(); +#[cfg(not(windows))] +#[tokio::test] +async fn test_partition_commands_update_rest_metadata_and_directories() { + let temp_dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(temp_dir.path().join("dt=2026-07-21")).unwrap(); + let schema = format_table_schema(&[("dt", DataType::VarChar(VarCharType::new(255).unwrap()))]); + let (_server, context) = setup_rest_table(&temp_dir, schema).await; context .sql("ALTER TABLE paimon.default.events ADD PARTITION (dt = '2026-07-22')") @@ -113,41 +140,15 @@ async fn rest_managed_format_partition_sql_end_to_end() { #[cfg(not(windows))] #[tokio::test] -async fn rest_managed_format_partition_sql_normalizes_typed_and_default_values() { +async fn test_partition_commands_normalize_typed_and_null_values() { let temp_dir = tempfile::tempdir().unwrap(); - let table_path = format!("file://{}", temp_dir.path().display()); - - let prefix = "mock-test"; - let server = start_mock_server( - "test_warehouse".to_string(), - temp_dir.path().to_string_lossy().into_owned(), - ConfigResponse::new(HashMap::from([("prefix".to_string(), prefix.to_string())])), - vec!["default".to_string()], - ) - .await; - let schema = Schema::builder() - .column("dt", DataType::Date(DateType::new())) - .column("month", DataType::Int(IntType::new())) - .column("active", DataType::Boolean(BooleanType::new())) - .column("label", DataType::VarChar(VarCharType::new(255).unwrap())) - .column("id", DataType::BigInt(BigIntType::new())) - .partition_keys(["dt", "month", "active", "label"]) - .option("type", "format-table") - .option("file.format", "parquet") - .option("metastore.partitioned-table", "true") - .build() - .unwrap(); - server.add_table_with_schema("default", "events", schema, &table_path); - server.set_table_external("default", "events", false); - - let mut options = Options::new(); - options.set("uri", server.url().unwrap()); - options.set("warehouse", "test_warehouse"); - options.set("token.provider", "bear"); - options.set("token", "test-token"); - let catalog = Arc::new(RESTCatalog::new(options, true).await.unwrap()); - let mut context = SQLContext::new(); - context.register_catalog("paimon", catalog).await.unwrap(); + let schema = format_table_schema(&[ + ("dt", DataType::Date(DateType::new())), + ("month", DataType::Int(IntType::new())), + ("active", DataType::Boolean(BooleanType::new())), + ("label", DataType::VarChar(VarCharType::new(255).unwrap())), + ]); + let (_server, context) = setup_rest_table(&temp_dir, schema).await; context .sql( @@ -193,35 +194,16 @@ async fn rest_managed_format_partition_sql_normalizes_typed_and_default_values() #[cfg(not(windows))] #[tokio::test] -async fn rest_drop_batch_fallback_preserves_exact_partition_aliases() { +async fn test_drop_partition_preserves_exact_registered_values() { let temp_dir = tempfile::tempdir().unwrap(); for month in ["1", "01", "02"] { std::fs::create_dir_all(temp_dir.path().join(format!("month={month}"))).unwrap(); } - let table_path = format!("file://{}", temp_dir.path().display()); - - let prefix = "mock-test"; - let server = start_mock_server( - "test_warehouse".to_string(), - temp_dir.path().to_string_lossy().into_owned(), - ConfigResponse::new(HashMap::from([("prefix".to_string(), prefix.to_string())])), - vec!["default".to_string()], - ) - .await; - let schema = Schema::builder() - .column("month", DataType::Int(IntType::new())) - .column("id", DataType::BigInt(BigIntType::new())) - .partition_keys(["month"]) - .option("type", "format-table") - .option("file.format", "parquet") - .option("metastore.partitioned-table", "true") - .build() - .unwrap(); - server.add_table_with_schema("default", "events", schema, &table_path); - server.set_table_external("default", "events", false); + let schema = format_table_schema(&[("month", DataType::Int(IntType::new()))]); + let (server, context) = setup_rest_table(&temp_dir, schema).await; server.set_table_partitions( - "default", - "events", + DATABASE, + TABLE, vec![ HashMap::from([("month".to_string(), "1".to_string())]), HashMap::from([("month".to_string(), "01".to_string())]), @@ -229,15 +211,6 @@ async fn rest_drop_batch_fallback_preserves_exact_partition_aliases() { ], ); - let mut options = Options::new(); - options.set("uri", server.url().unwrap()); - options.set("warehouse", "test_warehouse"); - options.set("token.provider", "bear"); - options.set("token", "test-token"); - let catalog = Arc::new(RESTCatalog::new(options, true).await.unwrap()); - let mut context = SQLContext::new(); - context.register_catalog("paimon", catalog).await.unwrap(); - context .sql( "ALTER TABLE paimon.default.events \ @@ -264,7 +237,7 @@ async fn rest_drop_batch_fallback_preserves_exact_partition_aliases() { #[cfg(not(windows))] #[tokio::test] -async fn rest_drop_mixed_complete_and_partial_specs_keep_complete_match_exact() { +async fn test_drop_partition_preserves_complete_specs_when_expanding_partial_specs() { let temp_dir = tempfile::tempdir().unwrap(); for (region, month) in [("us", "1"), ("us", "01"), ("eu", "02")] { std::fs::create_dir_all( @@ -274,31 +247,14 @@ async fn rest_drop_mixed_complete_and_partial_specs_keep_complete_match_exact() ) .unwrap(); } - let table_path = format!("file://{}", temp_dir.path().display()); - - let prefix = "mock-test"; - let server = start_mock_server( - "test_warehouse".to_string(), - temp_dir.path().to_string_lossy().into_owned(), - ConfigResponse::new(HashMap::from([("prefix".to_string(), prefix.to_string())])), - vec!["default".to_string()], - ) - .await; - let schema = Schema::builder() - .column("region", DataType::VarChar(VarCharType::new(32).unwrap())) - .column("month", DataType::Int(IntType::new())) - .column("id", DataType::BigInt(BigIntType::new())) - .partition_keys(["region", "month"]) - .option("type", "format-table") - .option("file.format", "parquet") - .option("metastore.partitioned-table", "true") - .build() - .unwrap(); - server.add_table_with_schema("default", "events", schema, &table_path); - server.set_table_external("default", "events", false); + let schema = format_table_schema(&[ + ("region", DataType::VarChar(VarCharType::new(32).unwrap())), + ("month", DataType::Int(IntType::new())), + ]); + let (server, context) = setup_rest_table(&temp_dir, schema).await; server.set_table_partitions( - "default", - "events", + DATABASE, + TABLE, vec![ HashMap::from([ ("region".to_string(), "us".to_string()), @@ -315,15 +271,6 @@ async fn rest_drop_mixed_complete_and_partial_specs_keep_complete_match_exact() ], ); - let mut options = Options::new(); - options.set("uri", server.url().unwrap()); - options.set("warehouse", "test_warehouse"); - options.set("token.provider", "bear"); - options.set("token", "test-token"); - let catalog = Arc::new(RESTCatalog::new(options, true).await.unwrap()); - let mut context = SQLContext::new(); - context.register_catalog("paimon", catalog).await.unwrap(); - context .sql( "ALTER TABLE paimon.default.events \ @@ -361,17 +308,14 @@ async fn show_partitions(context: &SQLContext) -> Vec { .collect() .await .unwrap(); - batches - .iter() - .flat_map(|batch| { - let values = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - (0..batch.num_rows()) - .map(|index| values.value(index).to_string()) - .collect::>() - }) - .collect() + let mut partitions = Vec::new(); + for batch in batches { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + partitions.extend((0..batch.num_rows()).map(|index| values.value(index).to_string())); + } + partitions } diff --git a/crates/paimon/src/api/api_request.rs b/crates/paimon/src/api/api_request.rs index 2d2f6bf9..16d04c31 100644 --- a/crates/paimon/src/api/api_request.rs +++ b/crates/paimon/src/api/api_request.rs @@ -19,7 +19,7 @@ //! //! This module contains all request structures used in REST API calls. -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use std::collections::HashMap; use crate::{ @@ -174,12 +174,17 @@ pub struct CreatePartitionsRequest { /// Partition specs to register. pub partition_specs: Vec>, /// Whether already registered partitions should be ignored. - #[serde(default = "default_true")] + /// + /// Defaults to `true` when omitted or null. + #[serde( + default = "default_true", + deserialize_with = "deserialize_null_as_true" + )] pub ignore_if_exists: bool, } impl CreatePartitionsRequest { - /// Create a new CreatePartitionsRequest. + /// Create a request to register partitions. pub fn new(partition_specs: Vec>, ignore_if_exists: bool) -> Self { Self { partition_specs, @@ -188,19 +193,24 @@ impl CreatePartitionsRequest { } } -/// Request to drop table partitions. +/// Request to drop (unregister) table partitions. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct DropPartitionsRequest { /// Partition specs to unregister. pub partition_specs: Vec>, /// Whether missing partitions should be ignored. - #[serde(default = "default_true")] + /// + /// Defaults to `true` when omitted or null. + #[serde( + default = "default_true", + deserialize_with = "deserialize_null_as_true" + )] pub ignore_if_not_exists: bool, } impl DropPartitionsRequest { - /// Create a new DropPartitionsRequest. + /// Create a request to unregister partitions. pub fn new(partition_specs: Vec>, ignore_if_not_exists: bool) -> Self { Self { partition_specs, @@ -219,7 +229,7 @@ pub struct ListPartitionsByNamesRequest { } impl ListPartitionsByNamesRequest { - /// Create a new ListPartitionsByNamesRequest. + /// Create a request to list partitions by exact specs. pub fn new(partition_specs: Vec>) -> Self { Self { partition_specs } } @@ -229,6 +239,13 @@ fn default_true() -> bool { true } +fn deserialize_null_as_true<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + Ok(Option::::deserialize(deserializer)?.unwrap_or(true)) +} + /// Request for auth table query: the projected columns of the query. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct AuthTableQueryRequest { @@ -305,13 +322,24 @@ mod tests { } #[test] - fn test_create_partitions_request_defaults_ignore_if_exists() { - let req: CreatePartitionsRequest = serde_json::from_value(serde_json::json!({ + fn test_create_partitions_request_defaults_missing_ignore_if_exists() { + let request: CreatePartitionsRequest = serde_json::from_value(serde_json::json!({ "partitionSpecs": [{"dt": "2026-07-22"}] })) .unwrap(); - assert!(req.ignore_if_exists); + assert!(request.ignore_if_exists); + } + + #[test] + fn test_create_partitions_request_defaults_null_ignore_if_exists() { + let request: CreatePartitionsRequest = serde_json::from_value(serde_json::json!({ + "partitionSpecs": [{"dt": "2026-07-22"}], + "ignoreIfExists": null + })) + .unwrap(); + + assert!(request.ignore_if_exists); } #[test] @@ -334,13 +362,24 @@ mod tests { } #[test] - fn test_drop_partitions_request_defaults_ignore_if_not_exists() { - let req: DropPartitionsRequest = serde_json::from_value(serde_json::json!({ + fn test_drop_partitions_request_defaults_missing_ignore_if_not_exists() { + let request: DropPartitionsRequest = serde_json::from_value(serde_json::json!({ "partitionSpecs": [{"dt": "2026-07-22"}] })) .unwrap(); - assert!(req.ignore_if_not_exists); + assert!(request.ignore_if_not_exists); + } + + #[test] + fn test_drop_partitions_request_defaults_null_ignore_if_not_exists() { + let request: DropPartitionsRequest = serde_json::from_value(serde_json::json!({ + "partitionSpecs": [{"dt": "2026-07-22"}], + "ignoreIfNotExists": null + })) + .unwrap(); + + assert!(request.ignore_if_not_exists); } #[test] diff --git a/crates/paimon/src/api/api_response.rs b/crates/paimon/src/api/api_response.rs index 0f866178..16d8951c 100644 --- a/crates/paimon/src/api/api_response.rs +++ b/crates/paimon/src/api/api_response.rs @@ -426,22 +426,18 @@ impl ListTablesResponse { #[serde(rename_all = "camelCase")] pub struct CreatePartitionsResponse { /// Partition specs created by the request. - #[serde(default, deserialize_with = "deserialize_null_to_default")] pub created: Vec>, - /// Partition specs which already existed. - #[serde(default, deserialize_with = "deserialize_null_to_default")] + /// Partition specs that already existed. pub existed: Vec>, } -/// Response for dropping table partitions. +/// Response for dropping (unregistering) table partitions. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct DropPartitionsResponse { - /// Partition specs dropped by the request. - #[serde(default, deserialize_with = "deserialize_null_to_default")] + /// Partition specs unregistered by the request. pub dropped: Vec>, - /// Partition specs which did not exist. - #[serde(default, deserialize_with = "deserialize_null_to_default")] + /// Partition specs that did not exist. pub missing: Vec>, } @@ -710,12 +706,23 @@ mod tests { } #[test] - fn test_create_partitions_response_defaults_null_and_missing_lists() { - let response: CreatePartitionsResponse = - serde_json::from_value(serde_json::json!({"created": null})).unwrap(); - + fn test_create_partitions_response_requires_non_null_lists() { + let response: CreatePartitionsResponse = serde_json::from_value(serde_json::json!({ + "created": [], + "existed": [] + })) + .unwrap(); assert!(response.created.is_empty()); assert!(response.existed.is_empty()); + + for invalid in [ + serde_json::json!({"created": []}), + serde_json::json!({"existed": []}), + serde_json::json!({"created": null, "existed": []}), + serde_json::json!({"created": [], "existed": null}), + ] { + assert!(serde_json::from_value::(invalid).is_err()); + } } #[test] @@ -743,12 +750,23 @@ mod tests { } #[test] - fn test_drop_partitions_response_defaults_null_and_missing_lists() { - let response: DropPartitionsResponse = - serde_json::from_value(serde_json::json!({"dropped": null})).unwrap(); - + fn test_drop_partitions_response_requires_non_null_lists() { + let response: DropPartitionsResponse = serde_json::from_value(serde_json::json!({ + "dropped": [], + "missing": [] + })) + .unwrap(); assert!(response.dropped.is_empty()); assert!(response.missing.is_empty()); + + for invalid in [ + serde_json::json!({"dropped": []}), + serde_json::json!({"missing": []}), + serde_json::json!({"dropped": null, "missing": []}), + serde_json::json!({"dropped": [], "missing": null}), + ] { + assert!(serde_json::from_value::(invalid).is_err()); + } } #[test] diff --git a/crates/paimon/src/api/resource_paths.rs b/crates/paimon/src/api/resource_paths.rs index 7ad46325..1be104d3 100644 --- a/crates/paimon/src/api/resource_paths.rs +++ b/crates/paimon/src/api/resource_paths.rs @@ -217,12 +217,12 @@ impl ResourcePaths { ) } - /// Get the drop partitions endpoint path for a table. + /// Get the endpoint path for dropping table partitions. pub fn drop_partitions(&self, database_name: &str, table_name: &str) -> String { format!("{}/drop", self.partitions(database_name, table_name)) } - /// Get the list partitions by names endpoint path for a table. + /// Get the endpoint path for listing table partitions by name. pub fn list_partitions_by_names(&self, database_name: &str, table_name: &str) -> String { format!( "{}/list-by-names", diff --git a/crates/paimon/src/api/rest_api.rs b/crates/paimon/src/api/rest_api.rs index 8c77bbf7..14aa0582 100644 --- a/crates/paimon/src/api/rest_api.rs +++ b/crates/paimon/src/api/rest_api.rs @@ -74,16 +74,16 @@ fn validate_non_empty_multi(values: &[(&str, &str)]) -> Result<()> { Ok(()) } -fn validate_partition_request_size( +fn validate_list_partitions_by_names_request_size( identifier: &Identifier, partition_spec_count: usize, ) -> Result<()> { - if partition_spec_count > RESTApi::MAX_PARTITION_SPECS_PER_REQUEST { + if partition_spec_count > RESTApi::PARTITION_REQUEST_SIZE { return Err(crate::Error::DataInvalid { message: format!( - "REST partition requests for table {} accept at most {} specs, got {}", + "List partitions by names accepts at most {} partition specs for table {}, got {}", + RESTApi::PARTITION_REQUEST_SIZE, identifier.full_name(), - RESTApi::MAX_PARTITION_SPECS_PER_REQUEST, partition_spec_count ), source: None, @@ -106,8 +106,10 @@ impl RESTApi { // Constants for query parameters and headers pub const HEADER_PREFIX: &'static str = "header."; pub const MAX_RESULTS: &'static str = "maxResults"; - /// Maximum partition specs accepted by one REST request. - pub const MAX_PARTITION_SPECS_PER_REQUEST: usize = 1000; + /// Partition batch size used by `RESTCatalog`. + /// + /// The list-by-names endpoint also limits each request to this many specs. + pub(crate) const PARTITION_REQUEST_SIZE: usize = 1000; pub const PAGE_TOKEN: &'static str = "pageToken"; pub const DATABASE_NAME_PATTERN: &'static str = "databaseNamePattern"; pub const TABLE_NAME_PATTERN: &'static str = "tableNamePattern"; @@ -578,11 +580,7 @@ impl RESTApi { // ==================== Partition Operations ==================== - /// Create table partitions with one REST request. - /// - /// Requests larger than [`Self::MAX_PARTITION_SPECS_PER_REQUEST`] are - /// rejected locally. Use [`crate::catalog::RESTCatalog`] when idempotent - /// batches should be split automatically. + /// Create table partitions in a single REST request. pub async fn create_partitions( &self, identifier: &Identifier, @@ -592,17 +590,15 @@ impl RESTApi { let database = identifier.database(); let table = identifier.object(); validate_non_empty_multi(&[(database, "database name"), (table, "table name")])?; - validate_partition_request_size(identifier, partition_specs.len())?; let path = self.resource_paths.partitions(database, table); let request = CreatePartitionsRequest::new(partition_specs, ignore_if_exists); self.client.post(&path, &request).await } - /// Drop table partitions with one REST request. + /// Unregister table partitions in a single REST request. /// - /// Requests larger than [`Self::MAX_PARTITION_SPECS_PER_REQUEST`] are - /// rejected locally. Use [`crate::catalog::RESTCatalog`] to split larger - /// idempotent batches. + /// The REST service removes metadata only; it does not delete partition + /// directories or data files. pub async fn drop_partitions( &self, identifier: &Identifier, @@ -612,17 +608,15 @@ impl RESTApi { let database = identifier.database(); let table = identifier.object(); validate_non_empty_multi(&[(database, "database name"), (table, "table name")])?; - validate_partition_request_size(identifier, partition_specs.len())?; let path = self.resource_paths.drop_partitions(database, table); let request = DropPartitionsRequest::new(partition_specs, ignore_if_not_exists); self.client.post(&path, &request).await } - /// List table partitions by exact specs with one REST request. + /// List table partitions by exact specs in a single REST request. /// - /// Requests larger than [`Self::MAX_PARTITION_SPECS_PER_REQUEST`] are - /// rejected locally. Use [`crate::catalog::RESTCatalog`] to split larger - /// batches. + /// At most 1,000 specs may be sent per request. Use + /// [`crate::catalog::RESTCatalog`] to split larger batches. pub async fn list_partitions_by_names( &self, identifier: &Identifier, @@ -631,7 +625,7 @@ impl RESTApi { let database = identifier.database(); let table = identifier.object(); validate_non_empty_multi(&[(database, "database name"), (table, "table name")])?; - validate_partition_request_size(identifier, partition_specs.len())?; + validate_list_partitions_by_names_request_size(identifier, partition_specs.len())?; let path = self .resource_paths .list_partitions_by_names(database, table); @@ -655,21 +649,22 @@ impl RESTApi { .list_partitions_paged(identifier, None, page_token.as_deref()) .await?; results.extend(paged.elements); - page_token = paged.next_page_token.filter(|token| !token.is_empty()); - match page_token.as_ref() { - Some(token) if !seen_page_tokens.insert(token.clone()) => { - return Err(crate::Error::UnexpectedError { - message: format!( - "REST catalog returned repeated partition page token '{token}' \ - for table {}", - identifier.full_name() - ), - source: None, - }); - } - Some(_) => {} - None => break, + + let Some(next_page_token) = paged.next_page_token.filter(|token| !token.is_empty()) + else { + break; + }; + if !seen_page_tokens.insert(next_page_token.clone()) { + return Err(crate::Error::UnexpectedError { + message: format!( + "REST catalog returned partition page token '{next_page_token}' more than \ + once for table {}", + identifier.full_name() + ), + source: None, + }); } + page_token = Some(next_page_token); } Ok(results) diff --git a/crates/paimon/src/catalog/mod.rs b/crates/paimon/src/catalog/mod.rs index 95fe7701..ffb0ec16 100644 --- a/crates/paimon/src/catalog/mod.rs +++ b/crates/paimon/src/catalog/mod.rs @@ -445,11 +445,13 @@ pub trait Catalog: Send + Sync { }) } + // ======================= partition methods =============================== + /// Register table partition specs in the catalog. /// - /// When `ignore_if_exists` is false, an existing spec must reject the - /// entire supplied batch without registering any of it. When true, - /// existing specs are ignored and retries must converge. + /// When `ignore_if_exists` is false, implementations must reject the + /// entire request if any supplied spec already exists. When true, existing + /// specs are ignored so callers can safely retry the request. async fn create_partitions( &self, _identifier: &Identifier, @@ -463,15 +465,14 @@ pub trait Catalog: Send + Sync { /// Unregister table partition specs from the catalog. /// - /// Missing specs are ignored so callers can safely retry a partially - /// applied batch. - async fn drop_partitions( + /// Missing specs are ignored so callers can safely retry the request. + async fn unregister_partitions( &self, _identifier: &Identifier, _partition_specs: Vec>, ) -> Result<()> { Err(Error::Unsupported { - message: "Catalog does not support dropping partitions".to_string(), + message: "Catalog does not support unregistering partitions".to_string(), }) } diff --git a/crates/paimon/src/catalog/rest/rest_catalog.rs b/crates/paimon/src/catalog/rest/rest_catalog.rs index cad07039..c1d5a567 100644 --- a/crates/paimon/src/catalog/rest/rest_catalog.rs +++ b/crates/paimon/src/catalog/rest/rest_catalog.rs @@ -378,7 +378,7 @@ impl Catalog for RESTCatalog { .map_err(|error| map_rest_error_for_create_partitions(error, identifier)); } - for batch in partition_specs.chunks(RESTApi::MAX_PARTITION_SPECS_PER_REQUEST) { + for batch in partition_specs.chunks(RESTApi::PARTITION_REQUEST_SIZE) { self.api .create_partitions(identifier, batch.to_vec(), true) .await @@ -387,12 +387,12 @@ impl Catalog for RESTCatalog { Ok(()) } - async fn drop_partitions( + async fn unregister_partitions( &self, identifier: &Identifier, partition_specs: Vec>, ) -> Result<()> { - for batch in partition_specs.chunks(RESTApi::MAX_PARTITION_SPECS_PER_REQUEST) { + for batch in partition_specs.chunks(RESTApi::PARTITION_REQUEST_SIZE) { self.api .drop_partitions(identifier, batch.to_vec(), true) .await @@ -406,21 +406,40 @@ impl Catalog for RESTCatalog { identifier: &Identifier, partition_specs: Vec>, ) -> Result> { - let mut found = Vec::new(); - for batch in partition_specs.chunks(RESTApi::MAX_PARTITION_SPECS_PER_REQUEST) { - found.extend( - self.api - .list_partitions_by_names(identifier, batch.to_vec()) - .await - .map_err(|error| map_rest_error_for_partition_request(error, identifier))?, - ); + let mut partitions = Vec::new(); + for batch in partition_specs.chunks(RESTApi::PARTITION_REQUEST_SIZE) { + match self + .api + .list_partitions_by_names(identifier, batch.to_vec()) + .await + { + Ok(batch_partitions) => partitions.extend(batch_partitions), + Err( + error @ Error::RestApi { + source: RestError::NotImplemented { .. }, + }, + ) => { + let table = self.get_table(identifier).await?; + if table.has_catalog_managed_partitions() { + return Err(error); + } + return Ok(list_partitions_from_file_system(&table) + .await? + .into_iter() + .filter(|partition| partition_specs.contains(&partition.spec)) + .collect()); + } + Err(error) => { + return Err(map_rest_error_for_partition_request(error, identifier)); + } + } } - Ok(found) + Ok(partitions) } async fn list_partitions(&self, identifier: &Identifier) -> Result> { match self.api.list_partitions(identifier).await { - Ok(parts) => Ok(parts), + Ok(partitions) => Ok(partitions), Err( error @ Error::RestApi { source: RestError::NotImplemented { .. }, @@ -432,7 +451,7 @@ impl Catalog for RESTCatalog { } list_partitions_from_file_system(&table).await } - Err(e) => Err(map_rest_error_for_table(e, identifier)), + Err(error) => Err(map_rest_error_for_table(error, identifier)), } } @@ -457,13 +476,14 @@ impl Catalog for RESTCatalog { if table.has_catalog_managed_partitions() { return Err(error); } - let parts = list_partitions_from_file_system(&table).await?; - Ok(PagedList::new(parts, None)) + let partitions = list_partitions_from_file_system(&table).await?; + Ok(PagedList::new(partitions, None)) } - Err(e) => Err(map_rest_error_for_table(e, identifier)), + Err(error) => Err(map_rest_error_for_table(error, identifier)), } } } + // ============================================================================ // Error mapping helpers // ============================================================================ @@ -516,7 +536,7 @@ fn map_rest_error_for_create_partitions(err: Error, identifier: &Identifier) -> source: RestError::AlreadyExists { message, .. }, } => Error::DataInvalid { message: format!( - "Some partitions of table {} already exist: {message}", + "One or more partitions already exist for table {}: {message}", identifier.full_name() ), source: None, diff --git a/crates/paimon/src/spec/core_options.rs b/crates/paimon/src/spec/core_options.rs index aacfd759..e705f708 100644 --- a/crates/paimon/src/spec/core_options.rs +++ b/crates/paimon/src/spec/core_options.rs @@ -34,8 +34,8 @@ const SOURCE_SPLIT_OPEN_FILE_COST_OPTION: &str = "source.split.open-file-cost"; const PARTITION_DEFAULT_NAME_OPTION: &str = "partition.default-name"; const PARTITION_LEGACY_NAME_OPTION: &str = "partition.legacy-name"; pub(crate) const METASTORE_PARTITIONED_TABLE_OPTION: &str = "metastore.partitioned-table"; -const FORMAT_TABLE_IMPLEMENTATION_OPTION: &str = "format-table.implementation"; -const FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE_OPTION: &str = +pub(crate) const FORMAT_TABLE_IMPLEMENTATION_OPTION: &str = "format-table.implementation"; +pub(crate) const FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE_OPTION: &str = "format-table.partition-path-only-value"; pub(crate) const BUCKET_KEY_OPTION: &str = "bucket-key"; const BUCKET_FUNCTION_TYPE_OPTION: &str = "bucket-function.type"; @@ -126,6 +126,12 @@ pub(crate) const BLOB_VIEW_FIELD_OPTION: &str = "blob-view-field"; pub const BLOB_VIEW_RESOLVE_ENABLED_OPTION: &str = "blob-view.resolve.enabled"; const PK_VECTOR_INDEX_COLUMNS_OPTION: &str = "pk-vector.index.columns"; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum FormatTableImplementation { + Paimon, + Engine, +} + /// Merge engine for primary-key tables. /// /// Reference: Java `CoreOptions.MergeEngine`. @@ -538,17 +544,41 @@ impl<'a> CoreOptions<'a> { .unwrap_or(false) } - pub fn partitioned_table_in_metastore(&self) -> bool { - self.options - .get(METASTORE_PARTITIONED_TABLE_OPTION) - .map(|value| value.eq_ignore_ascii_case("true")) - .unwrap_or(false) + pub(crate) fn try_format_table_partition_only_value_in_path(&self) -> crate::Result { + self.try_boolean_option(FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE_OPTION) } - pub fn format_table_uses_engine_implementation(&self) -> bool { - self.options + pub(crate) fn partitioned_table_in_metastore(&self) -> crate::Result { + self.try_boolean_option(METASTORE_PARTITIONED_TABLE_OPTION) + } + + pub(crate) fn format_table_implementation(&self) -> crate::Result { + match self + .options .get(FORMAT_TABLE_IMPLEMENTATION_OPTION) - .is_some_and(|value| value.eq_ignore_ascii_case("engine")) + .map(String::as_str) + .unwrap_or("paimon") + { + value if value.eq_ignore_ascii_case("paimon") => Ok(FormatTableImplementation::Paimon), + value if value.eq_ignore_ascii_case("engine") => Ok(FormatTableImplementation::Engine), + value => Err(crate::Error::ConfigInvalid { + message: format!( + "Invalid value '{value}' for {FORMAT_TABLE_IMPLEMENTATION_OPTION}; \ + expected 'paimon' or 'engine'" + ), + }), + } + } + + fn try_boolean_option(&self, key: &str) -> crate::Result { + match self.options.get(key).map(String::as_str) { + None => Ok(false), + Some(value) if value.eq_ignore_ascii_case("true") => Ok(true), + Some(value) if value.eq_ignore_ascii_case("false") => Ok(false), + Some(value) => Err(crate::Error::ConfigInvalid { + message: format!("Invalid value '{value}' for {key}; expected 'true' or 'false'"), + }), + } } pub fn global_index_enabled(&self) -> bool { @@ -1556,13 +1586,17 @@ mod tests { #[test] fn test_partitioned_table_in_metastore_option() { let empty = HashMap::new(); - assert!(!CoreOptions::new(&empty).partitioned_table_in_metastore()); + assert!(!CoreOptions::new(&empty) + .partitioned_table_in_metastore() + .unwrap()); let options = HashMap::from([( - "metastore.partitioned-table".to_string(), + METASTORE_PARTITIONED_TABLE_OPTION.to_string(), "TrUe".to_string(), )]); - assert!(CoreOptions::new(&options).partitioned_table_in_metastore()); + assert!(CoreOptions::new(&options) + .partitioned_table_in_metastore() + .unwrap()); } #[test] diff --git a/crates/paimon/src/spec/mod.rs b/crates/paimon/src/spec/mod.rs index 71d39244..9c962cd2 100644 --- a/crates/paimon/src/spec/mod.rs +++ b/crates/paimon/src/spec/mod.rs @@ -94,7 +94,7 @@ pub use types::*; mod partition; pub use partition::Partition; mod partition_utils; -pub(crate) use partition_utils::{escape_path_name, PartitionComputer}; +pub(crate) use partition_utils::{escape_path_name, unescape_path_name, PartitionComputer}; mod predicate; pub(crate) use predicate::datum_cmp; pub(crate) use predicate::eval_row; diff --git a/crates/paimon/src/spec/partition_utils.rs b/crates/paimon/src/spec/partition_utils.rs index f81bc941..c6659f46 100644 --- a/crates/paimon/src/spec/partition_utils.rs +++ b/crates/paimon/src/spec/partition_utils.rs @@ -488,6 +488,29 @@ pub(crate) fn escape_path_name(path: &str) -> String { sb } +/// Unescape a path component following Java `PartitionPathUtils.unescapePathName`. +/// +/// Invalid or incomplete percent escapes are preserved literally. +pub(crate) fn unescape_path_name(path: &str) -> String { + let mut result = String::with_capacity(path.len()); + let mut chars = path.chars().peekable(); + while let Some(character) = chars.next() { + if character == '%' { + let mut lookahead = chars.clone(); + if let (Some(high), Some(low)) = (lookahead.next(), lookahead.next()) { + if let (Some(high), Some(low)) = (high.to_digit(16), low.to_digit(16)) { + chars.next(); + chars.next(); + result.push(char::from(((high << 4) | low) as u8)); + continue; + } + } + } + result.push(character); + } + result +} + /// Check if a character needs escaping in partition path names. /// /// Matches Java `PartitionPathUtils.CHAR_TO_ESCAPE`: @@ -657,6 +680,14 @@ mod tests { assert_eq!(escape_path_name("a\x7Fb"), "a%7Fb"); } + #[test] + fn test_unescape_path_name_matches_java_lenient_decoding() { + assert_eq!(unescape_path_name("a%2Fb"), "a/b"); + assert_eq!(unescape_path_name("a%2fb"), "a/b"); + assert_eq!(unescape_path_name("a%ZZb"), "a%ZZb"); + assert_eq!(unescape_path_name("a%b"), "a%b"); + } + // ======================== PartitionComputer tests ======================== #[test] diff --git a/crates/paimon/src/table/format_partition.rs b/crates/paimon/src/table/format_partition.rs index 3d2c2528..ba0f888d 100644 --- a/crates/paimon/src/table/format_partition.rs +++ b/crates/paimon/src/table/format_partition.rs @@ -19,8 +19,12 @@ use std::collections::HashMap; +use chrono::{Datelike, NaiveDate}; + use crate::io::FileIO; -use crate::spec::escape_path_name; +use crate::spec::{escape_path_name, unescape_path_name, DataType, Datum}; + +const UNIX_EPOCH_DAYS_FROM_CE: i32 = 719_163; /// Generates canonical names and physical paths for Format Table partitions. #[derive(Debug, Clone, PartialEq, Eq)] @@ -30,7 +34,7 @@ pub struct FormatTablePartitionPaths { } impl FormatTablePartitionPaths { - /// Create helpers for the declared partition-key order and physical layout. + /// Create a helper for the declared partition-key order and physical layout. pub fn new(partition_keys: I, only_value_in_path: bool) -> Self where I: IntoIterator, @@ -72,7 +76,7 @@ impl FormatTablePartitionPaths { /// Hidden directories whose names begin with `.` or `_`, segments that do /// not match the configured layout, and paths shallower than the declared /// partition depth are ignored. In value-only layouts, the configured - /// default-partition directory is the only underscore-prefixed exception. + /// default-partition directory is the only hidden-name exception. /// A matching segment that is malformed or not canonically escaped returns /// an error rather than being skipped, because its catalog spec would not /// resolve back to the same physical path. @@ -83,11 +87,19 @@ impl FormatTablePartitionPaths { table_path: &str, default_partition_name: &str, ) -> crate::Result>> { + let default_partition_path_name = self + .only_value_in_path + .then(|| escape_path_name(default_partition_name)); let mut frontier = vec![(table_path.trim_end_matches('/').to_string(), HashMap::new())]; for key in &self.partition_keys { let mut next = Vec::new(); for (path, spec) in frontier { - for status in file_io.list_status(&path).await? { + let statuses = match file_io.list_status(&path).await { + Ok(statuses) => statuses, + Err(error) if is_not_found(&error) => continue, + Err(error) => return Err(error), + }; + for status in statuses { if !status.is_dir { continue; } @@ -95,9 +107,9 @@ impl FormatTablePartitionPaths { continue; }; let is_value_only_default = - self.only_value_in_path && segment == default_partition_name; - if segment.starts_with('.') - || (segment.starts_with('_') && !is_value_only_default) + default_partition_path_name.as_deref() == Some(segment); + if (segment.starts_with('.') || segment.starts_with('_')) + && !is_value_only_default { continue; } @@ -122,24 +134,21 @@ impl FormatTablePartitionPaths { } fn ordered_values<'a>(&self, spec: &'a HashMap) -> crate::Result> { - if spec.len() != self.partition_keys.len() - || self - .partition_keys - .iter() - .any(|key| !spec.contains_key(key)) - { + if spec.len() != self.partition_keys.len() { return Err(crate::Error::DataInvalid { - message: format!( - "Partition spec {spec:?} must contain exactly keys {:?}", - self.partition_keys - ), + message: self.invalid_partition_keys_message(spec), source: None, }); } let mut values = Vec::with_capacity(self.partition_keys.len()); for key in &self.partition_keys { - let value = spec[key].as_str(); + let Some(value) = spec.get(key).map(String::as_str) else { + return Err(crate::Error::DataInvalid { + message: self.invalid_partition_keys_message(spec), + source: None, + }); + }; if value.is_empty() || (self.only_value_in_path && matches!(value, "." | "..")) { return Err(crate::Error::DataInvalid { message: format!( @@ -153,6 +162,15 @@ impl FormatTablePartitionPaths { Ok(values) } + fn invalid_partition_keys_message(&self, spec: &HashMap) -> String { + let mut actual_keys = spec.keys().collect::>(); + actual_keys.sort(); + format!( + "Partition spec must contain exactly keys {:?}, but contains {actual_keys:?}", + self.partition_keys + ) + } + fn partition_value_from_segment( &self, key: &str, @@ -164,13 +182,7 @@ impl FormatTablePartitionPaths { let Some((segment_key, value)) = segment.split_once('=') else { return Ok(None); }; - let decoded_key = - unescape_path_name(segment_key).ok_or_else(|| crate::Error::DataInvalid { - message: format!( - "Partition path segment {segment_key:?} is not valid percent-encoded text" - ), - source: None, - })?; + let decoded_key = unescape_path_name(segment_key); if decoded_key != key { return Ok(None); } @@ -182,10 +194,7 @@ impl FormatTablePartitionPaths { } fn unescape_canonical_path_name(value: &str) -> crate::Result { - let decoded = unescape_path_name(value).ok_or_else(|| crate::Error::DataInvalid { - message: format!("Partition path segment {value:?} is not valid percent-encoded text"), - source: None, - })?; + let decoded = unescape_path_name(value); ensure_canonical_path_name(value, &decoded)?; Ok(decoded) } @@ -204,27 +213,214 @@ fn ensure_canonical_path_name(value: &str, decoded: &str) -> crate::Result<()> { Ok(()) } -pub(crate) fn unescape_path_name(value: &str) -> Option { - let mut out = String::with_capacity(value.len()); - let mut chars = value.chars().peekable(); - while let Some(character) = chars.next() { - if character == '%' { - let mut lookahead = chars.clone(); - if let (Some(high), Some(low)) = (lookahead.next(), lookahead.next()) { - if let (Some(high), Some(low)) = (high.to_digit(16), low.to_digit(16)) { - let decoded = char::from_u32((high << 4) | low)?; - chars.next(); - chars.next(); - out.push(decoded); - continue; - } - } +/// Parse a raw Format Table partition value from a path or catalog registration. +/// +/// Boolean and integer values follow Java metadata parsing and do not accept +/// surrounding whitespace. Character values are preserved exactly. Numeric +/// DATE values are epoch days; textual DATE values accept `yyyy-MM` and +/// `yyyy-MM-dd`, optionally followed by a space and time text. Invalid or +/// unsupported values return `None`. +pub fn parse_format_partition_value(value: &str, data_type: &DataType) -> Option { + match data_type { + DataType::Boolean(_) => match value.to_ascii_lowercase().as_str() { + "t" | "true" | "y" | "yes" | "1" => Some(Datum::Bool(true)), + "f" | "false" | "n" | "no" | "0" => Some(Datum::Bool(false)), + _ => None, + }, + DataType::TinyInt(_) => value.parse::().ok().map(Datum::TinyInt), + DataType::SmallInt(_) => value.parse::().ok().map(Datum::SmallInt), + DataType::Int(_) => value.parse::().ok().map(Datum::Int), + DataType::BigInt(_) => value.parse::().ok().map(Datum::Long), + DataType::Char(char_type) if value.chars().count() <= char_type.length() => { + Some(Datum::String(value.to_string())) + } + DataType::VarChar(varchar_type) + if value.chars().count() <= varchar_type.length() as usize => + { + Some(Datum::String(value.to_string())) } - out.push(character); + DataType::Char(_) | DataType::VarChar(_) => None, + DataType::Date(_) => parse_partition_date(value).map(Datum::Date), + DataType::Time(_) => value.parse::().ok().map(Datum::Time), + _ => None, + } +} + +fn parse_partition_date(value: &str) -> Option { + let digits = value.strip_prefix('-').unwrap_or(value); + if !digits.is_empty() && digits.chars().all(|character| character.is_ascii_digit()) { + let epoch_days = value.parse::().ok()?; + return is_supported_epoch_day(epoch_days).then_some(epoch_days); + } + + let date_value = value + .find(' ') + .filter(|index| *index > 0) + .map(|index| &value[..index]) + .unwrap_or(value); + let mut segments = date_value.split('-'); + let year = parse_date_segment(segments.next()?)?; + if !(0..=9999).contains(&year) { + return None; + } + let month = match segments.next() { + Some(value) => parse_date_segment(value)?, + None => 1, + }; + let day = match segments.next() { + Some(value) => parse_date_segment(value)?, + None => 1, + }; + if segments.next().is_some() { + return None; + } + let date = NaiveDate::from_ymd_opt(year, month as u32, day as u32)?; + let epoch = NaiveDate::from_num_days_from_ce_opt(UNIX_EPOCH_DAYS_FROM_CE)?; + date.signed_duration_since(epoch).num_days().try_into().ok() +} + +fn parse_date_segment(value: &str) -> Option { + if value.is_empty() || !value.chars().all(|character| character.is_ascii_digit()) { + return None; } - Some(out) + value.parse().ok() +} + +fn is_supported_epoch_day(epoch_days: i32) -> bool { + epoch_days + .checked_add(UNIX_EPOCH_DAYS_FROM_CE) + .and_then(NaiveDate::from_num_days_from_ce_opt) + .is_some_and(|date| (0..=9999).contains(&date.year())) +} + +pub(crate) fn format_partition_date(epoch_days: i32) -> Option { + NaiveDate::from_num_days_from_ce_opt(epoch_days.checked_add(UNIX_EPOCH_DAYS_FROM_CE)?) + .map(|date| date.format("%Y-%m-%d").to_string()) +} + +fn is_not_found(error: &crate::Error) -> bool { + matches!( + error, + crate::Error::IoUnexpected { source, .. } + if source.kind() == opendal::ErrorKind::NotFound + ) } fn last_path_segment(path: &str) -> Option<&str> { path.trim_end_matches('/').rsplit('/').next() } + +#[cfg(test)] +mod tests { + use super::*; + use crate::spec::{BooleanType, CharType, DateType, VarCharType}; + + #[test] + fn test_parse_format_partition_boolean_uses_java_spellings() { + let data_type = DataType::Boolean(BooleanType::new()); + + assert_eq!( + parse_format_partition_value("1", &data_type), + Some(Datum::Bool(true)) + ); + assert_eq!( + parse_format_partition_value("yes", &data_type), + Some(Datum::Bool(true)) + ); + assert_eq!( + parse_format_partition_value("0", &data_type), + Some(Datum::Bool(false)) + ); + assert_eq!( + parse_format_partition_value("no", &data_type), + Some(Datum::Bool(false)) + ); + assert_eq!(parse_format_partition_value(" yes ", &data_type), None); + } + + #[test] + fn test_parse_format_partition_date_accepts_java_partial_month() { + let data_type = DataType::Date(DateType::new()); + + assert_eq!( + parse_format_partition_value("2026-07", &data_type), + parse_format_partition_value("2026-07-01", &data_type) + ); + } + + #[test] + fn test_parse_format_partition_numeric_date_uses_epoch_days() { + let data_type = DataType::Date(DateType::new()); + + assert_eq!( + parse_format_partition_value("2026", &data_type), + Some(Datum::Date(2026)) + ); + } + + #[test] + fn test_parse_format_partition_date_rejects_outside_paimon_range() { + let data_type = DataType::Date(DateType::new()); + let epoch = NaiveDate::from_num_days_from_ce_opt(UNIX_EPOCH_DAYS_FROM_CE).unwrap(); + let min_epoch_day = i32::try_from( + NaiveDate::from_ymd_opt(0, 1, 1) + .unwrap() + .signed_duration_since(epoch) + .num_days(), + ) + .unwrap(); + let max_epoch_day = i32::try_from( + NaiveDate::from_ymd_opt(9999, 12, 31) + .unwrap() + .signed_duration_since(epoch) + .num_days(), + ) + .unwrap(); + + assert_eq!( + parse_format_partition_value("10000-01-01", &data_type), + None + ); + assert_eq!( + parse_format_partition_value(&min_epoch_day.to_string(), &data_type), + Some(Datum::Date(min_epoch_day)) + ); + assert_eq!( + parse_format_partition_value(&max_epoch_day.to_string(), &data_type), + Some(Datum::Date(max_epoch_day)) + ); + assert_eq!( + parse_format_partition_value(&(min_epoch_day - 1).to_string(), &data_type), + None + ); + assert_eq!( + parse_format_partition_value(&(max_epoch_day + 1).to_string(), &data_type), + None + ); + assert_eq!( + parse_format_partition_value(&i32::MIN.to_string(), &data_type), + None + ); + assert_eq!( + parse_format_partition_value(&i32::MAX.to_string(), &data_type), + None + ); + } + + #[test] + fn test_parse_format_partition_string_enforces_declared_character_length() { + let char_type = DataType::Char(CharType::new(3).unwrap()); + let varchar_type = DataType::VarChar(VarCharType::new(3).unwrap()); + + assert_eq!( + parse_format_partition_value("中文a", &char_type), + Some(Datum::String("中文a".to_string())) + ); + assert_eq!(parse_format_partition_value("中文ab", &char_type), None); + assert_eq!( + parse_format_partition_value("中文a", &varchar_type), + Some(Datum::String("中文a".to_string())) + ); + assert_eq!(parse_format_partition_value("中文ab", &varchar_type), None); + } +} diff --git a/crates/paimon/src/table/format_table_scan.rs b/crates/paimon/src/table/format_table_scan.rs index ea735bc6..56f195e8 100644 --- a/crates/paimon/src/table/format_table_scan.rs +++ b/crates/paimon/src/table/format_table_scan.rs @@ -17,17 +17,19 @@ //! Scan implementation for Java-compatible `type=format-table` metadata. -use super::format_partition::{unescape_path_name, FormatTablePartitionPaths}; +use std::collections::{HashMap, HashSet}; + +use super::format_partition::{ + format_partition_date, parse_format_partition_value, FormatTablePartitionPaths, +}; use super::{CatalogManagedPartitionOptions, Plan, ScanTrace, Table}; use crate::spec::stats::BinaryTableStats; use crate::spec::{ - escape_path_name, extract_datum, BinaryRow, BinaryRowBuilder, CoreOptions, DataField, - DataFileMeta, DataType, Datum, PartitionComputer, Predicate, PredicateOperator, + escape_path_name, extract_datum, unescape_path_name, BinaryRow, BinaryRowBuilder, CoreOptions, + DataField, DataFileMeta, DataType, Datum, PartitionComputer, Predicate, PredicateOperator, }; use crate::table::partition_filter::PartitionFilter; use crate::table::source::DataSplitBuilder; -use chrono::NaiveDate; -use std::collections::HashSet; #[derive(Debug, Clone)] pub(crate) struct FormatTableScan<'a> { @@ -131,19 +133,9 @@ impl<'a> FormatTableScan<'a> { partition: BinaryRow::new(0), }]); } - if self.table.has_catalog_managed_partitions() { - let managed_options = - self.table - .catalog_managed_partition_options() - .ok_or_else(|| crate::Error::DataInvalid { - message: format!( - "Format table {} is missing catalog-managed partition options", - self.table.identifier().full_name() - ), - source: None, - })?; + if let Some(managed_options) = self.table.catalog_managed_partition_options() { return self - .managed_scan_roots( + .catalog_managed_scan_roots( table_path, partition_keys, &partition_fields, @@ -205,7 +197,7 @@ impl<'a> FormatTableScan<'a> { Ok(roots) } - async fn managed_scan_roots( + async fn catalog_managed_scan_roots( &self, table_path: &str, partition_keys: &[String], @@ -232,35 +224,20 @@ impl<'a> FormatTableScan<'a> { let mut seen_paths = HashSet::with_capacity(partitions.len()); let mut roots = Vec::with_capacity(partitions.len()); for partition in partitions { - let partition_path = - partition_paths - .relative_path(&partition.spec) - .map_err(|error| crate::Error::DataInvalid { - message: format!( - "Catalog returned invalid partition metadata for format table {}", - self.table.identifier().full_name() - ), - source: Some(Box::new(error)), - })?; + let partition_path = partition_paths + .relative_path(&partition.spec) + .map_err(|error| self.invalid_catalog_partition_metadata(error))?; if !seen_paths.insert(partition_path.clone()) { continue; } let path = join_path(table_path, &partition_path); - let partition = partition_row_from_path( - table_path, - &path, + let partition = partition_row_from_catalog_spec( + &partition.spec, partition_fields, partition_keys, &managed_options.default_partition_name, - only_value_in_path, - )? - .ok_or_else(|| crate::Error::DataInvalid { - message: format!( - "Catalog returned corrupt partition metadata for format table {}", - self.table.identifier().full_name() - ), - source: None, - })?; + ) + .map_err(|error| self.invalid_catalog_partition_metadata(error))?; if self.partition_matches(&partition)? { roots.push(ScanRoot { path, partition }); } @@ -269,6 +246,16 @@ impl<'a> FormatTableScan<'a> { Ok(roots) } + fn invalid_catalog_partition_metadata(&self, source: crate::Error) -> crate::Error { + crate::Error::DataInvalid { + message: format!( + "Catalog returned invalid partition metadata for Format Table {}", + self.table.identifier().full_name() + ), + source: Some(Box::new(source)), + } + } + async fn list_status_recursive_if_exists( &self, path: &str, @@ -519,7 +506,7 @@ fn partition_value_from_datum( if legacy_partition_name { Some(value.to_string()) } else { - Some(format_partition_date(*value)) + format_partition_date(*value) } } (Datum::Time(value), DataType::Time(_)) => Some(value.to_string()), @@ -527,12 +514,6 @@ fn partition_value_from_datum( } } -fn format_partition_date(epoch_days: i32) -> String { - let date = NaiveDate::from_num_days_from_ce_opt(epoch_days + 719_163) - .unwrap_or(NaiveDate::from_ymd_opt(1970, 1, 1).unwrap()); - date.format("%Y-%m-%d").to_string() -} - fn partition_row_from_path( table_path: &str, file_parent: &str, @@ -559,10 +540,7 @@ fn partition_row_from_path( .filter(|segment| !segment.is_empty()) .take(partition_keys.len()) { - let Some(value) = unescape_path_name(segment) else { - return Ok(None); - }; - values.push(value); + values.push(unescape_path_name(segment)); } if values.len() != partition_keys.len() { return Ok(None); @@ -585,7 +563,8 @@ fn partition_row_from_path( builder.set_null_at(idx); continue; } - let Some(datum) = parse_partition_datum(value, partition_fields[idx].data_type()) else { + let Some(datum) = parse_format_partition_value(value, partition_fields[idx].data_type()) + else { return Ok(None); }; builder.write_datum(idx, &datum, partition_fields[idx].data_type()); @@ -593,86 +572,43 @@ fn partition_row_from_path( Ok(Some(builder.build())) } -fn partition_segment_value(segment: &str, key: &str) -> Option { - let (segment_key, segment_value) = segment.split_once('=')?; - if unescape_path_name(segment_key)? == key { - unescape_path_name(segment_value) - } else { - None - } -} - -fn parse_partition_datum(value: &str, data_type: &DataType) -> Option { - match data_type { - DataType::Boolean(_) => match value.to_ascii_lowercase().as_str() { - "t" | "true" | "y" | "yes" | "1" => Some(Datum::Bool(true)), - "f" | "false" | "n" | "no" | "0" => Some(Datum::Bool(false)), - _ => None, - }, - DataType::TinyInt(_) => value.parse::().ok().map(Datum::TinyInt), - DataType::SmallInt(_) => value.parse::().ok().map(Datum::SmallInt), - DataType::Int(_) => value.parse::().ok().map(Datum::Int), - DataType::BigInt(_) => value.parse::().ok().map(Datum::Long), - DataType::Char(char_type) if value.chars().count() <= char_type.length() => { - Some(Datum::String(value.to_string())) - } - DataType::VarChar(varchar_type) - if value.chars().count() <= varchar_type.length() as usize => - { - Some(Datum::String(value.to_string())) +fn partition_row_from_catalog_spec( + spec: &HashMap, + partition_fields: &[DataField], + partition_keys: &[String], + default_partition_name: &str, +) -> crate::Result { + let mut builder = BinaryRowBuilder::new(partition_fields.len() as i32); + for (index, (key, field)) in partition_keys.iter().zip(partition_fields).enumerate() { + let value = spec.get(key).ok_or_else(|| crate::Error::DataInvalid { + message: format!("Catalog partition is missing column '{key}'"), + source: None, + })?; + if value == default_partition_name { + builder.set_null_at(index); + continue; } - DataType::Char(_) | DataType::VarChar(_) => None, - DataType::Date(_) => parse_partition_date(value).map(Datum::Date), - DataType::Time(_) => value.parse::().ok().map(Datum::Time), - _ => None, - } -} - -fn parse_partition_date(value: &str) -> Option { - let is_numeric = value - .strip_prefix('-') - .unwrap_or(value) - .chars() - .all(|character| character.is_ascii_digit()) - && value != "-"; - if is_numeric { - return value.parse::().ok(); - } - - let date_value = value - .find(' ') - .filter(|index| *index > 0) - .map(|index| &value[..index]) - .unwrap_or(value); - let mut segments = date_value.split('-'); - let year = parse_date_segment(segments.next()?)?; - if !(0..=9999).contains(&year) { - return None; - } - let month = match segments.next() { - Some(value) => parse_date_segment(value)?, - None => 1, - }; - let day = match segments.next() { - Some(value) => parse_date_segment(value)?, - None => 1, - }; - if segments.next().is_some() { - return None; + let datum = parse_format_partition_value(value, field.data_type()).ok_or_else(|| { + crate::Error::DataInvalid { + message: format!( + "Invalid catalog partition value {value:?} for column '{key}' with type {:?}", + field.data_type() + ), + source: None, + } + })?; + builder.write_datum(index, &datum, field.data_type()); } - let date = NaiveDate::from_ymd_opt(year, month as u32, day as u32)?; - date.signed_duration_since(NaiveDate::from_ymd_opt(1970, 1, 1).unwrap()) - .num_days() - .try_into() - .ok() + Ok(builder.build()) } -fn parse_date_segment(value: &str) -> Option { - let value = value.trim(); - if value.is_empty() || !value.chars().all(|character| character.is_ascii_digit()) { - return None; +fn partition_segment_value(segment: &str, key: &str) -> Option { + let (segment_key, segment_value) = segment.split_once('=')?; + if unescape_path_name(segment_key) == key { + Some(unescape_path_name(segment_value)) + } else { + None } - value.parse().ok() } fn supported_format_table_extension(format: &str) -> crate::Result<&'static str> { @@ -720,66 +656,9 @@ fn data_file_meta(file_name: String, file_size: i64, schema_id: i64) -> DataFile #[cfg(test)] mod tests { use super::*; - use crate::spec::{BooleanType, CharType, DateType, VarCharType}; - - #[test] - fn test_parse_partition_boolean_uses_java_spellings() { - let data_type = DataType::Boolean(BooleanType::new()); - - assert_eq!( - parse_partition_datum("1", &data_type), - Some(Datum::Bool(true)) - ); - assert_eq!( - parse_partition_datum("yes", &data_type), - Some(Datum::Bool(true)) - ); - assert_eq!( - parse_partition_datum("0", &data_type), - Some(Datum::Bool(false)) - ); - assert_eq!( - parse_partition_datum("no", &data_type), - Some(Datum::Bool(false)) - ); - } - - #[test] - fn test_parse_partition_date_accepts_java_partial_month() { - let data_type = DataType::Date(DateType::new()); - - assert_eq!( - parse_partition_datum("2026-07", &data_type), - parse_partition_datum("2026-07-01", &data_type) - ); - } - - #[test] - fn test_parse_partition_date_rejects_outside_paimon_range() { - let data_type = DataType::Date(DateType::new()); - - assert_eq!(parse_partition_datum("10000-01-01", &data_type), None); - } - - #[test] - fn test_parse_partition_string_enforces_declared_character_length() { - let char_type = DataType::Char(CharType::new(3).unwrap()); - let varchar_type = DataType::VarChar(VarCharType::new(3).unwrap()); - - assert_eq!( - parse_partition_datum("中文a", &char_type), - Some(Datum::String("中文a".to_string())) - ); - assert_eq!(parse_partition_datum("中文ab", &char_type), None); - assert_eq!( - parse_partition_datum("中文a", &varchar_type), - Some(Datum::String("中文a".to_string())) - ); - assert_eq!(parse_partition_datum("中文ab", &varchar_type), None); - } #[test] - fn test_only_not_found_listing_errors_mean_missing_partition_directory() { + fn test_missing_partition_directory_requires_not_found_error() { let not_found = crate::Error::IoUnexpected { message: "missing".to_string(), source: Box::new(opendal::Error::new(opendal::ErrorKind::NotFound, "missing")), diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index 536e014b..050954d5 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -95,7 +95,7 @@ pub use btree_global_index_build_builder::BTreeGlobalIndexBuildBuilder; pub use commit_message::CommitMessage; pub use cow_writer::{CopyOnWriteMergeWriter, FileInfo}; pub use data_evolution_writer::{DataEvolutionDeleteWriter, DataEvolutionWriter}; -pub use format_partition::FormatTablePartitionPaths; +pub use format_partition::{parse_format_partition_value, FormatTablePartitionPaths}; #[cfg(feature = "fulltext")] pub use full_text_search_builder::FullTextSearchBuilder; use futures::stream::BoxStream; @@ -179,7 +179,6 @@ impl Table { let catalog_managed_partition_options = rest_env .as_ref() .filter(|rest_env| rest_env.has_catalog_managed_partitions()) - .filter(|_| options.is_format_table() && options.partitioned_table_in_metastore()) .map(|_| CatalogManagedPartitionOptions { table_path: options.path().unwrap_or(&location).to_string(), file_format: options.file_format().to_string(), @@ -279,8 +278,7 @@ impl Table { || CoreOptions::new(self.schema.options()).is_format_table() } - /// Whether metadata accepted at the REST catalog trust boundary marked - /// this as an internal Format Table with catalog-managed partitions. + /// Whether this table uses catalog-managed Format Table partitions. pub fn has_catalog_managed_partitions(&self) -> bool { self.catalog_managed_partition_options.is_some() } @@ -351,10 +349,8 @@ impl Table { /// [`Table::copy_with_time_travel`] when the options may select a /// historical snapshot whose schema should be used for reading. /// - /// Catalog-managed Format Tables retain the validated REST provenance and - /// physical partition layout captured when they were loaded. Dynamic - /// options therefore cannot redirect those scans to a different partition - /// source, table path, file format, or value-only layout. + /// Catalog-managed Format Table scans keep the partition source, table + /// path, file format, and path layout loaded from REST metadata. pub fn copy_with_options(&self, extra: HashMap) -> Self { // Changing the time-travel selector invalidates the resolved snapshot // (a time-travelled schema then has no matching snapshot anymore, and @@ -399,6 +395,9 @@ impl Table { /// schema (the `if let Ok` below swallows them); an invalid selector /// still fails later at scan planning. pub async fn copy_with_time_travel(&self, extra: HashMap) -> Result { + if let Some(rest_env) = &self.rest_env { + rest_env.validate_dynamic_format_table_partition_options(&extra)?; + } let mut table = self.copy_with_options(extra); // Reject unimplemented scan options on the merged view before any IO, so // both table-level and per-read options are covered. diff --git a/crates/paimon/src/table/rest_env.rs b/crates/paimon/src/table/rest_env.rs index 076d1f38..b603716c 100644 --- a/crates/paimon/src/table/rest_env.rs +++ b/crates/paimon/src/table/rest_env.rs @@ -17,17 +17,29 @@ //! REST environment for REST-backed table operations. +use std::collections::HashMap; +use std::sync::Arc; + use crate::api::rest_api::RESTApi; use crate::api::rest_error::RestError; use crate::catalog::{Identifier, RESTTokenFileIO}; use crate::common::Options; use crate::error::Error; use crate::io::FileIO; -use crate::spec::{CoreOptions, TableSchema, PATH_OPTION}; +use crate::spec::{ + CoreOptions, FormatTableImplementation, TableSchema, FORMAT_TABLE_IMPLEMENTATION_OPTION, + FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE_OPTION, METASTORE_PARTITIONED_TABLE_OPTION, PATH_OPTION, +}; use crate::table::snapshot_commit::{RESTSnapshotCommit, SnapshotCommit}; use crate::table::Table; use crate::Result; -use std::sync::Arc; + +#[derive(Clone, Debug)] +struct LoadedFormatTablePartitionOptions { + partitioned_table_in_metastore: bool, + only_value_in_path: bool, + implementation: FormatTableImplementation, +} /// REST environment that holds the REST API client, identifier, and uuid /// needed to create a `RESTSnapshotCommit`. @@ -38,7 +50,7 @@ pub struct RESTEnv { api: Arc, options: Options, data_token_enabled: bool, - catalog_managed_partitions: bool, + loaded_format_table_partition_options: Option, } impl std::fmt::Debug for RESTEnv { @@ -65,23 +77,15 @@ impl RESTEnv { api, options, data_token_enabled, - catalog_managed_partitions: false, + loaded_format_table_partition_options: None, } } - /// Create a REST environment from table metadata returned by a REST catalog. - /// - /// Catalog-managed Format Table partitions are enabled only when the - /// supplied schema requests them and the REST table is internal. Callers - /// must pass the server-provided schema and `is_external` value unchanged; - /// [`RESTEnv::new`] never enables this capability. + /// Create a REST environment for table metadata returned by a REST catalog. /// - /// # Trust boundary - /// - /// This constructor is public so custom REST catalog implementations can - /// attach their loaded table metadata. It does not fetch or authenticate - /// that metadata itself. Supplying fabricated schema or ownership metadata - /// can incorrectly grant catalog-managed partition capability. + /// `table_schema` and `is_external` must come from the same table response. + /// They determine whether catalog-managed Format Table partitions are + /// enabled. [`RESTEnv::new`] never enables this capability. pub fn new_for_table( identifier: Identifier, uuid: String, @@ -92,40 +96,91 @@ impl RESTEnv { is_external: bool, ) -> Result { let core_options = CoreOptions::new(table_schema.options()); - let catalog_managed_partitions = - core_options.is_format_table() && core_options.partitioned_table_in_metastore(); - if catalog_managed_partitions && core_options.format_table_uses_engine_implementation() { - return Err(Error::DataInvalid { - message: format!( - "Cannot combine catalog-managed partitions \ - (metastore.partitioned-table=true) with \ - format-table.implementation=engine for format table {}", - identifier.full_name() - ), - source: None, - }); - } - if catalog_managed_partitions && is_external { - return Err(Error::DataInvalid { - message: format!( - "Catalog-managed partitions are only supported for internal tables, but format table {} is external", - identifier.full_name() - ), - source: None, - }); - } + let loaded_format_table_partition_options = if core_options.is_format_table() { + let partitioned_table_in_metastore = core_options.partitioned_table_in_metastore()?; + let implementation = core_options.format_table_implementation()?; + let only_value_in_path = + core_options.try_format_table_partition_only_value_in_path()?; + + if partitioned_table_in_metastore && implementation == FormatTableImplementation::Engine + { + return Err(Error::DataInvalid { + message: format!( + "Format Table {} cannot set metastore.partitioned-table=true when \ + format-table.implementation=engine", + identifier.full_name() + ), + source: None, + }); + } + if partitioned_table_in_metastore && is_external { + return Err(Error::DataInvalid { + message: format!( + "Catalog-managed partitions require an internal Format Table, but {} is external", + identifier.full_name() + ), + source: None, + }); + } + + Some(LoadedFormatTablePartitionOptions { + partitioned_table_in_metastore, + only_value_in_path, + implementation, + }) + } else { + None + }; + Ok(Self { identifier, uuid, api, options, data_token_enabled, - catalog_managed_partitions, + loaded_format_table_partition_options, }) } pub(crate) fn has_catalog_managed_partitions(&self) -> bool { - self.catalog_managed_partitions + self.loaded_format_table_partition_options + .as_ref() + .is_some_and(|options| options.partitioned_table_in_metastore) + } + + pub(crate) fn validate_dynamic_format_table_partition_options( + &self, + extra: &HashMap, + ) -> Result<()> { + let Some(loaded_options) = &self.loaded_format_table_partition_options else { + return Ok(()); + }; + let options = CoreOptions::new(extra); + if extra.contains_key(METASTORE_PARTITIONED_TABLE_OPTION) { + ensure_partition_option_unchanged( + &self.identifier, + METASTORE_PARTITIONED_TABLE_OPTION, + loaded_options.partitioned_table_in_metastore, + options.partitioned_table_in_metastore()?, + )?; + } + if extra.contains_key(FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE_OPTION) { + ensure_partition_option_unchanged( + &self.identifier, + FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE_OPTION, + loaded_options.only_value_in_path, + options.try_format_table_partition_only_value_in_path()?, + )?; + } + if extra.contains_key(FORMAT_TABLE_IMPLEMENTATION_OPTION) { + ensure_partition_option_unchanged( + &self.identifier, + FORMAT_TABLE_IMPLEMENTATION_OPTION, + loaded_options.implementation, + options.format_table_implementation()?, + )?; + } + Ok(()) } /// Get the REST API client. @@ -240,6 +295,27 @@ impl RESTEnv { } } +fn ensure_partition_option_unchanged( + identifier: &Identifier, + key: &str, + loaded_value: T, + dynamic_value: T, +) -> Result<()> +where + T: PartialEq, +{ + if loaded_value == dynamic_value { + return Ok(()); + } + Err(Error::DataInvalid { + message: format!( + "Dynamic option '{key}' must match the value returned by the REST catalog for Format Table {}", + identifier.full_name() + ), + source: None, + }) +} + fn map_rest_error_for_table(err: Error, identifier: &Identifier) -> Error { match err { Error::RestApi { diff --git a/crates/paimon/tests/format_partition_test.rs b/crates/paimon/tests/format_partition_test.rs index 32ea9236..923dcc3a 100644 --- a/crates/paimon/tests/format_partition_test.rs +++ b/crates/paimon/tests/format_partition_test.rs @@ -93,6 +93,22 @@ async fn discover_format_partitions_returns_complete_sorted_specs() { ); } +#[cfg(not(windows))] +#[tokio::test] +async fn discover_format_partitions_treats_missing_root_as_empty() { + let tmp = tempfile::tempdir().unwrap(); + let table_path = format!("file://{}", tmp.path().join("missing").display()); + let file_io = FileIO::from_path(&table_path).unwrap().build().unwrap(); + let paths = FormatTablePartitionPaths::new(["dt"], false); + + let partitions = paths + .discover(&file_io, &table_path, "__DEFAULT_PARTITION__") + .await + .unwrap(); + + assert!(partitions.is_empty()); +} + #[cfg(not(windows))] #[tokio::test] async fn discover_format_partitions_rejects_non_round_tripping_percent_escape() { @@ -130,7 +146,7 @@ async fn discover_format_partitions_ignores_nonmatching_malformed_key_segment() assert_eq!( partitions, - vec![HashMap::from([("dt".to_string(), "valid".to_string(),)])] + vec![HashMap::from([("dt".to_string(), "valid".to_string())])] ); } @@ -176,3 +192,45 @@ async fn discover_value_only_partitions_keeps_default_partition_directory() { )])] ); } + +#[cfg(not(windows))] +#[tokio::test] +async fn discover_value_only_partitions_keeps_dot_prefixed_default_partition_directory() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(tmp.path().join(".NULL")).unwrap(); + std::fs::create_dir_all(tmp.path().join(".staging")).unwrap(); + let table_path = format!("file://{}", tmp.path().display()); + let file_io = FileIO::from_path(&table_path).unwrap().build().unwrap(); + let paths = FormatTablePartitionPaths::new(["dt"], true); + + let partitions = paths + .discover(&file_io, &table_path, ".NULL") + .await + .unwrap(); + + assert_eq!( + partitions, + vec![HashMap::from([("dt".to_string(), ".NULL".to_string())])] + ); +} + +#[cfg(not(windows))] +#[tokio::test] +async fn discover_value_only_partitions_keeps_escaped_default_partition_directory() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(tmp.path().join("_NULL%2F%25NA")).unwrap(); + std::fs::create_dir_all(tmp.path().join("_temporary")).unwrap(); + let table_path = format!("file://{}", tmp.path().display()); + let file_io = FileIO::from_path(&table_path).unwrap().build().unwrap(); + let paths = FormatTablePartitionPaths::new(["dt"], true); + + let partitions = paths + .discover(&file_io, &table_path, "_NULL/%NA") + .await + .unwrap(); + + assert_eq!( + partitions, + vec![HashMap::from([("dt".to_string(), "_NULL/%NA".to_string())])] + ); +} diff --git a/crates/paimon/tests/mock_server.rs b/crates/paimon/tests/mock_server.rs index 9f767237..bf0f1bdc 100644 --- a/crates/paimon/tests/mock_server.rs +++ b/crates/paimon/tests/mock_server.rs @@ -44,7 +44,8 @@ use paimon::api::{ use paimon::catalog::{Function, Identifier}; use paimon::spec::Partition; -type PartitionPageResponse = (Vec>, Option); +type PartitionPageResponse = (Vec, Option); +type PartitionSpecPageResponse = (Vec>, Option); #[derive(Clone, Debug, Default)] struct MockState { @@ -53,20 +54,16 @@ struct MockState { views: HashMap, functions: HashMap, partitions: HashMap>, - partition_pages: HashMap>>, - partition_page_tokens: HashMap>>, + partition_page_responses: HashMap>, partition_list_calls: HashMap, view_function_endpoints_unsupported: bool, drop_view_error_status: Option, list_page_size: Option, no_permission_databases: HashSet, no_permission_tables: HashSet, - last_create_partitions_call: Option<(String, String, CreatePartitionsRequest)>, - create_partitions_call_sizes: Vec, - last_drop_partitions_call: Option<(String, String, DropPartitionsRequest)>, - drop_partitions_call_sizes: Vec, - last_list_partitions_by_names_call: Option<(String, String, ListPartitionsByNamesRequest)>, - list_partitions_by_names_call_sizes: Vec, + create_partitions_calls: Vec<(String, String, CreatePartitionsRequest)>, + drop_partitions_calls: Vec<(String, String, DropPartitionsRequest)>, + list_partitions_by_names_calls: Vec<(String, String, ListPartitionsByNamesRequest)>, create_partitions_error_status: Option, drop_partitions_error_status: Option, list_partitions_error_status: Option, @@ -317,6 +314,11 @@ impl RESTServer { // Also remove all tables in this database let prefix = format!("{name}."); s.tables.retain(|key, _| !key.starts_with(&prefix)); + s.partitions.retain(|key, _| !key.starts_with(&prefix)); + s.partition_page_responses + .retain(|key, _| !key.starts_with(&prefix)); + s.partition_list_calls + .retain(|key, _| !key.starts_with(&prefix)); s.no_permission_tables .retain(|key| !key.starts_with(&prefix)); (StatusCode::OK, Json(serde_json::json!(""))).into_response() @@ -759,6 +761,9 @@ impl RESTServer { } if s.tables.remove(&key).is_some() { + s.partitions.remove(&key); + s.partition_page_responses.remove(&key); + s.partition_list_calls.remove(&key); s.no_permission_tables.remove(&key); (StatusCode::OK, Json(serde_json::json!(""))).into_response() } else { @@ -812,27 +817,27 @@ impl RESTServer { Extension(state): Extension>, Json(request): Json, ) -> impl IntoResponse { - let (error_status, table_exists) = { - let mut inner = state.inner.lock().unwrap(); - inner - .create_partitions_call_sizes - .push(request.partition_specs.len()); - inner.last_create_partitions_call = Some((db.clone(), table.clone(), request.clone())); - ( - inner.create_partitions_error_status, - inner.tables.contains_key(&format!("{db}.{table}")), - ) - }; - if let Some(status) = error_status { + let mut inner = state.inner.lock().unwrap(); + inner + .create_partitions_calls + .push((db.clone(), table.clone(), request.clone())); + if let Some(status) = inner.create_partitions_error_status { + let message = if status == StatusCode::CONFLICT { + "Some partitions already exist" + } else { + "Invalid partition request" + }; let error = ErrorResponse::new( Some("partition".to_string()), Some(table.clone()), - Some("Some partitions already exist".to_string()), + Some(message.to_string()), Some(status.as_u16() as i32), ); return (status, Json(error)).into_response(); } - if !table_exists { + + let key = format!("{db}.{table}"); + if !inner.tables.contains_key(&key) { let error = ErrorResponse::new( Some("table".to_string()), Some(table), @@ -842,17 +847,18 @@ impl RESTServer { return (StatusCode::NOT_FOUND, Json(error)).into_response(); } - let mut inner = state.inner.lock().unwrap(); - let key = format!("{db}.{table}"); - let registered = inner.partitions.entry(key.clone()).or_default(); - let mut seen = Vec::new(); - let conflicts = request.partition_specs.iter().any(|spec| { - registered.iter().any(|partition| partition.spec == *spec) || seen.contains(spec) || { - seen.push(spec.clone()); - false - } - }); - if conflicts && !request.ignore_if_exists { + let registered_partitions = inner.partitions.entry(key.clone()).or_default(); + let has_conflict = request + .partition_specs + .iter() + .enumerate() + .any(|(index, spec)| { + registered_partitions + .iter() + .any(|partition| partition.spec == *spec) + || request.partition_specs[..index].contains(spec) + }); + if has_conflict && !request.ignore_if_exists { let error = ErrorResponse::new( Some("partition".to_string()), Some(table), @@ -865,14 +871,17 @@ impl RESTServer { let mut created = Vec::new(); let mut existed = Vec::new(); for spec in request.partition_specs { - if registered.iter().any(|partition| partition.spec == spec) { + if registered_partitions + .iter() + .any(|partition| partition.spec == spec) + { existed.push(spec); } else { - registered.push(partition_from_spec(spec.clone())); + registered_partitions.push(partition_from_spec(spec.clone())); created.push(spec); } } - inner.partition_pages.remove(&key); + inner.partition_page_responses.remove(&key); let response = CreatePartitionsResponse { created, existed }; (StatusCode::OK, Json(response)).into_response() } @@ -903,28 +912,31 @@ impl RESTServer { ); return (status, Json(error)).into_response(); } - if let Some(pages) = inner.partition_pages.get(&key) { - let pages = pages.clone(); - let scripted_tokens = inner.partition_page_tokens.get(&key).cloned(); + if let Some(responses) = inner.partition_page_responses.get(&key) { + let responses = responses.clone(); let request_index = { let calls = inner.partition_list_calls.entry(key.clone()).or_default(); let request_index = *calls; *calls += 1; request_index }; - let page_index = if scripted_tokens.is_some() { - request_index - } else { - params - .get("pageToken") - .and_then(|token| token.parse::().ok()) - .unwrap_or(0) - }; - let partitions = pages.get(page_index).cloned().unwrap_or_default(); - let next_page_token = scripted_tokens.map_or_else( - || (page_index + 1 < pages.len()).then(|| (page_index + 1).to_string()), - |tokens| tokens.get(page_index).cloned().flatten(), - ); + let expected_token = request_index + .checked_sub(1) + .and_then(|index| responses.get(index)) + .and_then(|(_, token)| token.clone()); + let actual_token = params.get("pageToken").cloned(); + if actual_token != expected_token || request_index >= responses.len() { + let error = ErrorResponse::new( + Some("partition".to_string()), + Some(table), + Some(format!( + "Invalid page token: expected {expected_token:?}, got {actual_token:?}" + )), + Some(StatusCode::BAD_REQUEST.as_u16() as i32), + ); + return (StatusCode::BAD_REQUEST, Json(error)).into_response(); + } + let (partitions, next_page_token) = responses[request_index].clone(); return ( StatusCode::OK, Json(ListPartitionsResponse::new( @@ -948,18 +960,11 @@ impl RESTServer { Extension(state): Extension>, Json(request): Json, ) -> impl IntoResponse { - let (error_status, table_exists) = { - let mut inner = state.inner.lock().unwrap(); - inner - .drop_partitions_call_sizes - .push(request.partition_specs.len()); - inner.last_drop_partitions_call = Some((db.clone(), table.clone(), request.clone())); - ( - inner.drop_partitions_error_status, - inner.tables.contains_key(&format!("{db}.{table}")), - ) - }; - if let Some(status) = error_status { + let mut inner = state.inner.lock().unwrap(); + inner + .drop_partitions_calls + .push((db.clone(), table.clone(), request.clone())); + if let Some(status) = inner.drop_partitions_error_status { let error = ErrorResponse::new( Some("partition".to_string()), Some(table.clone()), @@ -968,7 +973,9 @@ impl RESTServer { ); return (status, Json(error)).into_response(); } - if !table_exists { + + let key = format!("{db}.{table}"); + if !inner.tables.contains_key(&key) { let error = ErrorResponse::new( Some("table".to_string()), Some(table), @@ -978,13 +985,15 @@ impl RESTServer { return (StatusCode::NOT_FOUND, Json(error)).into_response(); } - let mut inner = state.inner.lock().unwrap(); - let key = format!("{db}.{table}"); - let registered = inner.partitions.entry(key.clone()).or_default(); + let registered_partitions = inner.partitions.entry(key.clone()).or_default(); let missing = request .partition_specs .iter() - .filter(|spec| !registered.iter().any(|partition| partition.spec == **spec)) + .filter(|spec| { + !registered_partitions + .iter() + .any(|partition| partition.spec == **spec) + }) .cloned() .collect::>(); if !missing.is_empty() && !request.ignore_if_not_exists { @@ -998,7 +1007,7 @@ impl RESTServer { } let mut dropped = Vec::new(); - registered.retain(|partition| { + registered_partitions.retain(|partition| { if request.partition_specs.contains(&partition.spec) { dropped.push(partition.spec.clone()); false @@ -1006,7 +1015,7 @@ impl RESTServer { true } }); - inner.partition_pages.remove(&key); + inner.partition_page_responses.remove(&key); let response = DropPartitionsResponse { dropped, missing }; (StatusCode::OK, Json(response)).into_response() } @@ -1017,19 +1026,11 @@ impl RESTServer { Extension(state): Extension>, Json(request): Json, ) -> impl IntoResponse { - let (error_status, table_exists) = { - let mut inner = state.inner.lock().unwrap(); - inner - .list_partitions_by_names_call_sizes - .push(request.partition_specs.len()); - inner.last_list_partitions_by_names_call = - Some((db.clone(), table.clone(), request.clone())); - ( - inner.list_partitions_by_names_error_status, - inner.tables.contains_key(&format!("{db}.{table}")), - ) - }; - if let Some(status) = error_status { + let mut inner = state.inner.lock().unwrap(); + inner + .list_partitions_by_names_calls + .push((db.clone(), table.clone(), request.clone())); + if let Some(status) = inner.list_partitions_by_names_error_status { let error = ErrorResponse::new( Some("partition".to_string()), Some(table.clone()), @@ -1038,7 +1039,9 @@ impl RESTServer { ); return (status, Json(error)).into_response(); } - if !table_exists { + + let key = format!("{db}.{table}"); + if !inner.tables.contains_key(&key) { let error = ErrorResponse::new( Some("table".to_string()), Some(table), @@ -1048,10 +1051,9 @@ impl RESTServer { return (StatusCode::NOT_FOUND, Json(error)).into_response(); } - let inner = state.inner.lock().unwrap(); let partitions = inner .partitions - .get(&format!("{db}.{table}")) + .get(&key) .into_iter() .flatten() .filter(|partition| request.partition_specs.contains(&partition.spec)) @@ -1117,6 +1119,16 @@ impl RESTServer { if s.no_permission_tables.remove(&source_key) { s.no_permission_tables.insert(dest_key.clone()); } + if let Some(partitions) = s.partitions.remove(&source_key) { + s.partitions.insert(dest_key.clone(), partitions); + } + if let Some(responses) = s.partition_page_responses.remove(&source_key) { + s.partition_page_responses + .insert(dest_key.clone(), responses); + } + if let Some(calls) = s.partition_list_calls.remove(&source_key) { + s.partition_list_calls.insert(dest_key, calls); + } (StatusCode::OK, Json(serde_json::json!(""))).into_response() } else { @@ -1306,26 +1318,17 @@ impl RESTServer { ) { let partitions = partition_specs .into_iter() - .map(|spec| Partition { - spec, - record_count: 0, - file_size_in_bytes: 0, - file_count: 0, - last_file_creation_time: 0, - total_buckets: 0, - done: false, - created_at: None, - created_by: None, - updated_at: None, - updated_by: None, - options: None, - }) + .map(partition_from_spec) .collect(); - self.inner - .lock() - .unwrap() - .partitions - .insert(format!("{database}.{table}"), partitions); + let key = format!("{database}.{table}"); + let mut inner = self.inner.lock().unwrap(); + assert!( + inner.tables.contains_key(&key), + "table {key} does not exist" + ); + inner.partitions.insert(key.clone(), partitions); + inner.partition_page_responses.remove(&key); + inner.partition_list_calls.remove(&key); } /// Set explicit pages returned by the list-partitions endpoint. @@ -1335,33 +1338,26 @@ impl RESTServer { table: &str, pages: Vec>>, ) { - let pages = pages + let page_count = pages.len(); + let responses = pages .into_iter() - .map(|specs| { - specs - .into_iter() - .map(|spec| Partition { - spec, - record_count: 0, - file_size_in_bytes: 0, - file_count: 0, - last_file_creation_time: 0, - total_buckets: 0, - done: false, - created_at: None, - created_by: None, - updated_at: None, - updated_by: None, - options: None, - }) - .collect() + .enumerate() + .map(|(index, specs)| { + let partitions = specs.into_iter().map(partition_from_spec).collect(); + let next_page_token = (index + 1 < page_count).then(|| (index + 1).to_string()); + (partitions, next_page_token) }) .collect(); - self.inner - .lock() - .unwrap() - .partition_pages - .insert(format!("{database}.{table}"), pages); + let key = format!("{database}.{table}"); + let mut inner = self.inner.lock().unwrap(); + assert!( + inner.tables.contains_key(&key), + "table {key} does not exist" + ); + inner + .partition_page_responses + .insert(key.clone(), responses); + inner.partition_list_calls.remove(&key); } /// Set explicit list-partitions pages and response tokens in request order. @@ -1369,16 +1365,21 @@ impl RESTServer { &self, database: &str, table: &str, - responses: Vec, + responses: Vec, ) { - let (pages, tokens): (Vec<_>, Vec<_>) = responses + let responses = responses .into_iter() .map(|(specs, token)| (specs.into_iter().map(partition_from_spec).collect(), token)) - .unzip(); + .collect(); let key = format!("{database}.{table}"); let mut inner = self.inner.lock().unwrap(); - inner.partition_pages.insert(key.clone(), pages); - inner.partition_page_tokens.insert(key.clone(), tokens); + assert!( + inner.tables.contains_key(&key), + "table {key} does not exist" + ); + inner + .partition_page_responses + .insert(key.clone(), responses); inner.partition_list_calls.remove(&key); } @@ -1398,8 +1399,9 @@ impl RESTServer { self.inner .lock() .unwrap() - .last_create_partitions_call - .clone() + .create_partitions_calls + .last() + .cloned() } /// Return the partition counts of all create-partitions calls. @@ -1407,13 +1409,25 @@ impl RESTServer { self.inner .lock() .unwrap() - .create_partitions_call_sizes - .clone() + .create_partitions_calls + .iter() + .map(|(_, _, request)| request.partition_specs.len()) + .collect() + } + + /// Return all create-partitions calls received by the server. + pub fn create_partitions_calls(&self) -> Vec<(String, String, CreatePartitionsRequest)> { + self.inner.lock().unwrap().create_partitions_calls.clone() } /// Return the last drop-partitions call received by the server. pub fn last_drop_partitions_call(&self) -> Option<(String, String, DropPartitionsRequest)> { - self.inner.lock().unwrap().last_drop_partitions_call.clone() + self.inner + .lock() + .unwrap() + .drop_partitions_calls + .last() + .cloned() } /// Return the partition counts of all drop-partitions calls. @@ -1421,8 +1435,15 @@ impl RESTServer { self.inner .lock() .unwrap() - .drop_partitions_call_sizes - .clone() + .drop_partitions_calls + .iter() + .map(|(_, _, request)| request.partition_specs.len()) + .collect() + } + + /// Return all drop-partitions calls received by the server. + pub fn drop_partitions_calls(&self) -> Vec<(String, String, DropPartitionsRequest)> { + self.inner.lock().unwrap().drop_partitions_calls.clone() } /// Return the last list-partitions-by-names call received by the server. @@ -1432,8 +1453,9 @@ impl RESTServer { self.inner .lock() .unwrap() - .last_list_partitions_by_names_call - .clone() + .list_partitions_by_names_calls + .last() + .cloned() } /// Return the partition counts of all list-partitions-by-names calls. @@ -1441,7 +1463,20 @@ impl RESTServer { self.inner .lock() .unwrap() - .list_partitions_by_names_call_sizes + .list_partitions_by_names_calls + .iter() + .map(|(_, _, request)| request.partition_specs.len()) + .collect() + } + + /// Return all list-partitions-by-names calls received by the server. + pub fn list_partitions_by_names_calls( + &self, + ) -> Vec<(String, String, ListPartitionsByNamesRequest)> { + self.inner + .lock() + .unwrap() + .list_partitions_by_names_calls .clone() } diff --git a/crates/paimon/tests/rest_api_test.rs b/crates/paimon/tests/rest_api_test.rs index fc876974..94af57bb 100644 --- a/crates/paimon/tests/rest_api_test.rs +++ b/crates/paimon/tests/rest_api_test.rs @@ -613,16 +613,19 @@ async fn test_list_partitions_by_names_rejects_more_than_1000_specs() { .await .unwrap_err(); - assert!( - matches!(error, paimon::Error::DataInvalid { .. }) - && error.to_string().contains("at most 1000"), - "expected a request-size validation error, got: {error}" + let paimon::Error::DataInvalid { message, .. } = error else { + panic!("expected a request-size validation error, got: {error}"); + }; + assert_eq!( + message, + "List partitions by names accepts at most 1000 partition specs for table \ + default.managed_table, got 1001" ); assert!(ctx.server.last_list_partitions_by_names_call().is_none()); } #[tokio::test] -async fn test_create_partitions_posts_java_compatible_request() { +async fn test_create_partitions_posts_expected_request() { let ctx = setup_test_server(vec!["default"]).await; let identifier = Identifier::new("default", "managed_table"); ctx.server.add_table("default", "managed_table"); @@ -650,7 +653,26 @@ async fn test_create_partitions_posts_java_compatible_request() { } #[tokio::test] -async fn test_drop_partitions_posts_java_compatible_request() { +async fn test_create_partitions_allows_more_than_1000_specs() { + let ctx = setup_test_server(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + let partition_specs = (0..=1000) + .map(|day| HashMap::from([("day".to_string(), day.to_string())])) + .collect::>(); + + let response = ctx + .api + .create_partitions(&identifier, partition_specs.clone(), false) + .await + .unwrap(); + + assert_eq!(response.created, partition_specs); + assert_eq!(ctx.server.create_partitions_call_sizes(), vec![1001]); +} + +#[tokio::test] +async fn test_drop_partitions_posts_expected_request() { let ctx = setup_test_server(vec!["default"]).await; let identifier = Identifier::new("default", "managed_table"); ctx.server.add_table("default", "managed_table"); @@ -680,7 +702,26 @@ async fn test_drop_partitions_posts_java_compatible_request() { } #[tokio::test] -async fn test_list_partitions_by_names_posts_java_compatible_request() { +async fn test_drop_partitions_allows_more_than_1000_specs() { + let ctx = setup_test_server(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + let partition_specs = (0..=1000) + .map(|day| HashMap::from([("day".to_string(), day.to_string())])) + .collect::>(); + + let response = ctx + .api + .drop_partitions(&identifier, partition_specs.clone(), true) + .await + .unwrap(); + + assert_eq!(response.missing, partition_specs); + assert_eq!(ctx.server.drop_partitions_call_sizes(), vec![1001]); +} + +#[tokio::test] +async fn test_list_partitions_by_names_posts_expected_request() { let ctx = setup_test_server(vec!["default"]).await; let identifier = Identifier::new("default", "managed_table"); ctx.server.add_table("default", "managed_table"); @@ -715,7 +756,7 @@ async fn test_list_partitions_by_names_posts_java_compatible_request() { } #[tokio::test] -async fn test_list_partitions_follows_token_after_empty_page() { +async fn test_list_partitions_follows_non_empty_token_after_empty_page() { let ctx = setup_test_server(vec!["default"]).await; let identifier = Identifier::new("default", "managed_table"); ctx.server.add_table("default", "managed_table"); @@ -800,15 +841,13 @@ async fn test_list_partitions_rejects_repeated_page_token() { let error = ctx.api.list_partitions(&identifier).await.unwrap_err(); - assert!( - error - .to_string() - .contains("repeated partition page token 'a'"), - "expected repeated-token error, got: {error}" - ); - assert!( - error.to_string().contains("default.managed_table"), - "expected table context, got: {error}" + let paimon::Error::UnexpectedError { message, .. } = error else { + panic!("expected a repeated-page-token error, got: {error}"); + }; + assert_eq!( + message, + "REST catalog returned partition page token 'a' more than once for table \ + default.managed_table" ); assert_eq!( ctx.server diff --git a/crates/paimon/tests/rest_catalog_test.rs b/crates/paimon/tests/rest_catalog_test.rs index ccfa908e..7a5342b4 100644 --- a/crates/paimon/tests/rest_catalog_test.rs +++ b/crates/paimon/tests/rest_catalog_test.rs @@ -86,6 +86,20 @@ fn test_schema() -> Schema { .expect("Failed to build schema") } +fn catalog_managed_format_table_schema(options: &[(&str, &str)]) -> Schema { + let mut builder = Schema::builder() + .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) + .column("id", DataType::BigInt(BigIntType::new())) + .partition_keys(["dt"]) + .option("type", "format-table") + .option("file.format", "parquet") + .option("metastore.partitioned-table", "true"); + for (key, value) in options { + builder = builder.option(*key, *value); + } + builder.build().unwrap() +} + fn blob_schema(options: &[(&str, &str)]) -> Schema { let mut builder = Schema::builder() .column("id", DataType::Int(IntType::new())) @@ -189,7 +203,7 @@ async fn test_non_rest_catalog_partition_administration_is_unsupported() { )); assert!(matches!( catalog - .drop_partitions(&identifier, partition_specs.clone()) + .unregister_partitions(&identifier, partition_specs.clone()) .await .unwrap_err(), paimon::Error::Unsupported { .. } @@ -238,34 +252,43 @@ async fn test_rest_catalog_batches_idempotent_create_partitions() { .collect::>(); ctx.catalog - .create_partitions(&identifier, partition_specs, true) + .create_partitions(&identifier, partition_specs.clone(), true) .await .unwrap(); - assert_eq!(ctx.server.create_partitions_call_sizes(), vec![1000, 1]); + let calls = ctx.server.create_partitions_calls(); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].2.partition_specs, partition_specs[..1000]); + assert_eq!(calls[1].2.partition_specs, partition_specs[1000..]); + assert!(calls.iter().all(|(_, _, request)| request.ignore_if_exists)); + let listed = ctx.catalog.list_partitions(&identifier).await.unwrap(); + assert_eq!( + listed + .into_iter() + .map(|partition| partition.spec) + .collect::>(), + partition_specs + ); } #[tokio::test] -async fn test_rest_catalog_rejects_large_strict_create_before_request() { +async fn test_rest_catalog_keeps_non_idempotent_create_in_one_request() { let ctx = setup_catalog(vec!["default"]).await; let identifier = Identifier::new("default", "managed_table"); ctx.server.add_table("default", "managed_table"); - let partition_specs = (0..1001) + let partition_specs = (0..2500) .map(|value| HashMap::from([("dt".to_string(), value.to_string())])) .collect::>(); - let error = ctx - .catalog - .create_partitions(&identifier, partition_specs, false) + ctx.catalog + .create_partitions(&identifier, partition_specs.clone(), false) .await - .unwrap_err(); + .unwrap(); - assert!( - matches!(error, paimon::Error::DataInvalid { .. }) - && error.to_string().contains("at most 1000"), - "expected a request-size validation error, got: {error}" - ); - assert!(ctx.server.create_partitions_call_sizes().is_empty()); + let calls = ctx.server.create_partitions_calls(); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].2.partition_specs, partition_specs); + assert!(!calls[0].2.ignore_if_exists); } #[tokio::test] @@ -294,7 +317,7 @@ async fn test_rest_catalog_created_partitions_are_visible_to_list() { } #[tokio::test] -async fn test_rest_catalog_maps_strict_create_partition_conflict() { +async fn test_rest_catalog_maps_create_partition_conflict() { let ctx = setup_catalog(vec!["default"]).await; let identifier = Identifier::new("default", "managed_table"); ctx.server @@ -349,7 +372,7 @@ async fn test_rest_catalog_maps_invalid_partition_request() { } #[tokio::test] -async fn test_rest_catalog_drop_partitions_delegates_to_rest_metadata_only() { +async fn test_rest_catalog_unregister_partitions_delegates_to_rest_metadata_only() { let ctx = setup_catalog(vec!["default"]).await; let identifier = Identifier::new("default", "managed_table"); ctx.server.add_table("default", "managed_table"); @@ -359,7 +382,7 @@ async fn test_rest_catalog_drop_partitions_delegates_to_rest_metadata_only() { )])]; ctx.catalog - .drop_partitions(&identifier, partition_specs.clone()) + .unregister_partitions(&identifier, partition_specs.clone()) .await .unwrap(); @@ -374,7 +397,7 @@ async fn test_rest_catalog_drop_partitions_delegates_to_rest_metadata_only() { } #[tokio::test] -async fn test_rest_catalog_maps_invalid_drop_partition_request() { +async fn test_rest_catalog_maps_invalid_unregister_partition_request() { let ctx = setup_catalog(vec!["default"]).await; let identifier = Identifier::new("default", "managed_table"); ctx.server.add_table("default", "managed_table"); @@ -383,7 +406,7 @@ async fn test_rest_catalog_maps_invalid_drop_partition_request() { let error = ctx .catalog - .drop_partitions( + .unregister_partitions( &identifier, vec![HashMap::from([( "unknown".to_string(), @@ -401,7 +424,7 @@ async fn test_rest_catalog_maps_invalid_drop_partition_request() { } #[tokio::test] -async fn test_rest_catalog_batches_drop_partitions() { +async fn test_rest_catalog_batches_unregister_partitions() { let ctx = setup_catalog(vec!["default"]).await; let identifier = Identifier::new("default", "managed_table"); ctx.server.add_table("default", "managed_table"); @@ -412,15 +435,27 @@ async fn test_rest_catalog_batches_drop_partitions() { .set_table_partitions("default", "managed_table", partition_specs.clone()); ctx.catalog - .drop_partitions(&identifier, partition_specs) + .unregister_partitions(&identifier, partition_specs.clone()) .await .unwrap(); - assert_eq!(ctx.server.drop_partitions_call_sizes(), vec![1000, 1]); + let calls = ctx.server.drop_partitions_calls(); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].2.partition_specs, partition_specs[..1000]); + assert_eq!(calls[1].2.partition_specs, partition_specs[1000..]); + assert!(calls + .iter() + .all(|(_, _, request)| request.ignore_if_not_exists)); + assert!(ctx + .catalog + .list_partitions(&identifier) + .await + .unwrap() + .is_empty()); } #[tokio::test] -async fn test_rest_catalog_dropped_partitions_are_removed_from_list() { +async fn test_rest_catalog_unregistered_partitions_are_removed_from_list() { let ctx = setup_catalog(vec!["default"]).await; let identifier = Identifier::new("default", "managed_table"); ctx.server.add_table("default", "managed_table"); @@ -432,7 +467,7 @@ async fn test_rest_catalog_dropped_partitions_are_removed_from_list() { .set_table_partitions("default", "managed_table", partition_specs.clone()); ctx.catalog - .drop_partitions(&identifier, partition_specs) + .unregister_partitions(&identifier, partition_specs) .await .unwrap(); @@ -508,16 +543,56 @@ async fn test_rest_catalog_batches_list_partitions_by_names() { let partition_specs = (0..1001) .map(|value| HashMap::from([("dt".to_string(), value.to_string())])) .collect::>(); + ctx.server + .set_table_partitions("default", "managed_table", partition_specs.clone()); - ctx.catalog - .list_partitions_by_names(&identifier, partition_specs) + let listed = ctx + .catalog + .list_partitions_by_names(&identifier, partition_specs.clone()) .await .unwrap(); - assert_eq!( - ctx.server.list_partitions_by_names_call_sizes(), - vec![1000, 1] + let calls = ctx.server.list_partitions_by_names_calls(); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].2.partition_specs, partition_specs[..1000]); + assert_eq!(calls[1].2.partition_specs, partition_specs[1000..]); + let listed_specs = listed + .into_iter() + .map(|partition| partition.spec) + .collect::>(); + assert_eq!(listed_specs, partition_specs); +} + +#[tokio::test] +async fn test_rest_catalog_rejects_invalid_or_out_of_range_partition_page_token() { + let ctx = setup_catalog(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + ctx.server.set_table_partition_pages( + "default", + "managed_table", + vec![vec![HashMap::from([( + "dt".to_string(), + "2026-07-22".to_string(), + )])]], ); + + for token in ["invalid", "1"] { + let error = ctx + .catalog + .list_partitions_paged(&identifier, None, Some(token)) + .await + .unwrap_err(); + assert!( + matches!( + error, + paimon::Error::RestApi { + source: paimon::api::RestError::BadRequest { .. } + } + ), + "expected bad page token error for {token}, got: {error}" + ); + } } #[tokio::test] @@ -536,7 +611,7 @@ async fn test_rest_catalog_partition_operations_map_missing_table() { .unwrap_err(); let drop_error = ctx .catalog - .drop_partitions(&identifier, partition_specs.clone()) + .unregister_partitions(&identifier, partition_specs.clone()) .await .unwrap_err(); let list_error = ctx @@ -968,15 +1043,7 @@ async fn test_rest_catalog_reads_format_table() { #[tokio::test] async fn test_rest_catalog_marks_internal_format_table_as_catalog_managed() { let ctx = setup_catalog(vec!["default"]).await; - let schema = Schema::builder() - .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) - .column("id", DataType::BigInt(BigIntType::new())) - .partition_keys(["dt"]) - .option("type", "format-table") - .option("file.format", "parquet") - .option("metastore.partitioned-table", "true") - .build() - .unwrap(); + let schema = catalog_managed_format_table_schema(&[]); let identifier = Identifier::new("default", "managed_format"); ctx.server.add_table_with_schema( "default", @@ -999,26 +1066,116 @@ async fn test_rest_catalog_marks_internal_format_table_as_catalog_managed() { } #[tokio::test] -async fn test_plain_rest_env_does_not_assert_internal_catalog_managed_table_provenance() { +async fn test_rest_catalog_rejects_invalid_metastore_partitioned_table_option() { + let ctx = setup_catalog(vec!["default"]).await; + let schema = Schema::builder() + .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) + .partition_keys(["dt"]) + .option("type", "format-table") + .option("file.format", "parquet") + .option("metastore.partitioned-table", "tru") + .build() + .unwrap(); + let identifier = Identifier::new("default", "invalid_partition_source"); + ctx.server.add_table_with_schema( + "default", + "invalid_partition_source", + schema, + "memory:/invalid_partition_source", + ); + ctx.server + .set_table_external("default", "invalid_partition_source", false); + + let error = ctx.catalog.get_table(&identifier).await.unwrap_err(); + + assert!(matches!( + error, + paimon::Error::ConfigInvalid { message } + if message.contains("metastore.partitioned-table") + && message.contains("tru") + )); +} + +#[tokio::test] +async fn test_rest_catalog_rejects_invalid_format_table_implementation() { let ctx = setup_catalog(vec!["default"]).await; let schema = Schema::builder() .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) - .column("id", DataType::BigInt(BigIntType::new())) .partition_keys(["dt"]) .option("type", "format-table") .option("file.format", "parquet") .option("metastore.partitioned-table", "true") + .option("format-table.implementation", "unknown") .build() .unwrap(); - let identifier = Identifier::new("default", "managed_provenance"); + let identifier = Identifier::new("default", "invalid_format_implementation"); + ctx.server.add_table_with_schema( + "default", + "invalid_format_implementation", + schema, + "memory:/invalid_format_implementation", + ); + ctx.server + .set_table_external("default", "invalid_format_implementation", false); + + let error = ctx.catalog.get_table(&identifier).await.unwrap_err(); + + assert!(matches!( + error, + paimon::Error::ConfigInvalid { message } + if message.contains("format-table.implementation") + && message.contains("unknown") + )); +} + +#[tokio::test] +async fn test_rest_catalog_accepts_supported_partition_options() { + let ctx = setup_catalog(vec!["default"]).await; + for (table, partitioned, implementation, expected_managed) in [ + ("managed_paimon", "true", "paimon", true), + ("unmanaged_paimon", "false", "paimon", false), + ("unmanaged_engine", "false", "engine", false), + ] { + let schema = Schema::builder() + .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) + .partition_keys(["dt"]) + .option("type", "format-table") + .option("file.format", "parquet") + .option("metastore.partitioned-table", partitioned) + .option("format-table.implementation", implementation) + .build() + .unwrap(); + ctx.server + .add_table_with_schema("default", table, schema, &format!("memory:/{table}")); + ctx.server.set_table_external("default", table, false); + + let loaded = ctx + .catalog + .get_table(&Identifier::new("default", table)) + .await + .unwrap(); + + assert_eq!( + loaded.has_catalog_managed_partitions(), + expected_managed, + "{table}" + ); + } +} + +#[tokio::test] +async fn test_rest_env_new_does_not_enable_catalog_managed_partitions() { + let ctx = setup_catalog(vec!["default"]).await; + let schema = catalog_managed_format_table_schema(&[]); + let identifier = Identifier::new("default", "manual_rest_env"); ctx.server.add_table_with_schema( "default", - "managed_provenance", + "manual_rest_env", schema, - "memory:/managed_provenance", + "memory:/manual_rest_env", ); ctx.server - .set_table_external("default", "managed_provenance", false); + .set_table_external("default", "manual_rest_env", false); let loaded = ctx.catalog.get_table(&identifier).await.unwrap(); let plain_rest_env = RESTEnv::new( @@ -1041,27 +1198,21 @@ async fn test_plain_rest_env_does_not_assert_internal_catalog_managed_table_prov #[cfg(not(windows))] #[tokio::test] -async fn test_managed_format_partition_listing_does_not_fallback_to_filesystem() { +async fn test_managed_format_partition_listing_requires_rest_endpoint() { let temp_dir = tempfile::tempdir().unwrap(); std::fs::create_dir_all(temp_dir.path().join("dt=2026-07-22")).unwrap(); let table_path = format!("file://{}", temp_dir.path().display()); let ctx = setup_catalog(vec!["default"]).await; - let schema = Schema::builder() - .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) - .column("id", DataType::BigInt(BigIntType::new())) - .partition_keys(["dt"]) - .option("type", "format-table") - .option("file.format", "parquet") - .option("metastore.partitioned-table", "true") - .build() - .unwrap(); - let identifier = Identifier::new("default", "managed_no_fallback"); + let schema = catalog_managed_format_table_schema(&[]); + let identifier = Identifier::new("default", "managed_partition_endpoint"); ctx.server - .add_table_with_schema("default", "managed_no_fallback", schema, &table_path); + .add_table_with_schema("default", "managed_partition_endpoint", schema, &table_path); ctx.server - .set_table_external("default", "managed_no_fallback", false); + .set_table_external("default", "managed_partition_endpoint", false); ctx.server .set_list_partitions_error_status(Some(StatusCode::NOT_IMPLEMENTED)); + ctx.server + .set_list_partitions_by_names_error_status(Some(StatusCode::NOT_IMPLEMENTED)); let error = ctx.catalog.list_partitions(&identifier).await.unwrap_err(); @@ -1083,20 +1234,89 @@ async fn test_managed_format_partition_listing_does_not_fallback_to_filesystem() source: paimon::api::RestError::NotImplemented { .. } } )); + + let by_names_error = ctx + .catalog + .list_partitions_by_names( + &identifier, + vec![HashMap::from([( + "dt".to_string(), + "2026-07-22".to_string(), + )])], + ) + .await + .unwrap_err(); + assert!(matches!( + by_names_error, + paimon::Error::RestApi { + source: paimon::api::RestError::NotImplemented { .. } + } + )); } +#[cfg(not(windows))] #[tokio::test] -async fn test_rest_catalog_rejects_external_catalog_managed_format_table() { - let ctx = setup_catalog(vec!["default"]).await; +async fn test_unmanaged_list_partitions_by_names_uses_exact_filesystem_match_when_unsupported() { + let tmp = tempfile::tempdir().unwrap(); + let warehouse = format!("file://{}", tmp.path().display()); + let mut fs_options = Options::new(); + fs_options.set(CatalogOptions::WAREHOUSE, &warehouse); + let fs_catalog = FileSystemCatalog::new(fs_options).unwrap(); + fs_catalog + .create_database("default", true, HashMap::new()) + .await + .unwrap(); + let identifier = Identifier::new("default", "filesystem_partitions"); let schema = Schema::builder() .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) .column("id", DataType::BigInt(BigIntType::new())) .partition_keys(["dt"]) - .option("type", "format-table") - .option("file.format", "parquet") - .option("metastore.partitioned-table", "true") .build() .unwrap(); + fs_catalog + .create_table(&identifier, schema.clone(), false) + .await + .unwrap(); + let fs_table = fs_catalog.get_table(&identifier).await.unwrap(); + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("dt", ArrowDataType::Utf8, true), + ArrowField::new("id", ArrowDataType::Int64, true), + ])), + vec![ + Arc::new(StringArray::from(vec!["2026-07-22"])), + Arc::new(Int64Array::from(vec![1_i64])), + ], + ) + .unwrap(); + write_batch(&fs_table, batch, "partition-writer").await; + + let ctx = setup_catalog(vec!["default"]).await; + ctx.server.add_table_with_schema( + "default", + "filesystem_partitions", + schema, + fs_table.location(), + ); + ctx.server + .set_list_partitions_by_names_error_status(Some(StatusCode::NOT_IMPLEMENTED)); + let exact = HashMap::from([("dt".to_string(), "2026-07-22".to_string())]); + let alias = HashMap::from([("dt".to_string(), "2026-07-22 ".to_string())]); + + let partitions = ctx + .catalog + .list_partitions_by_names(&identifier, vec![exact.clone(), alias]) + .await + .unwrap(); + + assert_eq!(partitions.len(), 1); + assert_eq!(partitions[0].spec, exact); +} + +#[tokio::test] +async fn test_rest_catalog_rejects_external_catalog_managed_format_table() { + let ctx = setup_catalog(vec!["default"]).await; + let schema = catalog_managed_format_table_schema(&[]); let identifier = Identifier::new("default", "external_managed_format"); ctx.server.add_table_with_schema( "default", @@ -1118,16 +1338,7 @@ async fn test_rest_catalog_rejects_external_catalog_managed_format_table() { #[tokio::test] async fn test_rest_catalog_rejects_engine_managed_format_table() { let ctx = setup_catalog(vec!["default"]).await; - let schema = Schema::builder() - .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) - .column("id", DataType::BigInt(BigIntType::new())) - .partition_keys(["dt"]) - .option("type", "format-table") - .option("file.format", "parquet") - .option("metastore.partitioned-table", "true") - .option("format-table.implementation", "engine") - .build() - .unwrap(); + let schema = catalog_managed_format_table_schema(&[("format-table.implementation", "engine")]); let identifier = Identifier::new("default", "engine_managed_format"); ctx.server.add_table_with_schema( "default", @@ -1160,15 +1371,7 @@ async fn test_managed_format_scan_hides_unregistered_partition() { let table_path = format!("file://{}", tmp.path().display()); let ctx = setup_catalog(vec!["default"]).await; - let schema = Schema::builder() - .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) - .column("id", DataType::BigInt(BigIntType::new())) - .partition_keys(["dt"]) - .option("type", "format-table") - .option("file.format", "parquet") - .option("metastore.partitioned-table", "true") - .build() - .unwrap(); + let schema = catalog_managed_format_table_schema(&[]); let identifier = Identifier::new("default", "managed_visibility"); ctx.server .add_table_with_schema("default", "managed_visibility", schema, &table_path); @@ -1190,52 +1393,99 @@ async fn test_managed_format_scan_hides_unregistered_partition() { assert!(plan.splits()[0].bucket_path().ends_with("/dt=2026-07-22")); } -#[cfg(not(windows))] #[tokio::test] -async fn test_managed_format_scan_preserves_loaded_partition_layout_in_dynamic_copy() { - let tmp = tempfile::tempdir().unwrap(); - let partition_dir = tmp.path().join("dt=2026-07-22"); - std::fs::create_dir_all(&partition_dir).unwrap(); - std::fs::write(partition_dir.join("part-0.parquet"), b"data").unwrap(); - let table_path = format!("file://{}", tmp.path().display()); +async fn test_rest_format_table_rejects_partition_option_changes_in_dynamic_copy() { + let ctx = setup_catalog(vec!["default"]).await; + let schema = catalog_managed_format_table_schema(&[]); + let identifier = Identifier::new("default", "managed_dynamic_layout"); + ctx.server.add_table_with_schema( + "default", + "managed_dynamic_layout", + schema, + "memory:/managed_dynamic_layout", + ); + ctx.server + .set_table_external("default", "managed_dynamic_layout", false); + + let table = ctx.catalog.get_table(&identifier).await.unwrap(); + for (key, value) in [ + ("metastore.partitioned-table", "false"), + ("format-table.partition-path-only-value", "true"), + ("format-table.implementation", "engine"), + ] { + let error = table + .copy_with_time_travel(HashMap::from([(key.to_string(), value.to_string())])) + .await + .unwrap_err(); + assert!( + matches!(error, paimon::Error::DataInvalid { .. }) && error.to_string().contains(key), + "expected {key} validation error, got: {error}" + ); + } + + let copied = table + .copy_with_time_travel(HashMap::from([ + ( + "metastore.partitioned-table".to_string(), + "TRUE".to_string(), + ), + ( + "format-table.partition-path-only-value".to_string(), + "false".to_string(), + ), + ( + "format-table.implementation".to_string(), + "PAIMON".to_string(), + ), + ("read.batch-size".to_string(), "2048".to_string()), + ("type".to_string(), "table".to_string()), + ("path".to_string(), "memory:/dynamic-path".to_string()), + ("file.format".to_string(), "orc".to_string()), + ])) + .await + .unwrap(); + assert!(copied.has_catalog_managed_partitions()); + assert_eq!( + copied.schema().options().get("read.batch-size"), + Some(&"2048".to_string()) + ); +} +#[tokio::test] +async fn test_rest_format_table_rejects_enabling_catalog_partitions_in_dynamic_copy() { let ctx = setup_catalog(vec!["default"]).await; let schema = Schema::builder() .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) - .column("id", DataType::BigInt(BigIntType::new())) .partition_keys(["dt"]) .option("type", "format-table") .option("file.format", "parquet") - .option("metastore.partitioned-table", "true") + .option("metastore.partitioned-table", "false") .build() .unwrap(); - let identifier = Identifier::new("default", "managed_dynamic_layout"); - ctx.server - .add_table_with_schema("default", "managed_dynamic_layout", schema, &table_path); - ctx.server - .set_table_external("default", "managed_dynamic_layout", false); - ctx.server.set_table_partitions( + let identifier = Identifier::new("default", "unmanaged_dynamic_source"); + ctx.server.add_table_with_schema( "default", - "managed_dynamic_layout", - vec![HashMap::from([( - "dt".to_string(), - "2026-07-22".to_string(), - )])], + "unmanaged_dynamic_source", + schema, + "memory:/unmanaged_dynamic_source", ); + ctx.server + .set_table_external("default", "unmanaged_dynamic_source", false); let table = ctx.catalog.get_table(&identifier).await.unwrap(); - let copied = table.copy_with_options(HashMap::from([ - ( - "format-table.partition-path-only-value".to_string(), + let error = table + .copy_with_time_travel(HashMap::from([( + "metastore.partitioned-table".to_string(), "true".to_string(), - ), - ("path".to_string(), "memory:/wrong-path".to_string()), - ("type".to_string(), "table".to_string()), - ])); - let plan = copied.new_read_builder().new_scan().plan().await.unwrap(); + )])) + .await + .unwrap_err(); - assert_eq!(plan.splits().len(), 1); - assert!(plan.splits()[0].bucket_path().ends_with("/dt=2026-07-22")); + assert!( + matches!(error, paimon::Error::DataInvalid { .. }) + && error.to_string().contains("metastore.partitioned-table"), + "expected partition option validation error, got: {error}" + ); } #[cfg(not(windows))] @@ -1245,15 +1495,7 @@ async fn test_managed_format_scan_treats_registered_missing_directory_as_empty() let table_path = format!("file://{}", tmp.path().display()); let ctx = setup_catalog(vec!["default"]).await; - let schema = Schema::builder() - .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) - .column("id", DataType::BigInt(BigIntType::new())) - .partition_keys(["dt"]) - .option("type", "format-table") - .option("file.format", "parquet") - .option("metastore.partitioned-table", "true") - .build() - .unwrap(); + let schema = catalog_managed_format_table_schema(&[]); let identifier = Identifier::new("default", "managed_missing_directory"); ctx.server .add_table_with_schema("default", "managed_missing_directory", schema, &table_path); @@ -1284,15 +1526,7 @@ async fn test_managed_format_scan_propagates_partition_listing_error() { let table_path = format!("file://{}", tmp.path().display()); let ctx = setup_catalog(vec!["default"]).await; - let schema = Schema::builder() - .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) - .column("id", DataType::BigInt(BigIntType::new())) - .partition_keys(["dt"]) - .option("type", "format-table") - .option("file.format", "parquet") - .option("metastore.partitioned-table", "true") - .build() - .unwrap(); + let schema = catalog_managed_format_table_schema(&[]); let identifier = Identifier::new("default", "managed_scan_error"); ctx.server .add_table_with_schema("default", "managed_scan_error", schema, &table_path); @@ -1324,15 +1558,7 @@ async fn test_managed_format_scan_reports_table_for_malformed_partition_metadata let table_path = format!("file://{}", tmp.path().display()); let ctx = setup_catalog(vec!["default"]).await; - let schema = Schema::builder() - .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) - .column("id", DataType::BigInt(BigIntType::new())) - .partition_keys(["dt"]) - .option("type", "format-table") - .option("file.format", "parquet") - .option("metastore.partitioned-table", "true") - .build() - .unwrap(); + let schema = catalog_managed_format_table_schema(&[]); let identifier = Identifier::new("default", "managed_corrupt_metadata"); ctx.server .add_table_with_schema("default", "managed_corrupt_metadata", schema, &table_path); @@ -1355,12 +1581,19 @@ async fn test_managed_format_scan_reports_table_for_malformed_partition_metadata .await .unwrap_err(); - assert!( - error - .to_string() - .contains("default.managed_corrupt_metadata"), - "expected table context, got: {error}" - ); + match error { + paimon::Error::DataInvalid { + message, + source: Some(source), + } => { + assert!(message.contains("invalid partition metadata")); + assert!(message.contains("default.managed_corrupt_metadata")); + let cause = source.to_string(); + assert!(cause.contains("unexpected"), "unexpected cause: {cause}"); + assert!(cause.contains("dt"), "unexpected cause: {cause}"); + } + other => panic!("expected invalid partition metadata, got: {other}"), + } } #[cfg(not(windows))] diff --git a/crates/paimon/tests/rest_object_models_test.rs b/crates/paimon/tests/rest_object_models_test.rs index 522d10c1..11492dd1 100644 --- a/crates/paimon/tests/rest_object_models_test.rs +++ b/crates/paimon/tests/rest_object_models_test.rs @@ -26,7 +26,7 @@ use paimon::spec::DataField; use serde_json::json; #[test] -fn partition_rest_models_are_public_api() { +fn partition_rest_model_contract() { let spec = HashMap::from([("dt".to_string(), "2026-07-22".to_string())]); let create = CreatePartitionsRequest::new(vec![spec.clone()], false); @@ -38,16 +38,20 @@ fn partition_rest_models_are_public_api() { })) .unwrap(); let dropped: DropPartitionsResponse = serde_json::from_value(json!({ - "dropped": [spec], + "dropped": [spec.clone()], "missing": [] })) .unwrap(); assert!(!create.ignore_if_exists); + assert_eq!(create.partition_specs, vec![spec.clone()]); assert!(!drop.ignore_if_not_exists); - assert_eq!(list.partition_specs.len(), 1); - assert_eq!(created.created.len(), 1); - assert_eq!(dropped.dropped.len(), 1); + assert_eq!(drop.partition_specs, vec![spec.clone()]); + assert_eq!(list.partition_specs, vec![spec.clone()]); + assert_eq!(created.created, vec![spec.clone()]); + assert!(created.existed.is_empty()); + assert_eq!(dropped.dropped, vec![spec]); + assert!(dropped.missing.is_empty()); } #[test] diff --git a/docs/src/sql.md b/docs/src/sql.md index 5f8169d3..e1ed1924 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -40,7 +40,7 @@ Mosaic support is always available and currently read-only. SQL queries can read SQL support has two layers: - DataFusion provides the parser, query planner, optimizer, execution engine, expressions, scalar functions, aggregate functions, and window functions. SQL statements that `SQLContext` does not intercept are delegated to DataFusion. This includes the DataFusion SQL surface for `SELECT` queries, CTEs (including recursive CTEs), subqueries, joins including `LATERAL` joins, SQL lambda functions, grouping, `HAVING`, window clauses, `QUALIFY`, set operations, `ORDER BY`, `LIMIT`/`OFFSET`, `EXPLAIN`, information-schema commands such as `SHOW TABLES`, `DESCRIBE`, `COPY`, and ordinary `INSERT`. -- Paimon-specific table management and row-level writes are implemented by `SQLContext`. This includes Paimon `CREATE TABLE`, `ALTER TABLE`, `DROP TABLE`, `CREATE TEMPORARY TABLE`, `CREATE TEMPORARY VIEW`, REST Catalog persistent `CREATE VIEW`, `DROP VIEW`, and `CREATE FUNCTION`, REST-managed Format Table partition commands (`SHOW PARTITIONS`, `ADD/DROP PARTITION`, and `MSCK REPAIR TABLE`), `DROP TEMPORARY TABLE` / `VIEW`, `INSERT OVERWRITE ... PARTITION`, `UPDATE`, `DELETE`, `MERGE INTO`, `TRUNCATE TABLE`, `CALL sys.*`, Paimon time travel, and `SET` / `RESET 'paimon.*'`. +- Paimon-specific table management and row-level writes are implemented by `SQLContext`. This includes Paimon `CREATE TABLE`, `ALTER TABLE`, `DROP TABLE`, `CREATE TEMPORARY TABLE`, `CREATE TEMPORARY VIEW`, REST Catalog persistent `CREATE VIEW`, `DROP VIEW`, and `CREATE FUNCTION`, catalog-managed Format Table partition commands for internal tables loaded from REST Catalog (`SHOW PARTITIONS`, `ADD/DROP PARTITION`, and `MSCK REPAIR TABLE`), `DROP TEMPORARY TABLE` / `VIEW`, `INSERT OVERWRITE ... PARTITION`, `UPDATE`, `DELETE`, `MERGE INTO`, `TRUNCATE TABLE`, `CALL sys.*`, Paimon time travel, and `SET` / `RESET 'paimon.*'`. Not every DataFusion DDL/DML statement maps to a Paimon table operation. For Paimon catalogs, `CREATE EXTERNAL TABLE`, `LOCATION`, `CREATE MATERIALIZED VIEW`, and persistent `CREATE TABLE AS SELECT` are rejected or not implemented. Persistent `CREATE FUNCTION` is supported only for the REST Catalog SQL scalar form documented below. DataFusion `COPY` can export query results to files; it does not create or commit Paimon table files. @@ -703,26 +703,28 @@ ALTER TABLE paimon.my_db.users SET TBLPROPERTIES('data-evolution.enabled' = 'tru ALTER TABLE IF EXISTS paimon.my_db.users ADD COLUMN age INT; ``` -### REST-managed Format Table partitions +### Catalog-managed Format Table partitions -`SQLContext` supports Spark-compatible partition administration for a -catalog-managed Format Table. The target must satisfy all of these conditions: +`SQLContext` supports Spark-compatible partition commands for an internal +Format Table loaded from REST Catalog. The table must satisfy all of these +conditions: -- it was loaded from a REST Catalog; +- it was loaded from REST Catalog; - it is an internal table (`isExternal=false`); - it has `type=format-table`; - it is partitioned; - it has `metastore.partitioned-table=true`; - it does not use `format-table.implementation=engine`. -For such a table, REST partition registrations are authoritative for both +For these tables, REST partition registrations are authoritative for both `SHOW PARTITIONS` and scans. A directory that is not registered is invisible, while a registered partition whose directory is absent reads as empty when storage reports `NotFound`. Other partition-directory listing failures propagate instead of being treated as empty. Filesystem-discovered Format Tables and non-REST catalogs do not support these commands. Dynamic query -options cannot change the validated partition source, table path, file format, -or value-only partition layout of a loaded managed table. +options cannot override `metastore.partitioned-table`, +`format-table.partition-path-only-value`, or `format-table.implementation` for +a loaded table. #### SHOW PARTITIONS @@ -735,18 +737,18 @@ PARTITION (region = 'us'); ``` The optional filter may be partial and does not need to be a leading partition -prefix. Results contain one non-null string column named `partition`. Each name +prefix. The result has one non-null string column named `partition`. Each value uses declared partition-key order, and rows are sorted lexicographically by the -complete name, for example `dt=2026-07-22/region=us`. The result contains one +complete value, for example `dt=2026-07-22/region=us`. The result contains one row per REST registration, so type-equivalent raw registrations can produce -duplicate displayed names. +duplicate displayed values. -REST values are cast through their partition-column types before display and -filtering. Default-partition values display as `null`, dates as `yyyy-MM-dd`, +REST partition values are cast through their column types before display and +filtering. Default-partition values are shown as `null`, dates as `yyyy-MM-dd`, and an integer directory value such as `01` displays as `1`. Type-equivalent -raw values registered by MSCK are normalized for display and filtering; for -example, surrounding whitespace is ignored for numeric, boolean, and date -casts, while character values preserve their whitespace. +raw values registered by MSCK are normalized for display and filtering. Raw +numeric and boolean metadata must not contain surrounding whitespace. `CHAR` +and `VARCHAR` metadata is preserved exactly. #### ADD PARTITION @@ -760,14 +762,15 @@ PARTITION (dt = '2026-07-22', region = 'us') PARTITION (dt = '2026-07-23', region = 'eu'); ``` -Every ADD specification must contain all partition columns. The complete batch -is registered through REST before its partition directories are created. -Strict ADD leaves duplicate detection to the REST catalog; `IF NOT EXISTS` -makes retries converge. SQL constants are converted to strings and cast through -the partition-column types, so quoted numeric/boolean values and numeric values -for string columns are accepted. Integer types from `TINYINT` through `BIGINT`, -`BOOLEAN`, `CHAR`/`VARCHAR`, and `DATE` are supported. SQL `NULL` and -blank character values map to `partition.default-name`. +Each `ADD PARTITION` specification must contain all partition columns. The +complete batch is registered through REST before its partition directories are +created. Without `IF NOT EXISTS`, the REST catalog reports duplicate +registrations as conflicts. With `IF NOT EXISTS`, existing registrations are +ignored. SQL constants are converted to strings and cast through the partition +column types, so quoted numeric or boolean values and numeric values for string +columns are accepted. Integer types from `TINYINT` through `BIGINT`, `BOOLEAN`, +`CHAR`/`VARCHAR`, and `DATE` are supported. SQL `NULL` and blank character +values map to `partition.default-name`. Normalization follows Spark partition-literal behavior for the supported types. Integer SQL literals lose redundant leading zeros even when the target @@ -784,14 +787,19 @@ or day. Typed literals are normalized before the partition-column cast, so column. Compact numeric dates such as `20260722` are rejected instead of being interpreted as `yyyyMMdd`. Paimon `DATE` partition values are limited to `0000-01-01` through `9999-12-31`. Typed literals other than `DATE` are -rejected until their Spark source-type formatting is implemented. +not supported. -Strict ADD is sent as one REST request to preserve whole-batch conflict -semantics and therefore accepts at most 1000 specs. `IF NOT EXISTS` ADD is -split into requests of at most 1000 specs. -Partition values use Java-compatible path escaping: Unicode is preserved and -reserved characters such as `/`, `=`, `%`, and `}` are percent-encoded. -`LOCATION` is not supported. +With the default `partition.legacy-name=true`, `dt = '2026'` is stored in the +directory `dt=20454` and displayed by `SHOW PARTITIONS` as `dt=2026-01-01`. +With `partition.legacy-name=false`, the directory is `dt=2026-01-01`. +Numeric DATE values already present in REST metadata are always interpreted as +epoch days, independently of this option. + +`ADD PARTITION` without `IF NOT EXISTS` is sent as one REST request. +`ADD IF NOT EXISTS PARTITION` is split into requests of at most 1000 +specifications. Partition values use Java-compatible path escaping: Unicode is +preserved and reserved characters such as `/`, `=`, `%`, and `}` are +percent-encoded. `LOCATION` is not supported. #### DROP PARTITION @@ -814,17 +822,14 @@ DROP PARTITION (dt = '2026-07-22', region = 'us'), PARTITION (dt = '2026-07-23', region = 'eu'); ``` -Complete specifications first use the REST list-by-names lookup. If MSCK -registered a type-equivalent raw value such as integer `01`, whitespace-padded -numeric text, or an unpadded `CHAR` value, DROP falls back to a full -registration traversal so the normalized SQL specification still matches it. -Fallback is tracked per requested specification: an exact complete match stays -exact even when another complete miss or partial specification requires a full -traversal. Partial specifications are expanded against that full registration -traversal. Matching registrations are removed before their directories are -deleted. Therefore DROP never deletes an unregistered directory. A missing -complete specification fails unless partition-level `IF EXISTS` is present. -REST unregister calls are split into batches of at most 1000 specs. `PURGE` is +A complete specification selects the exact raw registration when it exists. +Otherwise, typed matching can select a type-equivalent registration, such as +integer metadata `01` for SQL value `1`. A partial specification selects every +registration whose typed values match the supplied keys, including non-leading +partition keys. Selected registrations are removed before their directories +are deleted, so DROP never deletes an unregistered directory. A missing +complete specification fails unless `IF EXISTS` is present. REST unregister +calls are split into batches of at most 1000 specifications. `PURGE` is rejected. #### MSCK REPAIR TABLE @@ -843,12 +848,10 @@ adding before dropping. Repair is metadata-only: it never creates or deletes partition directories. Filesystem values are preserved as written, so a path such as `month=01` is registered with the raw value `01`; `SHOW PARTITIONS` still displays its typed value as `month=1`. REST create/drop calls use batches -of at most 1000 specs. Every segment that matches the declared partition -layout must round-trip exactly through Java-compatible unescaping and canonical -escaping. Noncanonical spellings such as malformed `%ZZ`, lowercase percent -escapes, or over-escaped safe characters fail the repair before any REST -mutation because the REST partition model has no separate field in which to -retain the physical spelling. +of at most 1000 specifications. Partition path segments must use canonical +Java-compatible escaping. Repair fails before updating REST metadata for +malformed percent escapes such as `%ZZ`, lowercase percent escapes, or escaped +safe characters. ## DML @@ -1017,10 +1020,11 @@ Multiple partition key-value pairs can be specified: ALTER TABLE paimon.my_db.events DROP PARTITION (dt = '2024-01-01', region = 'us'); ``` -For a REST-managed Format Table, use the dedicated semantics documented in -[REST-managed Format Table partitions](#rest-managed-format-table-partitions): -REST metadata is unregistered before physical directories are deleted, and -multiple `PARTITION (...)` clauses are supported. +For a catalog-managed Format Table, use the semantics documented in +[Catalog-managed Format Table partitions](#catalog-managed-format-table-partitions). +These commands apply to internal Format Tables loaded from REST Catalog. REST +metadata is unregistered before physical directories are deleted, and multiple +`PARTITION (...)` clauses are supported. ## Procedures diff --git a/scripts/release_licenses.py b/scripts/release_licenses.py index 8c6e6323..5f83910f 100644 --- a/scripts/release_licenses.py +++ b/scripts/release_licenses.py @@ -61,6 +61,7 @@ class LicenseCorrection: license_path: str license_name: str anchor: str + license_from_repository: bool = False @dataclass(frozen=True) @@ -122,6 +123,14 @@ class BundledComponent: ) PYTHON_MIT_CORRECTIONS = ( + LicenseCorrection( + crates=("jieba-macros", "jieba-rs"), + license_crate="jieba-rs", + license_path="third-party-licenses/jieba-rs-0.10.3.LICENSE", + license_name="MIT License (jieba-rs workspace)", + anchor="mit-jieba-rs-workspace", + license_from_repository=True, + ), LicenseCorrection( crates=( "ownedbytes", @@ -342,14 +351,18 @@ def resolved_packages(metadata: dict) -> list[dict]: ] -def package_by_name( - metadata: dict, crate_name: str, required: bool = True -) -> dict | None: - matches = [ +def packages_by_name(metadata: dict, crate_name: str) -> list[dict]: + return [ package for package in resolved_packages(metadata) if package["name"] == crate_name ] + + +def package_by_name( + metadata: dict, crate_name: str, required: bool = True +) -> dict | None: + matches = packages_by_name(metadata, crate_name) if not matches and not required: return None if len(matches) != 1: @@ -371,25 +384,48 @@ def package_features(metadata: dict, package: dict) -> set[str]: return set(matches[0]) -def correction_html(metadata: dict, correction: LicenseCorrection) -> str: +def correction_html( + root: Path, metadata: dict, correction: LicenseCorrection +) -> str: used_by = [] for crate_name in correction.crates: - package = package_by_name(metadata, crate_name) - repository = package.get("repository") or ( - f"https://crates.io/crates/{crate_name}" - ) - used_by.append( - f'
  • ' - f"{html.escape(crate_name)} {html.escape(package['version'])}
    • " - ) + packages = packages_by_name(metadata, crate_name) + if not packages: + raise RuntimeError(f"expected at least one resolved {crate_name} package") + for package in packages: + repository = package.get("repository") or ( + f"https://crates.io/crates/{crate_name}" + ) + used_by.append( + f'
  • ' + f"{html.escape(crate_name)} {html.escape(package['version'])}
    • " + ) - license_package = package_by_name(metadata, correction.license_crate) - license_file = ( - Path(license_package["manifest_path"]).parent / correction.license_path - ) - if not license_file.is_file(): - raise RuntimeError(f"corrected license file is missing: {license_file}") - license_text = license_file.read_text(encoding="utf-8") + if correction.license_from_repository: + license_file = root / correction.license_path + if not license_file.is_file(): + raise RuntimeError(f"corrected license file is missing: {license_file}") + license_text = license_file.read_text(encoding="utf-8") + else: + license_files = [ + Path(package["manifest_path"]).parent / correction.license_path + for package in packages_by_name(metadata, correction.license_crate) + ] + if not license_files: + raise RuntimeError( + f"expected at least one resolved {correction.license_crate} package" + ) + missing = [path for path in license_files if not path.is_file()] + if missing: + raise RuntimeError(f"corrected license file is missing: {missing[0]}") + license_texts = { + path.read_text(encoding="utf-8") for path in license_files + } + if len(license_texts) != 1: + raise RuntimeError( + f"corrected license files differ for {correction.license_crate}" + ) + license_text = license_texts.pop() return "\n".join( [ @@ -407,6 +443,7 @@ def correction_html(metadata: dict, correction: LicenseCorrection) -> str: def replace_placeholder_entry( + root: Path, report_text: str, metadata: dict, marker: str, @@ -441,7 +478,7 @@ def replace_placeholder_entry( ) replacement = "\n".join( - correction_html(metadata, correction) for correction in corrections + correction_html(root, metadata, correction) for correction in corrections ) return report_text[:entry_start] + replacement + report_text[entry_end:] @@ -538,13 +575,13 @@ def complete_report( root: Path, base_report: str, report: Report, metadata: dict ) -> str: result = replace_placeholder_entry( - base_report, metadata, ALLOC_PLACEHOLDER, ALLOC_CORRECTIONS + root, base_report, metadata, ALLOC_PLACEHOLDER, ALLOC_CORRECTIONS ) mit_corrections = COMMON_MIT_CORRECTIONS if report.component == "python": mit_corrections += PYTHON_MIT_CORRECTIONS result = replace_placeholder_entry( - result, metadata, MIT_PLACEHOLDER, mit_corrections + root, result, metadata, MIT_PLACEHOLDER, mit_corrections ) for placeholder in LICENSE_PLACEHOLDERS: diff --git a/third-party-licenses/README.md b/third-party-licenses/README.md index 642d169a..449c9aa3 100644 --- a/third-party-licenses/README.md +++ b/third-party-licenses/README.md @@ -25,6 +25,11 @@ OpenSSL 1.1.1w on aarch64 Linux. Both versions use the same license text. The XZ Utils 5.8.3 license is read from the locked `liblzma-sys` crate at `xz/COPYING.0BSD`. +`jieba-rs-0.10.3.LICENSE` is the license file from the root of the upstream +`jieba-rs` workspace at commit +`c62e0df1f9dcc2cc1e014711c5aa4561ae260538`. It covers `jieba-rs` 0.10.3 and +`jieba-macros` 0.10.3. + Target-specific Python and Go reports are generated in their binary build jobs. They are intentionally not committed or included in the ASF source archive. @@ -32,3 +37,4 @@ Sources: - - +- diff --git a/third-party-licenses/jieba-rs-0.10.3.LICENSE b/third-party-licenses/jieba-rs-0.10.3.LICENSE new file mode 100644 index 00000000..a9c2c612 --- /dev/null +++ b/third-party-licenses/jieba-rs-0.10.3.LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2018 - 2019 messense +Copyright (c) 2019 Paul Meng + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 10ea96667b1a2bcb6c15e98b05dc64114f9b0c53 Mon Sep 17 00:00:00 2001 From: Dapeng Sun Date: Thu, 23 Jul 2026 08:29:22 +0800 Subject: [PATCH 5/7] build: exclude bundled jieba license from header check --- .licenserc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.licenserc.yaml b/.licenserc.yaml index c3cac61b..e147040a 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -28,6 +28,7 @@ header: - ".github/PULL_REQUEST_TEMPLATE.md" - "crates/paimon/tests/**/*.json" - "crates/paimon/testdata/**" + - "third-party-licenses/jieba-rs-0.10.3.LICENSE" - "third-party-licenses/openssl-1.1.1.LICENSE" - "**/go.sum" - "**/DEPENDENCIES.*.tsv" From 723d1cb0ec39a65c630d2fbec081a85f9e28eee7 Mon Sep 17 00:00:00 2001 From: Dapeng Sun Date: Thu, 23 Jul 2026 14:19:44 +0800 Subject: [PATCH 6/7] datafusion: simplify REST partition command changes --- .../datafusion/src/sql_context.rs | 1128 +++++------------ .../tests/rest_format_partition_sql.rs | 246 ++-- crates/paimon/src/api/api_request.rs | 60 +- crates/paimon/src/api/api_response.rs | 68 +- .../paimon/src/catalog/rest/rest_catalog.rs | 31 +- crates/paimon/src/table/format_partition.rs | 18 +- crates/paimon/src/table/format_table_scan.rs | 36 +- crates/paimon/tests/format_partition_test.rs | 146 +-- crates/paimon/tests/mock_server.rs | 109 -- crates/paimon/tests/rest_api_test.rs | 88 +- crates/paimon/tests/rest_catalog_test.rs | 470 ++----- .../paimon/tests/rest_object_models_test.rs | 34 +- docs/src/sql.md | 166 +-- 13 files changed, 655 insertions(+), 1945 deletions(-) diff --git a/crates/integrations/datafusion/src/sql_context.rs b/crates/integrations/datafusion/src/sql_context.rs index e87420d6..bc004438 100644 --- a/crates/integrations/datafusion/src/sql_context.rs +++ b/crates/integrations/datafusion/src/sql_context.rs @@ -4891,6 +4891,44 @@ mod tests { ) } + fn parse_partition_literal(literal: &str) -> SqlExpr { + Parser::new(&GenericDialect {}) + .try_with_sql(literal) + .unwrap() + .parse_expr() + .unwrap() + } + + fn format_partition_literal_for_test( + literal: &str, + data_type: &PaimonDataType, + extra_options: &[(&str, &str)], + ) -> DFResult { + let expr = parse_partition_literal(literal); + let options = extra_options + .iter() + .map(|(key, value)| ((*key).to_string(), (*value).to_string())) + .collect::>(); + let core_options = CoreOptions::new(&options); + match format_partition_literal_to_datum(&expr, data_type)? { + None => Ok(core_options.partition_default_name().to_string()), + Some(datum) => format_partition_datum_for_storage(&datum, data_type, &core_options), + } + } + + async fn show_partition_values(sql_context: &SQLContext, sql: &str) -> Vec { + let batches = sql_context.sql(sql).await.unwrap().collect().await.unwrap(); + assert_eq!(batches[0].schema().field(0).name(), "partition"); + let partitions = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + (0..batches[0].num_rows()) + .map(|index| partitions.value(index).to_string()) + .collect() + } + fn add_unary_sql_function( catalog: &MockCatalog, name: &str, @@ -7667,39 +7705,24 @@ mod tests { } #[tokio::test] - async fn test_msck_repair_table_reports_missing_catalog_table() { - let catalog = Arc::new(MockCatalog::new()); - let sql_context = make_sql_context(catalog).await; - - let error = sql_context - .sql("MSCK REPAIR TABLE mydb.missing") - .await - .unwrap_err(); - - assert!( - error - .to_string() - .contains("Table mydb.missing does not exist"), - "expected catalog table error, got: {error}" - ); - } - - #[tokio::test] - async fn test_repair_table_reports_missing_catalog_table() { + async fn test_partition_commands_report_missing_catalog_table() { let catalog = Arc::new(MockCatalog::new()); let sql_context = make_sql_context(catalog).await; - let error = sql_context - .sql("REPAIR TABLE mydb.missing") - .await - .unwrap_err(); - - assert!( - error - .to_string() - .contains("Table mydb.missing does not exist"), - "expected catalog table error, got: {error}" - ); + for sql in [ + "MSCK REPAIR TABLE mydb.missing", + "REPAIR TABLE mydb.missing", + "SHOW PARTITIONS mydb.missing", + "ALTER TABLE mydb.missing ADD PARTITION (dt = '2026-07-22')", + ] { + let error = sql_context.sql(sql).await.unwrap_err(); + assert!( + error + .to_string() + .contains("Table mydb.missing does not exist"), + "expected catalog table error for {sql}, got: {error}" + ); + } } #[tokio::test] @@ -7719,61 +7742,25 @@ mod tests { } #[tokio::test] - async fn test_msck_repair_table_rejects_branch_target_before_catalog_access() { - let catalog = Arc::new(MockCatalog::new()); - let sql_context = make_sql_context(catalog.clone()).await; - - let error = sql_context - .sql("MSCK REPAIR TABLE mydb.events$branch_dev") - .await - .unwrap_err(); - - assert!( - error - .to_string() - .contains("MSCK REPAIR TABLE on Paimon branch 'dev' is not supported"), - "expected branch write rejection, got: {error}" - ); - assert_eq!(catalog.get_table_calls(), 0); - } - - #[tokio::test] - async fn test_repair_table_rejects_branch_target_before_catalog_access() { + async fn test_repair_table_syntaxes_reject_branch_target_before_catalog_access() { let catalog = Arc::new(MockCatalog::new()); let sql_context = make_sql_context(catalog.clone()).await; - let error = sql_context - .sql("REPAIR TABLE mydb.events$branch_dev") - .await - .unwrap_err(); - - assert!( - error - .to_string() - .contains("MSCK REPAIR TABLE on Paimon branch 'dev' is not supported"), - "expected branch write rejection, got: {error}" - ); + for sql in [ + "MSCK REPAIR TABLE mydb.events$branch_dev", + "REPAIR TABLE mydb.events$branch_dev", + ] { + let error = sql_context.sql(sql).await.unwrap_err(); + assert!( + error + .to_string() + .contains("MSCK REPAIR TABLE on Paimon branch 'dev' is not supported"), + "expected branch write rejection for {sql}, got: {error}" + ); + } assert_eq!(catalog.get_table_calls(), 0); } - #[tokio::test] - async fn test_show_partitions_reports_missing_catalog_table() { - let catalog = Arc::new(MockCatalog::new()); - let sql_context = make_sql_context(catalog).await; - - let error = sql_context - .sql("SHOW PARTITIONS mydb.missing") - .await - .unwrap_err(); - - assert!( - error - .to_string() - .contains("Table mydb.missing does not exist"), - "expected catalog table error, got: {error}" - ); - } - #[test] fn test_parse_show_partitions_with_partition_filter() { let statement = parse_show_partitions( @@ -7828,208 +7815,56 @@ mod tests { } #[tokio::test] - async fn test_show_partitions_returns_sorted_catalog_registrations() { + async fn test_show_partitions_formats_sorts_and_filters_catalog_registrations() { let catalog = Arc::new(MockCatalog::new()); - let identifier = Identifier::new("mydb", "events"); catalog.set_table( - managed_format_table(identifier, "memory:/show_partitions", &["dt", "hour"]).await, + managed_format_table_with_partition_fields_and_options( + Identifier::new("mydb", "events"), + "memory:/show_partitions", + vec![ + ("dt", PaimonDataType::Date(DateType::new())), + ("month", PaimonDataType::Int(IntType::new())), + ], + &[("partition.legacy-name", "false")], + ) + .await, ); catalog.set_partition_specs(vec![ HashMap::from([ - ("dt".to_string(), "2026-07-22".to_string()), - ("hour".to_string(), "10".to_string()), + ("dt".to_string(), "20656".to_string()), + ("month".to_string(), "01".to_string()), ]), HashMap::from([ - ("dt".to_string(), "2026-07-21".to_string()), - ("hour".to_string(), "09".to_string()), + ("dt".to_string(), "20656".to_string()), + ("month".to_string(), "1".to_string()), ]), - ]); - let sql_context = make_sql_context(catalog).await; - - let batches = sql_context - .sql("SHOW PARTITIONS mydb.events") - .await - .unwrap() - .collect() - .await - .unwrap(); - - assert_eq!(batches[0].schema().field(0).name(), "partition"); - let partitions = batches[0] - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!( - (0..batches[0].num_rows()) - .map(|index| partitions.value(index)) - .collect::>(), - vec!["dt=2026-07-21/hour=09", "dt=2026-07-22/hour=10",] - ); - } - - #[tokio::test] - async fn test_show_partitions_filters_by_non_prefix_partition_key() { - let catalog = Arc::new(MockCatalog::new()); - let identifier = Identifier::new("mydb", "events"); - catalog.set_table( - managed_format_table( - identifier, - "memory:/show_partitions_filter", - &["dt", "hour"], - ) - .await, - ); - catalog.set_partition_specs(vec![ HashMap::from([ - ("dt".to_string(), "2026-07-22".to_string()), - ("hour".to_string(), "10".to_string()), + ("dt".to_string(), "__DEFAULT_PARTITION__".to_string()), + ("month".to_string(), "02".to_string()), ]), HashMap::from([ - ("dt".to_string(), "2026-07-21".to_string()), - ("hour".to_string(), "09".to_string()), + ("dt".to_string(), "20655".to_string()), + ("month".to_string(), "09".to_string()), ]), ]); let sql_context = make_sql_context(catalog).await; - let batches = sql_context - .sql("SHOW PARTITIONS mydb.events PARTITION (hour = '10')") - .await - .unwrap() - .collect() - .await - .unwrap(); - - let partitions = batches[0] - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(batches[0].num_rows(), 1); - assert_eq!(partitions.value(0), "dt=2026-07-22/hour=10"); - } - - #[tokio::test] - async fn test_show_partitions_formats_default_partition_as_null() { - let catalog = Arc::new(MockCatalog::new()); - let identifier = Identifier::new("mydb", "events"); - catalog.set_table( - managed_format_table(identifier, "memory:/show_default_partition", &["dt"]).await, - ); - catalog.set_partition_specs(vec![HashMap::from([( - "dt".to_string(), - "__DEFAULT_PARTITION__".to_string(), - )])]); - let sql_context = make_sql_context(catalog).await; - - let batches = sql_context - .sql("SHOW PARTITIONS mydb.events") - .await - .unwrap() - .collect() - .await - .unwrap(); - let partitions = batches[0] - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - - assert_eq!(partitions.value(0), "dt=null"); - } - - #[tokio::test] - async fn test_show_partitions_formats_date_and_integer_values_by_type() { - let catalog = Arc::new(MockCatalog::new()); - let identifier = Identifier::new("mydb", "events"); - catalog.set_table( - managed_format_table_with_partition_fields_and_options( - identifier, - "memory:/show_typed_partitions", - vec![ - ("dt", PaimonDataType::Date(DateType::new())), - ("month", PaimonDataType::Int(IntType::new())), - ], - &[("partition.legacy-name", "false")], - ) - .await, + assert_eq!( + show_partition_values(&sql_context, "SHOW PARTITIONS mydb.events").await, + vec![ + "dt=2026-07-21/month=9", + "dt=2026-07-22/month=1", + "dt=2026-07-22/month=1", + "dt=null/month=2", + ] ); - catalog.set_partition_specs(vec![HashMap::from([ - ("dt".to_string(), "20656".to_string()), - ("month".to_string(), "01".to_string()), - ])]); - let sql_context = make_sql_context(catalog).await; - - let batches = sql_context - .sql("SHOW PARTITIONS mydb.events PARTITION (month = '1')") - .await - .unwrap() - .collect() - .await - .unwrap(); - let partitions = batches[0] - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - - assert_eq!(partitions.value(0), "dt=2026-07-22/month=1"); - } - - #[tokio::test] - async fn test_show_partitions_preserves_duplicate_normalized_names() { - let catalog = Arc::new(MockCatalog::new()); - let identifier = Identifier::new("mydb", "events"); - catalog.set_table( - managed_format_table_with_partition_fields( - identifier, - "memory:/show_duplicate_typed_partitions", - vec![("month", PaimonDataType::Int(IntType::new()))], + assert_eq!( + show_partition_values( + &sql_context, + "SHOW PARTITIONS mydb.events PARTITION (month = '1')", ) .await, - ); - catalog.set_partition_specs(vec![ - HashMap::from([("month".to_string(), "01".to_string())]), - HashMap::from([("month".to_string(), "1".to_string())]), - ]); - let sql_context = make_sql_context(catalog).await; - - let batches = sql_context - .sql("SHOW PARTITIONS mydb.events") - .await - .unwrap() - .collect() - .await - .unwrap(); - let partitions = batches[0] - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - - assert_eq!( - (0..batches[0].num_rows()) - .map(|index| partitions.value(index)) - .collect::>(), - vec!["month=1", "month=1"] - ); - } - - #[tokio::test] - async fn test_add_partition_reports_missing_catalog_table() { - let catalog = Arc::new(MockCatalog::new()); - let sql_context = make_sql_context(catalog).await; - - let error = sql_context - .sql("ALTER TABLE mydb.missing ADD PARTITION (dt = '2026-07-22')") - .await - .unwrap_err(); - - assert!( - error - .to_string() - .contains("Table mydb.missing does not exist"), - "expected catalog table error, got: {error}" + vec!["dt=2026-07-22/month=1", "dt=2026-07-22/month=1"] ); } @@ -8098,174 +7933,202 @@ mod tests { assert_eq!(catalog.partition_mutation_calls(), vec!["create", "create"]); } - #[tokio::test] - async fn test_add_format_partition_normalizes_integer_literal() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![("month", PaimonDataType::Int(IntType::new()))], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("ALTER TABLE mydb.events ADD PARTITION (month = 01)") - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([("month".to_string(), "1".to_string())])] - ); - assert!(temp_dir.path().join("month=1").is_dir()); - assert!(!temp_dir.path().join("month=01").exists()); - } - - #[tokio::test] - async fn test_add_format_partition_normalizes_numeric_literal_for_string_column() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table(Identifier::new("mydb", "events"), &location, &["label"]).await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("ALTER TABLE mydb.events ADD PARTITION (label = 01)") - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([("label".to_string(), "1".to_string())])] - ); - assert!(temp_dir.path().join("label=1").is_dir()); - assert!(!temp_dir.path().join("label=01").exists()); - } - - #[tokio::test] - async fn test_add_format_partition_rejects_sign_on_string_literal() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table(Identifier::new("mydb", "events"), &location, &["label"]).await, - ); - let sql_context = make_sql_context(catalog.clone()).await; + #[test] + fn test_format_partition_literal_compatibility() { + let cases = [ + ( + "01", + PaimonDataType::VarChar(VarCharType::new(255).unwrap()), + &[][..], + "1", + ), + ( + "' us '", + PaimonDataType::VarChar(VarCharType::new(5).unwrap()), + &[][..], + " us ", + ), + ( + "'us'", + PaimonDataType::Char(CharType::new(2).unwrap()), + &[][..], + "us", + ), + ( + "'us'", + PaimonDataType::Char(CharType::new(4).unwrap()), + &[][..], + "us ", + ), + ( + "'数据'", + PaimonDataType::Char(CharType::new(2).unwrap()), + &[][..], + "数据", + ), + ( + "'a '", + PaimonDataType::Char(CharType::new(5).unwrap()), + &[][..], + "a ", + ), + ( + "'a '", + PaimonDataType::VarChar(VarCharType::new(5).unwrap()), + &[][..], + "a ", + ), + ( + "' '", + PaimonDataType::VarChar(VarCharType::new(5).unwrap()), + &[][..], + "__DEFAULT_PARTITION__", + ), + ("' 01 '", PaimonDataType::Int(IntType::new()), &[][..], "1"), + ( + "9223372036854775807", + PaimonDataType::BigInt(BigIntType::new()), + &[][..], + "9223372036854775807", + ), + ( + "-128", + PaimonDataType::TinyInt(TinyIntType::new()), + &[][..], + "-128", + ), + ( + "32767", + PaimonDataType::SmallInt(SmallIntType::new()), + &[][..], + "32767", + ), + ( + "'2026'", + PaimonDataType::Date(DateType::new()), + &[][..], + "20454", + ), + ( + "'2026-07'", + PaimonDataType::Date(DateType::new()), + &[][..], + "20635", + ), + ( + "DATE '2026-07-22'", + PaimonDataType::Date(DateType::new()), + &[][..], + "20656", + ), + ( + "DATE '2026'", + PaimonDataType::VarChar(VarCharType::new(10).unwrap()), + &[][..], + "2026-01-01", + ), + ( + "'2026-07-22'", + PaimonDataType::Date(DateType::new()), + &[("partition.legacy-name", "false")][..], + "2026-07-22", + ), + ]; - for literal in ["+'us'", "-'us'"] { - let error = sql_context - .sql(&format!( - "ALTER TABLE mydb.events ADD PARTITION (label = {literal})" - )) - .await - .unwrap_err(); - assert!( - error - .to_string() - .contains("Unary signs are supported only for numeric partition literals"), - "expected signed string rejection for {literal}, got: {error}" + for (literal, data_type, options, expected) in cases { + assert_eq!( + format_partition_literal_for_test(literal, &data_type, options).unwrap(), + expected, + "literal {literal} as {data_type:?}" ); } - assert!(catalog.partition_specs().is_empty()); - assert!(catalog.partition_mutation_calls().is_empty()); - } - - #[tokio::test] - async fn test_add_format_partition_accepts_char_literal() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![("region", PaimonDataType::Char(CharType::new(2).unwrap()))], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("ALTER TABLE mydb.events ADD PARTITION (region = 'us')") - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([("region".to_string(), "us".to_string())])] - ); - assert!(temp_dir.path().join("region=us").is_dir()); - } - - #[tokio::test] - async fn test_add_format_partition_pads_short_char_literal() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![("region", PaimonDataType::Char(CharType::new(4).unwrap()))], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("ALTER TABLE mydb.events ADD PARTITION (region = 'us')") - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([("region".to_string(), "us ".to_string())])] - ); - assert!(temp_dir.path().join("region=us ").is_dir()); - } - - #[tokio::test] - async fn test_add_format_partition_rejects_oversize_char_and_varchar_literals() { - for data_type in [ - PaimonDataType::Char(CharType::new(5).unwrap()), - PaimonDataType::VarChar(VarCharType::new(5).unwrap()), + for (literal, expected) in [ + ("'t'", "true"), + ("'TRUE'", "true"), + ("' y '", "true"), + ("'yes'", "true"), + ("1", "true"), + ("'f'", "false"), + ("'FALSE'", "false"), + ("' n '", "false"), + ("'no'", "false"), + ("0", "false"), ] { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![("region", data_type)], + assert_eq!( + format_partition_literal_for_test( + literal, + &PaimonDataType::Boolean(BooleanType::new()), + &[], ) - .await, + .unwrap(), + expected, + "boolean literal {literal}" ); - let sql_context = make_sql_context(catalog.clone()).await; + } + } - let error = sql_context - .sql("ALTER TABLE mydb.events ADD PARTITION (region = 'abcdef')") - .await - .unwrap_err(); + #[test] + fn test_invalid_format_partition_literals() { + let cases = [ + ( + "+'us'", + PaimonDataType::VarChar(VarCharType::new(8).unwrap()), + "Unary signs are supported only for numeric partition literals", + ), + ( + "-'us'", + PaimonDataType::VarChar(VarCharType::new(8).unwrap()), + "Unary signs are supported only for numeric partition literals", + ), + ( + "'abcdef'", + PaimonDataType::Char(CharType::new(5).unwrap()), + "exceeds maximum length 5", + ), + ( + "'abcdef'", + PaimonDataType::VarChar(VarCharType::new(5).unwrap()), + "exceeds maximum length 5", + ), + ( + "'数据流'", + PaimonDataType::VarChar(VarCharType::new(2).unwrap()), + "exceeds maximum length 2", + ), + ( + "TIMESTAMP '2026-07-22'", + PaimonDataType::VarChar(VarCharType::new(32).unwrap()), + "Unsupported typed partition literal", + ), + ( + "20260722", + PaimonDataType::Date(DateType::new()), + "Invalid DATE '20260722'", + ), + ( + "'-0001-01-01'", + PaimonDataType::Date(DateType::new()), + "outside Paimon DATE range 0000-01-01..9999-12-31", + ), + ( + "'10000-01-01'", + PaimonDataType::Date(DateType::new()), + "outside Paimon DATE range 0000-01-01..9999-12-31", + ), + ]; + for (literal, data_type, expected_error) in cases { + let error = format_partition_literal_for_test(literal, &data_type, &[]).unwrap_err(); assert!( - error.to_string().contains("exceeds maximum length 5"), - "unexpected error: {error}" + error.to_string().contains(expected_error), + "unexpected error for literal {literal} as {data_type:?}: {error}" ); - assert!(catalog.partition_specs().is_empty()); } } #[tokio::test] - async fn test_add_format_partition_trims_excess_trailing_spaces_for_char_and_varchar() { + async fn test_add_format_partition_casts_typed_literals_and_creates_directory() { let temp_dir = tempfile::tempdir().unwrap(); let location = format!("file://{}", temp_dir.path().display()); let catalog = Arc::new(MockCatalog::new()); @@ -8274,11 +8137,12 @@ mod tests { Identifier::new("mydb", "events"), &location, vec![ - ("fixed", PaimonDataType::Char(CharType::new(5).unwrap())), + ("month", PaimonDataType::Int(IntType::new())), ( - "variable", - PaimonDataType::VarChar(VarCharType::new(5).unwrap()), + "label", + PaimonDataType::VarChar(VarCharType::new(10).unwrap()), ), + ("dt", PaimonDataType::Date(DateType::new())), ], ) .await, @@ -8288,18 +8152,21 @@ mod tests { sql_context .sql( "ALTER TABLE mydb.events ADD PARTITION (\ - fixed = 'a ', variable = 'a ')", + month = '01', label = DATE '2026', dt = DATE '2026-07-22')", ) .await .unwrap(); - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([ - ("fixed".to_string(), "a ".to_string()), - ("variable".to_string(), "a ".to_string()), - ])] - ); + let expected = HashMap::from([ + ("month".to_string(), "1".to_string()), + ("label".to_string(), "2026-01-01".to_string()), + ("dt".to_string(), "20656".to_string()), + ]); + assert_eq!(catalog.partition_specs(), vec![expected]); + assert!(temp_dir + .path() + .join("month=1/label=2026-01-01/dt=20656") + .is_dir()); } #[tokio::test] @@ -8327,413 +8194,6 @@ mod tests { assert!(temp_dir.path().join("dt=__DEFAULT_PARTITION__").is_dir()); } - #[tokio::test] - async fn test_add_format_partition_maps_blank_string_to_default_partition_name() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("ALTER TABLE mydb.events ADD PARTITION (dt = ' ')") - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([( - "dt".to_string(), - "__DEFAULT_PARTITION__".to_string(), - )])] - ); - assert!(temp_dir.path().join("dt=__DEFAULT_PARTITION__").is_dir()); - } - - #[tokio::test] - async fn test_add_format_partition_normalizes_unquoted_partition_column_name() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("ALTER TABLE mydb.events ADD PARTITION (DT = '2026-07-22')") - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([( - "dt".to_string(), - "2026-07-22".to_string(), - )])] - ); - } - - #[tokio::test] - async fn test_add_format_partition_normalizes_common_typed_literals() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![ - ("shard", PaimonDataType::BigInt(BigIntType::new())), - ("active", PaimonDataType::Boolean(BooleanType::new())), - ("dt", PaimonDataType::Date(DateType::new())), - ], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql( - "ALTER TABLE mydb.events ADD PARTITION (\ - shard = 9223372036854775807, active = true, dt = '2026-07-22')", - ) - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([ - ("shard".to_string(), "9223372036854775807".to_string()), - ("active".to_string(), "true".to_string()), - ("dt".to_string(), "20656".to_string()), - ])] - ); - assert!(temp_dir - .path() - .join("shard=9223372036854775807/active=true/dt=20656") - .is_dir()); - } - - #[tokio::test] - async fn test_add_format_partition_casts_literals_through_partition_column_types() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![ - ("month", PaimonDataType::Int(IntType::new())), - ( - "label", - PaimonDataType::VarChar(VarCharType::new(255).unwrap()), - ), - ("active", PaimonDataType::Boolean(BooleanType::new())), - ], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql( - "ALTER TABLE mydb.events ADD PARTITION (\ - month = '01', label = 20260722, active = 'TRUE')", - ) - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([ - ("month".to_string(), "1".to_string()), - ("label".to_string(), "20260722".to_string()), - ("active".to_string(), "true".to_string()), - ])] - ); - } - - #[tokio::test] - async fn test_add_format_partition_trims_string_before_integer_cast() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![("month", PaimonDataType::Int(IntType::new()))], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("ALTER TABLE mydb.events ADD PARTITION (month = ' 01 ')") - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([("month".to_string(), "1".to_string())])] - ); - assert!(temp_dir.path().join("month=1").is_dir()); - } - - #[tokio::test] - async fn test_add_format_partition_accepts_spark_boolean_spellings() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![("active", PaimonDataType::Boolean(BooleanType::new()))], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql( - "ALTER TABLE mydb.events ADD \ - PARTITION (active = 'yes') \ - PARTITION (active = 0)", - ) - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![ - HashMap::from([("active".to_string(), "true".to_string())]), - HashMap::from([("active".to_string(), "false".to_string())]), - ] - ); - assert!(temp_dir.path().join("active=true").is_dir()); - assert!(temp_dir.path().join("active=false").is_dir()); - } - - #[tokio::test] - async fn test_add_format_partition_trims_string_before_boolean_cast() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![("active", PaimonDataType::Boolean(BooleanType::new()))], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("ALTER TABLE mydb.events ADD PARTITION (active = ' yes ')") - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([("active".to_string(), "true".to_string())])] - ); - assert!(temp_dir.path().join("active=true").is_dir()); - } - - #[tokio::test] - async fn test_add_format_partition_accepts_tinyint_and_smallint_literals() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![ - ("tiny", PaimonDataType::TinyInt(TinyIntType::new())), - ("small", PaimonDataType::SmallInt(SmallIntType::new())), - ], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql( - "ALTER TABLE mydb.events \ - ADD PARTITION (tiny = -128, small = 32767)", - ) - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([ - ("tiny".to_string(), "-128".to_string()), - ("small".to_string(), "32767".to_string()), - ])] - ); - assert!(temp_dir.path().join("tiny=-128/small=32767").is_dir()); - } - - #[tokio::test] - async fn test_add_format_partition_accepts_typed_date_literal() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![("dt", PaimonDataType::Date(DateType::new()))], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("ALTER TABLE mydb.events ADD PARTITION (dt = DATE '2026-07-22')") - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([("dt".to_string(), "20656".to_string())])] - ); - assert!(temp_dir.path().join("dt=20656").is_dir()); - } - - #[tokio::test] - async fn test_add_format_partition_normalizes_typed_date_before_varchar_cast() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![( - "label", - PaimonDataType::VarChar(VarCharType::new(10).unwrap()), - )], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("ALTER TABLE mydb.events ADD PARTITION (label = DATE '2026')") - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([( - "label".to_string(), - "2026-01-01".to_string() - )])] - ); - assert!(temp_dir.path().join("label=2026-01-01").is_dir()); - } - - #[tokio::test] - async fn test_add_format_partition_rejects_unsupported_typed_literal() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![( - "label", - PaimonDataType::VarChar(VarCharType::new(32).unwrap()), - )], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - let error = sql_context - .sql( - "ALTER TABLE mydb.events \ - ADD PARTITION (label = TIMESTAMP '2026-07-22')", - ) - .await - .unwrap_err(); - - assert!(error - .to_string() - .contains("Unsupported typed partition literal")); - assert!(catalog.partition_specs().is_empty()); - } - - #[tokio::test] - async fn test_add_format_partition_parses_spark_year_only_date_string() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![("dt", PaimonDataType::Date(DateType::new()))], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("ALTER TABLE mydb.events ADD PARTITION (dt = '2026')") - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([("dt".to_string(), "20454".to_string())])] - ); - assert!(temp_dir.path().join("dt=20454").is_dir()); - } - - #[tokio::test] - async fn test_add_format_partition_rejects_date_outside_paimon_range() { - for value in ["-0001-01-01", "10000-01-01"] { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![("dt", PaimonDataType::Date(DateType::new()))], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - let error = sql_context - .sql(&format!( - "ALTER TABLE mydb.events ADD PARTITION (dt = '{value}')" - )) - .await - .unwrap_err(); - - assert!( - error - .to_string() - .contains("outside Paimon DATE range 0000-01-01..9999-12-31"), - "unexpected error for {value}: {error}" - ); - assert!(catalog.partition_specs().is_empty()); - } - } - #[tokio::test] async fn test_add_format_partition_rejects_compact_numeric_date_without_mutation() { let temp_dir = tempfile::tempdir().unwrap(); diff --git a/crates/integrations/datafusion/tests/rest_format_partition_sql.rs b/crates/integrations/datafusion/tests/rest_format_partition_sql.rs index 1e9765aa..99694cf2 100644 --- a/crates/integrations/datafusion/tests/rest_format_partition_sql.rs +++ b/crates/integrations/datafusion/tests/rest_format_partition_sql.rs @@ -84,6 +84,23 @@ fn format_table_schema(partition_columns: &[(&str, DataType)]) -> Schema { .unwrap() } +fn partition_spec(values: &[(&str, &str)]) -> HashMap { + values + .iter() + .map(|(key, value)| ((*key).to_string(), (*value).to_string())) + .collect() +} + +fn assert_partition_directories(temp_dir: &TempDir, expected: &[(&str, bool)]) { + for (path, exists) in expected { + assert_eq!( + temp_dir.path().join(path).exists(), + *exists, + "partition directory {path}" + ); + } +} + #[cfg(not(windows))] #[tokio::test] async fn test_partition_commands_update_rest_metadata_and_directories() { @@ -96,30 +113,21 @@ async fn test_partition_commands_update_rest_metadata_and_directories() { .sql("ALTER TABLE paimon.default.events ADD PARTITION (dt = '2026-07-22')") .await .unwrap(); - assert_eq!( - show_partitions(&context).await, - vec!["dt=2026-07-22".to_string()] - ); - assert!(temp_dir.path().join("dt=2026-07-22").is_dir()); + assert_partitions(&context, &["dt=2026-07-22"]).await; + assert_partition_directories(&temp_dir, &[("dt=2026-07-22", true)]); context .sql("REPAIR TABLE paimon.default.events ADD PARTITIONS") .await .unwrap(); - assert_eq!( - show_partitions(&context).await, - vec!["dt=2026-07-21".to_string(), "dt=2026-07-22".to_string()] - ); + assert_partitions(&context, &["dt=2026-07-21", "dt=2026-07-22"]).await; std::fs::remove_dir_all(temp_dir.path().join("dt=2026-07-22")).unwrap(); context .sql("MSCK REPAIR TABLE paimon.default.events SYNC PARTITIONS") .await .unwrap(); - assert_eq!( - show_partitions(&context).await, - vec!["dt=2026-07-21".to_string()] - ); + assert_partitions(&context, &["dt=2026-07-21"]).await; context .sql("ALTER TABLE paimon.default.events ADD PARTITION (dt = '2026-07-22')") @@ -133,119 +141,65 @@ async fn test_partition_commands_update_rest_metadata_and_directories() { ) .await .unwrap(); - assert!(show_partitions(&context).await.is_empty()); - assert!(!temp_dir.path().join("dt=2026-07-21").exists()); - assert!(!temp_dir.path().join("dt=2026-07-22").exists()); -} - -#[cfg(not(windows))] -#[tokio::test] -async fn test_partition_commands_normalize_typed_and_null_values() { - let temp_dir = tempfile::tempdir().unwrap(); - let schema = format_table_schema(&[ - ("dt", DataType::Date(DateType::new())), - ("month", DataType::Int(IntType::new())), - ("active", DataType::Boolean(BooleanType::new())), - ("label", DataType::VarChar(VarCharType::new(255).unwrap())), - ]); - let (_server, context) = setup_rest_table(&temp_dir, schema).await; - - context - .sql( - "ALTER TABLE paimon.default.events ADD PARTITION (\ - dt = '2026', month = '01', active = 'yes', label = 01)", - ) - .await - .unwrap(); - assert_eq!( - show_partitions(&context).await, - vec!["dt=2026-01-01/month=1/active=true/label=1".to_string()] - ); - assert!(temp_dir - .path() - .join("dt=20454/month=1/active=true/label=1") - .is_dir()); - - context - .sql("ALTER TABLE paimon.default.events DROP PARTITION (active = 'yes')") - .await - .unwrap(); - assert!(show_partitions(&context).await.is_empty()); - - context - .sql( - "ALTER TABLE paimon.default.events ADD PARTITION (\ - dt = NULL, month = NULL, active = NULL, label = NULL)", - ) - .await - .unwrap(); - assert_eq!( - show_partitions(&context).await, - vec!["dt=null/month=null/active=null/label=null".to_string()] + assert_partitions(&context, &[]).await; + assert_partition_directories( + &temp_dir, + &[("dt=2026-07-21", false), ("dt=2026-07-22", false)], ); - assert!(temp_dir - .path() - .join( - "dt=__DEFAULT_PARTITION__/month=__DEFAULT_PARTITION__/\ - active=__DEFAULT_PARTITION__/label=__DEFAULT_PARTITION__", - ) - .is_dir()); } #[cfg(not(windows))] #[tokio::test] -async fn test_drop_partition_preserves_exact_registered_values() { - let temp_dir = tempfile::tempdir().unwrap(); - for month in ["1", "01", "02"] { - std::fs::create_dir_all(temp_dir.path().join(format!("month={month}"))).unwrap(); - } - let schema = format_table_schema(&[("month", DataType::Int(IntType::new()))]); - let (server, context) = setup_rest_table(&temp_dir, schema).await; - server.set_table_partitions( - DATABASE, - TABLE, - vec![ - HashMap::from([("month".to_string(), "1".to_string())]), - HashMap::from([("month".to_string(), "01".to_string())]), - HashMap::from([("month".to_string(), "02".to_string())]), - ], - ); +async fn test_partition_values_and_drop_matching() { + { + let temp_dir = tempfile::tempdir().unwrap(); + let schema = format_table_schema(&[ + ("dt", DataType::Date(DateType::new())), + ("month", DataType::Int(IntType::new())), + ("active", DataType::Boolean(BooleanType::new())), + ("label", DataType::VarChar(VarCharType::new(255).unwrap())), + ]); + let (_server, context) = setup_rest_table(&temp_dir, schema).await; - context - .sql( - "ALTER TABLE paimon.default.events \ - DROP PARTITION (month = 1), \ - PARTITION (month = 2)", - ) - .await - .unwrap(); - - assert_eq!(show_partitions(&context).await, vec!["month=1".to_string()]); - assert!(!temp_dir.path().join("month=1").exists()); - assert!(temp_dir.path().join("month=01").is_dir()); - assert!(!temp_dir.path().join("month=02").exists()); + context + .sql( + "ALTER TABLE paimon.default.events ADD PARTITION (\ + dt = '2026', month = '01', active = 'yes', label = 01)", + ) + .await + .unwrap(); + assert_partitions(&context, &["dt=2026-01-01/month=1/active=true/label=1"]).await; + assert_partition_directories(&temp_dir, &[("dt=20454/month=1/active=true/label=1", true)]); - let (_, _, request) = server.last_drop_partitions_call().unwrap(); - assert_eq!(request.partition_specs.len(), 2); - assert!(request - .partition_specs - .contains(&HashMap::from([("month".to_string(), "1".to_string())]))); - assert!(request - .partition_specs - .contains(&HashMap::from([("month".to_string(), "02".to_string())]))); -} + context + .sql("ALTER TABLE paimon.default.events DROP PARTITION (active = 'yes')") + .await + .unwrap(); + context + .sql( + "ALTER TABLE paimon.default.events ADD PARTITION (\ + dt = NULL, month = NULL, active = NULL, label = NULL)", + ) + .await + .unwrap(); + assert_partitions(&context, &["dt=null/month=null/active=null/label=null"]).await; + assert_partition_directories( + &temp_dir, + &[( + "dt=__DEFAULT_PARTITION__/month=__DEFAULT_PARTITION__/\ + active=__DEFAULT_PARTITION__/label=__DEFAULT_PARTITION__", + true, + )], + ); + } -#[cfg(not(windows))] -#[tokio::test] -async fn test_drop_partition_preserves_complete_specs_when_expanding_partial_specs() { let temp_dir = tempfile::tempdir().unwrap(); - for (region, month) in [("us", "1"), ("us", "01"), ("eu", "02")] { - std::fs::create_dir_all( - temp_dir - .path() - .join(format!("region={region}/month={month}")), - ) - .unwrap(); + for path in [ + "region=us/month=1", + "region=us/month=01", + "region=eu/month=02", + ] { + std::fs::create_dir_all(temp_dir.path().join(path)).unwrap(); } let schema = format_table_schema(&[ ("region", DataType::VarChar(VarCharType::new(32).unwrap())), @@ -256,18 +210,9 @@ async fn test_drop_partition_preserves_complete_specs_when_expanding_partial_spe DATABASE, TABLE, vec![ - HashMap::from([ - ("region".to_string(), "us".to_string()), - ("month".to_string(), "1".to_string()), - ]), - HashMap::from([ - ("region".to_string(), "us".to_string()), - ("month".to_string(), "01".to_string()), - ]), - HashMap::from([ - ("region".to_string(), "eu".to_string()), - ("month".to_string(), "02".to_string()), - ]), + partition_spec(&[("region", "us"), ("month", "1")]), + partition_spec(&[("region", "us"), ("month", "01")]), + partition_spec(&[("region", "eu"), ("month", "02")]), ], ); @@ -280,24 +225,25 @@ async fn test_drop_partition_preserves_complete_specs_when_expanding_partial_spe .await .unwrap(); - assert_eq!( - show_partitions(&context).await, - vec!["region=us/month=1".to_string()] + assert_partitions(&context, &["region=us/month=1"]).await; + assert_partition_directories( + &temp_dir, + &[ + ("region=us/month=1", false), + ("region=us/month=01", true), + ("region=eu/month=02", false), + ], ); - assert!(!temp_dir.path().join("region=us/month=1").exists()); - assert!(temp_dir.path().join("region=us/month=01").is_dir()); - assert!(!temp_dir.path().join("region=eu/month=02").exists()); - let (_, _, request) = server.last_drop_partitions_call().unwrap(); - assert_eq!(request.partition_specs.len(), 2); - assert!(request.partition_specs.contains(&HashMap::from([ - ("region".to_string(), "us".to_string()), - ("month".to_string(), "1".to_string()), - ]))); - assert!(request.partition_specs.contains(&HashMap::from([ - ("region".to_string(), "eu".to_string()), - ("month".to_string(), "02".to_string()), - ]))); + let (_, _, request) = server.drop_partitions_calls().pop().unwrap(); + let expected = [ + partition_spec(&[("region", "us"), ("month", "1")]), + partition_spec(&[("region", "eu"), ("month", "02")]), + ]; + assert_eq!(request.partition_specs.len(), expected.len()); + assert!(expected + .iter() + .all(|spec| request.partition_specs.contains(spec))); } async fn show_partitions(context: &SQLContext) -> Vec { @@ -319,3 +265,13 @@ async fn show_partitions(context: &SQLContext) -> Vec { } partitions } + +async fn assert_partitions(context: &SQLContext, expected: &[&str]) { + assert_eq!( + show_partitions(context).await, + expected + .iter() + .map(|partition| (*partition).to_string()) + .collect::>() + ); +} diff --git a/crates/paimon/src/api/api_request.rs b/crates/paimon/src/api/api_request.rs index 16d04c31..891a281d 100644 --- a/crates/paimon/src/api/api_request.rs +++ b/crates/paimon/src/api/api_request.rs @@ -319,27 +319,17 @@ mod tests { "ignoreIfExists": false }) ); - } - - #[test] - fn test_create_partitions_request_defaults_missing_ignore_if_exists() { - let request: CreatePartitionsRequest = serde_json::from_value(serde_json::json!({ - "partitionSpecs": [{"dt": "2026-07-22"}] - })) - .unwrap(); - assert!(request.ignore_if_exists); - } - - #[test] - fn test_create_partitions_request_defaults_null_ignore_if_exists() { - let request: CreatePartitionsRequest = serde_json::from_value(serde_json::json!({ - "partitionSpecs": [{"dt": "2026-07-22"}], - "ignoreIfExists": null - })) - .unwrap(); - - assert!(request.ignore_if_exists); + for value in [ + serde_json::json!({"partitionSpecs": []}), + serde_json::json!({"partitionSpecs": [], "ignoreIfExists": null}), + ] { + assert!( + serde_json::from_value::(value) + .unwrap() + .ignore_if_exists + ); + } } #[test] @@ -359,27 +349,17 @@ mod tests { "ignoreIfNotExists": false }) ); - } - - #[test] - fn test_drop_partitions_request_defaults_missing_ignore_if_not_exists() { - let request: DropPartitionsRequest = serde_json::from_value(serde_json::json!({ - "partitionSpecs": [{"dt": "2026-07-22"}] - })) - .unwrap(); - assert!(request.ignore_if_not_exists); - } - - #[test] - fn test_drop_partitions_request_defaults_null_ignore_if_not_exists() { - let request: DropPartitionsRequest = serde_json::from_value(serde_json::json!({ - "partitionSpecs": [{"dt": "2026-07-22"}], - "ignoreIfNotExists": null - })) - .unwrap(); - - assert!(request.ignore_if_not_exists); + for value in [ + serde_json::json!({"partitionSpecs": []}), + serde_json::json!({"partitionSpecs": [], "ignoreIfNotExists": null}), + ] { + assert!( + serde_json::from_value::(value) + .unwrap() + .ignore_if_not_exists + ); + } } #[test] diff --git a/crates/paimon/src/api/api_response.rs b/crates/paimon/src/api/api_response.rs index 16d8951c..01b793a8 100644 --- a/crates/paimon/src/api/api_response.rs +++ b/crates/paimon/src/api/api_response.rs @@ -683,37 +683,21 @@ mod tests { #[test] fn test_create_partitions_response_deserialization() { + let created = HashMap::from([("dt".to_string(), "2026-07-22".to_string())]); + let existed = HashMap::from([("dt".to_string(), "2026-07-21".to_string())]); let response: CreatePartitionsResponse = serde_json::from_value(serde_json::json!({ - "created": [{"dt": "2026-07-22"}], - "existed": [{"dt": "2026-07-21"}] + "created": [created.clone()], + "existed": [existed.clone()] })) .unwrap(); assert_eq!( - response.created, - vec![HashMap::from([( - "dt".to_string(), - "2026-07-22".to_string() - )])] - ); - assert_eq!( - response.existed, - vec![HashMap::from([( - "dt".to_string(), - "2026-07-21".to_string() - )])] + response, + CreatePartitionsResponse { + created: vec![created], + existed: vec![existed], + } ); - } - - #[test] - fn test_create_partitions_response_requires_non_null_lists() { - let response: CreatePartitionsResponse = serde_json::from_value(serde_json::json!({ - "created": [], - "existed": [] - })) - .unwrap(); - assert!(response.created.is_empty()); - assert!(response.existed.is_empty()); for invalid in [ serde_json::json!({"created": []}), @@ -727,37 +711,21 @@ mod tests { #[test] fn test_drop_partitions_response_deserialization() { + let dropped = HashMap::from([("dt".to_string(), "2026-07-22".to_string())]); + let missing = HashMap::from([("dt".to_string(), "2026-07-21".to_string())]); let response: DropPartitionsResponse = serde_json::from_value(serde_json::json!({ - "dropped": [{"dt": "2026-07-22"}], - "missing": [{"dt": "2026-07-21"}] + "dropped": [dropped.clone()], + "missing": [missing.clone()] })) .unwrap(); assert_eq!( - response.dropped, - vec![HashMap::from([( - "dt".to_string(), - "2026-07-22".to_string() - )])] - ); - assert_eq!( - response.missing, - vec![HashMap::from([( - "dt".to_string(), - "2026-07-21".to_string() - )])] + response, + DropPartitionsResponse { + dropped: vec![dropped], + missing: vec![missing], + } ); - } - - #[test] - fn test_drop_partitions_response_requires_non_null_lists() { - let response: DropPartitionsResponse = serde_json::from_value(serde_json::json!({ - "dropped": [], - "missing": [] - })) - .unwrap(); - assert!(response.dropped.is_empty()); - assert!(response.missing.is_empty()); for invalid in [ serde_json::json!({"dropped": []}), diff --git a/crates/paimon/src/catalog/rest/rest_catalog.rs b/crates/paimon/src/catalog/rest/rest_catalog.rs index c1d5a567..c17dde73 100644 --- a/crates/paimon/src/catalog/rest/rest_catalog.rs +++ b/crates/paimon/src/catalog/rest/rest_catalog.rs @@ -408,31 +408,12 @@ impl Catalog for RESTCatalog { ) -> Result> { let mut partitions = Vec::new(); for batch in partition_specs.chunks(RESTApi::PARTITION_REQUEST_SIZE) { - match self - .api - .list_partitions_by_names(identifier, batch.to_vec()) - .await - { - Ok(batch_partitions) => partitions.extend(batch_partitions), - Err( - error @ Error::RestApi { - source: RestError::NotImplemented { .. }, - }, - ) => { - let table = self.get_table(identifier).await?; - if table.has_catalog_managed_partitions() { - return Err(error); - } - return Ok(list_partitions_from_file_system(&table) - .await? - .into_iter() - .filter(|partition| partition_specs.contains(&partition.spec)) - .collect()); - } - Err(error) => { - return Err(map_rest_error_for_partition_request(error, identifier)); - } - } + partitions.extend( + self.api + .list_partitions_by_names(identifier, batch.to_vec()) + .await + .map_err(|error| map_rest_error_for_partition_request(error, identifier))?, + ); } Ok(partitions) } diff --git a/crates/paimon/src/table/format_partition.rs b/crates/paimon/src/table/format_partition.rs index ba0f888d..dc67b7ac 100644 --- a/crates/paimon/src/table/format_partition.rs +++ b/crates/paimon/src/table/format_partition.rs @@ -96,7 +96,7 @@ impl FormatTablePartitionPaths { for (path, spec) in frontier { let statuses = match file_io.list_status(&path).await { Ok(statuses) => statuses, - Err(error) if is_not_found(&error) => continue, + Err(error) if is_storage_not_found(&error) => continue, Err(error) => return Err(error), }; for status in statuses { @@ -298,7 +298,7 @@ pub(crate) fn format_partition_date(epoch_days: i32) -> Option { .map(|date| date.format("%Y-%m-%d").to_string()) } -fn is_not_found(error: &crate::Error) -> bool { +pub(crate) fn is_storage_not_found(error: &crate::Error) -> bool { matches!( error, crate::Error::IoUnexpected { source, .. } @@ -423,4 +423,18 @@ mod tests { ); assert_eq!(parse_format_partition_value("中文ab", &varchar_type), None); } + + #[test] + fn test_storage_not_found_matches_only_not_found() { + for (kind, expected) in [ + (opendal::ErrorKind::NotFound, true), + (opendal::ErrorKind::PermissionDenied, false), + ] { + let error = crate::Error::IoUnexpected { + message: "list partition directory".to_string(), + source: Box::new(opendal::Error::new(kind, "test")), + }; + assert_eq!(is_storage_not_found(&error), expected); + } + } } diff --git a/crates/paimon/src/table/format_table_scan.rs b/crates/paimon/src/table/format_table_scan.rs index 56f195e8..a8826f20 100644 --- a/crates/paimon/src/table/format_table_scan.rs +++ b/crates/paimon/src/table/format_table_scan.rs @@ -20,7 +20,8 @@ use std::collections::{HashMap, HashSet}; use super::format_partition::{ - format_partition_date, parse_format_partition_value, FormatTablePartitionPaths, + format_partition_date, is_storage_not_found, parse_format_partition_value, + FormatTablePartitionPaths, }; use super::{CatalogManagedPartitionOptions, Plan, ScanTrace, Table}; use crate::spec::stats::BinaryTableStats; @@ -262,7 +263,7 @@ impl<'a> FormatTableScan<'a> { ) -> crate::Result> { match self.table.file_io().list_status_recursive(path).await { Ok(statuses) => Ok(statuses), - Err(err) if is_missing_partition_directory_error(&err) => Ok(Vec::new()), + Err(err) if is_storage_not_found(&err) => Ok(Vec::new()), Err(err) => Err(err), } } @@ -383,14 +384,6 @@ fn join_path(parent: &str, child: &str) -> String { } } -fn is_missing_partition_directory_error(error: &crate::Error) -> bool { - matches!( - error, - crate::Error::IoUnexpected { source, .. } - if source.kind() == opendal::ErrorKind::NotFound - ) -} - fn leading_equality_partition_path( table_path: &str, partition_keys: &[String], @@ -652,26 +645,3 @@ fn data_file_meta(file_name: String, file_size: i64, schema_id: i64) -> DataFile write_cols: None, } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_missing_partition_directory_requires_not_found_error() { - let not_found = crate::Error::IoUnexpected { - message: "missing".to_string(), - source: Box::new(opendal::Error::new(opendal::ErrorKind::NotFound, "missing")), - }; - let permission_denied = crate::Error::IoUnexpected { - message: "denied".to_string(), - source: Box::new(opendal::Error::new( - opendal::ErrorKind::PermissionDenied, - "denied", - )), - }; - - assert!(is_missing_partition_directory_error(¬_found)); - assert!(!is_missing_partition_directory_error(&permission_denied)); - } -} diff --git a/crates/paimon/tests/format_partition_test.rs b/crates/paimon/tests/format_partition_test.rs index 923dcc3a..a76ed6c5 100644 --- a/crates/paimon/tests/format_partition_test.rs +++ b/crates/paimon/tests/format_partition_test.rs @@ -16,10 +16,26 @@ // under the License. use std::collections::HashMap; +#[cfg(not(windows))] +use std::path::Path; use paimon::io::FileIO; use paimon::table::FormatTablePartitionPaths; +#[cfg(not(windows))] +async fn discover_partitions( + root: &Path, + partition_keys: &[&str], + only_value_in_path: bool, + default_partition_name: &str, +) -> paimon::Result>> { + let table_path = format!("file://{}", root.display()); + let file_io = FileIO::from_path(&table_path)?.build()?; + FormatTablePartitionPaths::new(partition_keys.iter().copied(), only_value_in_path) + .discover(&file_io, &table_path, default_partition_name) + .await +} + #[test] fn format_partition_paths_escape_values_and_honor_layout() { let spec = HashMap::from([ @@ -69,14 +85,11 @@ async fn discover_format_partitions_returns_complete_sorted_specs() { ] { std::fs::create_dir_all(tmp.path().join(relative)).unwrap(); } - let table_path = format!("file://{}", tmp.path().display()); - let file_io = FileIO::from_path(&table_path).unwrap().build().unwrap(); - let paths = FormatTablePartitionPaths::new(["dt", "hour"], false); - let partitions = paths - .discover(&file_io, &table_path, "__DEFAULT_PARTITION__") - .await - .unwrap(); + let partitions = + discover_partitions(tmp.path(), &["dt", "hour"], false, "__DEFAULT_PARTITION__") + .await + .unwrap(); assert_eq!( partitions, @@ -97,14 +110,15 @@ async fn discover_format_partitions_returns_complete_sorted_specs() { #[tokio::test] async fn discover_format_partitions_treats_missing_root_as_empty() { let tmp = tempfile::tempdir().unwrap(); - let table_path = format!("file://{}", tmp.path().join("missing").display()); - let file_io = FileIO::from_path(&table_path).unwrap().build().unwrap(); - let paths = FormatTablePartitionPaths::new(["dt"], false); - let partitions = paths - .discover(&file_io, &table_path, "__DEFAULT_PARTITION__") - .await - .unwrap(); + let partitions = discover_partitions( + &tmp.path().join("missing"), + &["dt"], + false, + "__DEFAULT_PARTITION__", + ) + .await + .unwrap(); assert!(partitions.is_empty()); } @@ -114,12 +128,8 @@ async fn discover_format_partitions_treats_missing_root_as_empty() { async fn discover_format_partitions_rejects_non_round_tripping_percent_escape() { let tmp = tempfile::tempdir().unwrap(); std::fs::create_dir_all(tmp.path().join("dt=value%ZZ")).unwrap(); - let table_path = format!("file://{}", tmp.path().display()); - let file_io = FileIO::from_path(&table_path).unwrap().build().unwrap(); - let paths = FormatTablePartitionPaths::new(["dt"], false); - let error = paths - .discover(&file_io, &table_path, "__DEFAULT_PARTITION__") + let error = discover_partitions(tmp.path(), &["dt"], false, "__DEFAULT_PARTITION__") .await .unwrap_err(); @@ -135,12 +145,8 @@ async fn discover_format_partitions_ignores_nonmatching_malformed_key_segment() let tmp = tempfile::tempdir().unwrap(); std::fs::create_dir_all(tmp.path().join("wrong%ZZ=value")).unwrap(); std::fs::create_dir_all(tmp.path().join("dt=valid")).unwrap(); - let table_path = format!("file://{}", tmp.path().display()); - let file_io = FileIO::from_path(&table_path).unwrap().build().unwrap(); - let paths = FormatTablePartitionPaths::new(["dt"], false); - let partitions = paths - .discover(&file_io, &table_path, "__DEFAULT_PARTITION__") + let partitions = discover_partitions(tmp.path(), &["dt"], false, "__DEFAULT_PARTITION__") .await .unwrap(); @@ -155,12 +161,8 @@ async fn discover_format_partitions_ignores_nonmatching_malformed_key_segment() async fn discover_value_only_partitions_rejects_parent_traversal_value() { let tmp = tempfile::tempdir().unwrap(); std::fs::create_dir_all(tmp.path().join("%2E%2E")).unwrap(); - let table_path = format!("file://{}", tmp.path().display()); - let file_io = FileIO::from_path(&table_path).unwrap().build().unwrap(); - let paths = FormatTablePartitionPaths::new(["dt"], true); - let error = paths - .discover(&file_io, &table_path, "__DEFAULT_PARTITION__") + let error = discover_partitions(tmp.path(), &["dt"], true, "__DEFAULT_PARTITION__") .await .unwrap_err(); @@ -172,65 +174,29 @@ async fn discover_value_only_partitions_rejects_parent_traversal_value() { #[cfg(not(windows))] #[tokio::test] -async fn discover_value_only_partitions_keeps_default_partition_directory() { - let tmp = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(tmp.path().join("__DEFAULT_PARTITION__")).unwrap(); - let table_path = format!("file://{}", tmp.path().display()); - let file_io = FileIO::from_path(&table_path).unwrap().build().unwrap(); - let paths = FormatTablePartitionPaths::new(["dt"], true); - - let partitions = paths - .discover(&file_io, &table_path, "__DEFAULT_PARTITION__") - .await - .unwrap(); - - assert_eq!( - partitions, - vec![HashMap::from([( - "dt".to_string(), - "__DEFAULT_PARTITION__".to_string(), - )])] - ); -} - -#[cfg(not(windows))] -#[tokio::test] -async fn discover_value_only_partitions_keeps_dot_prefixed_default_partition_directory() { - let tmp = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(tmp.path().join(".NULL")).unwrap(); - std::fs::create_dir_all(tmp.path().join(".staging")).unwrap(); - let table_path = format!("file://{}", tmp.path().display()); - let file_io = FileIO::from_path(&table_path).unwrap().build().unwrap(); - let paths = FormatTablePartitionPaths::new(["dt"], true); - - let partitions = paths - .discover(&file_io, &table_path, ".NULL") - .await - .unwrap(); - - assert_eq!( - partitions, - vec![HashMap::from([("dt".to_string(), ".NULL".to_string())])] - ); -} - -#[cfg(not(windows))] -#[tokio::test] -async fn discover_value_only_partitions_keeps_escaped_default_partition_directory() { - let tmp = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(tmp.path().join("_NULL%2F%25NA")).unwrap(); - std::fs::create_dir_all(tmp.path().join("_temporary")).unwrap(); - let table_path = format!("file://{}", tmp.path().display()); - let file_io = FileIO::from_path(&table_path).unwrap().build().unwrap(); - let paths = FormatTablePartitionPaths::new(["dt"], true); - - let partitions = paths - .discover(&file_io, &table_path, "_NULL/%NA") - .await - .unwrap(); - - assert_eq!( - partitions, - vec![HashMap::from([("dt".to_string(), "_NULL/%NA".to_string())])] - ); +async fn discover_value_only_partitions_keeps_hidden_default_directory() { + for (default_name, directory, ignored_directory) in [ + ("__DEFAULT_PARTITION__", "__DEFAULT_PARTITION__", None), + (".NULL", ".NULL", Some(".staging")), + ("_NULL/%NA", "_NULL%2F%25NA", Some("_temporary")), + ] { + let tmp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(tmp.path().join(directory)).unwrap(); + if let Some(ignored_directory) = ignored_directory { + std::fs::create_dir_all(tmp.path().join(ignored_directory)).unwrap(); + } + + let partitions = discover_partitions(tmp.path(), &["dt"], true, default_name) + .await + .unwrap(); + + assert_eq!( + partitions, + vec![HashMap::from([( + "dt".to_string(), + default_name.to_string() + )])], + "{default_name}" + ); + } } diff --git a/crates/paimon/tests/mock_server.rs b/crates/paimon/tests/mock_server.rs index bf0f1bdc..c8bf388a 100644 --- a/crates/paimon/tests/mock_server.rs +++ b/crates/paimon/tests/mock_server.rs @@ -65,7 +65,6 @@ struct MockState { drop_partitions_calls: Vec<(String, String, DropPartitionsRequest)>, list_partitions_by_names_calls: Vec<(String, String, ListPartitionsByNamesRequest)>, create_partitions_error_status: Option, - drop_partitions_error_status: Option, list_partitions_error_status: Option, list_partitions_by_names_error_status: Option, /// ECS metadata role name (for token loader testing) @@ -964,15 +963,6 @@ impl RESTServer { inner .drop_partitions_calls .push((db.clone(), table.clone(), request.clone())); - if let Some(status) = inner.drop_partitions_error_status { - let error = ErrorResponse::new( - Some("partition".to_string()), - Some(table.clone()), - Some("Invalid partition request".to_string()), - Some(status.as_u16() as i32), - ); - return (status, Json(error)).into_response(); - } let key = format!("{db}.{table}"); if !inner.tables.contains_key(&key) { @@ -1237,11 +1227,6 @@ impl RESTServer { self.inner.lock().unwrap().create_partitions_error_status = status; } - /// Make the drop-partitions endpoint return the given status. - pub fn set_drop_partitions_error_status(&self, status: Option) { - self.inner.lock().unwrap().drop_partitions_error_status = status; - } - /// Make the list-partitions endpoint return the given status. pub fn set_list_partitions_error_status(&self, status: Option) { self.inner.lock().unwrap().list_partitions_error_status = status; @@ -1331,35 +1316,6 @@ impl RESTServer { inner.partition_list_calls.remove(&key); } - /// Set explicit pages returned by the list-partitions endpoint. - pub fn set_table_partition_pages( - &self, - database: &str, - table: &str, - pages: Vec>>, - ) { - let page_count = pages.len(); - let responses = pages - .into_iter() - .enumerate() - .map(|(index, specs)| { - let partitions = specs.into_iter().map(partition_from_spec).collect(); - let next_page_token = (index + 1 < page_count).then(|| (index + 1).to_string()); - (partitions, next_page_token) - }) - .collect(); - let key = format!("{database}.{table}"); - let mut inner = self.inner.lock().unwrap(); - assert!( - inner.tables.contains_key(&key), - "table {key} does not exist" - ); - inner - .partition_page_responses - .insert(key.clone(), responses); - inner.partition_list_calls.remove(&key); - } - /// Set explicit list-partitions pages and response tokens in request order. pub fn set_table_partition_page_responses( &self, @@ -1394,81 +1350,16 @@ impl RESTServer { .unwrap_or_default() } - /// Return the last create-partitions call received by the server. - pub fn last_create_partitions_call(&self) -> Option<(String, String, CreatePartitionsRequest)> { - self.inner - .lock() - .unwrap() - .create_partitions_calls - .last() - .cloned() - } - - /// Return the partition counts of all create-partitions calls. - pub fn create_partitions_call_sizes(&self) -> Vec { - self.inner - .lock() - .unwrap() - .create_partitions_calls - .iter() - .map(|(_, _, request)| request.partition_specs.len()) - .collect() - } - /// Return all create-partitions calls received by the server. pub fn create_partitions_calls(&self) -> Vec<(String, String, CreatePartitionsRequest)> { self.inner.lock().unwrap().create_partitions_calls.clone() } - /// Return the last drop-partitions call received by the server. - pub fn last_drop_partitions_call(&self) -> Option<(String, String, DropPartitionsRequest)> { - self.inner - .lock() - .unwrap() - .drop_partitions_calls - .last() - .cloned() - } - - /// Return the partition counts of all drop-partitions calls. - pub fn drop_partitions_call_sizes(&self) -> Vec { - self.inner - .lock() - .unwrap() - .drop_partitions_calls - .iter() - .map(|(_, _, request)| request.partition_specs.len()) - .collect() - } - /// Return all drop-partitions calls received by the server. pub fn drop_partitions_calls(&self) -> Vec<(String, String, DropPartitionsRequest)> { self.inner.lock().unwrap().drop_partitions_calls.clone() } - /// Return the last list-partitions-by-names call received by the server. - pub fn last_list_partitions_by_names_call( - &self, - ) -> Option<(String, String, ListPartitionsByNamesRequest)> { - self.inner - .lock() - .unwrap() - .list_partitions_by_names_calls - .last() - .cloned() - } - - /// Return the partition counts of all list-partitions-by-names calls. - pub fn list_partitions_by_names_call_sizes(&self) -> Vec { - self.inner - .lock() - .unwrap() - .list_partitions_by_names_calls - .iter() - .map(|(_, _, request)| request.partition_specs.len()) - .collect() - } - /// Return all list-partitions-by-names calls received by the server. pub fn list_partitions_by_names_calls( &self, diff --git a/crates/paimon/tests/rest_api_test.rs b/crates/paimon/tests/rest_api_test.rs index 94af57bb..8ab6d5da 100644 --- a/crates/paimon/tests/rest_api_test.rs +++ b/crates/paimon/tests/rest_api_test.rs @@ -73,6 +73,13 @@ async fn setup_test_server(initial_dbs: Vec<&str>) -> TestContext { TestContext { server, api, url } } +async fn setup_partition_api() -> (TestContext, Identifier) { + let ctx = setup_test_server(vec!["default"]).await; + let identifier = Identifier::new("default", "managed_table"); + ctx.server.add_table("default", "managed_table"); + (ctx, identifier) +} + // ==================== Database Tests ==================== #[tokio::test] async fn test_list_databases() { @@ -600,9 +607,7 @@ async fn test_drop_table_no_permission() { #[tokio::test] async fn test_list_partitions_by_names_rejects_more_than_1000_specs() { - let ctx = setup_test_server(vec!["default"]).await; - let identifier = Identifier::new("default", "managed_table"); - ctx.server.add_table("default", "managed_table"); + let (ctx, identifier) = setup_partition_api().await; let partition_specs = (0..=1000) .map(|day| HashMap::from([("day".to_string(), day.to_string())])) .collect::>(); @@ -621,14 +626,12 @@ async fn test_list_partitions_by_names_rejects_more_than_1000_specs() { "List partitions by names accepts at most 1000 partition specs for table \ default.managed_table, got 1001" ); - assert!(ctx.server.last_list_partitions_by_names_call().is_none()); + assert!(ctx.server.list_partitions_by_names_calls().is_empty()); } #[tokio::test] async fn test_create_partitions_posts_expected_request() { - let ctx = setup_test_server(vec!["default"]).await; - let identifier = Identifier::new("default", "managed_table"); - ctx.server.add_table("default", "managed_table"); + let (ctx, identifier) = setup_partition_api().await; let partition_specs = vec![HashMap::from([ ("dt".to_string(), "2026-07-22".to_string()), ("hour".to_string(), "10".to_string()), @@ -643,7 +646,7 @@ async fn test_create_partitions_posts_expected_request() { assert_eq!(response.created, partition_specs.clone()); assert!(response.existed.is_empty()); assert_eq!( - ctx.server.last_create_partitions_call(), + ctx.server.create_partitions_calls().pop(), Some(( "default".to_string(), "managed_table".to_string(), @@ -652,30 +655,9 @@ async fn test_create_partitions_posts_expected_request() { ); } -#[tokio::test] -async fn test_create_partitions_allows_more_than_1000_specs() { - let ctx = setup_test_server(vec!["default"]).await; - let identifier = Identifier::new("default", "managed_table"); - ctx.server.add_table("default", "managed_table"); - let partition_specs = (0..=1000) - .map(|day| HashMap::from([("day".to_string(), day.to_string())])) - .collect::>(); - - let response = ctx - .api - .create_partitions(&identifier, partition_specs.clone(), false) - .await - .unwrap(); - - assert_eq!(response.created, partition_specs); - assert_eq!(ctx.server.create_partitions_call_sizes(), vec![1001]); -} - #[tokio::test] async fn test_drop_partitions_posts_expected_request() { - let ctx = setup_test_server(vec!["default"]).await; - let identifier = Identifier::new("default", "managed_table"); - ctx.server.add_table("default", "managed_table"); + let (ctx, identifier) = setup_partition_api().await; let partition_specs = vec![HashMap::from([( "dt".to_string(), "2026-07-22".to_string(), @@ -692,7 +674,7 @@ async fn test_drop_partitions_posts_expected_request() { assert_eq!(response.dropped, partition_specs.clone()); assert!(response.missing.is_empty()); assert_eq!( - ctx.server.last_drop_partitions_call(), + ctx.server.drop_partitions_calls().pop(), Some(( "default".to_string(), "managed_table".to_string(), @@ -701,30 +683,9 @@ async fn test_drop_partitions_posts_expected_request() { ); } -#[tokio::test] -async fn test_drop_partitions_allows_more_than_1000_specs() { - let ctx = setup_test_server(vec!["default"]).await; - let identifier = Identifier::new("default", "managed_table"); - ctx.server.add_table("default", "managed_table"); - let partition_specs = (0..=1000) - .map(|day| HashMap::from([("day".to_string(), day.to_string())])) - .collect::>(); - - let response = ctx - .api - .drop_partitions(&identifier, partition_specs.clone(), true) - .await - .unwrap(); - - assert_eq!(response.missing, partition_specs); - assert_eq!(ctx.server.drop_partitions_call_sizes(), vec![1001]); -} - #[tokio::test] async fn test_list_partitions_by_names_posts_expected_request() { - let ctx = setup_test_server(vec!["default"]).await; - let identifier = Identifier::new("default", "managed_table"); - ctx.server.add_table("default", "managed_table"); + let (ctx, identifier) = setup_partition_api().await; let partition_specs = vec![ HashMap::from([("dt".to_string(), "2026-07-22".to_string())]), HashMap::from([("dt".to_string(), "2026-07-21".to_string())]), @@ -746,7 +707,7 @@ async fn test_list_partitions_by_names_posts_expected_request() { partition_specs.clone() ); assert_eq!( - ctx.server.last_list_partitions_by_names_call(), + ctx.server.list_partitions_by_names_calls().pop(), Some(( "default".to_string(), "managed_table".to_string(), @@ -757,14 +718,15 @@ async fn test_list_partitions_by_names_posts_expected_request() { #[tokio::test] async fn test_list_partitions_follows_non_empty_token_after_empty_page() { - let ctx = setup_test_server(vec!["default"]).await; - let identifier = Identifier::new("default", "managed_table"); - ctx.server.add_table("default", "managed_table"); + let (ctx, identifier) = setup_partition_api().await; let expected = HashMap::from([("dt".to_string(), "2026-07-22".to_string())]); - ctx.server.set_table_partition_pages( + ctx.server.set_table_partition_page_responses( "default", "managed_table", - vec![Vec::new(), vec![expected.clone()]], + vec![ + (Vec::new(), Some("1".to_string())), + (vec![expected.clone()], None), + ], ); let partitions = ctx.api.list_partitions(&identifier).await.unwrap(); @@ -775,9 +737,7 @@ async fn test_list_partitions_follows_non_empty_token_after_empty_page() { #[tokio::test] async fn test_list_partitions_stops_on_empty_next_page_token() { - let ctx = setup_test_server(vec!["default"]).await; - let identifier = Identifier::new("default", "managed_table"); - ctx.server.add_table("default", "managed_table"); + let (ctx, identifier) = setup_partition_api().await; let expected = HashMap::from([("dt".to_string(), "2026-07-22".to_string())]); let unexpected = HashMap::from([("dt".to_string(), "2026-07-23".to_string())]); ctx.server.set_table_partition_page_responses( @@ -807,9 +767,7 @@ async fn test_list_partitions_stops_on_empty_next_page_token() { #[tokio::test] async fn test_list_partitions_rejects_repeated_page_token() { - let ctx = setup_test_server(vec!["default"]).await; - let identifier = Identifier::new("default", "managed_table"); - ctx.server.add_table("default", "managed_table"); + let (ctx, identifier) = setup_partition_api().await; ctx.server.set_table_partition_page_responses( "default", "managed_table", diff --git a/crates/paimon/tests/rest_catalog_test.rs b/crates/paimon/tests/rest_catalog_test.rs index 7a5342b4..f42626fc 100644 --- a/crates/paimon/tests/rest_catalog_test.rs +++ b/crates/paimon/tests/rest_catalog_test.rs @@ -27,9 +27,7 @@ use arrow_array::{Array, BinaryArray, Int32Array, Int64Array, RecordBatch, Strin use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema}; use axum::http::StatusCode; use futures::TryStreamExt; -use paimon::api::{ - ConfigResponse, CreatePartitionsRequest, DropPartitionsRequest, ListPartitionsByNamesRequest, -}; +use paimon::api::ConfigResponse; use paimon::catalog::{Catalog, Function, FunctionDefinition, Identifier, RESTCatalog, ViewSchema}; use paimon::common::Options; use paimon::spec::{ @@ -100,6 +98,16 @@ fn catalog_managed_format_table_schema(options: &[(&str, &str)]) -> Schema { builder.build().unwrap() } +fn add_internal_table_with_schema( + server: &RESTServer, + table: &str, + schema: Schema, + location: &str, +) { + server.add_table_with_schema("default", table, schema, location); + server.set_table_external("default", table, false); +} + fn blob_schema(options: &[(&str, &str)]) -> Schema { let mut builder = Schema::builder() .column("id", DataType::Int(IntType::new())) @@ -183,65 +191,6 @@ fn collect_blob_rows(batches: &[RecordBatch]) -> Vec<(i32, String, Option>(), - partition_specs - ); } #[tokio::test] @@ -291,31 +232,6 @@ async fn test_rest_catalog_keeps_non_idempotent_create_in_one_request() { assert!(!calls[0].2.ignore_if_exists); } -#[tokio::test] -async fn test_rest_catalog_created_partitions_are_visible_to_list() { - let ctx = setup_catalog(vec!["default"]).await; - let identifier = Identifier::new("default", "managed_table"); - ctx.server.add_table("default", "managed_table"); - let partition_specs = vec![HashMap::from([( - "dt".to_string(), - "2026-07-22".to_string(), - )])]; - - ctx.catalog - .create_partitions(&identifier, partition_specs.clone(), false) - .await - .unwrap(); - - let listed = ctx.catalog.list_partitions(&identifier).await.unwrap(); - assert_eq!( - listed - .into_iter() - .map(|partition| partition.spec) - .collect::>(), - partition_specs - ); -} - #[tokio::test] async fn test_rest_catalog_maps_create_partition_conflict() { let ctx = setup_catalog(vec!["default"]).await; @@ -371,58 +287,6 @@ async fn test_rest_catalog_maps_invalid_partition_request() { )); } -#[tokio::test] -async fn test_rest_catalog_unregister_partitions_delegates_to_rest_metadata_only() { - let ctx = setup_catalog(vec!["default"]).await; - let identifier = Identifier::new("default", "managed_table"); - ctx.server.add_table("default", "managed_table"); - let partition_specs = vec![HashMap::from([( - "dt".to_string(), - "2026-07-22".to_string(), - )])]; - - ctx.catalog - .unregister_partitions(&identifier, partition_specs.clone()) - .await - .unwrap(); - - assert_eq!( - ctx.server.last_drop_partitions_call(), - Some(( - "default".to_string(), - "managed_table".to_string(), - DropPartitionsRequest::new(partition_specs, true), - )) - ); -} - -#[tokio::test] -async fn test_rest_catalog_maps_invalid_unregister_partition_request() { - let ctx = setup_catalog(vec!["default"]).await; - let identifier = Identifier::new("default", "managed_table"); - ctx.server.add_table("default", "managed_table"); - ctx.server - .set_drop_partitions_error_status(Some(StatusCode::BAD_REQUEST)); - - let error = ctx - .catalog - .unregister_partitions( - &identifier, - vec![HashMap::from([( - "unknown".to_string(), - "value".to_string(), - )])], - ) - .await - .unwrap_err(); - - assert!(matches!( - error, - paimon::Error::DataInvalid { message, .. } - if message.contains("default.managed_table") - )); -} - #[tokio::test] async fn test_rest_catalog_batches_unregister_partitions() { let ctx = setup_catalog(vec!["default"]).await; @@ -446,93 +310,6 @@ async fn test_rest_catalog_batches_unregister_partitions() { assert!(calls .iter() .all(|(_, _, request)| request.ignore_if_not_exists)); - assert!(ctx - .catalog - .list_partitions(&identifier) - .await - .unwrap() - .is_empty()); -} - -#[tokio::test] -async fn test_rest_catalog_unregistered_partitions_are_removed_from_list() { - let ctx = setup_catalog(vec!["default"]).await; - let identifier = Identifier::new("default", "managed_table"); - ctx.server.add_table("default", "managed_table"); - let partition_specs = vec![HashMap::from([( - "dt".to_string(), - "2026-07-22".to_string(), - )])]; - ctx.server - .set_table_partitions("default", "managed_table", partition_specs.clone()); - - ctx.catalog - .unregister_partitions(&identifier, partition_specs) - .await - .unwrap(); - - assert!(ctx - .catalog - .list_partitions(&identifier) - .await - .unwrap() - .is_empty()); -} - -#[tokio::test] -async fn test_rest_catalog_lists_partitions_by_names_through_rest() { - let ctx = setup_catalog(vec!["default"]).await; - let identifier = Identifier::new("default", "managed_table"); - ctx.server.add_table("default", "managed_table"); - let existing = HashMap::from([("dt".to_string(), "2026-07-22".to_string())]); - let missing = HashMap::from([("dt".to_string(), "2026-07-23".to_string())]); - ctx.server - .set_table_partitions("default", "managed_table", vec![existing.clone()]); - let partition_specs = vec![existing.clone(), missing]; - - let partitions = ctx - .catalog - .list_partitions_by_names(&identifier, partition_specs.clone()) - .await - .unwrap(); - - assert_eq!(partitions.len(), 1); - assert_eq!(partitions[0].spec, existing); - assert_eq!( - ctx.server.last_list_partitions_by_names_call(), - Some(( - "default".to_string(), - "managed_table".to_string(), - ListPartitionsByNamesRequest::new(partition_specs), - )) - ); -} - -#[tokio::test] -async fn test_rest_catalog_maps_invalid_list_partitions_by_names_request() { - let ctx = setup_catalog(vec!["default"]).await; - let identifier = Identifier::new("default", "managed_table"); - ctx.server.add_table("default", "managed_table"); - ctx.server - .set_list_partitions_by_names_error_status(Some(StatusCode::BAD_REQUEST)); - - let error = ctx - .catalog - .list_partitions_by_names( - &identifier, - vec![HashMap::from([( - "unknown".to_string(), - "value".to_string(), - )])], - ) - .await - .unwrap_err(); - - assert!(matches!( - error, - paimon::Error::DataInvalid { message, .. } - if message.contains("default.managed_table") - )); } #[tokio::test] @@ -568,13 +345,16 @@ async fn test_rest_catalog_rejects_invalid_or_out_of_range_partition_page_token( let ctx = setup_catalog(vec!["default"]).await; let identifier = Identifier::new("default", "managed_table"); ctx.server.add_table("default", "managed_table"); - ctx.server.set_table_partition_pages( + ctx.server.set_table_partition_page_responses( "default", "managed_table", - vec![vec![HashMap::from([( - "dt".to_string(), - "2026-07-22".to_string(), - )])]], + vec![( + vec![HashMap::from([( + "dt".to_string(), + "2026-07-22".to_string(), + )])], + None, + )], ); for token in ["invalid", "1"] { @@ -1045,14 +825,12 @@ async fn test_rest_catalog_marks_internal_format_table_as_catalog_managed() { let ctx = setup_catalog(vec!["default"]).await; let schema = catalog_managed_format_table_schema(&[]); let identifier = Identifier::new("default", "managed_format"); - ctx.server.add_table_with_schema( - "default", + add_internal_table_with_schema( + &ctx.server, "managed_format", schema, "memory:/managed_format", ); - ctx.server - .set_table_external("default", "managed_format", false); let table = ctx.catalog.get_table(&identifier).await.unwrap(); @@ -1066,66 +844,36 @@ async fn test_rest_catalog_marks_internal_format_table_as_catalog_managed() { } #[tokio::test] -async fn test_rest_catalog_rejects_invalid_metastore_partitioned_table_option() { +async fn test_rest_catalog_rejects_invalid_catalog_managed_partition_options() { let ctx = setup_catalog(vec!["default"]).await; - let schema = Schema::builder() - .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) - .partition_keys(["dt"]) - .option("type", "format-table") - .option("file.format", "parquet") - .option("metastore.partitioned-table", "tru") - .build() - .unwrap(); - let identifier = Identifier::new("default", "invalid_partition_source"); - ctx.server.add_table_with_schema( - "default", - "invalid_partition_source", - schema, - "memory:/invalid_partition_source", - ); - ctx.server - .set_table_external("default", "invalid_partition_source", false); - - let error = ctx.catalog.get_table(&identifier).await.unwrap_err(); - - assert!(matches!( - error, - paimon::Error::ConfigInvalid { message } - if message.contains("metastore.partitioned-table") - && message.contains("tru") - )); -} - -#[tokio::test] -async fn test_rest_catalog_rejects_invalid_format_table_implementation() { - let ctx = setup_catalog(vec!["default"]).await; - let schema = Schema::builder() - .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) - .partition_keys(["dt"]) - .option("type", "format-table") - .option("file.format", "parquet") - .option("metastore.partitioned-table", "true") - .option("format-table.implementation", "unknown") - .build() - .unwrap(); - let identifier = Identifier::new("default", "invalid_format_implementation"); - ctx.server.add_table_with_schema( - "default", - "invalid_format_implementation", - schema, - "memory:/invalid_format_implementation", - ); - ctx.server - .set_table_external("default", "invalid_format_implementation", false); + for (table, option, bad_value) in [ + ( + "invalid_partition_source", + "metastore.partitioned-table", + "tru", + ), + ( + "invalid_format_implementation", + "format-table.implementation", + "unknown", + ), + ] { + let schema = catalog_managed_format_table_schema(&[(option, bad_value)]); + add_internal_table_with_schema(&ctx.server, table, schema, &format!("memory:/{table}")); - let error = ctx.catalog.get_table(&identifier).await.unwrap_err(); + let error = ctx + .catalog + .get_table(&Identifier::new("default", table)) + .await + .unwrap_err(); - assert!(matches!( - error, - paimon::Error::ConfigInvalid { message } - if message.contains("format-table.implementation") - && message.contains("unknown") - )); + assert!( + matches!(error, paimon::Error::ConfigInvalid { .. }) + && error.to_string().contains(option) + && error.to_string().contains(bad_value), + "expected {option}={bad_value} validation error, got: {error}" + ); + } } #[tokio::test] @@ -1145,9 +893,7 @@ async fn test_rest_catalog_accepts_supported_partition_options() { .option("format-table.implementation", implementation) .build() .unwrap(); - ctx.server - .add_table_with_schema("default", table, schema, &format!("memory:/{table}")); - ctx.server.set_table_external("default", table, false); + add_internal_table_with_schema(&ctx.server, table, schema, &format!("memory:/{table}")); let loaded = ctx .catalog @@ -1168,14 +914,12 @@ async fn test_rest_env_new_does_not_enable_catalog_managed_partitions() { let ctx = setup_catalog(vec!["default"]).await; let schema = catalog_managed_format_table_schema(&[]); let identifier = Identifier::new("default", "manual_rest_env"); - ctx.server.add_table_with_schema( - "default", + add_internal_table_with_schema( + &ctx.server, "manual_rest_env", schema, "memory:/manual_rest_env", ); - ctx.server - .set_table_external("default", "manual_rest_env", false); let loaded = ctx.catalog.get_table(&identifier).await.unwrap(); let plain_rest_env = RESTEnv::new( @@ -1205,10 +949,12 @@ async fn test_managed_format_partition_listing_requires_rest_endpoint() { let ctx = setup_catalog(vec!["default"]).await; let schema = catalog_managed_format_table_schema(&[]); let identifier = Identifier::new("default", "managed_partition_endpoint"); - ctx.server - .add_table_with_schema("default", "managed_partition_endpoint", schema, &table_path); - ctx.server - .set_table_external("default", "managed_partition_endpoint", false); + add_internal_table_with_schema( + &ctx.server, + "managed_partition_endpoint", + schema, + &table_path, + ); ctx.server .set_list_partitions_error_status(Some(StatusCode::NOT_IMPLEMENTED)); ctx.server @@ -1254,65 +1000,6 @@ async fn test_managed_format_partition_listing_requires_rest_endpoint() { )); } -#[cfg(not(windows))] -#[tokio::test] -async fn test_unmanaged_list_partitions_by_names_uses_exact_filesystem_match_when_unsupported() { - let tmp = tempfile::tempdir().unwrap(); - let warehouse = format!("file://{}", tmp.path().display()); - let mut fs_options = Options::new(); - fs_options.set(CatalogOptions::WAREHOUSE, &warehouse); - let fs_catalog = FileSystemCatalog::new(fs_options).unwrap(); - fs_catalog - .create_database("default", true, HashMap::new()) - .await - .unwrap(); - let identifier = Identifier::new("default", "filesystem_partitions"); - let schema = Schema::builder() - .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) - .column("id", DataType::BigInt(BigIntType::new())) - .partition_keys(["dt"]) - .build() - .unwrap(); - fs_catalog - .create_table(&identifier, schema.clone(), false) - .await - .unwrap(); - let fs_table = fs_catalog.get_table(&identifier).await.unwrap(); - let batch = RecordBatch::try_new( - Arc::new(ArrowSchema::new(vec![ - ArrowField::new("dt", ArrowDataType::Utf8, true), - ArrowField::new("id", ArrowDataType::Int64, true), - ])), - vec![ - Arc::new(StringArray::from(vec!["2026-07-22"])), - Arc::new(Int64Array::from(vec![1_i64])), - ], - ) - .unwrap(); - write_batch(&fs_table, batch, "partition-writer").await; - - let ctx = setup_catalog(vec!["default"]).await; - ctx.server.add_table_with_schema( - "default", - "filesystem_partitions", - schema, - fs_table.location(), - ); - ctx.server - .set_list_partitions_by_names_error_status(Some(StatusCode::NOT_IMPLEMENTED)); - let exact = HashMap::from([("dt".to_string(), "2026-07-22".to_string())]); - let alias = HashMap::from([("dt".to_string(), "2026-07-22 ".to_string())]); - - let partitions = ctx - .catalog - .list_partitions_by_names(&identifier, vec![exact.clone(), alias]) - .await - .unwrap(); - - assert_eq!(partitions.len(), 1); - assert_eq!(partitions[0].spec, exact); -} - #[tokio::test] async fn test_rest_catalog_rejects_external_catalog_managed_format_table() { let ctx = setup_catalog(vec!["default"]).await; @@ -1340,14 +1027,12 @@ async fn test_rest_catalog_rejects_engine_managed_format_table() { let ctx = setup_catalog(vec!["default"]).await; let schema = catalog_managed_format_table_schema(&[("format-table.implementation", "engine")]); let identifier = Identifier::new("default", "engine_managed_format"); - ctx.server.add_table_with_schema( - "default", + add_internal_table_with_schema( + &ctx.server, "engine_managed_format", schema, "memory:/engine_managed_format", ); - ctx.server - .set_table_external("default", "engine_managed_format", false); let error = ctx.catalog.get_table(&identifier).await.unwrap_err(); @@ -1373,10 +1058,7 @@ async fn test_managed_format_scan_hides_unregistered_partition() { let ctx = setup_catalog(vec!["default"]).await; let schema = catalog_managed_format_table_schema(&[]); let identifier = Identifier::new("default", "managed_visibility"); - ctx.server - .add_table_with_schema("default", "managed_visibility", schema, &table_path); - ctx.server - .set_table_external("default", "managed_visibility", false); + add_internal_table_with_schema(&ctx.server, "managed_visibility", schema, &table_path); ctx.server.set_table_partitions( "default", "managed_visibility", @@ -1398,14 +1080,12 @@ async fn test_rest_format_table_rejects_partition_option_changes_in_dynamic_copy let ctx = setup_catalog(vec!["default"]).await; let schema = catalog_managed_format_table_schema(&[]); let identifier = Identifier::new("default", "managed_dynamic_layout"); - ctx.server.add_table_with_schema( - "default", + add_internal_table_with_schema( + &ctx.server, "managed_dynamic_layout", schema, "memory:/managed_dynamic_layout", ); - ctx.server - .set_table_external("default", "managed_dynamic_layout", false); let table = ctx.catalog.get_table(&identifier).await.unwrap(); for (key, value) in [ @@ -1463,14 +1143,12 @@ async fn test_rest_format_table_rejects_enabling_catalog_partitions_in_dynamic_c .build() .unwrap(); let identifier = Identifier::new("default", "unmanaged_dynamic_source"); - ctx.server.add_table_with_schema( - "default", + add_internal_table_with_schema( + &ctx.server, "unmanaged_dynamic_source", schema, "memory:/unmanaged_dynamic_source", ); - ctx.server - .set_table_external("default", "unmanaged_dynamic_source", false); let table = ctx.catalog.get_table(&identifier).await.unwrap(); let error = table @@ -1497,10 +1175,12 @@ async fn test_managed_format_scan_treats_registered_missing_directory_as_empty() let ctx = setup_catalog(vec!["default"]).await; let schema = catalog_managed_format_table_schema(&[]); let identifier = Identifier::new("default", "managed_missing_directory"); - ctx.server - .add_table_with_schema("default", "managed_missing_directory", schema, &table_path); - ctx.server - .set_table_external("default", "managed_missing_directory", false); + add_internal_table_with_schema( + &ctx.server, + "managed_missing_directory", + schema, + &table_path, + ); ctx.server.set_table_partitions( "default", "managed_missing_directory", @@ -1528,10 +1208,7 @@ async fn test_managed_format_scan_propagates_partition_listing_error() { let ctx = setup_catalog(vec!["default"]).await; let schema = catalog_managed_format_table_schema(&[]); let identifier = Identifier::new("default", "managed_scan_error"); - ctx.server - .add_table_with_schema("default", "managed_scan_error", schema, &table_path); - ctx.server - .set_table_external("default", "managed_scan_error", false); + add_internal_table_with_schema(&ctx.server, "managed_scan_error", schema, &table_path); ctx.server .set_list_partitions_error_status(Some(StatusCode::NOT_IMPLEMENTED)); @@ -1560,10 +1237,7 @@ async fn test_managed_format_scan_reports_table_for_malformed_partition_metadata let ctx = setup_catalog(vec!["default"]).await; let schema = catalog_managed_format_table_schema(&[]); let identifier = Identifier::new("default", "managed_corrupt_metadata"); - ctx.server - .add_table_with_schema("default", "managed_corrupt_metadata", schema, &table_path); - ctx.server - .set_table_external("default", "managed_corrupt_metadata", false); + add_internal_table_with_schema(&ctx.server, "managed_corrupt_metadata", schema, &table_path); ctx.server.set_table_partitions( "default", "managed_corrupt_metadata", diff --git a/crates/paimon/tests/rest_object_models_test.rs b/crates/paimon/tests/rest_object_models_test.rs index 11492dd1..f4e3af1e 100644 --- a/crates/paimon/tests/rest_object_models_test.rs +++ b/crates/paimon/tests/rest_object_models_test.rs @@ -17,43 +17,11 @@ use std::collections::HashMap; -use paimon::api::{ - CreateFunctionRequest, CreatePartitionsRequest, CreatePartitionsResponse, CreateViewRequest, - DropPartitionsRequest, DropPartitionsResponse, ListPartitionsByNamesRequest, -}; +use paimon::api::{CreateFunctionRequest, CreateViewRequest}; use paimon::catalog::{Function, FunctionDefinition, Identifier, View, ViewSchema}; use paimon::spec::DataField; use serde_json::json; -#[test] -fn partition_rest_model_contract() { - let spec = HashMap::from([("dt".to_string(), "2026-07-22".to_string())]); - - let create = CreatePartitionsRequest::new(vec![spec.clone()], false); - let drop = DropPartitionsRequest::new(vec![spec.clone()], false); - let list = ListPartitionsByNamesRequest::new(vec![spec.clone()]); - let created: CreatePartitionsResponse = serde_json::from_value(json!({ - "created": [spec.clone()], - "existed": [] - })) - .unwrap(); - let dropped: DropPartitionsResponse = serde_json::from_value(json!({ - "dropped": [spec.clone()], - "missing": [] - })) - .unwrap(); - - assert!(!create.ignore_if_exists); - assert_eq!(create.partition_specs, vec![spec.clone()]); - assert!(!drop.ignore_if_not_exists); - assert_eq!(drop.partition_specs, vec![spec.clone()]); - assert_eq!(list.partition_specs, vec![spec.clone()]); - assert_eq!(created.created, vec![spec.clone()]); - assert!(created.existed.is_empty()); - assert_eq!(dropped.dropped, vec![spec]); - assert!(dropped.missing.is_empty()); -} - #[test] fn construct_view_schema_contract() { let fields: Vec = serde_json::from_value(json!([ diff --git a/docs/src/sql.md b/docs/src/sql.md index e1ed1924..14c11f66 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -705,132 +705,56 @@ ALTER TABLE IF EXISTS paimon.my_db.users ADD COLUMN age INT; ### Catalog-managed Format Table partitions -`SQLContext` supports Spark-compatible partition commands for an internal -Format Table loaded from REST Catalog. The table must satisfy all of these -conditions: - -- it was loaded from REST Catalog; -- it is an internal table (`isExternal=false`); -- it has `type=format-table`; -- it is partitioned; -- it has `metastore.partitioned-table=true`; -- it does not use `format-table.implementation=engine`. - -For these tables, REST partition registrations are authoritative for both -`SHOW PARTITIONS` and scans. A directory that is not registered is invisible, -while a registered partition whose directory is absent reads as empty when -storage reports `NotFound`. Other partition-directory listing failures -propagate instead of being treated as empty. Filesystem-discovered Format -Tables and non-REST catalogs do not support these commands. Dynamic query -options cannot override `metastore.partitioned-table`, -`format-table.partition-path-only-value`, or `format-table.implementation` for -a loaded table. +`SQLContext` supports partition commands for internal REST Catalog Format +Tables that are partitioned, set `metastore.partitioned-table=true`, and do +not set `format-table.implementation=engine`. + +REST registrations determine the partitions returned by `SHOW PARTITIONS` and +included in scans. Unregistered directories are ignored. A registered +partition with a missing directory reads as empty; other storage errors are +reported. #### SHOW PARTITIONS ```sql SHOW PARTITIONS paimon.my_db.events; -SHOW PARTITIONS paimon.my_db.events -PARTITION (dt = '2026-07-22'); -SHOW PARTITIONS paimon.my_db.events -PARTITION (region = 'us'); -``` - -The optional filter may be partial and does not need to be a leading partition -prefix. The result has one non-null string column named `partition`. Each value -uses declared partition-key order, and rows are sorted lexicographically by the -complete value, for example `dt=2026-07-22/region=us`. The result contains one -row per REST registration, so type-equivalent raw registrations can produce -duplicate displayed values. - -REST partition values are cast through their column types before display and -filtering. Default-partition values are shown as `null`, dates as `yyyy-MM-dd`, -and an integer directory value such as `01` displays as `1`. Type-equivalent -raw values registered by MSCK are normalized for display and filtering. Raw -numeric and boolean metadata must not contain surrounding whitespace. `CHAR` -and `VARCHAR` metadata is preserved exactly. +SHOW PARTITIONS paimon.my_db.events PARTITION (region = 'us'); +``` + +The filter may contain any subset of partition keys. Results use declared key +order, are sorted lexicographically, and normalize values through their column +types. Default values are shown as `null`. #### ADD PARTITION ```sql -ALTER TABLE paimon.my_db.events -ADD PARTITION (dt = '2026-07-22', region = 'us'); - -ALTER TABLE paimon.my_db.events -ADD IF NOT EXISTS -PARTITION (dt = '2026-07-22', region = 'us') -PARTITION (dt = '2026-07-23', region = 'eu'); -``` - -Each `ADD PARTITION` specification must contain all partition columns. The -complete batch is registered through REST before its partition directories are -created. Without `IF NOT EXISTS`, the REST catalog reports duplicate -registrations as conflicts. With `IF NOT EXISTS`, existing registrations are -ignored. SQL constants are converted to strings and cast through the partition -column types, so quoted numeric or boolean values and numeric values for string -columns are accepted. Integer types from `TINYINT` through `BIGINT`, `BOOLEAN`, -`CHAR`/`VARCHAR`, and `DATE` are supported. SQL `NULL` and blank character -values map to `partition.default-name`. - -Normalization follows Spark partition-literal behavior for the supported -types. Integer SQL literals lose redundant leading zeros even when the target -column is character data (`VARCHAR label = 01` becomes `label=1`; a `CHAR` -target also applies its fixed-width padding). String inputs to integer, -boolean, and date columns ignore surrounding whitespace, while `CHAR` and -`VARCHAR` values preserve it. `CHAR(n)` right-pads short values with ASCII -spaces. Both `CHAR(n)` and `VARCHAR(n)` count Unicode characters, reject -overlong non-space content, and trim only the excess ASCII trailing spaces -needed to fit `n`. Boolean values accept `t`/`true`/`y`/`yes`/`1` and -`f`/`false`/`n`/`no`/`0`, case-insensitively. Date strings may omit the month -or day. Typed literals are normalized before the partition-column cast, so -`DATE '2026'` becomes `2026-01-01` even for a `CHAR`/`VARCHAR` partition -column. Compact numeric dates such as `20260722` are rejected instead of being -interpreted as `yyyyMMdd`. Paimon `DATE` partition values are limited to -`0000-01-01` through `9999-12-31`. Typed literals other than `DATE` are -not supported. - -With the default `partition.legacy-name=true`, `dt = '2026'` is stored in the -directory `dt=20454` and displayed by `SHOW PARTITIONS` as `dt=2026-01-01`. -With `partition.legacy-name=false`, the directory is `dt=2026-01-01`. -Numeric DATE values already present in REST metadata are always interpreted as -epoch days, independently of this option. - -`ADD PARTITION` without `IF NOT EXISTS` is sent as one REST request. -`ADD IF NOT EXISTS PARTITION` is split into requests of at most 1000 -specifications. Partition values use Java-compatible path escaping: Unicode is -preserved and reserved characters such as `/`, `=`, `%`, and `}` are -percent-encoded. `LOCATION` is not supported. +ALTER TABLE paimon.my_db.events ADD PARTITION (dt = '2026-07-22', region = 'us'); -#### DROP PARTITION +ALTER TABLE paimon.my_db.events ADD IF NOT EXISTS PARTITION (dt = '2026-07-22', region = 'us'); +``` -```sql --- Complete specification -ALTER TABLE paimon.my_db.events -DROP PARTITION (dt = '2026-07-22', region = 'us'); +Each specification must include every partition key. Partitions are registered +in REST before their directories are created. `IF NOT EXISTS` ignores existing +registrations; otherwise duplicates are conflicts. --- Partial, including a non-leading key -ALTER TABLE paimon.my_db.events -DROP PARTITION (region = 'us'); +Supported types are integers, `BOOLEAN`, `CHAR`, `VARCHAR`, and `DATE`. Values +are normalized through the target type. `NULL` and blank character values use +`partition.default-name`; dates honor `partition.legacy-name`. Paths use +Java-compatible escaping. --- Ignore a missing complete registration -ALTER TABLE paimon.my_db.events -DROP IF EXISTS PARTITION (dt = '2026-07-22', region = 'us'); +#### DROP PARTITION + +```sql +ALTER TABLE paimon.my_db.events DROP PARTITION (dt = '2026-07-22', region = 'us'); --- Spark-compatible multi-partition DROP -ALTER TABLE paimon.my_db.events -DROP PARTITION (dt = '2026-07-22', region = 'us'), - PARTITION (dt = '2026-07-23', region = 'eu'); +ALTER TABLE paimon.my_db.events DROP IF EXISTS PARTITION (region = 'us'); ``` -A complete specification selects the exact raw registration when it exists. -Otherwise, typed matching can select a type-equivalent registration, such as -integer metadata `01` for SQL value `1`. A partial specification selects every -registration whose typed values match the supplied keys, including non-leading -partition keys. Selected registrations are removed before their directories -are deleted, so DROP never deletes an unregistered directory. A missing -complete specification fails unless `IF EXISTS` is present. REST unregister -calls are split into batches of at most 1000 specifications. `PURGE` is -rejected. +A complete specification selects its exact raw registration when present; +otherwise typed matching may select an equivalent value such as integer +metadata `01` for SQL value `1`. A partial specification selects every match, +including non-leading keys. Registrations are removed before directories are +deleted. Missing complete specifications fail unless `IF EXISTS` is used. #### MSCK REPAIR TABLE @@ -841,17 +765,17 @@ MSCK REPAIR TABLE paimon.my_db.events DROP PARTITIONS; MSCK REPAIR TABLE paimon.my_db.events SYNC PARTITIONS; ``` -Plain `MSCK REPAIR TABLE` and `ADD PARTITIONS` register partitions that exist -on the filesystem but not in REST. `DROP PARTITIONS` unregisters REST -partitions whose directories are absent. `SYNC PARTITIONS` performs both, -adding before dropping. Repair is metadata-only: it never creates or deletes -partition directories. Filesystem values are preserved as written, so a path -such as `month=01` is registered with the raw value `01`; `SHOW PARTITIONS` -still displays its typed value as `month=1`. REST create/drop calls use batches -of at most 1000 specifications. Partition path segments must use canonical -Java-compatible escaping. Repair fails before updating REST metadata for -malformed percent escapes such as `%ZZ`, lowercase percent escapes, or escaped -safe characters. +Plain repair and `ADD PARTITIONS` register filesystem partitions missing from +REST. `DROP PARTITIONS` unregisters entries whose directories are absent. +`SYNC PARTITIONS` adds and then drops. Repair only changes REST metadata; it +never creates or deletes directories. + +Filesystem values are registered as written. Paths must use canonical +Java-compatible escaping; malformed or non-canonical escapes fail before REST +metadata is changed. + +`LOCATION`, `DROP ... PURGE`, branch-qualified table names, and these commands +on non-REST or unsupported Format Tables are not supported. ## DML From 78f19a52739a544c05585f5a64c54e3043743ee7 Mon Sep 17 00:00:00 2001 From: Dapeng Sun Date: Thu, 23 Jul 2026 17:05:42 +0800 Subject: [PATCH 7/7] datafusion: align REST partition commands with project style --- .../datafusion/src/format_partition_repair.rs | 10 +- .../datafusion/src/sql_context.rs | 2575 +++-------------- .../tests/rest_format_partition_sql.rs | 136 +- crates/paimon/src/api/api_request.rs | 74 +- crates/paimon/src/api/api_response.rs | 76 - crates/paimon/src/api/mod.rs | 11 +- crates/paimon/src/api/resource_paths.rs | 14 +- crates/paimon/src/api/rest_api.rs | 62 +- crates/paimon/src/catalog/mod.rs | 20 +- .../paimon/src/catalog/rest/rest_catalog.rs | 32 +- crates/paimon/src/spec/partition_utils.rs | 51 +- crates/paimon/src/table/format_partition.rs | 226 +- crates/paimon/src/table/format_table_scan.rs | 94 +- crates/paimon/src/table/mod.rs | 37 +- crates/paimon/src/table/rest_env.rs | 152 +- crates/paimon/tests/format_partition_test.rs | 42 - crates/paimon/tests/mock_server.rs | 158 +- crates/paimon/tests/rest_api_test.rs | 103 +- crates/paimon/tests/rest_catalog_test.rs | 415 +-- docs/src/sql.md | 84 +- 20 files changed, 787 insertions(+), 3585 deletions(-) diff --git a/crates/integrations/datafusion/src/format_partition_repair.rs b/crates/integrations/datafusion/src/format_partition_repair.rs index 7271bf1d..f0b1738f 100644 --- a/crates/integrations/datafusion/src/format_partition_repair.rs +++ b/crates/integrations/datafusion/src/format_partition_repair.rs @@ -41,6 +41,12 @@ pub(crate) async fn repair( ); let table_path = table.location(); + if matches!(mode, RepairMode::Drop | RepairMode::Sync) { + // Discovery treats a missing root as empty. Destructive repair must fail + // instead, or it could unregister every catalog partition. + table.file_io().list_status(table_path).await?; + } + // Load both views before changing catalog metadata so a listing failure leaves // metadata unchanged. Discovery preserves raw directory values such as month=01. let discovered_specs = partition_paths @@ -86,9 +92,7 @@ pub(crate) async fn repair( .await?; } if !to_unregister.is_empty() { - catalog - .unregister_partitions(identifier, to_unregister) - .await?; + catalog.drop_partitions(identifier, to_unregister).await?; } Ok(()) } diff --git a/crates/integrations/datafusion/src/sql_context.rs b/crates/integrations/datafusion/src/sql_context.rs index bc004438..0102e5f5 100644 --- a/crates/integrations/datafusion/src/sql_context.rs +++ b/crates/integrations/datafusion/src/sql_context.rs @@ -30,16 +30,16 @@ //! - `ALTER TABLE db.t RENAME COLUMN old TO new` //! - `ALTER TABLE db.t RENAME TO new_name` //! - `ALTER TABLE db.t ADD [IF NOT EXISTS] PARTITION (...) [PARTITION (...)]` -//! - `ALTER TABLE db.t DROP [IF EXISTS] PARTITION (...) [, PARTITION (...)]` +//! - `ALTER TABLE db.t DROP [IF EXISTS] PARTITION (...)` //! - `SHOW PARTITIONS db.t [PARTITION (...)]` -//! - `[MSCK] REPAIR TABLE db.t [{ADD|DROP|SYNC} PARTITIONS]` +//! - `MSCK REPAIR TABLE db.t [{ADD|DROP|SYNC} PARTITIONS]` //! - `CREATE VIEW [IF NOT EXISTS] view [(col, ...)] AS query` //! - `DROP VIEW [IF EXISTS] view` //! - `CREATE FUNCTION name(args) RETURNS type [LANGUAGE SQL] RETURN expression` //! - `TRUNCATE TABLE db.t` //! - `TRUNCATE TABLE db.t PARTITION (col = val, ...)` -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use datafusion::arrow::array::{ @@ -50,7 +50,7 @@ use datafusion::arrow::compute::cast; use datafusion::arrow::datatypes::{DataType as ArrowDataType, Field, Schema}; use datafusion::arrow::record_batch::RecordBatch; use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; -use datafusion::common::TableReference; +use datafusion::common::{ScalarValue, TableReference}; use datafusion::datasource::{MemTable, TableProvider}; use datafusion::error::{DataFusionError, Result as DFResult}; use datafusion::execution::SessionStateBuilder; @@ -68,9 +68,9 @@ use datafusion::sql::sqlparser::ast::{ use datafusion::sql::sqlparser::dialect::GenericDialect; use datafusion::sql::sqlparser::keywords::Keyword; use datafusion::sql::sqlparser::parser::Parser; -use datafusion::sql::sqlparser::tokenizer::{Token, TokenWithSpan, Tokenizer}; +use datafusion::sql::sqlparser::tokenizer::{Token, Tokenizer}; use futures::StreamExt; -use paimon::catalog::{parse_object_name, Catalog, Identifier}; +use paimon::catalog::{parse_object_name, Catalog, Identifier, ParsedObjectName}; use paimon::spec::{ ArrayType as PaimonArrayType, BigIntType, BinaryType, BlobType, BooleanType, CharType, CoreOptions, DataField as PaimonDataField, DataType as PaimonDataType, DateType, Datum, @@ -78,9 +78,12 @@ use paimon::spec::{ RowType as PaimonRowType, SchemaChange, SmallIntType, TimestampType, TinyIntType, VarBinaryType, VarCharType, VariantType, }; -use paimon::table::{parse_format_partition_value, FormatTablePartitionPaths}; +use paimon::table::{ + format_partition_value, parse_format_partition_value, FormatTablePartitionPaths, +}; use crate::error::to_datafusion_error; +use crate::filter_pushdown::scalar_to_datum; use crate::table_loader::load_table_for_read; use crate::{BlobReaderRegistry, DynamicOptions}; @@ -371,12 +374,6 @@ impl SQLContext { if let Some(show_partitions) = parse_show_partitions(&rewritten_sql)? { return self.handle_show_partitions(&show_partitions).await; } - if is_drop_partition_purge(&rewritten_sql)? { - return Err(DataFusionError::Plan( - "DROP PARTITION PURGE is not supported".to_string(), - )); - } - let statements = parse_sql_statements(&rewritten_sql)?; if statements.len() != 1 { @@ -1050,30 +1047,44 @@ impl SQLContext { operations: &[AlterTableOperation], if_exists: bool, ) -> DFResult { - Self::ensure_main_branch_write_target(name, "ALTER TABLE")?; + let has_partition_operation = operations.iter().any(|operation| { + matches!( + operation, + AlterTableOperation::AddPartitions { .. } + | AlterTableOperation::DropPartitions { .. } + ) + }); + if has_partition_operation { + Self::ensure_partition_command_target(name, "ALTER TABLE")?; + } else { + Self::ensure_main_branch_write_target(name, "ALTER TABLE")?; + } let identifier = self.resolve_table_name(name)?; - let drop_partitions = operations - .iter() - .filter_map(|operation| match operation { - AlterTableOperation::DropPartitions { + if let [AlterTableOperation::DropPartitions { + partitions, + if_exists: partition_if_exists, + }] = operations + { + return self + .handle_drop_partition( + catalog, + &identifier, partitions, + *partition_if_exists, if_exists, - } => Some((partitions.as_slice(), *if_exists)), - _ => None, - }) - .collect::>(); - if !drop_partitions.is_empty() { - if drop_partitions.len() != operations.len() { - return Err(DataFusionError::Plan( - "ALTER TABLE DROP PARTITION cannot be combined with other operations" - .to_string(), - )); - } - return self - .handle_drop_partitions(catalog, &identifier, &drop_partitions, if_exists) + ) .await; } + if operations + .iter() + .any(|operation| matches!(operation, AlterTableOperation::DropPartitions { .. })) + { + return Err(DataFusionError::Plan( + "ALTER TABLE DROP PARTITION supports one partition specification per statement" + .to_string(), + )); + } if operations .iter() .any(|operation| matches!(operation, AlterTableOperation::AddPartitions { .. })) @@ -1141,12 +1152,6 @@ impl SQLContext { ) .await; } - AlterTableOperation::DropPartitions { .. } => { - return Err(DataFusionError::Internal( - "DROP PARTITION should have been dispatched before ALTER processing" - .to_string(), - )); - } other => { return Err(DataFusionError::Plan(format!( "Unsupported ALTER TABLE operation: {other}" @@ -1703,149 +1708,88 @@ impl SQLContext { Ok(()) } - async fn handle_drop_partitions( + async fn handle_drop_partition( &self, catalog: &Arc, identifier: &Identifier, - partition_specs: &[(&[SqlExpr], bool)], + expressions: &[SqlExpr], + ignore_if_not_exists: bool, ignore_if_table_not_exists: bool, ) -> DFResult { - if partition_specs.is_empty() || partition_specs.iter().any(|(spec, _)| spec.is_empty()) { + if expressions.is_empty() { return Err(DataFusionError::Plan( - "DROP PARTITIONS requires at least one partition specification".to_string(), + "DROP PARTITION requires a partition specification".to_string(), )); } let table = match catalog.get_table(identifier).await { - Ok(t) => t, - Err(e) if ignore_if_table_not_exists && is_table_not_exist(&e) => { + Ok(table) => table, + Err(error) if ignore_if_table_not_exists && is_table_not_exist(&error) => { return ok_result(&self.ctx); } - Err(e) => return Err(to_datafusion_error(e)), + Err(error) => return Err(to_datafusion_error(error)), }; if table.has_catalog_managed_partitions() { ensure_catalog_managed_format_table(&table, "ALTER TABLE DROP PARTITION")?; - let requested = partition_specs - .iter() - .map(|(expressions, ignore_if_not_exists)| { - Ok(( - parse_format_partition_spec(expressions, &table, false)?, - *ignore_if_not_exists, - )) + let requested = parse_format_partition_spec(expressions, &table, true)?; + let requested_normalized = normalize_catalog_partition_spec(&requested, &table, true)?; + let registered = catalog + .list_partitions(identifier) + .await + .map_err(to_datafusion_error)? + .into_iter() + .map(|partition| { + let normalized = + normalize_catalog_partition_spec(&partition.spec, &table, true)?; + Ok((partition.spec, normalized)) }) .collect::>>()?; + let has_exact_match = registered.iter().any(|(spec, _)| spec == &requested); let core_options = CoreOptions::new(table.schema().options()); let partition_paths = FormatTablePartitionPaths::new( table.schema().partition_keys().iter().cloned(), core_options.format_table_partition_only_value_in_path(), ); - let table_path = table.location(); - let partition_key_count = table.schema().partition_keys().len(); - let complete_specs = requested - .iter() - .filter(|(spec, _)| spec.len() == partition_key_count) - .map(|(spec, _)| spec.clone()) - .collect::>(); - let exact_registered = if complete_specs.is_empty() { - Vec::new() - } else { - catalog - .list_partitions_by_names(identifier, complete_specs) - .await - .map_err(to_datafusion_error)? - }; - let exact_matches = requested - .iter() - .map(|(requested, _)| { - requested.len() == partition_key_count - && exact_registered - .iter() - .any(|partition| partition.spec == *requested) - }) - .collect::>(); - let needs_full_listing = requested.iter().enumerate().any(|(index, (spec, _))| { - spec.len() != partition_key_count || !exact_matches[index] - }); - let registered_partitions = if needs_full_listing { - catalog - .list_partitions(identifier) - .await - .map_err(to_datafusion_error)? - } else { - exact_registered - }; - - let normalized_partitions = registered_partitions - .into_iter() - .map(|partition| { - let normalized = normalize_format_partition_spec_values( - &partition.spec, - &table, - FormatPartitionValueSource::CatalogMetadata, - true, - )?; - Ok((partition, normalized)) - }) - .collect::>>()?; - let mut selected_partitions = BTreeMap::new(); - for (index, (requested_spec, ignore_if_not_exists)) in requested.iter().enumerate() { - let requested_normalized = normalize_format_partition_spec_values( - requested_spec, - &table, - FormatPartitionValueSource::SqlLiteral, - false, - )?; - let mut matched = false; - for (partition, normalized) in &normalized_partitions { - let partition_matches = if exact_matches[index] { - partition.spec == *requested_spec - } else { - requested_normalized - .iter() - .all(|(key, value)| normalized.get(key) == Some(value)) - }; - if partition_matches { - matched = true; - let name = partition_paths - .partition_name(&partition.spec) - .map_err(to_datafusion_error)?; - let relative_path = partition_paths - .relative_path(&partition.spec) - .map_err(to_datafusion_error)?; - selected_partitions.entry(name).or_insert_with(|| { - ( - partition.spec.clone(), - format!("{}/{}", table_path.trim_end_matches('/'), relative_path), - ) - }); - } - } - if !matched - && requested_spec.len() == table.schema().partition_keys().len() - && !ignore_if_not_exists - { - return Err(DataFusionError::Plan(format!( - "Partition {requested_spec:?} does not exist in table {}", - identifier.full_name() - ))); + let table_path = table.location().trim_end_matches('/'); + let mut selected = Vec::new(); + for (spec, normalized) in registered { + let matches = if has_exact_match { + spec == requested + } else { + normalized == requested_normalized + }; + if matches { + let relative_path = partition_paths + .relative_path(&spec) + .map_err(to_datafusion_error)?; + selected.push((spec, format!("{table_path}/{relative_path}"))); } } + if !has_exact_match && selected.len() > 1 { + return Err(DataFusionError::Plan(format!( + "Partition {requested:?} matches multiple catalog registrations in table {}; use an exact catalog value", + identifier.full_name() + ))); + } - if selected_partitions.is_empty() { - return ok_result(&self.ctx); + if selected.is_empty() { + if ignore_if_not_exists { + return ok_result(&self.ctx); + } + return Err(DataFusionError::Plan(format!( + "Partition {requested:?} does not exist in table {}", + identifier.full_name() + ))); } catalog - .unregister_partitions( + .drop_partitions( identifier, - selected_partitions - .values() - .map(|(spec, _)| spec.clone()) - .collect(), + selected.iter().map(|(spec, _)| spec.clone()).collect(), ) .await .map_err(to_datafusion_error)?; - for (_, path) in selected_partitions.into_values() { + for (_, path) in selected { table .file_io() .delete_dir(&path) @@ -1859,21 +1803,14 @@ impl SQLContext { } let partition_values = parse_partition_values( - partition_specs[0].0, + expressions, table.schema().fields(), table.schema().partition_keys(), )?; - let mut partition_values = partition_values; - for (expressions, _) in &partition_specs[1..] { - partition_values.extend(parse_partition_values( - expressions, - table.schema().fields(), - table.schema().partition_keys(), - )?); - } - - let wb = table.new_write_builder(); - let commit = wb.try_new_commit().map_err(to_datafusion_error)?; + let commit = table + .new_write_builder() + .try_new_commit() + .map_err(to_datafusion_error)?; commit .truncate_partitions(partition_values) .await @@ -1953,7 +1890,7 @@ impl SQLContext { "MSCK requires the REPAIR keyword".to_string(), )); } - Self::ensure_main_branch_write_target(&msck.table_name, "MSCK REPAIR TABLE")?; + Self::ensure_partition_command_target(&msck.table_name, "MSCK REPAIR TABLE")?; let (catalog, _catalog_name, identifier) = self.resolve_catalog_and_table(&msck.table_name)?; let table = catalog @@ -1976,6 +1913,7 @@ impl SQLContext { &self, show_partitions: &ShowPartitionsStatement, ) -> DFResult { + Self::ensure_partition_command_target(&show_partitions.table_name, "SHOW PARTITIONS")?; let (catalog, _catalog_name, identifier) = self.resolve_catalog_and_table(&show_partitions.table_name)?; let table = catalog @@ -1988,12 +1926,7 @@ impl SQLContext { } else { let spec = parse_format_partition_spec(&show_partitions.partition_filter, &table, false)?; - Some(normalize_format_partition_spec_values( - &spec, - &table, - FormatPartitionValueSource::SqlLiteral, - false, - )?) + Some(normalize_catalog_partition_spec(&spec, &table, false)?) }; let partition_paths = FormatTablePartitionPaths::new( @@ -2006,12 +1939,7 @@ impl SQLContext { .await .map_err(to_datafusion_error)? { - let normalized = normalize_format_partition_spec_values( - &partition.spec, - &table, - FormatPartitionValueSource::CatalogMetadata, - true, - )?; + let normalized = normalize_catalog_partition_spec(&partition.spec, &table, true)?; if !filter.as_ref().is_none_or(|filter| { filter .iter() @@ -2019,10 +1947,7 @@ impl SQLContext { }) { continue; } - let display_spec = normalized - .into_iter() - .map(|(key, value)| (key, value.unwrap_or_else(|| "null".to_string()))) - .collect(); + let display_spec = display_partition_spec(&normalized, &table)?; let name = partition_paths .partition_name(&display_spec) .map_err(to_datafusion_error)?; @@ -2191,21 +2116,40 @@ impl SQLContext { } fn ensure_main_branch_write_target(name: &ObjectName, operation: &str) -> DFResult<()> { - let object = name - .0 - .last() - .and_then(|part| part.as_ident()) - .map(|ident| ident.value.as_str()) - .ok_or_else(|| DataFusionError::Plan(format!("Invalid table reference: {name}")))?; - let parsed = parse_object_name(object).map_err(to_datafusion_error)?; + let parsed = Self::parse_target_object_name(name)?; + if let Some(branch) = parsed.branch() { + return Err(DataFusionError::NotImplemented(format!( + "{operation} on Paimon branch '{branch}' is not supported" + ))); + } + Ok(()) + } + + fn ensure_partition_command_target(name: &ObjectName, operation: &str) -> DFResult<()> { + let parsed = Self::parse_target_object_name(name)?; if let Some(branch) = parsed.branch() { return Err(DataFusionError::NotImplemented(format!( "{operation} on Paimon branch '{branch}' is not supported" ))); } + if let Some(system_table) = parsed.system_table() { + return Err(DataFusionError::NotImplemented(format!( + "{operation} on Paimon system table '{system_table}' is not supported" + ))); + } Ok(()) } + fn parse_target_object_name(name: &ObjectName) -> DFResult { + let object = name + .0 + .last() + .and_then(|part| part.as_ident()) + .map(|ident| ident.value.as_str()) + .ok_or_else(|| DataFusionError::Plan(format!("Invalid table reference: {name}")))?; + parse_object_name(object).map_err(to_datafusion_error) + } + /// Resolve an ObjectName to just the Identifier (for backward compat in handle_alter_table). fn resolve_table_name(&self, name: &ObjectName) -> DFResult { let (_catalog, _catalog_name, identifier) = self.resolve_catalog_and_table(name)?; @@ -2329,7 +2273,7 @@ fn parse_show_partitions(sql: &str) -> DFResult> let dialect = GenericDialect {}; let tokens = Tokenizer::new(&dialect, sql) .tokenize_with_location() - .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; + .map_err(sql_parse_error)?; let significant = tokens .iter() .filter_map(|token| match &token.token { @@ -2346,23 +2290,21 @@ fn parse_show_partitions(sql: &str) -> DFResult> let mut parser = Parser::new(&dialect).with_tokens_with_locations(tokens); parser .expect_keyword_is(Keyword::SHOW) - .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; + .map_err(sql_parse_error)?; parser .expect_keyword_is(Keyword::PARTITIONS) - .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; - let table_name = parser - .parse_object_name(false) - .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; + .map_err(sql_parse_error)?; + let table_name = parser.parse_object_name(false).map_err(sql_parse_error)?; let partition_filter = if parser.parse_keyword(Keyword::PARTITION) { parser .expect_token(&Token::LParen) - .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; + .map_err(sql_parse_error)?; let expressions = parser .parse_comma_separated(Parser::parse_expr) - .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; + .map_err(sql_parse_error)?; parser .expect_token(&Token::RParen) - .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; + .map_err(sql_parse_error)?; expressions } else { Vec::new() @@ -2380,12 +2322,15 @@ fn parse_show_partitions(sql: &str) -> DFResult> })) } +fn sql_parse_error(error: impl std::fmt::Display) -> DataFusionError { + DataFusionError::Plan(format!("SQL parse error: {error}")) +} + fn parse_sql_statements(sql: &str) -> DFResult> { let dialect = GenericDialect {}; let mut tokens = Tokenizer::new(&dialect, sql) .tokenize_with_location() - .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; - tokens = rewrite_spark_multi_drop_partition_tokens(tokens); + .map_err(sql_parse_error)?; let significant = tokens .iter() .enumerate() @@ -2396,9 +2341,6 @@ fn parse_sql_statements(sql: &str) -> DFResult> { }) .take(5) .collect::>(); - if matches!(significant.first(), Some((_, Keyword::REPAIR))) { - return parse_sql_statements(&format!("MSCK {sql}")); - } let create_function_if_not_exists = matches!( significant.as_slice(), [ @@ -2434,7 +2376,7 @@ fn parse_sql_statements(sql: &str) -> DFResult> { let mut statements = Parser::new(&dialect) .with_tokens_with_locations(tokens) .parse_statements() - .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; + .map_err(sql_parse_error)?; if create_function_if_not_exists { let Some(Statement::CreateFunction(create_function)) = statements.first_mut() else { return Err(DataFusionError::Plan( @@ -2454,136 +2396,6 @@ fn parse_sql_statements(sql: &str) -> DFResult> { Ok(statements) } -fn rewrite_spark_multi_drop_partition_tokens(tokens: Vec) -> Vec { - // sqlparser requires DROP before every partition clause. Spark allows - // `DROP IF EXISTS PARTITION (...), PARTITION (...)`. - let first_keywords = tokens - .iter() - .filter_map(|token| match &token.token { - Token::Whitespace(_) => None, - Token::Word(word) => Some(word.keyword), - _ => Some(Keyword::NoKeyword), - }) - .take(2) - .collect::>(); - if !matches!(first_keywords.as_slice(), [Keyword::ALTER, Keyword::TABLE]) { - return tokens; - } - - let next_significant = |start: usize| { - (start..tokens.len()).find(|index| !matches!(tokens[*index].token, Token::Whitespace(_))) - }; - let mut insertions = HashMap::new(); - let mut depth = 0_usize; - let mut in_drop_partitions = false; - let mut propagate_if_exists = false; - let mut index = 0; - while index < tokens.len() { - match &tokens[index].token { - Token::LParen => depth += 1, - Token::RParen => depth = depth.saturating_sub(1), - Token::Word(word) if depth == 0 && word.keyword == Keyword::DROP => { - let Some(mut next) = next_significant(index + 1) else { - break; - }; - let mut if_exists = false; - if matches!( - &tokens[next].token, - Token::Word(word) if word.keyword == Keyword::IF - ) { - let Some(exists) = next_significant(next + 1) else { - break; - }; - if matches!( - &tokens[exists].token, - Token::Word(word) if word.keyword == Keyword::EXISTS - ) { - if_exists = true; - let Some(after_exists) = next_significant(exists + 1) else { - break; - }; - next = after_exists; - } - } - in_drop_partitions = matches!( - &tokens[next].token, - Token::Word(word) if word.keyword == Keyword::PARTITION - ); - propagate_if_exists = if_exists; - } - Token::Comma if depth == 0 && in_drop_partitions => { - let Some(next) = next_significant(index + 1) else { - break; - }; - match &tokens[next].token { - Token::Word(word) if word.keyword == Keyword::PARTITION => { - insertions.insert(next, propagate_if_exists); - } - Token::Word(word) if word.keyword == Keyword::DROP => {} - _ => in_drop_partitions = false, - } - } - _ => {} - } - index += 1; - } - - if insertions.is_empty() { - return tokens; - } - let mut rewritten = Vec::with_capacity(tokens.len() + insertions.len() * 3); - for (index, token) in tokens.into_iter().enumerate() { - if let Some(if_exists) = insertions.get(&index) { - rewritten.push(TokenWithSpan::wrap(Token::make_keyword("DROP"))); - if *if_exists { - rewritten.push(TokenWithSpan::wrap(Token::make_keyword("IF"))); - rewritten.push(TokenWithSpan::wrap(Token::make_keyword("EXISTS"))); - } - } - rewritten.push(token); - } - rewritten -} - -fn is_drop_partition_purge(sql: &str) -> DFResult { - let dialect = GenericDialect {}; - let mut tokens = Tokenizer::new(&dialect, sql) - .tokenize_with_location() - .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; - let Some(purge_index) = tokens.iter().enumerate().rev().find_map(|(index, token)| { - if matches!(token.token, Token::Whitespace(_) | Token::SemiColon) { - None - } else { - Some(index) - } - }) else { - return Ok(false); - }; - if !matches!( - &tokens[purge_index].token, - Token::Word(word) if word.keyword == Keyword::PURGE - ) { - return Ok(false); - } - - tokens.remove(purge_index); - tokens = rewrite_spark_multi_drop_partition_tokens(tokens); - let Ok(statements) = Parser::new(&dialect) - .with_tokens_with_locations(tokens) - .parse_statements() - else { - return Ok(false); - }; - Ok(matches!( - statements.as_slice(), - [Statement::AlterTable(alter_table)] - if alter_table - .operations - .iter() - .any(|operation| matches!(operation, AlterTableOperation::DropPartitions { .. })) - )) -} - fn validate_persistent_create_view(create_view: &CreateView) -> DFResult<()> { let unsupported = if create_view.or_alter { Some("CREATE OR ALTER VIEW is not supported") @@ -3228,81 +3040,66 @@ fn parse_format_partition_spec( table: &paimon::Table, require_complete: bool, ) -> DFResult> { - let field_map: HashMap<&str, &PaimonDataField> = table + let fields = table .schema() .fields() .iter() .map(|field| (field.name(), field)) - .collect(); + .collect::>(); let partition_keys = table.schema().partition_keys(); - let core_options = CoreOptions::new(table.schema().options()); - let mut partition = HashMap::with_capacity(exprs.len()); + let options = CoreOptions::new(table.schema().options()); + let mut spec = HashMap::with_capacity(exprs.len()); for expr in exprs { - let (column_name, value_expr) = match expr { - SqlExpr::BinaryOp { - left, - op: datafusion::sql::sqlparser::ast::BinaryOperator::Eq, - right, - } => { - let column_name = match left.as_ref() { - SqlExpr::Identifier(identifier) => { - IdentNormalizer::default().normalize(identifier.clone()) - } - other => { - return Err(DataFusionError::Plan(format!( - "Expected column name in partition spec, got: {other}" - ))) - } - }; - (column_name, right.as_ref()) - } - other => { - return Err(DataFusionError::Plan(format!( - "Expected 'column = value' in partition spec, got: {other}" - ))) - } - }; - - if !partition_keys.contains(&column_name) { + let (column, value) = partition_assignment(expr)?; + if !partition_keys.contains(&column) { return Err(DataFusionError::Plan(format!( - "Column '{column_name}' is not a partition column" + "Column '{column}' is not a partition column" ))); } - if partition.contains_key(&column_name) { + if spec.contains_key(&column) { return Err(DataFusionError::Plan(format!( - "Duplicate partition column '{column_name}'" + "Duplicate partition column '{column}'" ))); } - let field = field_map.get(column_name.as_str()).ok_or_else(|| { - DataFusionError::Plan(format!("Column '{column_name}' not found in table schema")) + let field = fields.get(column.as_str()).ok_or_else(|| { + DataFusionError::Plan(format!("Column '{column}' not found in table schema")) })?; - let value = match format_partition_literal_to_datum(value_expr, field.data_type())? { - None => core_options.partition_default_name().to_string(), - Some(datum) => { - format_partition_datum_for_storage(&datum, field.data_type(), &core_options)? - } + let value = match parse_format_partition_literal(value, field.data_type())? { + None => options.partition_default_name().to_string(), + Some(datum) => format_partition_value( + &datum, + field.data_type(), + options.partition_default_name(), + options.legacy_partition_name(), + ) + .ok_or_else(|| { + DataFusionError::NotImplemented(format!( + "Partition literals of type {:?} are not supported", + field.data_type() + )) + })?, }; - partition.insert(column_name, value); + spec.insert(column, value); } if require_complete { let missing = partition_keys .iter() - .filter(|key| !partition.contains_key(key.as_str())) + .filter(|key| !spec.contains_key(key.as_str())) .cloned() .collect::>(); if !missing.is_empty() { return Err(DataFusionError::Plan(format!( - "Incomplete partition spec: missing keys [{}]. All partition columns must be specified.", + "Incomplete partition spec: missing keys [{}]", missing.join(", ") ))); } } - Ok(partition) + Ok(spec) } -fn format_partition_literal_to_datum( +fn parse_format_partition_literal( expr: &SqlExpr, data_type: &PaimonDataType, ) -> DFResult> { @@ -3312,94 +3109,48 @@ fn format_partition_literal_to_datum( ) { return Ok(None); } - let raw = format_partition_literal_as_string(expr)?; - let datum = match data_type { - PaimonDataType::Char(char_type) => Datum::String(normalize_format_partition_string( - &raw, - char_type.length(), - true, - "CHAR", - )?), - PaimonDataType::VarChar(varchar_type) => Datum::String(normalize_format_partition_string( - &raw, - varchar_type.length() as usize, - false, - "VARCHAR", - )?), - PaimonDataType::Boolean(_) => Datum::Bool(parse_format_partition_bool(&raw)?), - PaimonDataType::TinyInt(_) => { - Datum::TinyInt(raw.trim().parse::().map_err(|error| { - DataFusionError::Plan(format!("Invalid TINYINT partition value '{raw}': {error}")) - })?) - } - PaimonDataType::SmallInt(_) => { - Datum::SmallInt(raw.trim().parse::().map_err(|error| { - DataFusionError::Plan(format!("Invalid SMALLINT partition value '{raw}': {error}")) - })?) - } - PaimonDataType::Int(_) => Datum::Int(raw.trim().parse::().map_err(|error| { - DataFusionError::Plan(format!("Invalid INT partition value '{raw}': {error}")) - })?), - PaimonDataType::BigInt(_) => Datum::Long(raw.trim().parse::().map_err(|error| { - DataFusionError::Plan(format!("Invalid BIGINT partition value '{raw}': {error}")) - })?), - PaimonDataType::Date(_) => Datum::Date(parse_spark_partition_date(&raw)?), - other => { - return Err(DataFusionError::NotImplemented(format!( - "Partition literals of type {other:?} are not implemented yet" - ))) - } - }; - Ok(Some(datum)) -} -fn normalize_format_partition_string( - value: &str, - length: usize, - pad_to_length: bool, - type_name: &str, -) -> DFResult { - let mut normalized = value.to_string(); - let mut character_count = normalized.chars().count(); - if character_count > length { - for _ in 0..(character_count - length) { - if !normalized.ends_with(' ') { - break; - } - normalized.pop(); - character_count -= 1; - } - if character_count > length { - return Err(DataFusionError::Plan(format!( - "{type_name} partition value '{value}' exceeds maximum length {length}" - ))); - } - } - if pad_to_length && character_count < length { - normalized.push_str(&" ".repeat(length - character_count)); - Ok(normalized) - } else { - Ok(normalized) + let mut value = format_partition_literal_as_string(expr)?; + if !matches!( + data_type, + PaimonDataType::Char(_) | PaimonDataType::VarChar(_) + ) { + value = value.trim().to_string(); } + let arrow_type = paimon::arrow::paimon_type_to_arrow(data_type).map_err(to_datafusion_error)?; + let scalar = ScalarValue::Utf8(Some(value)) + .cast_to(&arrow_type) + .map_err(|error| { + DataFusionError::Plan(format!( + "Cannot convert partition literal {expr} to {data_type:?}: {error}" + )) + })?; + scalar_to_datum(&scalar, data_type) + .map(Some) + .ok_or_else(|| { + DataFusionError::Plan(format!( + "Partition literals of type {data_type:?} are not supported" + )) + }) } fn format_partition_literal_as_string(expr: &SqlExpr) -> DFResult { match expr { - SqlExpr::TypedString(typed) => { - let value = typed.value.value.clone().into_string().ok_or_else(|| { - DataFusionError::Plan(format!("Unsupported typed partition literal: {expr}")) - })?; - match &typed.data_type { - datafusion::sql::sqlparser::ast::DataType::Date => { - format_partition_date(parse_spark_partition_date(&value)?) - } - _ => Err(DataFusionError::Plan(format!( - "Unsupported typed partition literal: {expr}" - ))), - } + SqlExpr::TypedString(typed) + if matches!( + typed.data_type, + datafusion::sql::sqlparser::ast::DataType::Date + ) => + { + typed + .value + .value + .clone() + .into_string() + .ok_or_else(|| DataFusionError::Plan(format!("Invalid partition literal: {expr}"))) } SqlExpr::Value(value) => match &value.value { - SqlValue::Number(value, _) => Ok(canonicalize_integer_literal_if_possible(value)), + SqlValue::Number(value, _) => Ok(canonicalize_partition_number(value)), SqlValue::Boolean(value) => Ok(value.to_string()), SqlValue::Null => Err(DataFusionError::Internal( "NULL partition literal reached string conversion".to_string(), @@ -3408,210 +3159,96 @@ fn format_partition_literal_as_string(expr: &SqlExpr) -> DFResult { DataFusionError::Plan(format!("Unsupported partition literal: {expr}")) }), }, - SqlExpr::UnaryOp { - op: datafusion::sql::sqlparser::ast::UnaryOperator::Plus, - expr: inner, - } => format_signed_partition_number(inner, ""), - SqlExpr::UnaryOp { - op: datafusion::sql::sqlparser::ast::UnaryOperator::Minus, - expr: inner, - } => format_signed_partition_number(inner, "-"), - other => Err(DataFusionError::Plan(format!( - "Unsupported partition value expression: {other}" + SqlExpr::UnaryOp { op, expr } + if matches!( + op, + datafusion::sql::sqlparser::ast::UnaryOperator::Plus + | datafusion::sql::sqlparser::ast::UnaryOperator::Minus + ) => + { + let SqlExpr::Value(value) = expr.as_ref() else { + return Err(DataFusionError::Plan(format!( + "Unary signs are supported only for numeric partition literals: {expr}" + ))); + }; + let SqlValue::Number(value, _) = &value.value else { + return Err(DataFusionError::Plan(format!( + "Unary signs are supported only for numeric partition literals: {expr}" + ))); + }; + let sign = if matches!(op, datafusion::sql::sqlparser::ast::UnaryOperator::Minus) { + "-" + } else { + "" + }; + Ok(format!("{sign}{}", canonicalize_partition_number(value))) + } + _ => Err(DataFusionError::Plan(format!( + "Unsupported partition value expression: {expr}" ))), } } -fn format_signed_partition_number(expr: &SqlExpr, sign: &str) -> DFResult { - let SqlExpr::Value(value) = expr else { +fn canonicalize_partition_number(value: &str) -> String { + value + .parse::() + .map(|value| value.to_string()) + .or_else(|_| value.parse::().map(|value| value.to_string())) + .unwrap_or_else(|_| value.to_string()) +} + +fn partition_assignment(expr: &SqlExpr) -> DFResult<(String, &SqlExpr)> { + let SqlExpr::BinaryOp { + left, + op: datafusion::sql::sqlparser::ast::BinaryOperator::Eq, + right, + } = expr + else { return Err(DataFusionError::Plan(format!( - "Unary signs are supported only for numeric partition literals, got: {expr}" + "Expected 'column = value' in partition spec, got: {expr}" ))); }; - let SqlValue::Number(value, _) = &value.value else { + let SqlExpr::Identifier(identifier) = left.as_ref() else { return Err(DataFusionError::Plan(format!( - "Unary signs are supported only for numeric partition literals, got: {expr}" + "Expected column name in partition spec, got: {left}" ))); }; - Ok(format!( - "{sign}{}", - canonicalize_integer_literal_if_possible(value) + Ok(( + IdentNormalizer::default().normalize(identifier.clone()), + right.as_ref(), )) } -fn canonicalize_integer_literal_if_possible(value: &str) -> String { - if let Ok(value) = value.parse::() { - value.to_string() - } else if let Ok(value) = value.parse::() { - value.to_string() - } else { - value.to_string() - } -} - -fn parse_format_partition_bool(value: &str) -> DFResult { - match value.trim().to_ascii_lowercase().as_str() { - "t" | "true" | "y" | "yes" | "1" => Ok(true), - "f" | "false" | "n" | "no" | "0" => Ok(false), - _ => Err(DataFusionError::Plan(format!( - "Invalid BOOLEAN partition value '{value}'" - ))), - } -} - -fn parse_spark_partition_date(value: &str) -> DFResult { - let value = value.trim(); - let bytes = value.as_bytes(); - if bytes.is_empty() { - return Err(DataFusionError::Plan("Invalid DATE ''".to_string())); - } - - let mut sign = 1_i32; - let mut offset = 0; - if matches!(bytes[0], b'+' | b'-') { - sign = if bytes[0] == b'-' { -1 } else { 1 }; - offset = 1; - } - - let mut segments = [1_i32, 1_i32, 1_i32]; - let mut segment_index = 0; - let mut segment_value = 0_i32; - let mut segment_digits = 0_usize; - while offset < bytes.len() && segment_index < 3 && !matches!(bytes[offset], b' ' | b'T') { - let byte = bytes[offset]; - if segment_index < 2 && byte == b'-' { - if !valid_spark_date_segment(segment_index, segment_digits) { - return Err(DataFusionError::Plan(format!("Invalid DATE '{value}'"))); - } - segments[segment_index] = segment_value; - segment_index += 1; - segment_value = 0; - segment_digits = 0; - } else if byte.is_ascii_digit() { - segment_value = segment_value - .checked_mul(10) - .and_then(|current| current.checked_add(i32::from(byte - b'0'))) - .ok_or_else(|| DataFusionError::Plan(format!("Invalid DATE '{value}'")))?; - segment_digits += 1; - } else { - return Err(DataFusionError::Plan(format!("Invalid DATE '{value}'"))); - } - offset += 1; - } - - if !valid_spark_date_segment(segment_index, segment_digits) - || (segment_index < 2 && offset < bytes.len()) - { - return Err(DataFusionError::Plan(format!("Invalid DATE '{value}'"))); - } - segments[segment_index] = segment_value; - - let year = sign * segments[0]; - if !(0..=9999).contains(&year) { - return Err(DataFusionError::Plan(format!( - "DATE '{value}' is outside Paimon DATE range 0000-01-01..9999-12-31" - ))); - } - let date = chrono::NaiveDate::from_ymd_opt(year, segments[1] as u32, segments[2] as u32) - .ok_or_else(|| DataFusionError::Plan(format!("Invalid DATE '{value}'")))?; - i32::try_from((date - chrono::NaiveDate::default()).num_days()).map_err(|error| { - DataFusionError::Plan(format!( - "DATE '{value}' is outside the supported partition range: {error}" - )) - }) -} - -fn valid_spark_date_segment(segment_index: usize, digits: usize) -> bool { - if segment_index == 0 { - (4..=7).contains(&digits) - } else { - (1..=2).contains(&digits) - } -} - -fn format_partition_date(epoch_days: i32) -> DFResult { - let ce_days = epoch_days.checked_add(719_163).ok_or_else(|| { - DataFusionError::Plan(format!( - "DATE partition value {epoch_days} is outside the supported range" - )) - })?; - Ok(chrono::NaiveDate::from_num_days_from_ce_opt(ce_days) - .ok_or_else(|| { - DataFusionError::Plan(format!( - "DATE partition value {epoch_days} is outside the supported range" - )) - })? - .format("%Y-%m-%d") - .to_string()) -} - -fn format_partition_datum_for_storage( - datum: &Datum, - data_type: &PaimonDataType, - core_options: &CoreOptions<'_>, -) -> DFResult { - match (datum, data_type) { - (Datum::String(value), PaimonDataType::Char(_) | PaimonDataType::VarChar(_)) => { - if value.trim().is_empty() { - Ok(core_options.partition_default_name().to_string()) - } else { - Ok(value.clone()) - } - } - (Datum::Bool(value), PaimonDataType::Boolean(_)) => Ok(value.to_string()), - (Datum::TinyInt(value), PaimonDataType::TinyInt(_)) => Ok(value.to_string()), - (Datum::SmallInt(value), PaimonDataType::SmallInt(_)) => Ok(value.to_string()), - (Datum::Int(value), PaimonDataType::Int(_)) => Ok(value.to_string()), - (Datum::Long(value), PaimonDataType::BigInt(_)) => Ok(value.to_string()), - (Datum::Date(value), PaimonDataType::Date(_)) => { - if core_options.legacy_partition_name() { - Ok(value.to_string()) - } else { - format_partition_date(*value) - } - } - _ => Err(DataFusionError::NotImplemented(format!( - "Partition literals of type {data_type:?} are not implemented yet" - ))), - } -} - -#[derive(Debug, Clone, Copy)] -enum FormatPartitionValueSource { - SqlLiteral, - CatalogMetadata, -} - -fn normalize_format_partition_spec_values( - spec: &HashMap, - table: &paimon::Table, - source: FormatPartitionValueSource, - require_complete: bool, -) -> DFResult>> { - let partition_keys = table.schema().partition_keys(); - if spec.keys().any(|key| !partition_keys.contains(key)) { - return Err(DataFusionError::Plan(format!( - "Invalid partition spec {spec:?} for table {}: partition keys are {partition_keys:?}", - table.identifier().full_name() - ))); +fn normalize_catalog_partition_spec( + spec: &HashMap, + table: &paimon::Table, + require_complete: bool, +) -> DFResult>> { + let partition_keys = table.schema().partition_keys(); + if spec.keys().any(|key| !partition_keys.contains(key)) { + return Err(DataFusionError::Plan(format!( + "Invalid partition spec {spec:?} for table {}", + table.identifier().full_name() + ))); } if require_complete && (spec.len() != partition_keys.len() || partition_keys.iter().any(|key| !spec.contains_key(key))) { return Err(DataFusionError::Plan(format!( - "Invalid partition spec {spec:?} for table {}: expected exactly {partition_keys:?}", + "Invalid partition spec {spec:?} for table {}: expected keys {partition_keys:?}", table.identifier().full_name() ))); } + let fields = table .schema() .partition_fields() .into_iter() .map(|field| (field.name().to_string(), field)) .collect::>(); - let core_options = CoreOptions::new(table.schema().options()); - let default_partition_name = core_options.partition_default_name(); + let options = CoreOptions::new(table.schema().options()); + let default_partition_name = options.partition_default_name(); let mut normalized = HashMap::with_capacity(spec.len()); for key in partition_keys.iter().filter(|key| spec.contains_key(*key)) { let raw = &spec[key]; @@ -3620,133 +3257,57 @@ fn normalize_format_partition_spec_values( } else { let field = fields.get(key).ok_or_else(|| { DataFusionError::Plan(format!( - "Partition column '{key}' is missing from table {} schema", - table.identifier().full_name() + "Partition column '{key}' is missing from table schema" )) })?; - Some(match source { - FormatPartitionValueSource::CatalogMetadata => { - normalize_catalog_format_partition_value(raw, key, field.data_type())? - } - FormatPartitionValueSource::SqlLiteral => normalize_sql_format_partition_value( - raw, - key, - field.data_type(), - &core_options, - )?, - }) + Some( + parse_format_partition_value(raw, field.data_type()).ok_or_else(|| { + DataFusionError::Plan(format!( + "Invalid catalog partition value {raw:?} for column '{key}' with type {:?}", + field.data_type() + )) + })?, + ) }; normalized.insert(key.clone(), value); } Ok(normalized) } -fn normalize_sql_format_partition_value( - raw: &str, - key: &str, - data_type: &PaimonDataType, - core_options: &CoreOptions<'_>, -) -> DFResult { - match data_type { - PaimonDataType::Char(char_type) => { - normalize_format_partition_string(raw, char_type.length(), true, "CHAR") - } - PaimonDataType::VarChar(varchar_type) => { - normalize_format_partition_string(raw, varchar_type.length() as usize, false, "VARCHAR") - } - PaimonDataType::Boolean(_) => Ok(parse_format_partition_bool(raw)?.to_string()), - PaimonDataType::TinyInt(_) => raw - .trim() - .parse::() - .map(|value| value.to_string()) - .map_err(|error| { - DataFusionError::Plan(format!( - "Invalid TINYINT partition value '{raw}' for column '{key}': {error}" - )) - }), - PaimonDataType::SmallInt(_) => raw - .trim() - .parse::() - .map(|value| value.to_string()) - .map_err(|error| { - DataFusionError::Plan(format!( - "Invalid SMALLINT partition value '{raw}' for column '{key}': {error}" - )) - }), - PaimonDataType::Int(_) => raw - .trim() - .parse::() - .map(|value| value.to_string()) - .map_err(|error| { - DataFusionError::Plan(format!( - "Invalid INT partition value '{raw}' for column '{key}': {error}" - )) - }), - PaimonDataType::BigInt(_) => raw - .trim() - .parse::() - .map(|value| value.to_string()) - .map_err(|error| { - DataFusionError::Plan(format!( - "Invalid BIGINT partition value '{raw}' for column '{key}': {error}" - )) - }), - PaimonDataType::Date(_) => { - let epoch_days = if core_options.legacy_partition_name() { - raw.trim().parse::().map_err(|error| { - DataFusionError::Plan(format!( - "Invalid legacy DATE partition value '{raw}' for column '{key}': {error}" - )) - })? - } else { - parse_spark_partition_date(raw)? +fn display_partition_spec( + spec: &HashMap>, + table: &paimon::Table, +) -> DFResult> { + let fields = table + .schema() + .partition_fields() + .into_iter() + .map(|field| (field.name().to_string(), field)) + .collect::>(); + let options = CoreOptions::new(table.schema().options()); + let default_partition_name = options.partition_default_name(); + spec.iter() + .map(|(key, value)| { + let value = match value { + None => "null".to_string(), + Some(datum) => { + let field = fields.get(key).ok_or_else(|| { + DataFusionError::Plan(format!( + "Partition column '{key}' is missing from table schema" + )) + })?; + format_partition_value(datum, field.data_type(), default_partition_name, false) + .ok_or_else(|| { + DataFusionError::NotImplemented(format!( + "SHOW PARTITIONS does not support type {:?}", + field.data_type() + )) + })? + } }; - format_partition_date(epoch_days) - } - other => Err(DataFusionError::NotImplemented(format!( - "SHOW/DROP PARTITION values of type {other:?} are not implemented yet" - ))), - } -} - -fn normalize_catalog_format_partition_value( - raw: &str, - key: &str, - data_type: &PaimonDataType, -) -> DFResult { - match data_type { - PaimonDataType::Char(_) - | PaimonDataType::VarChar(_) - | PaimonDataType::Boolean(_) - | PaimonDataType::TinyInt(_) - | PaimonDataType::SmallInt(_) - | PaimonDataType::Int(_) - | PaimonDataType::BigInt(_) - | PaimonDataType::Date(_) => {} - other => { - return Err(DataFusionError::NotImplemented(format!( - "SHOW/DROP PARTITION values of type {other:?} are not implemented yet" - ))) - } - } - - match (data_type, parse_format_partition_value(raw, data_type)) { - (PaimonDataType::Char(_) | PaimonDataType::VarChar(_), Some(Datum::String(value))) => { - Ok(value) - } - (PaimonDataType::Boolean(_), Some(Datum::Bool(value))) => Ok(value.to_string()), - (PaimonDataType::TinyInt(_), Some(Datum::TinyInt(value))) => Ok(value.to_string()), - (PaimonDataType::SmallInt(_), Some(Datum::SmallInt(value))) => Ok(value.to_string()), - (PaimonDataType::Int(_), Some(Datum::Int(value))) => Ok(value.to_string()), - (PaimonDataType::BigInt(_), Some(Datum::Long(value))) => Ok(value.to_string()), - (PaimonDataType::Date(_), Some(Datum::Date(value))) => format_partition_date(value), - (_, None) => Err(DataFusionError::Plan(format!( - "Invalid catalog partition value {raw:?} for column '{key}' with type {data_type:?}" - ))), - (_, Some(datum)) => Err(DataFusionError::Internal(format!( - "Format partition parser returned {datum:?} for column '{key}' with type {data_type:?}" - ))), - } + Ok((key.clone(), value)) + }) + .collect() } /// Parse partition expressions (`col = val, ...`) into partition value maps @@ -3764,30 +3325,9 @@ fn parse_partition_values( let mut partition = HashMap::new(); for expr in exprs { - let (col_name, val_expr) = match expr { - SqlExpr::BinaryOp { - left, - op: datafusion::sql::sqlparser::ast::BinaryOperator::Eq, - right, - } => { - let col = match left.as_ref() { - SqlExpr::Identifier(ident) => ident.value.clone(), - other => { - return Err(DataFusionError::Plan(format!( - "Expected column name in partition spec, got: {other}" - ))) - } - }; - (col, right.as_ref()) - } - other => { - return Err(DataFusionError::Plan(format!( - "Expected 'column = value' in partition spec, got: {other}" - ))) - } - }; + let (col_name, val_expr) = partition_assignment(expr)?; - if !partition_keys.iter().any(|k| k == &col_name) { + if !partition_keys.contains(&col_name) { return Err(DataFusionError::Plan(format!( "Column '{col_name}' is not a partition column" ))); @@ -3940,12 +3480,13 @@ fn sql_expr_to_datum(expr: &SqlExpr, data_type: &PaimonDataType) -> DFResult DFResult { let date = chrono::NaiveDate::parse_from_str(value, "%Y-%m-%d") .map_err(|e| DataFusionError::Plan(format!("Invalid DATE '{value}': {e}")))?; - let epoch_days = - i32::try_from((date - chrono::NaiveDate::default()).num_days()).map_err(|e| { - DataFusionError::Plan(format!( - "DATE '{value}' is outside the supported partition range: {e}" - )) - })?; + let unix_epoch = + chrono::NaiveDate::from_ymd_opt(1970, 1, 1).expect("1970-01-01 is a valid date"); + let epoch_days = i32::try_from((date - unix_epoch).num_days()).map_err(|e| { + DataFusionError::Plan(format!( + "DATE '{value}' is outside the supported partition range: {e}" + )) + })?; Ok(Datum::Date(epoch_days)) } @@ -4398,14 +3939,10 @@ mod tests { use async_trait::async_trait; use datafusion::arrow::array::StringViewArray; use paimon::catalog::Database; - use paimon::common::Options; - use paimon::io::FileIO; use paimon::spec::{ - DataField as PaimonDataField, DataType as PaimonDataType, IntType, - Partition as PaimonPartition, Schema as PaimonSchema, TableSchema, TimeType, + DataField as PaimonDataField, DataType as PaimonDataType, IntType, Schema as PaimonSchema, }; - use paimon::table::{RESTEnv, Table}; - use paimon::{CatalogOptions, RESTApi}; + use paimon::table::Table; // ==================== Mock Catalog ==================== @@ -4434,12 +3971,6 @@ mod tests { existing_table: Mutex>, functions: Mutex>, views: Mutex>, - partition_specs: Mutex>>, - get_table_calls: Mutex, - list_partitions_calls: Mutex, - list_partitions_by_names_calls: Mutex>>>, - partition_mutation_calls: Mutex>, - fail_list_partitions: Mutex, drop_view_supported: bool, } @@ -4450,12 +3981,6 @@ mod tests { existing_table: Mutex::new(None), functions: Mutex::new(HashMap::new()), views: Mutex::new(HashMap::new()), - partition_specs: Mutex::new(Vec::new()), - get_table_calls: Mutex::new(0), - list_partitions_calls: Mutex::new(0), - list_partitions_by_names_calls: Mutex::new(Vec::new()), - partition_mutation_calls: Mutex::new(Vec::new()), - fail_list_partitions: Mutex::new(false), drop_view_supported: true, } } @@ -4484,55 +4009,6 @@ mod tests { .unwrap() .insert(view.identifier().clone(), view); } - - fn set_table(&self, table: Table) { - *self.existing_table.lock().unwrap() = Some(table); - } - - fn set_partition_specs(&self, specs: Vec>) { - *self.partition_specs.lock().unwrap() = specs; - } - - fn partition_specs(&self) -> Vec> { - self.partition_specs.lock().unwrap().clone() - } - - fn get_table_calls(&self) -> usize { - *self.get_table_calls.lock().unwrap() - } - - fn list_partitions_calls(&self) -> usize { - *self.list_partitions_calls.lock().unwrap() - } - - fn list_partitions_by_names_calls(&self) -> Vec>> { - self.list_partitions_by_names_calls.lock().unwrap().clone() - } - - fn partition_mutation_calls(&self) -> Vec<&'static str> { - self.partition_mutation_calls.lock().unwrap().clone() - } - - fn fail_list_partitions(&self) { - *self.fail_list_partitions.lock().unwrap() = true; - } - } - - fn partition_from_spec(spec: HashMap) -> PaimonPartition { - PaimonPartition { - spec, - record_count: 0, - file_size_in_bytes: 0, - file_count: 0, - last_file_creation_time: 0, - total_buckets: 0, - done: false, - created_at: None, - created_by: None, - updated_at: None, - updated_by: None, - options: None, - } } #[async_trait] @@ -4559,13 +4035,12 @@ mod tests { ) -> paimon::Result<()> { Ok(()) } - async fn get_table(&self, identifier: &Identifier) -> paimon::Result
    { - *self.get_table_calls.lock().unwrap() += 1; + async fn get_table(&self, _identifier: &Identifier) -> paimon::Result
    { if let Some(table) = self.existing_table.lock().unwrap().clone() { return Ok(table); } Err(paimon::Error::TableNotExist { - full_name: identifier.to_string(), + full_name: _identifier.to_string(), }) } async fn list_tables(&self, _database_name: &str) -> paimon::Result> { @@ -4618,82 +4093,6 @@ mod tests { Ok(()) } - async fn create_partitions( - &self, - _identifier: &Identifier, - partitions: Vec>, - ignore_if_exists: bool, - ) -> paimon::Result<()> { - self.partition_mutation_calls.lock().unwrap().push("create"); - let mut current = self.partition_specs.lock().unwrap(); - if !ignore_if_exists && partitions.iter().any(|spec| current.contains(spec)) { - return Err(paimon::Error::DataInvalid { - message: "Some partitions already exist".to_string(), - source: None, - }); - } - for spec in partitions { - if !current.contains(&spec) { - current.push(spec); - } - } - Ok(()) - } - - async fn unregister_partitions( - &self, - _identifier: &Identifier, - partitions: Vec>, - ) -> paimon::Result<()> { - self.partition_mutation_calls.lock().unwrap().push("drop"); - self.partition_specs - .lock() - .unwrap() - .retain(|spec| !partitions.contains(spec)); - Ok(()) - } - - async fn list_partitions_by_names( - &self, - _identifier: &Identifier, - partitions: Vec>, - ) -> paimon::Result> { - self.list_partitions_by_names_calls - .lock() - .unwrap() - .push(partitions.clone()); - Ok(self - .partition_specs - .lock() - .unwrap() - .iter() - .filter(|spec| partitions.contains(spec)) - .cloned() - .map(partition_from_spec) - .collect()) - } - - async fn list_partitions( - &self, - _identifier: &Identifier, - ) -> paimon::Result> { - *self.list_partitions_calls.lock().unwrap() += 1; - if *self.fail_list_partitions.lock().unwrap() { - return Err(paimon::Error::UnexpectedError { - message: "Injected partition listing failure".to_string(), - source: None, - }); - } - Ok(self - .partition_specs - .lock() - .unwrap() - .iter() - .cloned() - .map(partition_from_spec) - .collect()) - } - async fn list_functions(&self, database_name: &str) -> paimon::Result> { Ok(self .functions @@ -4807,128 +4206,6 @@ mod tests { ctx } - async fn managed_format_table( - identifier: Identifier, - location: &str, - partition_keys: &[&str], - ) -> Table { - managed_format_table_with_partition_fields( - identifier, - location, - partition_keys - .iter() - .map(|key| { - ( - *key, - PaimonDataType::VarChar(VarCharType::new(255).unwrap()), - ) - }) - .collect(), - ) - .await - } - - async fn managed_format_table_with_partition_fields( - identifier: Identifier, - location: &str, - partition_fields: Vec<(&str, PaimonDataType)>, - ) -> Table { - managed_format_table_with_partition_fields_and_options( - identifier, - location, - partition_fields, - &[], - ) - .await - } - - async fn managed_format_table_with_partition_fields_and_options( - identifier: Identifier, - location: &str, - partition_fields: Vec<(&str, PaimonDataType)>, - extra_options: &[(&str, &str)], - ) -> Table { - let partition_keys = partition_fields - .iter() - .map(|(key, _)| *key) - .collect::>(); - let mut builder = PaimonSchema::builder() - .column("id", PaimonDataType::Int(IntType::new())) - .option("type", "format-table") - .option("metastore.partitioned-table", "true") - .option("path", location); - for (key, data_type) in partition_fields { - builder = builder.column(key, data_type); - } - for (key, value) in extra_options { - builder = builder.option(*key, *value); - } - let schema = builder.partition_keys(partition_keys).build().unwrap(); - let table_schema = TableSchema::new(0, &schema); - let file_io = FileIO::from_path(location).unwrap().build().unwrap(); - - let mut options = Options::new(); - options.set(CatalogOptions::URI, "http://127.0.0.1:1"); - options.set("token.provider", "bear"); - options.set("token", "test-token"); - let api = Arc::new(RESTApi::new(options.clone(), false).await.unwrap()); - let rest_env = RESTEnv::new_for_table( - identifier.clone(), - "test-uuid".to_string(), - api, - options, - false, - &table_schema, - false, - ) - .unwrap(); - Table::new( - file_io, - identifier, - location.to_string(), - table_schema, - Some(rest_env), - ) - } - - fn parse_partition_literal(literal: &str) -> SqlExpr { - Parser::new(&GenericDialect {}) - .try_with_sql(literal) - .unwrap() - .parse_expr() - .unwrap() - } - - fn format_partition_literal_for_test( - literal: &str, - data_type: &PaimonDataType, - extra_options: &[(&str, &str)], - ) -> DFResult { - let expr = parse_partition_literal(literal); - let options = extra_options - .iter() - .map(|(key, value)| ((*key).to_string(), (*value).to_string())) - .collect::>(); - let core_options = CoreOptions::new(&options); - match format_partition_literal_to_datum(&expr, data_type)? { - None => Ok(core_options.partition_default_name().to_string()), - Some(datum) => format_partition_datum_for_storage(&datum, data_type, &core_options), - } - } - - async fn show_partition_values(sql_context: &SQLContext, sql: &str) -> Vec { - let batches = sql_context.sql(sql).await.unwrap().collect().await.unwrap(); - assert_eq!(batches[0].schema().field(0).name(), "partition"); - let partitions = batches[0] - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - (0..batches[0].num_rows()) - .map(|index| partitions.value(index).to_string()) - .collect() - } - fn add_unary_sql_function( catalog: &MockCatalog, name: &str, @@ -7705,1195 +6982,21 @@ mod tests { } #[tokio::test] - async fn test_partition_commands_report_missing_catalog_table() { + async fn test_create_external_table_rejected() { let catalog = Arc::new(MockCatalog::new()); let sql_context = make_sql_context(catalog).await; - - for sql in [ - "MSCK REPAIR TABLE mydb.missing", - "REPAIR TABLE mydb.missing", - "SHOW PARTITIONS mydb.missing", - "ALTER TABLE mydb.missing ADD PARTITION (dt = '2026-07-22')", - ] { - let error = sql_context.sql(sql).await.unwrap_err(); - assert!( - error - .to_string() - .contains("Table mydb.missing does not exist"), - "expected catalog table error for {sql}, got: {error}" - ); - } + let result = sql_context + .sql("CREATE EXTERNAL TABLE mydb.t1 (id INT) STORED AS PARQUET") + .await; + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("CREATE EXTERNAL TABLE is not supported")); } #[tokio::test] - async fn test_msck_without_repair_is_rejected_before_catalog_access() { - let catalog = Arc::new(MockCatalog::new()); - let sql_context = make_sql_context(catalog.clone()).await; - - let error = sql_context.sql("MSCK TABLE mydb.events").await.unwrap_err(); - - assert!( - error - .to_string() - .contains("MSCK requires the REPAIR keyword"), - "expected missing REPAIR error, got: {error}" - ); - assert_eq!(catalog.get_table_calls(), 0); - } - - #[tokio::test] - async fn test_repair_table_syntaxes_reject_branch_target_before_catalog_access() { - let catalog = Arc::new(MockCatalog::new()); - let sql_context = make_sql_context(catalog.clone()).await; - - for sql in [ - "MSCK REPAIR TABLE mydb.events$branch_dev", - "REPAIR TABLE mydb.events$branch_dev", - ] { - let error = sql_context.sql(sql).await.unwrap_err(); - assert!( - error - .to_string() - .contains("MSCK REPAIR TABLE on Paimon branch 'dev' is not supported"), - "expected branch write rejection for {sql}, got: {error}" - ); - } - assert_eq!(catalog.get_table_calls(), 0); - } - - #[test] - fn test_parse_show_partitions_with_partition_filter() { - let statement = parse_show_partitions( - "SHOW PARTITIONS mydb.events PARTITION (dt = '2026-07-22', region = 3)", - ) - .unwrap() - .unwrap(); - - assert_eq!(statement.table_name.to_string(), "mydb.events"); - assert_eq!( - statement - .partition_filter - .iter() - .map(ToString::to_string) - .collect::>(), - vec!["dt = '2026-07-22'", "region = 3"] - ); - } - - #[test] - fn test_parse_spark_multi_drop_partitions() { - let statements = parse_sql_statements( - "ALTER TABLE mydb.events \ - DROP IF EXISTS PARTITION (dt = '2026-07-21'), \ - PARTITION (dt = '2026-07-22')", - ) - .unwrap(); - let [Statement::AlterTable(alter_table)] = statements.as_slice() else { - panic!("expected ALTER TABLE"); - }; - - assert_eq!(alter_table.operations.len(), 2); - assert!(alter_table.operations.iter().all(|operation| matches!( - operation, - AlterTableOperation::DropPartitions { - if_exists: true, - .. - } - ))); - } - - #[test] - fn test_catalog_partition_normalization_rejects_unsupported_type() { - let error = normalize_catalog_format_partition_value( - "1234", - "event_time", - &PaimonDataType::Time(TimeType::new(0).unwrap()), - ) - .unwrap_err(); - - assert!(matches!(error, DataFusionError::NotImplemented(_))); - } - - #[tokio::test] - async fn test_show_partitions_formats_sorts_and_filters_catalog_registrations() { - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields_and_options( - Identifier::new("mydb", "events"), - "memory:/show_partitions", - vec![ - ("dt", PaimonDataType::Date(DateType::new())), - ("month", PaimonDataType::Int(IntType::new())), - ], - &[("partition.legacy-name", "false")], - ) - .await, - ); - catalog.set_partition_specs(vec![ - HashMap::from([ - ("dt".to_string(), "20656".to_string()), - ("month".to_string(), "01".to_string()), - ]), - HashMap::from([ - ("dt".to_string(), "20656".to_string()), - ("month".to_string(), "1".to_string()), - ]), - HashMap::from([ - ("dt".to_string(), "__DEFAULT_PARTITION__".to_string()), - ("month".to_string(), "02".to_string()), - ]), - HashMap::from([ - ("dt".to_string(), "20655".to_string()), - ("month".to_string(), "09".to_string()), - ]), - ]); - let sql_context = make_sql_context(catalog).await; - - assert_eq!( - show_partition_values(&sql_context, "SHOW PARTITIONS mydb.events").await, - vec![ - "dt=2026-07-21/month=9", - "dt=2026-07-22/month=1", - "dt=2026-07-22/month=1", - "dt=null/month=2", - ] - ); - assert_eq!( - show_partition_values( - &sql_context, - "SHOW PARTITIONS mydb.events PARTITION (month = '1')", - ) - .await, - vec!["dt=2026-07-22/month=1", "dt=2026-07-22/month=1"] - ); - } - - #[tokio::test] - async fn test_add_format_partitions_registers_batch_and_creates_directories() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql( - "ALTER TABLE mydb.events ADD \ - PARTITION (dt = '2026-07-22') \ - PARTITION (dt = '2026-07-21')", - ) - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![ - HashMap::from([("dt".to_string(), "2026-07-22".to_string())]), - HashMap::from([("dt".to_string(), "2026-07-21".to_string())]), - ] - ); - assert!(temp_dir.path().join("dt=2026-07-22").is_dir()); - assert!(temp_dir.path().join("dt=2026-07-21").is_dir()); - } - - #[tokio::test] - async fn test_add_format_partition_respects_if_not_exists() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, - ); - let spec = HashMap::from([("dt".to_string(), "2026-07-22".to_string())]); - catalog.set_partition_specs(vec![spec.clone()]); - let sql_context = make_sql_context(catalog.clone()).await; - - let error = sql_context - .sql("ALTER TABLE mydb.events ADD PARTITION (dt = '2026-07-22')") - .await - .unwrap_err(); - assert!( - error.to_string().contains("already exist"), - "expected duplicate partition error, got: {error}" - ); - assert!(!temp_dir.path().join("dt=2026-07-22").exists()); - - sql_context - .sql( - "ALTER TABLE mydb.events \ - ADD IF NOT EXISTS PARTITION (dt = '2026-07-22')", - ) - .await - .unwrap(); - - assert_eq!(catalog.partition_specs(), vec![spec]); - assert!(temp_dir.path().join("dt=2026-07-22").is_dir()); - assert_eq!(catalog.partition_mutation_calls(), vec!["create", "create"]); - } - - #[test] - fn test_format_partition_literal_compatibility() { - let cases = [ - ( - "01", - PaimonDataType::VarChar(VarCharType::new(255).unwrap()), - &[][..], - "1", - ), - ( - "' us '", - PaimonDataType::VarChar(VarCharType::new(5).unwrap()), - &[][..], - " us ", - ), - ( - "'us'", - PaimonDataType::Char(CharType::new(2).unwrap()), - &[][..], - "us", - ), - ( - "'us'", - PaimonDataType::Char(CharType::new(4).unwrap()), - &[][..], - "us ", - ), - ( - "'数据'", - PaimonDataType::Char(CharType::new(2).unwrap()), - &[][..], - "数据", - ), - ( - "'a '", - PaimonDataType::Char(CharType::new(5).unwrap()), - &[][..], - "a ", - ), - ( - "'a '", - PaimonDataType::VarChar(VarCharType::new(5).unwrap()), - &[][..], - "a ", - ), - ( - "' '", - PaimonDataType::VarChar(VarCharType::new(5).unwrap()), - &[][..], - "__DEFAULT_PARTITION__", - ), - ("' 01 '", PaimonDataType::Int(IntType::new()), &[][..], "1"), - ( - "9223372036854775807", - PaimonDataType::BigInt(BigIntType::new()), - &[][..], - "9223372036854775807", - ), - ( - "-128", - PaimonDataType::TinyInt(TinyIntType::new()), - &[][..], - "-128", - ), - ( - "32767", - PaimonDataType::SmallInt(SmallIntType::new()), - &[][..], - "32767", - ), - ( - "'2026'", - PaimonDataType::Date(DateType::new()), - &[][..], - "20454", - ), - ( - "'2026-07'", - PaimonDataType::Date(DateType::new()), - &[][..], - "20635", - ), - ( - "DATE '2026-07-22'", - PaimonDataType::Date(DateType::new()), - &[][..], - "20656", - ), - ( - "DATE '2026'", - PaimonDataType::VarChar(VarCharType::new(10).unwrap()), - &[][..], - "2026-01-01", - ), - ( - "'2026-07-22'", - PaimonDataType::Date(DateType::new()), - &[("partition.legacy-name", "false")][..], - "2026-07-22", - ), - ]; - - for (literal, data_type, options, expected) in cases { - assert_eq!( - format_partition_literal_for_test(literal, &data_type, options).unwrap(), - expected, - "literal {literal} as {data_type:?}" - ); - } - - for (literal, expected) in [ - ("'t'", "true"), - ("'TRUE'", "true"), - ("' y '", "true"), - ("'yes'", "true"), - ("1", "true"), - ("'f'", "false"), - ("'FALSE'", "false"), - ("' n '", "false"), - ("'no'", "false"), - ("0", "false"), - ] { - assert_eq!( - format_partition_literal_for_test( - literal, - &PaimonDataType::Boolean(BooleanType::new()), - &[], - ) - .unwrap(), - expected, - "boolean literal {literal}" - ); - } - } - - #[test] - fn test_invalid_format_partition_literals() { - let cases = [ - ( - "+'us'", - PaimonDataType::VarChar(VarCharType::new(8).unwrap()), - "Unary signs are supported only for numeric partition literals", - ), - ( - "-'us'", - PaimonDataType::VarChar(VarCharType::new(8).unwrap()), - "Unary signs are supported only for numeric partition literals", - ), - ( - "'abcdef'", - PaimonDataType::Char(CharType::new(5).unwrap()), - "exceeds maximum length 5", - ), - ( - "'abcdef'", - PaimonDataType::VarChar(VarCharType::new(5).unwrap()), - "exceeds maximum length 5", - ), - ( - "'数据流'", - PaimonDataType::VarChar(VarCharType::new(2).unwrap()), - "exceeds maximum length 2", - ), - ( - "TIMESTAMP '2026-07-22'", - PaimonDataType::VarChar(VarCharType::new(32).unwrap()), - "Unsupported typed partition literal", - ), - ( - "20260722", - PaimonDataType::Date(DateType::new()), - "Invalid DATE '20260722'", - ), - ( - "'-0001-01-01'", - PaimonDataType::Date(DateType::new()), - "outside Paimon DATE range 0000-01-01..9999-12-31", - ), - ( - "'10000-01-01'", - PaimonDataType::Date(DateType::new()), - "outside Paimon DATE range 0000-01-01..9999-12-31", - ), - ]; - - for (literal, data_type, expected_error) in cases { - let error = format_partition_literal_for_test(literal, &data_type, &[]).unwrap_err(); - assert!( - error.to_string().contains(expected_error), - "unexpected error for literal {literal} as {data_type:?}: {error}" - ); - } - } - - #[tokio::test] - async fn test_add_format_partition_casts_typed_literals_and_creates_directory() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![ - ("month", PaimonDataType::Int(IntType::new())), - ( - "label", - PaimonDataType::VarChar(VarCharType::new(10).unwrap()), - ), - ("dt", PaimonDataType::Date(DateType::new())), - ], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql( - "ALTER TABLE mydb.events ADD PARTITION (\ - month = '01', label = DATE '2026', dt = DATE '2026-07-22')", - ) - .await - .unwrap(); - - let expected = HashMap::from([ - ("month".to_string(), "1".to_string()), - ("label".to_string(), "2026-01-01".to_string()), - ("dt".to_string(), "20656".to_string()), - ]); - assert_eq!(catalog.partition_specs(), vec![expected]); - assert!(temp_dir - .path() - .join("month=1/label=2026-01-01/dt=20656") - .is_dir()); - } - - #[tokio::test] - async fn test_add_format_partition_maps_null_to_default_partition_name() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("ALTER TABLE mydb.events ADD PARTITION (dt = NULL)") - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([( - "dt".to_string(), - "__DEFAULT_PARTITION__".to_string(), - )])] - ); - assert!(temp_dir.path().join("dt=__DEFAULT_PARTITION__").is_dir()); - } - - #[tokio::test] - async fn test_add_format_partition_rejects_compact_numeric_date_without_mutation() { - let temp_dir = tempfile::tempdir().unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![("dt", PaimonDataType::Date(DateType::new()))], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - let error = sql_context - .sql("ALTER TABLE mydb.events ADD PARTITION (dt = 20260722)") - .await - .unwrap_err(); - - assert!( - error.to_string().contains("Invalid DATE '20260722'"), - "expected invalid date error, got: {error}" - ); - assert!(catalog.partition_specs().is_empty()); - assert!(catalog.partition_mutation_calls().is_empty()); - assert!(!temp_dir.path().join("dt=20260722").exists()); - } - - #[tokio::test] - async fn test_add_format_partition_rejects_location_explicitly() { - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table( - Identifier::new("mydb", "events"), - "memory:/add_partition_location", - &["dt"], - ) - .await, - ); - let sql_context = make_sql_context(catalog).await; - - let error = sql_context - .sql( - "ALTER TABLE mydb.events ADD PARTITION (dt = '2026-07-22') \ - LOCATION 'memory:/other'", - ) - .await - .unwrap_err(); - - assert!( - error - .to_string() - .contains("LOCATION is not supported for Format Table partitions"), - "expected explicit LOCATION rejection, got: {error}" - ); - } - - #[tokio::test] - async fn test_drop_managed_format_partition_unregisters_then_deletes_directory() { - let temp_dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(temp_dir.path().join("dt=2026-07-22")).unwrap(); - std::fs::create_dir_all(temp_dir.path().join("dt=2026-07-21")).unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, - ); - catalog.set_partition_specs(vec![ - HashMap::from([("dt".to_string(), "2026-07-22".to_string())]), - HashMap::from([("dt".to_string(), "2026-07-21".to_string())]), - ]); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("ALTER TABLE mydb.events DROP PARTITION (dt = '2026-07-22')") - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([( - "dt".to_string(), - "2026-07-21".to_string() - )])] - ); - assert!(!temp_dir.path().join("dt=2026-07-22").exists()); - assert!(temp_dir.path().join("dt=2026-07-21").is_dir()); - } - - #[tokio::test] - async fn test_drop_complete_format_partition_uses_list_by_names() { - let temp_dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(temp_dir.path().join("dt=2026-07-22/hour=10")).unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table( - Identifier::new("mydb", "events"), - &location, - &["dt", "hour"], - ) - .await, - ); - let spec = HashMap::from([ - ("dt".to_string(), "2026-07-22".to_string()), - ("hour".to_string(), "10".to_string()), - ]); - catalog.set_partition_specs(vec![spec.clone()]); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql( - "ALTER TABLE mydb.events \ - DROP PARTITION (dt = '2026-07-22', hour = '10')", - ) - .await - .unwrap(); - - assert_eq!(catalog.list_partitions_calls(), 0); - assert_eq!(catalog.list_partitions_by_names_calls(), vec![vec![spec]]); - } - - #[tokio::test] - async fn test_drop_multiple_managed_format_partitions_uses_spark_syntax() { - let temp_dir = tempfile::tempdir().unwrap(); - for dt in ["2026-07-21", "2026-07-22"] { - std::fs::create_dir_all(temp_dir.path().join(format!("dt={dt}"))).unwrap(); - } - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, - ); - catalog.set_partition_specs(vec![ - HashMap::from([("dt".to_string(), "2026-07-21".to_string())]), - HashMap::from([("dt".to_string(), "2026-07-22".to_string())]), - ]); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql( - "ALTER TABLE mydb.events \ - DROP PARTITION (dt = '2026-07-21'), \ - PARTITION (dt = '2026-07-22')", - ) - .await - .unwrap(); - - assert!(catalog.partition_specs().is_empty()); - assert!(!temp_dir.path().join("dt=2026-07-21").exists()); - assert!(!temp_dir.path().join("dt=2026-07-22").exists()); - } - - #[tokio::test] - async fn test_drop_partial_managed_format_partition_expands_registered_matches() { - let temp_dir = tempfile::tempdir().unwrap(); - for (dt, region) in [ - ("2026-07-21", "us"), - ("2026-07-22", "us"), - ("2026-07-22", "eu"), - ] { - std::fs::create_dir_all(temp_dir.path().join(format!("dt={dt}/region={region}"))) - .unwrap(); - } - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table( - Identifier::new("mydb", "events"), - &location, - &["dt", "region"], - ) - .await, - ); - catalog.set_partition_specs(vec![ - HashMap::from([ - ("dt".to_string(), "2026-07-21".to_string()), - ("region".to_string(), "us".to_string()), - ]), - HashMap::from([ - ("dt".to_string(), "2026-07-22".to_string()), - ("region".to_string(), "us".to_string()), - ]), - HashMap::from([ - ("dt".to_string(), "2026-07-22".to_string()), - ("region".to_string(), "eu".to_string()), - ]), - ]); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("ALTER TABLE mydb.events DROP PARTITION (region = 'us')") - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([ - ("dt".to_string(), "2026-07-22".to_string()), - ("region".to_string(), "eu".to_string()), - ])] - ); - assert_eq!(catalog.list_partitions_calls(), 1); - assert!(!temp_dir.path().join("dt=2026-07-21/region=us").exists()); - assert!(!temp_dir.path().join("dt=2026-07-22/region=us").exists()); - assert!(temp_dir.path().join("dt=2026-07-22/region=eu").is_dir()); - } - - #[tokio::test] - async fn test_drop_complete_partition_matches_msck_value_by_type() { - let temp_dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(temp_dir.path().join("month=01")).unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![("month", PaimonDataType::Int(IntType::new()))], - ) - .await, - ); - catalog.set_partition_specs(vec![HashMap::from([( - "month".to_string(), - "01".to_string(), - )])]); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("ALTER TABLE mydb.events DROP PARTITION (month = 1)") - .await - .unwrap(); - - assert!(catalog.partition_specs().is_empty()); - assert_eq!(catalog.list_partitions_calls(), 1); - assert!(!temp_dir.path().join("month=01").exists()); - } - - #[tokio::test] - async fn test_drop_batch_preserves_exact_partition_aliases() { - let temp_dir = tempfile::tempdir().unwrap(); - for month in ["1", "01", "02"] { - std::fs::create_dir_all(temp_dir.path().join(format!("month={month}"))).unwrap(); - } - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![("month", PaimonDataType::Int(IntType::new()))], - ) - .await, - ); - catalog.set_partition_specs(vec![ - HashMap::from([("month".to_string(), "1".to_string())]), - HashMap::from([("month".to_string(), "01".to_string())]), - HashMap::from([("month".to_string(), "02".to_string())]), - ]); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql( - "ALTER TABLE mydb.events \ - DROP PARTITION (month = 1), \ - PARTITION (month = 2)", - ) - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([("month".to_string(), "01".to_string())])] - ); - assert!(!temp_dir.path().join("month=1").exists()); - assert!(temp_dir.path().join("month=01").is_dir()); - assert!(!temp_dir.path().join("month=02").exists()); - } - - #[tokio::test] - async fn test_drop_mixed_complete_and_partial_specs_keep_complete_match_exact() { - let temp_dir = tempfile::tempdir().unwrap(); - for (region, month) in [("us", "1"), ("us", "01"), ("eu", "02")] { - std::fs::create_dir_all( - temp_dir - .path() - .join(format!("region={region}/month={month}")), - ) - .unwrap(); - } - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![ - ( - "region", - PaimonDataType::VarChar(VarCharType::new(32).unwrap()), - ), - ("month", PaimonDataType::Int(IntType::new())), - ], - ) - .await, - ); - catalog.set_partition_specs(vec![ - HashMap::from([ - ("region".to_string(), "us".to_string()), - ("month".to_string(), "1".to_string()), - ]), - HashMap::from([ - ("region".to_string(), "us".to_string()), - ("month".to_string(), "01".to_string()), - ]), - HashMap::from([ - ("region".to_string(), "eu".to_string()), - ("month".to_string(), "02".to_string()), - ]), - ]); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql( - "ALTER TABLE mydb.events \ - DROP PARTITION (region = 'us', month = 1), \ - PARTITION (region = 'eu')", - ) - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([ - ("region".to_string(), "us".to_string()), - ("month".to_string(), "01".to_string()), - ])] - ); - assert!(!temp_dir.path().join("region=us/month=1").exists()); - assert!(temp_dir.path().join("region=us/month=01").is_dir()); - assert!(!temp_dir.path().join("region=eu/month=02").exists()); - } - - #[tokio::test] - async fn test_drop_rejects_whitespace_padded_catalog_integer() { - let temp_dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(temp_dir.path().join("month= 01")).unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![("month", PaimonDataType::Int(IntType::new()))], - ) - .await, - ); - catalog.set_partition_specs(vec![HashMap::from([( - "month".to_string(), - " 01".to_string(), - )])]); - let sql_context = make_sql_context(catalog.clone()).await; - - let error = sql_context - .sql("ALTER TABLE mydb.events DROP PARTITION (month = 1)") - .await - .unwrap_err(); - - assert!( - error - .to_string() - .contains("Invalid catalog partition value \" 01\" for column 'month'"), - "expected malformed catalog metadata error, got: {error}" - ); - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([("month".to_string(), " 01".to_string())])] - ); - assert_eq!(catalog.list_partitions_calls(), 1); - assert!(temp_dir.path().join("month= 01").is_dir()); - } - - #[tokio::test] - async fn test_drop_char_partition_requires_exact_catalog_value() { - let temp_dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(temp_dir.path().join("region=us")).unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![("region", PaimonDataType::Char(CharType::new(4).unwrap()))], - ) - .await, - ); - catalog.set_partition_specs(vec![HashMap::from([( - "region".to_string(), - "us".to_string(), - )])]); - let sql_context = make_sql_context(catalog.clone()).await; - - let error = sql_context - .sql("ALTER TABLE mydb.events DROP PARTITION (region = 'us')") - .await - .unwrap_err(); - - assert!( - error.to_string().contains("does not exist"), - "expected catalog CHAR value to remain distinct, got: {error}" - ); - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([("region".to_string(), "us".to_string())])] - ); - assert_eq!(catalog.list_partitions_calls(), 1); - assert!(temp_dir.path().join("region=us").is_dir()); - } - - #[tokio::test] - async fn test_drop_missing_complete_partition_requires_partition_if_exists() { - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table( - Identifier::new("mydb", "events"), - "memory:/drop_missing_partition", - &["dt"], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - let error = sql_context - .sql( - "ALTER TABLE IF EXISTS mydb.events \ - DROP PARTITION (dt = '2026-07-22')", - ) - .await - .unwrap_err(); - assert!( - error.to_string().contains("does not exist"), - "expected missing partition error, got: {error}" - ); - assert!(catalog.partition_mutation_calls().is_empty()); - - sql_context - .sql( - "ALTER TABLE mydb.events \ - DROP IF EXISTS PARTITION (dt = '2026-07-22')", - ) - .await - .unwrap(); - assert!(catalog.partition_mutation_calls().is_empty()); - } - - #[tokio::test] - async fn test_drop_format_partition_rejects_purge_before_catalog_access() { - let catalog = Arc::new(MockCatalog::new()); - let sql_context = make_sql_context(catalog.clone()).await; - - let error = sql_context - .sql( - "ALTER TABLE mydb.events \ - DROP PARTITION (dt = '2026-07-22') PURGE", - ) - .await - .unwrap_err(); - - assert!( - error - .to_string() - .contains("DROP PARTITION PURGE is not supported"), - "expected explicit PURGE rejection, got: {error}" - ); - assert_eq!(catalog.get_table_calls(), 0); - assert_eq!(catalog.list_partitions_calls(), 0); - assert!(catalog.list_partitions_by_names_calls().is_empty()); - } - - #[tokio::test] - async fn test_msck_add_preserves_filesystem_partition_values() { - let temp_dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(temp_dir.path().join("month=01")).unwrap(); - std::fs::create_dir_all(temp_dir.path().join("month=02")).unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![("month", PaimonDataType::Int(IntType::new()))], - ) - .await, - ); - catalog.set_partition_specs(vec![HashMap::from([( - "month".to_string(), - "02".to_string(), - )])]); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("MSCK REPAIR TABLE mydb.events") - .await - .unwrap(); - - let mut specs = catalog.partition_specs(); - specs.sort_by_key(|spec| spec["month"].clone()); - assert_eq!( - specs, - vec![ - HashMap::from([("month".to_string(), "01".to_string())]), - HashMap::from([("month".to_string(), "02".to_string())]), - ] - ); - assert!(temp_dir.path().join("month=01").is_dir()); - assert!(temp_dir.path().join("month=02").is_dir()); - } - - #[tokio::test] - async fn test_msck_rejects_non_round_tripping_partition_path_without_mutation() { - let temp_dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(temp_dir.path().join("dt=value%ZZ")).unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields( - Identifier::new("mydb", "events"), - &location, - vec![("dt", PaimonDataType::VarChar(VarCharType::new(32).unwrap()))], - ) - .await, - ); - let sql_context = make_sql_context(catalog.clone()).await; - - let error = sql_context - .sql("MSCK REPAIR TABLE mydb.events") - .await - .unwrap_err(); - - assert!( - error.to_string().contains("cannot round-trip"), - "expected a non-round-tripping path error, got: {error}" - ); - assert!(catalog.partition_specs().is_empty()); - assert!(catalog.partition_mutation_calls().is_empty()); - assert!(temp_dir.path().join("dt=value%ZZ").is_dir()); - } - - #[tokio::test] - async fn test_msck_listing_failure_performs_no_partition_mutation() { - let temp_dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(temp_dir.path().join("dt=2026-07-22")).unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, - ); - catalog.fail_list_partitions(); - let sql_context = make_sql_context(catalog.clone()).await; - - let error = sql_context - .sql("MSCK REPAIR TABLE mydb.events SYNC PARTITIONS") - .await - .unwrap_err(); - - assert!( - error - .to_string() - .contains("Injected partition listing failure"), - "expected listing failure, got: {error}" - ); - assert!(catalog.partition_mutation_calls().is_empty()); - assert!(catalog.partition_specs().is_empty()); - } - - #[tokio::test] - async fn test_msck_drop_removes_only_missing_filesystem_partitions() { - let temp_dir = tempfile::tempdir().unwrap(); - for dt in ["2026-07-21", "2026-07-22"] { - std::fs::create_dir_all(temp_dir.path().join(format!("dt={dt}"))).unwrap(); - } - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, - ); - catalog.set_partition_specs(vec![ - HashMap::from([("dt".to_string(), "2026-07-20".to_string())]), - HashMap::from([("dt".to_string(), "2026-07-21".to_string())]), - ]); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("MSCK REPAIR TABLE mydb.events DROP PARTITIONS") - .await - .unwrap(); - - assert_eq!( - catalog.partition_specs(), - vec![HashMap::from([( - "dt".to_string(), - "2026-07-21".to_string(), - )])] - ); - assert_eq!(catalog.partition_mutation_calls(), vec!["drop"]); - assert!(temp_dir.path().join("dt=2026-07-21").is_dir()); - assert!(temp_dir.path().join("dt=2026-07-22").is_dir()); - } - - #[tokio::test] - async fn test_msck_sync_adds_before_drop_and_rerun_is_idempotent() { - let temp_dir = tempfile::tempdir().unwrap(); - for dt in ["2026-07-21", "2026-07-22"] { - std::fs::create_dir_all(temp_dir.path().join(format!("dt={dt}"))).unwrap(); - } - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table(Identifier::new("mydb", "events"), &location, &["dt"]).await, - ); - catalog.set_partition_specs(vec![ - HashMap::from([("dt".to_string(), "2026-07-21".to_string())]), - HashMap::from([("dt".to_string(), "2026-07-20".to_string())]), - ]); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("MSCK REPAIR TABLE mydb.events SYNC PARTITIONS") - .await - .unwrap(); - let mut specs = catalog.partition_specs(); - specs.sort_by_key(|spec| spec["dt"].clone()); - assert_eq!( - specs, - vec![ - HashMap::from([("dt".to_string(), "2026-07-21".to_string())]), - HashMap::from([("dt".to_string(), "2026-07-22".to_string())]), - ] - ); - assert_eq!(catalog.partition_mutation_calls(), vec!["create", "drop"]); - assert!(temp_dir.path().join("dt=2026-07-21").is_dir()); - assert!(temp_dir.path().join("dt=2026-07-22").is_dir()); - - sql_context - .sql("MSCK REPAIR TABLE mydb.events SYNC PARTITIONS") - .await - .unwrap(); - assert_eq!(catalog.partition_mutation_calls(), vec!["create", "drop"]); - } - - #[tokio::test] - async fn test_msck_sync_keeps_value_only_default_partition_directory() { - let temp_dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(temp_dir.path().join("__DEFAULT_PARTITION__")).unwrap(); - let location = format!("file://{}", temp_dir.path().display()); - let catalog = Arc::new(MockCatalog::new()); - catalog.set_table( - managed_format_table_with_partition_fields_and_options( - Identifier::new("mydb", "events"), - &location, - vec![( - "dt", - PaimonDataType::VarChar(VarCharType::new(255).unwrap()), - )], - &[("format-table.partition-path-only-value", "true")], - ) - .await, - ); - let default_partition = - HashMap::from([("dt".to_string(), "__DEFAULT_PARTITION__".to_string())]); - catalog.set_partition_specs(vec![default_partition.clone()]); - let sql_context = make_sql_context(catalog.clone()).await; - - sql_context - .sql("MSCK REPAIR TABLE mydb.events SYNC PARTITIONS") - .await - .unwrap(); - - assert_eq!(catalog.partition_specs(), vec![default_partition]); - assert!(catalog.partition_mutation_calls().is_empty()); - assert!(temp_dir.path().join("__DEFAULT_PARTITION__").is_dir()); - } - - #[tokio::test] - async fn test_create_external_table_rejected() { - let catalog = Arc::new(MockCatalog::new()); - let sql_context = make_sql_context(catalog).await; - let result = sql_context - .sql("CREATE EXTERNAL TABLE mydb.t1 (id INT) STORED AS PARQUET") - .await; - assert!(result.is_err()); - assert!(result - .unwrap_err() - .to_string() - .contains("CREATE EXTERNAL TABLE is not supported")); - } - - #[tokio::test] - async fn test_non_ddl_delegates_to_datafusion() { + async fn test_non_ddl_delegates_to_datafusion() { let catalog = Arc::new(MockCatalog::new()); let sql_context = make_sql_context(catalog.clone()).await; // SELECT should be delegated to DataFusion, not intercepted @@ -9434,6 +7537,20 @@ mod tests { .unwrap(); } + #[tokio::test] + async fn test_drop_if_exists_partition_does_not_ignore_missing_table() { + let (_tmp, sql_context) = setup_fs_sql_context().await; + + let err = sql_context + .sql("ALTER TABLE paimon.test_db.nonexistent DROP IF EXISTS PARTITION (pt = 'a')") + .await + .unwrap_err(); + assert!( + err.to_string().contains("does not exist"), + "Expected table-not-exist error, got: {err}" + ); + } + #[tokio::test] async fn test_drop_partition_incomplete_spec() { let (_tmp, sql_context) = setup_fs_sql_context().await; diff --git a/crates/integrations/datafusion/tests/rest_format_partition_sql.rs b/crates/integrations/datafusion/tests/rest_format_partition_sql.rs index 99694cf2..cb0cdff9 100644 --- a/crates/integrations/datafusion/tests/rest_format_partition_sql.rs +++ b/crates/integrations/datafusion/tests/rest_format_partition_sql.rs @@ -84,13 +84,6 @@ fn format_table_schema(partition_columns: &[(&str, DataType)]) -> Schema { .unwrap() } -fn partition_spec(values: &[(&str, &str)]) -> HashMap { - values - .iter() - .map(|(key, value)| ((*key).to_string(), (*value).to_string())) - .collect() -} - fn assert_partition_directories(temp_dir: &TempDir, expected: &[(&str, bool)]) { for (path, exists) in expected { assert_eq!( @@ -117,7 +110,7 @@ async fn test_partition_commands_update_rest_metadata_and_directories() { assert_partition_directories(&temp_dir, &[("dt=2026-07-22", true)]); context - .sql("REPAIR TABLE paimon.default.events ADD PARTITIONS") + .sql("MSCK REPAIR TABLE paimon.default.events ADD PARTITIONS") .await .unwrap(); assert_partitions(&context, &["dt=2026-07-21", "dt=2026-07-22"]).await; @@ -134,11 +127,11 @@ async fn test_partition_commands_update_rest_metadata_and_directories() { .await .unwrap(); context - .sql( - "ALTER TABLE paimon.default.events \ - DROP PARTITION (dt = '2026-07-21'), \ - PARTITION (dt = '2026-07-22')", - ) + .sql("ALTER TABLE paimon.default.events DROP PARTITION (dt = '2026-07-21')") + .await + .unwrap(); + context + .sql("ALTER TABLE paimon.default.events DROP PARTITION (dt = '2026-07-22')") .await .unwrap(); assert_partitions(&context, &[]).await; @@ -150,100 +143,57 @@ async fn test_partition_commands_update_rest_metadata_and_directories() { #[cfg(not(windows))] #[tokio::test] -async fn test_partition_values_and_drop_matching() { - { - let temp_dir = tempfile::tempdir().unwrap(); - let schema = format_table_schema(&[ - ("dt", DataType::Date(DateType::new())), - ("month", DataType::Int(IntType::new())), - ("active", DataType::Boolean(BooleanType::new())), - ("label", DataType::VarChar(VarCharType::new(255).unwrap())), - ]); - let (_server, context) = setup_rest_table(&temp_dir, schema).await; - - context - .sql( - "ALTER TABLE paimon.default.events ADD PARTITION (\ - dt = '2026', month = '01', active = 'yes', label = 01)", - ) - .await - .unwrap(); - assert_partitions(&context, &["dt=2026-01-01/month=1/active=true/label=1"]).await; - assert_partition_directories(&temp_dir, &[("dt=20454/month=1/active=true/label=1", true)]); - - context - .sql("ALTER TABLE paimon.default.events DROP PARTITION (active = 'yes')") - .await - .unwrap(); - context - .sql( - "ALTER TABLE paimon.default.events ADD PARTITION (\ - dt = NULL, month = NULL, active = NULL, label = NULL)", - ) - .await - .unwrap(); - assert_partitions(&context, &["dt=null/month=null/active=null/label=null"]).await; - assert_partition_directories( - &temp_dir, - &[( - "dt=__DEFAULT_PARTITION__/month=__DEFAULT_PARTITION__/\ - active=__DEFAULT_PARTITION__/label=__DEFAULT_PARTITION__", - true, - )], - ); - } - +async fn test_partition_literals_and_default_path() { let temp_dir = tempfile::tempdir().unwrap(); - for path in [ - "region=us/month=1", - "region=us/month=01", - "region=eu/month=02", - ] { - std::fs::create_dir_all(temp_dir.path().join(path)).unwrap(); - } let schema = format_table_schema(&[ - ("region", DataType::VarChar(VarCharType::new(32).unwrap())), + ("dt", DataType::Date(DateType::new())), ("month", DataType::Int(IntType::new())), + ("active", DataType::Boolean(BooleanType::new())), + ("label", DataType::VarChar(VarCharType::new(255).unwrap())), ]); - let (server, context) = setup_rest_table(&temp_dir, schema).await; - server.set_table_partitions( - DATABASE, - TABLE, - vec![ - partition_spec(&[("region", "us"), ("month", "1")]), - partition_spec(&[("region", "us"), ("month", "01")]), - partition_spec(&[("region", "eu"), ("month", "02")]), - ], - ); + let (_server, context) = setup_rest_table(&temp_dir, schema).await; context .sql( - "ALTER TABLE paimon.default.events \ - DROP PARTITION (region = 'us', month = 1), \ - PARTITION (region = 'eu')", + "ALTER TABLE paimon.default.events ADD PARTITION (\ + dt = DATE '2026-07-22', month = '01', active = 'TRUE', label = 20260722)", ) .await .unwrap(); - - assert_partitions(&context, &["region=us/month=1"]).await; + assert_partitions( + &context, + &["dt=2026-07-22/month=1/active=true/label=20260722"], + ) + .await; + // Non-legacy DATE partition paths use Unix epoch days. assert_partition_directories( &temp_dir, - &[ - ("region=us/month=1", false), - ("region=us/month=01", true), - ("region=eu/month=02", false), - ], + &[("dt=20656/month=1/active=true/label=20260722", true)], ); - let (_, _, request) = server.drop_partitions_calls().pop().unwrap(); - let expected = [ - partition_spec(&[("region", "us"), ("month", "1")]), - partition_spec(&[("region", "eu"), ("month", "02")]), - ]; - assert_eq!(request.partition_specs.len(), expected.len()); - assert!(expected - .iter() - .all(|spec| request.partition_specs.contains(spec))); + context + .sql( + "ALTER TABLE paimon.default.events DROP PARTITION (\ + dt = DATE '2026-07-22', month = '01', active = 'TRUE', label = 20260722)", + ) + .await + .unwrap(); + context + .sql( + "ALTER TABLE paimon.default.events ADD PARTITION (\ + dt = NULL, month = NULL, active = NULL, label = NULL)", + ) + .await + .unwrap(); + assert_partitions(&context, &["dt=null/month=null/active=null/label=null"]).await; + assert_partition_directories( + &temp_dir, + &[( + "dt=__DEFAULT_PARTITION__/month=__DEFAULT_PARTITION__/\ + active=__DEFAULT_PARTITION__/label=__DEFAULT_PARTITION__", + true, + )], + ); } async fn show_partitions(context: &SQLContext) -> Vec { diff --git a/crates/paimon/src/api/api_request.rs b/crates/paimon/src/api/api_request.rs index 891a281d..ea292b03 100644 --- a/crates/paimon/src/api/api_request.rs +++ b/crates/paimon/src/api/api_request.rs @@ -174,11 +174,9 @@ pub struct CreatePartitionsRequest { /// Partition specs to register. pub partition_specs: Vec>, /// Whether already registered partitions should be ignored. - /// - /// Defaults to `true` when omitted or null. #[serde( default = "default_true", - deserialize_with = "deserialize_null_as_true" + deserialize_with = "deserialize_null_to_true" )] pub ignore_if_exists: bool, } @@ -200,11 +198,9 @@ pub struct DropPartitionsRequest { /// Partition specs to unregister. pub partition_specs: Vec>, /// Whether missing partitions should be ignored. - /// - /// Defaults to `true` when omitted or null. #[serde( default = "default_true", - deserialize_with = "deserialize_null_as_true" + deserialize_with = "deserialize_null_to_true" )] pub ignore_if_not_exists: bool, } @@ -219,27 +215,11 @@ impl DropPartitionsRequest { } } -/// Request to list table partitions by exact specs. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ListPartitionsByNamesRequest { - /// Partition specs to retrieve. - #[serde(rename = "specs")] - pub partition_specs: Vec>, -} - -impl ListPartitionsByNamesRequest { - /// Create a request to list partitions by exact specs. - pub fn new(partition_specs: Vec>) -> Self { - Self { partition_specs } - } -} - fn default_true() -> bool { true } -fn deserialize_null_as_true<'de, D>(deserializer: D) -> Result +fn deserialize_null_to_true<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, { @@ -319,17 +299,6 @@ mod tests { "ignoreIfExists": false }) ); - - for value in [ - serde_json::json!({"partitionSpecs": []}), - serde_json::json!({"partitionSpecs": [], "ignoreIfExists": null}), - ] { - assert!( - serde_json::from_value::(value) - .unwrap() - .ignore_if_exists - ); - } } #[test] @@ -349,37 +318,26 @@ mod tests { "ignoreIfNotExists": false }) ); + } - for value in [ + #[test] + fn test_partition_request_flags_default_to_true() { + for json in [ + serde_json::json!({"partitionSpecs": []}), + serde_json::json!({"partitionSpecs": [], "ignoreIfExists": null}), + ] { + let request: CreatePartitionsRequest = serde_json::from_value(json).unwrap(); + assert!(request.ignore_if_exists); + } + for json in [ serde_json::json!({"partitionSpecs": []}), serde_json::json!({"partitionSpecs": [], "ignoreIfNotExists": null}), ] { - assert!( - serde_json::from_value::(value) - .unwrap() - .ignore_if_not_exists - ); + let request: DropPartitionsRequest = serde_json::from_value(json).unwrap(); + assert!(request.ignore_if_not_exists); } } - #[test] - fn test_list_partitions_by_names_request_serialization() { - let req = ListPartitionsByNamesRequest::new(vec![HashMap::from([ - ("dt".to_string(), "2026-07-22".to_string()), - ("hour".to_string(), "10".to_string()), - ])]); - - assert_eq!( - serde_json::to_value(req).unwrap(), - serde_json::json!({ - "specs": [{ - "dt": "2026-07-22", - "hour": "10" - }] - }) - ); - } - #[test] fn test_rename_table_request_serialization() { let source = Identifier::new("db1".to_string(), "table1".to_string()); diff --git a/crates/paimon/src/api/api_response.rs b/crates/paimon/src/api/api_response.rs index 01b793a8..9d68cf3d 100644 --- a/crates/paimon/src/api/api_response.rs +++ b/crates/paimon/src/api/api_response.rs @@ -421,26 +421,6 @@ impl ListTablesResponse { } } -/// Response for creating table partitions. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CreatePartitionsResponse { - /// Partition specs created by the request. - pub created: Vec>, - /// Partition specs that already existed. - pub existed: Vec>, -} - -/// Response for dropping (unregistering) table partitions. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct DropPartitionsResponse { - /// Partition specs unregistered by the request. - pub dropped: Vec>, - /// Partition specs that did not exist. - pub missing: Vec>, -} - /// Response for listing partitions. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -681,62 +661,6 @@ mod tests { assert!(json.contains("\"nextPageToken\":\"token123\"")); } - #[test] - fn test_create_partitions_response_deserialization() { - let created = HashMap::from([("dt".to_string(), "2026-07-22".to_string())]); - let existed = HashMap::from([("dt".to_string(), "2026-07-21".to_string())]); - let response: CreatePartitionsResponse = serde_json::from_value(serde_json::json!({ - "created": [created.clone()], - "existed": [existed.clone()] - })) - .unwrap(); - - assert_eq!( - response, - CreatePartitionsResponse { - created: vec![created], - existed: vec![existed], - } - ); - - for invalid in [ - serde_json::json!({"created": []}), - serde_json::json!({"existed": []}), - serde_json::json!({"created": null, "existed": []}), - serde_json::json!({"created": [], "existed": null}), - ] { - assert!(serde_json::from_value::(invalid).is_err()); - } - } - - #[test] - fn test_drop_partitions_response_deserialization() { - let dropped = HashMap::from([("dt".to_string(), "2026-07-22".to_string())]); - let missing = HashMap::from([("dt".to_string(), "2026-07-21".to_string())]); - let response: DropPartitionsResponse = serde_json::from_value(serde_json::json!({ - "dropped": [dropped.clone()], - "missing": [missing.clone()] - })) - .unwrap(); - - assert_eq!( - response, - DropPartitionsResponse { - dropped: vec![dropped], - missing: vec![missing], - } - ); - - for invalid in [ - serde_json::json!({"dropped": []}), - serde_json::json!({"missing": []}), - serde_json::json!({"dropped": null, "missing": []}), - serde_json::json!({"dropped": [], "missing": null}), - ] { - assert!(serde_json::from_value::(invalid).is_err()); - } - } - #[test] fn test_audit_response_options() { let audit = AuditRESTResponse::new( diff --git a/crates/paimon/src/api/mod.rs b/crates/paimon/src/api/mod.rs index e5b5689e..5773a85a 100644 --- a/crates/paimon/src/api/mod.rs +++ b/crates/paimon/src/api/mod.rs @@ -33,16 +33,15 @@ mod api_response; pub use api_request::{ AlterDatabaseRequest, AlterTableRequest, AuthTableQueryRequest, CreateDatabaseRequest, CreateFunctionRequest, CreatePartitionsRequest, CreateTableRequest, CreateViewRequest, - DropPartitionsRequest, ListPartitionsByNamesRequest, RenameTableRequest, + DropPartitionsRequest, RenameTableRequest, }; // Re-export response types pub use api_response::{ - AuditRESTResponse, AuthTableQueryResponse, ConfigResponse, CreatePartitionsResponse, - DropPartitionsResponse, ErrorResponse, GetDatabaseResponse, GetFunctionResponse, - GetTableResponse, GetTableTokenResponse, GetViewResponse, ListDatabasesResponse, - ListFunctionsResponse, ListPartitionsResponse, ListTablesResponse, ListViewsResponse, - PagedList, + AuditRESTResponse, AuthTableQueryResponse, ConfigResponse, ErrorResponse, GetDatabaseResponse, + GetFunctionResponse, GetTableResponse, GetTableTokenResponse, GetViewResponse, + ListDatabasesResponse, ListFunctionsResponse, ListPartitionsResponse, ListTablesResponse, + ListViewsResponse, PagedList, }; // Re-export error types diff --git a/crates/paimon/src/api/resource_paths.rs b/crates/paimon/src/api/resource_paths.rs index 1be104d3..569d1b92 100644 --- a/crates/paimon/src/api/resource_paths.rs +++ b/crates/paimon/src/api/resource_paths.rs @@ -221,14 +221,6 @@ impl ResourcePaths { pub fn drop_partitions(&self, database_name: &str, table_name: &str) -> String { format!("{}/drop", self.partitions(database_name, table_name)) } - - /// Get the endpoint path for listing table partitions by name. - pub fn list_partitions_by_names(&self, database_name: &str, table_name: &str) -> String { - format!( - "{}/list-by-names", - self.partitions(database_name, table_name) - ) - } } #[cfg(test)] @@ -311,15 +303,11 @@ mod tests { } #[test] - fn test_partition_operation_paths_encode_names() { + fn test_drop_partitions_path_encodes_names() { let paths = ResourcePaths::new("catalog"); assert_eq!( paths.drop_partitions("analytics db", "user events"), "/v1/catalog/databases/analytics+db/tables/user+events/partitions/drop" ); - assert_eq!( - paths.list_partitions_by_names("analytics db", "user events"), - "/v1/catalog/databases/analytics+db/tables/user+events/partitions/list-by-names" - ); } } diff --git a/crates/paimon/src/api/rest_api.rs b/crates/paimon/src/api/rest_api.rs index 14aa0582..48a38105 100644 --- a/crates/paimon/src/api/rest_api.rs +++ b/crates/paimon/src/api/rest_api.rs @@ -31,13 +31,12 @@ use crate::Result; use super::api_request::{ AlterDatabaseRequest, AlterTableRequest, AuthTableQueryRequest, CreateDatabaseRequest, CreateFunctionRequest, CreatePartitionsRequest, CreateTableRequest, CreateViewRequest, - DropPartitionsRequest, ListPartitionsByNamesRequest, RenameTableRequest, + DropPartitionsRequest, RenameTableRequest, }; use super::api_response::{ - AuthTableQueryResponse, ConfigResponse, CreatePartitionsResponse, DropPartitionsResponse, - GetDatabaseResponse, GetFunctionResponse, GetTableResponse, GetViewResponse, - ListDatabasesResponse, ListFunctionsResponse, ListPartitionsResponse, ListTablesResponse, - ListViewsResponse, PagedList, + AuthTableQueryResponse, ConfigResponse, GetDatabaseResponse, GetFunctionResponse, + GetTableResponse, GetViewResponse, ListDatabasesResponse, ListFunctionsResponse, + ListPartitionsResponse, ListTablesResponse, ListViewsResponse, PagedList, }; use super::auth::{AuthProviderFactory, RESTAuthFunction}; use super::resource_paths::ResourcePaths; @@ -74,24 +73,6 @@ fn validate_non_empty_multi(values: &[(&str, &str)]) -> Result<()> { Ok(()) } -fn validate_list_partitions_by_names_request_size( - identifier: &Identifier, - partition_spec_count: usize, -) -> Result<()> { - if partition_spec_count > RESTApi::PARTITION_REQUEST_SIZE { - return Err(crate::Error::DataInvalid { - message: format!( - "List partitions by names accepts at most {} partition specs for table {}, got {}", - RESTApi::PARTITION_REQUEST_SIZE, - identifier.full_name(), - partition_spec_count - ), - source: None, - }); - } - Ok(()) -} - /// REST API wrapper for Paimon catalog operations. /// /// This struct provides methods for database and table CRUD operations @@ -106,10 +87,6 @@ impl RESTApi { // Constants for query parameters and headers pub const HEADER_PREFIX: &'static str = "header."; pub const MAX_RESULTS: &'static str = "maxResults"; - /// Partition batch size used by `RESTCatalog`. - /// - /// The list-by-names endpoint also limits each request to this many specs. - pub(crate) const PARTITION_REQUEST_SIZE: usize = 1000; pub const PAGE_TOKEN: &'static str = "pageToken"; pub const DATABASE_NAME_PATTERN: &'static str = "databaseNamePattern"; pub const TABLE_NAME_PATTERN: &'static str = "tableNamePattern"; @@ -586,13 +563,14 @@ impl RESTApi { identifier: &Identifier, partition_specs: Vec>, ignore_if_exists: bool, - ) -> Result { + ) -> Result<()> { let database = identifier.database(); let table = identifier.object(); validate_non_empty_multi(&[(database, "database name"), (table, "table name")])?; let path = self.resource_paths.partitions(database, table); let request = CreatePartitionsRequest::new(partition_specs, ignore_if_exists); - self.client.post(&path, &request).await + let _resp: serde_json::Value = self.client.post(&path, &request).await?; + Ok(()) } /// Unregister table partitions in a single REST request. @@ -604,34 +582,14 @@ impl RESTApi { identifier: &Identifier, partition_specs: Vec>, ignore_if_not_exists: bool, - ) -> Result { + ) -> Result<()> { let database = identifier.database(); let table = identifier.object(); validate_non_empty_multi(&[(database, "database name"), (table, "table name")])?; let path = self.resource_paths.drop_partitions(database, table); let request = DropPartitionsRequest::new(partition_specs, ignore_if_not_exists); - self.client.post(&path, &request).await - } - - /// List table partitions by exact specs in a single REST request. - /// - /// At most 1,000 specs may be sent per request. Use - /// [`crate::catalog::RESTCatalog`] to split larger batches. - pub async fn list_partitions_by_names( - &self, - identifier: &Identifier, - partition_specs: Vec>, - ) -> Result> { - let database = identifier.database(); - let table = identifier.object(); - validate_non_empty_multi(&[(database, "database name"), (table, "table name")])?; - validate_list_partitions_by_names_request_size(identifier, partition_specs.len())?; - let path = self - .resource_paths - .list_partitions_by_names(database, table); - let request = ListPartitionsByNamesRequest::new(partition_specs); - let response: ListPartitionsResponse = self.client.post(&path, &request).await?; - Ok(response.partitions.unwrap_or_default()) + let _resp: serde_json::Value = self.client.post(&path, &request).await?; + Ok(()) } /// List all partitions of a table, paging internally. diff --git a/crates/paimon/src/catalog/mod.rs b/crates/paimon/src/catalog/mod.rs index ffb0ec16..bc9487b6 100644 --- a/crates/paimon/src/catalog/mod.rs +++ b/crates/paimon/src/catalog/mod.rs @@ -463,27 +463,17 @@ pub trait Catalog: Send + Sync { }) } - /// Unregister table partition specs from the catalog. + /// Unregister table partition metadata from the catalog. /// - /// Missing specs are ignored so callers can safely retry the request. - async fn unregister_partitions( + /// This does not delete partition directories or data files. Missing specs + /// are ignored so callers can safely retry the request. + async fn drop_partitions( &self, _identifier: &Identifier, _partition_specs: Vec>, ) -> Result<()> { Err(Error::Unsupported { - message: "Catalog does not support unregistering partitions".to_string(), - }) - } - - /// List table partitions matching exact partition specs. - async fn list_partitions_by_names( - &self, - _identifier: &Identifier, - _partition_specs: Vec>, - ) -> Result> { - Err(Error::Unsupported { - message: "Catalog does not support listing partitions by names".to_string(), + message: "Catalog does not support dropping partitions".to_string(), }) } diff --git a/crates/paimon/src/catalog/rest/rest_catalog.rs b/crates/paimon/src/catalog/rest/rest_catalog.rs index c17dde73..6db8e13f 100644 --- a/crates/paimon/src/catalog/rest/rest_catalog.rs +++ b/crates/paimon/src/catalog/rest/rest_catalog.rs @@ -37,6 +37,8 @@ use crate::spec::{Partition, Schema, SchemaChange}; use crate::table::{RESTEnv, Table}; use crate::Result; +const PARTITION_BATCH_SIZE: usize = 1000; + /// REST catalog implementation. /// /// This catalog communicates with a Paimon REST catalog server @@ -369,16 +371,18 @@ impl Catalog for RESTCatalog { partition_specs: Vec>, ignore_if_exists: bool, ) -> Result<()> { + if partition_specs.is_empty() { + return Ok(()); + } if !ignore_if_exists { return self .api .create_partitions(identifier, partition_specs, false) .await - .map(|_| ()) .map_err(|error| map_rest_error_for_create_partitions(error, identifier)); } - for batch in partition_specs.chunks(RESTApi::PARTITION_REQUEST_SIZE) { + for batch in partition_specs.chunks(PARTITION_BATCH_SIZE) { self.api .create_partitions(identifier, batch.to_vec(), true) .await @@ -387,12 +391,15 @@ impl Catalog for RESTCatalog { Ok(()) } - async fn unregister_partitions( + async fn drop_partitions( &self, identifier: &Identifier, partition_specs: Vec>, ) -> Result<()> { - for batch in partition_specs.chunks(RESTApi::PARTITION_REQUEST_SIZE) { + if partition_specs.is_empty() { + return Ok(()); + } + for batch in partition_specs.chunks(PARTITION_BATCH_SIZE) { self.api .drop_partitions(identifier, batch.to_vec(), true) .await @@ -401,23 +408,6 @@ impl Catalog for RESTCatalog { Ok(()) } - async fn list_partitions_by_names( - &self, - identifier: &Identifier, - partition_specs: Vec>, - ) -> Result> { - let mut partitions = Vec::new(); - for batch in partition_specs.chunks(RESTApi::PARTITION_REQUEST_SIZE) { - partitions.extend( - self.api - .list_partitions_by_names(identifier, batch.to_vec()) - .await - .map_err(|error| map_rest_error_for_partition_request(error, identifier))?, - ); - } - Ok(partitions) - } - async fn list_partitions(&self, identifier: &Identifier) -> Result> { match self.api.list_partitions(identifier).await { Ok(partitions) => Ok(partitions), diff --git a/crates/paimon/src/spec/partition_utils.rs b/crates/paimon/src/spec/partition_utils.rs index c6659f46..56ab1d3d 100644 --- a/crates/paimon/src/spec/partition_utils.rs +++ b/crates/paimon/src/spec/partition_utils.rs @@ -489,26 +489,31 @@ pub(crate) fn escape_path_name(path: &str) -> String { } /// Unescape a path component following Java `PartitionPathUtils.unescapePathName`. -/// -/// Invalid or incomplete percent escapes are preserved literally. -pub(crate) fn unescape_path_name(path: &str) -> String { - let mut result = String::with_capacity(path.len()); - let mut chars = path.chars().peekable(); - while let Some(character) = chars.next() { - if character == '%' { - let mut lookahead = chars.clone(); - if let (Some(high), Some(low)) = (lookahead.next(), lookahead.next()) { - if let (Some(high), Some(low)) = (high.to_digit(16), low.to_digit(16)) { - chars.next(); - chars.next(); - result.push(char::from(((high << 4) | low) as u8)); - continue; - } - } +pub(crate) fn unescape_path_name(path: &str) -> Option { + let bytes = path.as_bytes(); + let mut result = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + if bytes[index] == b'%' { + let high = hex_value(*bytes.get(index + 1)?)?; + let low = hex_value(*bytes.get(index + 2)?)?; + result.push((high << 4) | low); + index += 3; + } else { + result.push(bytes[index]); + index += 1; } - result.push(character); } - result + String::from_utf8(result).ok() +} + +fn hex_value(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + b'A'..=b'F' => Some(byte - b'A' + 10), + _ => None, + } } /// Check if a character needs escaping in partition path names. @@ -681,11 +686,11 @@ mod tests { } #[test] - fn test_unescape_path_name_matches_java_lenient_decoding() { - assert_eq!(unescape_path_name("a%2Fb"), "a/b"); - assert_eq!(unescape_path_name("a%2fb"), "a/b"); - assert_eq!(unescape_path_name("a%ZZb"), "a%ZZb"); - assert_eq!(unescape_path_name("a%b"), "a%b"); + fn test_unescape_path_name() { + assert_eq!(unescape_path_name("a%2Fb"), Some("a/b".to_string())); + assert_eq!(unescape_path_name("%E4%B8%AD"), Some("中".to_string())); + assert_eq!(unescape_path_name("a%ZZb"), None); + assert_eq!(unescape_path_name("a%b"), None); } // ======================== PartitionComputer tests ======================== diff --git a/crates/paimon/src/table/format_partition.rs b/crates/paimon/src/table/format_partition.rs index dc67b7ac..7fdd1cc7 100644 --- a/crates/paimon/src/table/format_partition.rs +++ b/crates/paimon/src/table/format_partition.rs @@ -19,7 +19,7 @@ use std::collections::HashMap; -use chrono::{Datelike, NaiveDate}; +use chrono::NaiveDate; use crate::io::FileIO; use crate::spec::{escape_path_name, unescape_path_name, DataType, Datum}; @@ -177,30 +177,27 @@ impl FormatTablePartitionPaths { segment: &str, ) -> crate::Result> { let value = if self.only_value_in_path { - unescape_canonical_path_name(segment)? + decode_canonical_path_name(segment)? } else { let Some((segment_key, value)) = segment.split_once('=') else { return Ok(None); }; - let decoded_key = unescape_path_name(segment_key); + let decoded_key = decode_canonical_path_name(segment_key)?; if decoded_key != key { return Ok(None); } - ensure_canonical_path_name(segment_key, &decoded_key)?; - unescape_canonical_path_name(value)? + decode_canonical_path_name(value)? }; Ok(Some(value)) } } -fn unescape_canonical_path_name(value: &str) -> crate::Result { - let decoded = unescape_path_name(value); - ensure_canonical_path_name(value, &decoded)?; - Ok(decoded) -} - -fn ensure_canonical_path_name(value: &str, decoded: &str) -> crate::Result<()> { - let canonical = escape_path_name(decoded); +fn decode_canonical_path_name(value: &str) -> crate::Result { + let decoded = unescape_path_name(value).ok_or_else(|| crate::Error::DataInvalid { + message: format!("Invalid escaped partition path segment {value:?}"), + source: None, + })?; + let canonical = escape_path_name(&decoded); if canonical != value { return Err(crate::Error::DataInvalid { message: format!( @@ -210,89 +207,65 @@ fn ensure_canonical_path_name(value: &str, decoded: &str) -> crate::Result<()> { source: None, }); } - Ok(()) + Ok(decoded) } /// Parse a raw Format Table partition value from a path or catalog registration. -/// -/// Boolean and integer values follow Java metadata parsing and do not accept -/// surrounding whitespace. Character values are preserved exactly. Numeric -/// DATE values are epoch days; textual DATE values accept `yyyy-MM` and -/// `yyyy-MM-dd`, optionally followed by a space and time text. Invalid or -/// unsupported values return `None`. pub fn parse_format_partition_value(value: &str, data_type: &DataType) -> Option { match data_type { - DataType::Boolean(_) => match value.to_ascii_lowercase().as_str() { - "t" | "true" | "y" | "yes" | "1" => Some(Datum::Bool(true)), - "f" | "false" | "n" | "no" | "0" => Some(Datum::Bool(false)), - _ => None, - }, + DataType::Boolean(_) => value.parse::().ok().map(Datum::Bool), DataType::TinyInt(_) => value.parse::().ok().map(Datum::TinyInt), DataType::SmallInt(_) => value.parse::().ok().map(Datum::SmallInt), DataType::Int(_) => value.parse::().ok().map(Datum::Int), DataType::BigInt(_) => value.parse::().ok().map(Datum::Long), - DataType::Char(char_type) if value.chars().count() <= char_type.length() => { - Some(Datum::String(value.to_string())) - } - DataType::VarChar(varchar_type) - if value.chars().count() <= varchar_type.length() as usize => - { - Some(Datum::String(value.to_string())) - } - DataType::Char(_) | DataType::VarChar(_) => None, + DataType::Char(_) | DataType::VarChar(_) => Some(Datum::String(value.to_string())), DataType::Date(_) => parse_partition_date(value).map(Datum::Date), DataType::Time(_) => value.parse::().ok().map(Datum::Time), _ => None, } } -fn parse_partition_date(value: &str) -> Option { - let digits = value.strip_prefix('-').unwrap_or(value); - if !digits.is_empty() && digits.chars().all(|character| character.is_ascii_digit()) { - let epoch_days = value.parse::().ok()?; - return is_supported_epoch_day(epoch_days).then_some(epoch_days); +/// Format a typed value for Format Table partition metadata and paths. +pub fn format_partition_value( + datum: &Datum, + data_type: &DataType, + default_partition_name: &str, + legacy_partition_name: bool, +) -> Option { + match (datum, data_type) { + (Datum::Bool(value), DataType::Boolean(_)) => Some(value.to_string()), + (Datum::TinyInt(value), DataType::TinyInt(_)) => Some(value.to_string()), + (Datum::SmallInt(value), DataType::SmallInt(_)) => Some(value.to_string()), + (Datum::Int(value), DataType::Int(_)) => Some(value.to_string()), + (Datum::Long(value), DataType::BigInt(_)) => Some(value.to_string()), + (Datum::String(value), DataType::Char(_) | DataType::VarChar(_)) => { + if value.trim().is_empty() { + Some(default_partition_name.to_string()) + } else { + Some(value.clone()) + } + } + (Datum::Date(value), DataType::Date(_)) => { + if legacy_partition_name { + Some(value.to_string()) + } else { + format_partition_date(*value) + } + } + (Datum::Time(value), DataType::Time(_)) => Some(value.to_string()), + _ => None, } +} - let date_value = value - .find(' ') - .filter(|index| *index > 0) - .map(|index| &value[..index]) - .unwrap_or(value); - let mut segments = date_value.split('-'); - let year = parse_date_segment(segments.next()?)?; - if !(0..=9999).contains(&year) { - return None; - } - let month = match segments.next() { - Some(value) => parse_date_segment(value)?, - None => 1, - }; - let day = match segments.next() { - Some(value) => parse_date_segment(value)?, - None => 1, - }; - if segments.next().is_some() { - return None; +fn parse_partition_date(value: &str) -> Option { + if let Ok(epoch_days) = value.parse::() { + return Some(epoch_days); } - let date = NaiveDate::from_ymd_opt(year, month as u32, day as u32)?; + let date = NaiveDate::parse_from_str(value, "%Y-%m-%d").ok()?; let epoch = NaiveDate::from_num_days_from_ce_opt(UNIX_EPOCH_DAYS_FROM_CE)?; date.signed_duration_since(epoch).num_days().try_into().ok() } -fn parse_date_segment(value: &str) -> Option { - if value.is_empty() || !value.chars().all(|character| character.is_ascii_digit()) { - return None; - } - value.parse().ok() -} - -fn is_supported_epoch_day(epoch_days: i32) -> bool { - epoch_days - .checked_add(UNIX_EPOCH_DAYS_FROM_CE) - .and_then(NaiveDate::from_num_days_from_ce_opt) - .is_some_and(|date| (0..=9999).contains(&date.year())) -} - pub(crate) fn format_partition_date(epoch_days: i32) -> Option { NaiveDate::from_num_days_from_ce_opt(epoch_days.checked_add(UNIX_EPOCH_DAYS_FROM_CE)?) .map(|date| date.format("%Y-%m-%d").to_string()) @@ -313,115 +286,26 @@ fn last_path_segment(path: &str) -> Option<&str> { #[cfg(test)] mod tests { use super::*; - use crate::spec::{BooleanType, CharType, DateType, VarCharType}; + use crate::spec::{BooleanType, DateType}; #[test] - fn test_parse_format_partition_boolean_uses_java_spellings() { - let data_type = DataType::Boolean(BooleanType::new()); - + fn test_parse_format_partition_value() { assert_eq!( - parse_format_partition_value("1", &data_type), + parse_format_partition_value("true", &DataType::Boolean(BooleanType::new())), Some(Datum::Bool(true)) ); assert_eq!( - parse_format_partition_value("yes", &data_type), - Some(Datum::Bool(true)) - ); - assert_eq!( - parse_format_partition_value("0", &data_type), - Some(Datum::Bool(false)) - ); - assert_eq!( - parse_format_partition_value("no", &data_type), - Some(Datum::Bool(false)) - ); - assert_eq!(parse_format_partition_value(" yes ", &data_type), None); - } - - #[test] - fn test_parse_format_partition_date_accepts_java_partial_month() { - let data_type = DataType::Date(DateType::new()); - - assert_eq!( - parse_format_partition_value("2026-07", &data_type), - parse_format_partition_value("2026-07-01", &data_type) - ); - } - - #[test] - fn test_parse_format_partition_numeric_date_uses_epoch_days() { - let data_type = DataType::Date(DateType::new()); - - assert_eq!( - parse_format_partition_value("2026", &data_type), - Some(Datum::Date(2026)) - ); - } - - #[test] - fn test_parse_format_partition_date_rejects_outside_paimon_range() { - let data_type = DataType::Date(DateType::new()); - let epoch = NaiveDate::from_num_days_from_ce_opt(UNIX_EPOCH_DAYS_FROM_CE).unwrap(); - let min_epoch_day = i32::try_from( - NaiveDate::from_ymd_opt(0, 1, 1) - .unwrap() - .signed_duration_since(epoch) - .num_days(), - ) - .unwrap(); - let max_epoch_day = i32::try_from( - NaiveDate::from_ymd_opt(9999, 12, 31) - .unwrap() - .signed_duration_since(epoch) - .num_days(), - ) - .unwrap(); - - assert_eq!( - parse_format_partition_value("10000-01-01", &data_type), - None - ); - assert_eq!( - parse_format_partition_value(&min_epoch_day.to_string(), &data_type), - Some(Datum::Date(min_epoch_day)) + parse_format_partition_value("2026-07-22", &DataType::Date(DateType::new())), + Some(Datum::Date(20_656)) ); assert_eq!( - parse_format_partition_value(&max_epoch_day.to_string(), &data_type), - Some(Datum::Date(max_epoch_day)) + parse_format_partition_value("20656", &DataType::Date(DateType::new())), + Some(Datum::Date(20_656)) ); assert_eq!( - parse_format_partition_value(&(min_epoch_day - 1).to_string(), &data_type), + parse_format_partition_value("yes", &DataType::Boolean(BooleanType::new())), None ); - assert_eq!( - parse_format_partition_value(&(max_epoch_day + 1).to_string(), &data_type), - None - ); - assert_eq!( - parse_format_partition_value(&i32::MIN.to_string(), &data_type), - None - ); - assert_eq!( - parse_format_partition_value(&i32::MAX.to_string(), &data_type), - None - ); - } - - #[test] - fn test_parse_format_partition_string_enforces_declared_character_length() { - let char_type = DataType::Char(CharType::new(3).unwrap()); - let varchar_type = DataType::VarChar(VarCharType::new(3).unwrap()); - - assert_eq!( - parse_format_partition_value("中文a", &char_type), - Some(Datum::String("中文a".to_string())) - ); - assert_eq!(parse_format_partition_value("中文ab", &char_type), None); - assert_eq!( - parse_format_partition_value("中文a", &varchar_type), - Some(Datum::String("中文a".to_string())) - ); - assert_eq!(parse_format_partition_value("中文ab", &varchar_type), None); } #[test] diff --git a/crates/paimon/src/table/format_table_scan.rs b/crates/paimon/src/table/format_table_scan.rs index a8826f20..61856663 100644 --- a/crates/paimon/src/table/format_table_scan.rs +++ b/crates/paimon/src/table/format_table_scan.rs @@ -20,14 +20,15 @@ use std::collections::{HashMap, HashSet}; use super::format_partition::{ - format_partition_date, is_storage_not_found, parse_format_partition_value, + format_partition_value, is_storage_not_found, parse_format_partition_value, FormatTablePartitionPaths, }; -use super::{CatalogManagedPartitionOptions, Plan, ScanTrace, Table}; +use super::rest_env::LoadedFormatTablePartitionOptions; +use super::{Plan, RESTEnv, ScanTrace, Table}; use crate::spec::stats::BinaryTableStats; use crate::spec::{ escape_path_name, extract_datum, unescape_path_name, BinaryRow, BinaryRowBuilder, CoreOptions, - DataField, DataFileMeta, DataType, Datum, PartitionComputer, Predicate, PredicateOperator, + DataField, DataFileMeta, Datum, PartitionComputer, Predicate, PredicateOperator, }; use crate::table::partition_filter::PartitionFilter; use crate::table::source::DataSplitBuilder; @@ -70,7 +71,10 @@ impl<'a> FormatTableScan<'a> { async fn plan_inner(&self, trace: Option<&mut ScanTrace>) -> crate::Result { let core_options = CoreOptions::new(self.table.schema().options()); - let managed_options = self.table.catalog_managed_partition_options(); + let managed_options = self + .table + .rest_env() + .and_then(RESTEnv::catalog_managed_partition_options); let file_format = managed_options .map(|options| options.file_format.as_str()) .unwrap_or_else(|| core_options.file_format()); @@ -134,15 +138,18 @@ impl<'a> FormatTableScan<'a> { partition: BinaryRow::new(0), }]); } - if let Some(managed_options) = self.table.catalog_managed_partition_options() { - return self - .catalog_managed_scan_roots( - table_path, - partition_keys, - &partition_fields, - managed_options, - ) - .await; + if let Some(rest_env) = self.table.rest_env() { + if let Some(managed_options) = rest_env.catalog_managed_partition_options() { + return self + .catalog_managed_scan_roots( + rest_env, + table_path, + partition_keys, + &partition_fields, + managed_options, + ) + .await; + } } let Some(PartitionFilter::PartitionSet { partitions, .. }) = &self.partition_filter else { @@ -200,21 +207,12 @@ impl<'a> FormatTableScan<'a> { async fn catalog_managed_scan_roots( &self, + rest_env: &RESTEnv, table_path: &str, partition_keys: &[String], partition_fields: &[DataField], - managed_options: &CatalogManagedPartitionOptions, + managed_options: &LoadedFormatTablePartitionOptions, ) -> crate::Result> { - let rest_env = self - .table - .rest_env() - .ok_or_else(|| crate::Error::DataInvalid { - message: format!( - "Format table {} has catalog-managed partitions but no REST environment", - self.table.identifier().full_name() - ), - source: None, - })?; let partitions = rest_env .api() .list_partitions(rest_env.identifier()) @@ -222,6 +220,8 @@ impl<'a> FormatTableScan<'a> { let only_value_in_path = managed_options.only_value_in_path; let partition_paths = FormatTablePartitionPaths::new(partition_keys.iter().cloned(), only_value_in_path); + let core_options = CoreOptions::new(self.table.schema().options()); + let default_partition_name = core_options.partition_default_name(); let mut seen_paths = HashSet::with_capacity(partitions.len()); let mut roots = Vec::with_capacity(partitions.len()); for partition in partitions { @@ -236,7 +236,7 @@ impl<'a> FormatTableScan<'a> { &partition.spec, partition_fields, partition_keys, - &managed_options.default_partition_name, + default_partition_name, ) .map_err(|error| self.invalid_catalog_partition_metadata(error))?; if self.partition_matches(&partition)? { @@ -415,7 +415,7 @@ fn leading_equality_partition_path( let Some(datum) = values[idx] else { break; }; - let value = partition_value_from_datum( + let value = format_partition_value( datum, partition_fields[idx].data_type(), default_partition_name, @@ -450,7 +450,7 @@ fn partition_path_from_row( for (idx, field) in partition_fields.iter().enumerate() { let value = match extract_datum(row, idx, field.data_type())? { None => default_partition_name.to_string(), - Some(datum) => partition_value_from_datum( + Some(datum) => format_partition_value( &datum, field.data_type(), default_partition_name, @@ -476,37 +476,6 @@ fn partition_path_from_row( Ok(segments.join("/")) } -fn partition_value_from_datum( - datum: &Datum, - data_type: &DataType, - default_partition_name: &str, - legacy_partition_name: bool, -) -> Option { - match (datum, data_type) { - (Datum::Bool(value), DataType::Boolean(_)) => Some(value.to_string()), - (Datum::TinyInt(value), DataType::TinyInt(_)) => Some(value.to_string()), - (Datum::SmallInt(value), DataType::SmallInt(_)) => Some(value.to_string()), - (Datum::Int(value), DataType::Int(_)) => Some(value.to_string()), - (Datum::Long(value), DataType::BigInt(_)) => Some(value.to_string()), - (Datum::String(value), DataType::Char(_) | DataType::VarChar(_)) => { - if value.trim().is_empty() { - Some(default_partition_name.to_string()) - } else { - Some(value.clone()) - } - } - (Datum::Date(value), DataType::Date(_)) => { - if legacy_partition_name { - Some(value.to_string()) - } else { - format_partition_date(*value) - } - } - (Datum::Time(value), DataType::Time(_)) => Some(value.to_string()), - _ => None, - } -} - fn partition_row_from_path( table_path: &str, file_parent: &str, @@ -533,7 +502,10 @@ fn partition_row_from_path( .filter(|segment| !segment.is_empty()) .take(partition_keys.len()) { - values.push(unescape_path_name(segment)); + let Some(value) = unescape_path_name(segment) else { + return Ok(None); + }; + values.push(value); } if values.len() != partition_keys.len() { return Ok(None); @@ -597,8 +569,8 @@ fn partition_row_from_catalog_spec( fn partition_segment_value(segment: &str, key: &str) -> Option { let (segment_key, segment_value) = segment.split_once('=')?; - if unescape_path_name(segment_key) == key { - Some(unescape_path_name(segment_value)) + if unescape_path_name(segment_key)? == key { + unescape_path_name(segment_value) } else { None } diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index 6cafde1b..8b7298c1 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -98,7 +98,9 @@ pub use btree_global_index_build_builder::BTreeGlobalIndexBuildBuilder; pub use commit_message::CommitMessage; pub use cow_writer::{CopyOnWriteMergeWriter, FileInfo}; pub use data_evolution_writer::{DataEvolutionDeleteWriter, DataEvolutionWriter}; -pub use format_partition::{parse_format_partition_value, FormatTablePartitionPaths}; +pub use format_partition::{ + format_partition_value, parse_format_partition_value, FormatTablePartitionPaths, +}; #[cfg(feature = "fulltext")] pub use full_text_search_builder::FullTextSearchBuilder; use futures::stream::BoxStream; @@ -138,14 +140,6 @@ use crate::io::FileIO; use crate::spec::{CoreOptions, DataField, Snapshot, TableSchema}; use std::collections::HashMap; -#[derive(Debug, Clone)] -pub(crate) struct CatalogManagedPartitionOptions { - table_path: String, - file_format: String, - default_partition_name: String, - only_value_in_path: bool, -} - /// Table represents a table in the catalog. #[derive(Debug, Clone)] pub struct Table { @@ -157,7 +151,6 @@ pub struct Table { branch: String, branch_reference: bool, rest_env: Option, - catalog_managed_partition_options: Option, /// True when this table copy was switched to a historical schema by /// [`Table::copy_with_time_travel`]. Such a copy is read-only. time_traveled: bool, @@ -178,16 +171,6 @@ impl Table { ) -> Self { let schema_manager = SchemaManager::new(file_io.clone(), location.clone()); let branch = DEFAULT_MAIN_BRANCH.to_string(); - let options = CoreOptions::new(schema.options()); - let catalog_managed_partition_options = rest_env - .as_ref() - .filter(|rest_env| rest_env.has_catalog_managed_partitions()) - .map(|_| CatalogManagedPartitionOptions { - table_path: options.path().unwrap_or(&location).to_string(), - file_format: options.file_format().to_string(), - default_partition_name: options.partition_default_name().to_string(), - only_value_in_path: options.format_table_partition_only_value_in_path(), - }); Self { file_io, identifier, @@ -197,7 +180,6 @@ impl Table { branch, branch_reference: false, rest_env, - catalog_managed_partition_options, time_traveled: false, travel_snapshot: None, } @@ -283,13 +265,10 @@ impl Table { /// Whether this table uses catalog-managed Format Table partitions. pub fn has_catalog_managed_partitions(&self) -> bool { - self.catalog_managed_partition_options.is_some() - } - - pub(crate) fn catalog_managed_partition_options( - &self, - ) -> Option<&CatalogManagedPartitionOptions> { - self.catalog_managed_partition_options.as_ref() + self.rest_env + .as_ref() + .and_then(RESTEnv::catalog_managed_partition_options) + .is_some() } /// Create a read builder for scan/read. @@ -374,7 +353,6 @@ impl Table { branch: self.branch.clone(), branch_reference: self.branch_reference, rest_env: self.rest_env.clone(), - catalog_managed_partition_options: self.catalog_managed_partition_options.clone(), time_traveled: self.time_traveled, travel_snapshot: if selector_changed { None @@ -458,7 +436,6 @@ impl Table { branch, branch_reference: true, rest_env: self.rest_env.clone(), - catalog_managed_partition_options: self.catalog_managed_partition_options.clone(), time_traveled: false, travel_snapshot: None, }) diff --git a/crates/paimon/src/table/rest_env.rs b/crates/paimon/src/table/rest_env.rs index b603716c..b97c953a 100644 --- a/crates/paimon/src/table/rest_env.rs +++ b/crates/paimon/src/table/rest_env.rs @@ -35,14 +35,14 @@ use crate::table::Table; use crate::Result; #[derive(Clone, Debug)] -struct LoadedFormatTablePartitionOptions { +pub(crate) struct LoadedFormatTablePartitionOptions { + pub(crate) table_path: String, + pub(crate) file_format: String, + pub(crate) only_value_in_path: bool, partitioned_table_in_metastore: bool, - only_value_in_path: bool, - implementation: FormatTableImplementation, } -/// REST environment that holds the REST API client, identifier, and uuid -/// needed to create a `RESTSnapshotCommit`. +/// REST-backed table context used by snapshot commits and managed Format Table scans. #[derive(Clone)] pub struct RESTEnv { identifier: Identifier, @@ -81,71 +81,12 @@ impl RESTEnv { } } - /// Create a REST environment for table metadata returned by a REST catalog. - /// - /// `table_schema` and `is_external` must come from the same table response. - /// They determine whether catalog-managed Format Table partitions are - /// enabled. [`RESTEnv::new`] never enables this capability. - pub fn new_for_table( - identifier: Identifier, - uuid: String, - api: Arc, - options: Options, - data_token_enabled: bool, - table_schema: &TableSchema, - is_external: bool, - ) -> Result { - let core_options = CoreOptions::new(table_schema.options()); - let loaded_format_table_partition_options = if core_options.is_format_table() { - let partitioned_table_in_metastore = core_options.partitioned_table_in_metastore()?; - let implementation = core_options.format_table_implementation()?; - let only_value_in_path = - core_options.try_format_table_partition_only_value_in_path()?; - - if partitioned_table_in_metastore && implementation == FormatTableImplementation::Engine - { - return Err(Error::DataInvalid { - message: format!( - "Format Table {} cannot set metastore.partitioned-table=true when \ - format-table.implementation=engine", - identifier.full_name() - ), - source: None, - }); - } - if partitioned_table_in_metastore && is_external { - return Err(Error::DataInvalid { - message: format!( - "Catalog-managed partitions require an internal Format Table, but {} is external", - identifier.full_name() - ), - source: None, - }); - } - - Some(LoadedFormatTablePartitionOptions { - partitioned_table_in_metastore, - only_value_in_path, - implementation, - }) - } else { - None - }; - - Ok(Self { - identifier, - uuid, - api, - options, - data_token_enabled, - loaded_format_table_partition_options, - }) - } - - pub(crate) fn has_catalog_managed_partitions(&self) -> bool { + pub(crate) fn catalog_managed_partition_options( + &self, + ) -> Option<&LoadedFormatTablePartitionOptions> { self.loaded_format_table_partition_options .as_ref() - .is_some_and(|options| options.partitioned_table_in_metastore) + .filter(|options| options.partitioned_table_in_metastore) } pub(crate) fn validate_dynamic_format_table_partition_options( @@ -164,7 +105,9 @@ impl RESTEnv { options.partitioned_table_in_metastore()?, )?; } - if extra.contains_key(FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE_OPTION) { + if loaded_options.partitioned_table_in_metastore + && extra.contains_key(FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE_OPTION) + { ensure_partition_option_unchanged( &self.identifier, FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE_OPTION, @@ -172,11 +115,13 @@ impl RESTEnv { options.try_format_table_partition_only_value_in_path()?, )?; } - if extra.contains_key(FORMAT_TABLE_IMPLEMENTATION_OPTION) { + if loaded_options.partitioned_table_in_metastore + && extra.contains_key(FORMAT_TABLE_IMPLEMENTATION_OPTION) + { ensure_partition_option_unchanged( &self.identifier, FORMAT_TABLE_IMPLEMENTATION_OPTION, - loaded_options.implementation, + FormatTableImplementation::Paimon, options.format_table_implementation()?, )?; } @@ -266,15 +211,9 @@ impl RESTEnv { builder.build()? }; - let rest_env = RESTEnv::new_for_table( - identifier.clone(), - uuid, - api, - options, - data_token_enabled, - &table_schema, - is_external, - )?; + let mut rest_env = RESTEnv::new(identifier.clone(), uuid, api, options, data_token_enabled); + rest_env.loaded_format_table_partition_options = + load_format_table_partition_options(identifier, &table_schema, is_external)?; Ok(Table::new( file_io, @@ -295,6 +234,59 @@ impl RESTEnv { } } +fn load_format_table_partition_options( + identifier: &Identifier, + table_schema: &TableSchema, + is_external: bool, +) -> Result> { + let core_options = CoreOptions::new(table_schema.options()); + if !core_options.is_format_table() { + return Ok(None); + } + + let partitioned_table_in_metastore = core_options.partitioned_table_in_metastore()?; + let only_value_in_path = if partitioned_table_in_metastore { + if core_options.format_table_implementation()? == FormatTableImplementation::Engine { + return Err(Error::DataInvalid { + message: format!( + "Format Table {} cannot set metastore.partitioned-table=true when \ + format-table.implementation=engine", + identifier.full_name() + ), + source: None, + }); + } + if is_external { + return Err(Error::DataInvalid { + message: format!( + "Catalog-managed partitions require an internal Format Table, but {} is external", + identifier.full_name() + ), + source: None, + }); + } + core_options.try_format_table_partition_only_value_in_path()? + } else { + core_options.format_table_partition_only_value_in_path() + }; + + Ok(Some(LoadedFormatTablePartitionOptions { + table_path: core_options + .path() + .ok_or_else(|| Error::DataInvalid { + message: format!( + "REST Format Table {} is missing option '{PATH_OPTION}'", + identifier.full_name() + ), + source: None, + })? + .to_string(), + file_format: core_options.file_format().to_string(), + partitioned_table_in_metastore, + only_value_in_path, + })) +} + fn ensure_partition_option_unchanged( identifier: &Identifier, key: &str, diff --git a/crates/paimon/tests/format_partition_test.rs b/crates/paimon/tests/format_partition_test.rs index a76ed6c5..a0da749c 100644 --- a/crates/paimon/tests/format_partition_test.rs +++ b/crates/paimon/tests/format_partition_test.rs @@ -64,15 +64,6 @@ fn format_partition_paths_escape_values_and_honor_layout() { ); } -#[test] -fn format_partition_paths_match_java_escaping_for_unicode_and_closing_brace() { - let spec = HashMap::from([("region".to_string(), "杭州}".to_string())]); - let paths = FormatTablePartitionPaths::new(["region"], false); - - assert_eq!(paths.partition_name(&spec).unwrap(), "region=杭州%7D"); - assert_eq!(paths.relative_path(&spec).unwrap(), "region=杭州%7D"); -} - #[cfg(not(windows))] #[tokio::test] async fn discover_format_partitions_returns_complete_sorted_specs() { @@ -123,39 +114,6 @@ async fn discover_format_partitions_treats_missing_root_as_empty() { assert!(partitions.is_empty()); } -#[cfg(not(windows))] -#[tokio::test] -async fn discover_format_partitions_rejects_non_round_tripping_percent_escape() { - let tmp = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(tmp.path().join("dt=value%ZZ")).unwrap(); - - let error = discover_partitions(tmp.path(), &["dt"], false, "__DEFAULT_PARTITION__") - .await - .unwrap_err(); - - assert!( - error.to_string().contains("cannot round-trip"), - "expected a non-round-tripping path error, got: {error}" - ); -} - -#[cfg(not(windows))] -#[tokio::test] -async fn discover_format_partitions_ignores_nonmatching_malformed_key_segment() { - let tmp = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(tmp.path().join("wrong%ZZ=value")).unwrap(); - std::fs::create_dir_all(tmp.path().join("dt=valid")).unwrap(); - - let partitions = discover_partitions(tmp.path(), &["dt"], false, "__DEFAULT_PARTITION__") - .await - .unwrap(); - - assert_eq!( - partitions, - vec![HashMap::from([("dt".to_string(), "valid".to_string())])] - ); -} - #[cfg(not(windows))] #[tokio::test] async fn discover_value_only_partitions_rejects_parent_traversal_value() { diff --git a/crates/paimon/tests/mock_server.rs b/crates/paimon/tests/mock_server.rs index c8bf388a..b5ffb724 100644 --- a/crates/paimon/tests/mock_server.rs +++ b/crates/paimon/tests/mock_server.rs @@ -35,11 +35,10 @@ use tokio::task::JoinHandle; use paimon::api::{ AlterDatabaseRequest, AlterTableRequest, AuditRESTResponse, ConfigResponse, - CreateFunctionRequest, CreatePartitionsRequest, CreatePartitionsResponse, CreateViewRequest, - DropPartitionsRequest, DropPartitionsResponse, ErrorResponse, GetDatabaseResponse, - GetFunctionResponse, GetTableResponse, GetViewResponse, ListDatabasesResponse, - ListFunctionsResponse, ListPartitionsByNamesRequest, ListPartitionsResponse, - ListTablesResponse, ListViewsResponse, RenameTableRequest, ResourcePaths, + CreateFunctionRequest, CreatePartitionsRequest, CreateViewRequest, DropPartitionsRequest, + ErrorResponse, GetDatabaseResponse, GetFunctionResponse, GetTableResponse, GetViewResponse, + ListDatabasesResponse, ListFunctionsResponse, ListPartitionsResponse, ListTablesResponse, + ListViewsResponse, RenameTableRequest, ResourcePaths, }; use paimon::catalog::{Function, Identifier}; use paimon::spec::Partition; @@ -55,7 +54,7 @@ struct MockState { functions: HashMap, partitions: HashMap>, partition_page_responses: HashMap>, - partition_list_calls: HashMap, + partition_list_call_counts: HashMap, view_function_endpoints_unsupported: bool, drop_view_error_status: Option, list_page_size: Option, @@ -63,10 +62,8 @@ struct MockState { no_permission_tables: HashSet, create_partitions_calls: Vec<(String, String, CreatePartitionsRequest)>, drop_partitions_calls: Vec<(String, String, DropPartitionsRequest)>, - list_partitions_by_names_calls: Vec<(String, String, ListPartitionsByNamesRequest)>, create_partitions_error_status: Option, list_partitions_error_status: Option, - list_partitions_by_names_error_status: Option, /// ECS metadata role name (for token loader testing) ecs_role_name: Option, /// ECS metadata token (for token loader testing) @@ -316,7 +313,7 @@ impl RESTServer { s.partitions.retain(|key, _| !key.starts_with(&prefix)); s.partition_page_responses .retain(|key, _| !key.starts_with(&prefix)); - s.partition_list_calls + s.partition_list_call_counts .retain(|key, _| !key.starts_with(&prefix)); s.no_permission_tables .retain(|key| !key.starts_with(&prefix)); @@ -762,7 +759,7 @@ impl RESTServer { if s.tables.remove(&key).is_some() { s.partitions.remove(&key); s.partition_page_responses.remove(&key); - s.partition_list_calls.remove(&key); + s.partition_list_call_counts.remove(&key); s.no_permission_tables.remove(&key); (StatusCode::OK, Json(serde_json::json!(""))).into_response() } else { @@ -846,7 +843,7 @@ impl RESTServer { return (StatusCode::NOT_FOUND, Json(error)).into_response(); } - let registered_partitions = inner.partitions.entry(key.clone()).or_default(); + let registered_partitions = inner.partitions.entry(key).or_default(); let has_conflict = request .partition_specs .iter() @@ -867,21 +864,15 @@ impl RESTServer { return (StatusCode::CONFLICT, Json(error)).into_response(); } - let mut created = Vec::new(); - let mut existed = Vec::new(); for spec in request.partition_specs { - if registered_partitions + if !registered_partitions .iter() .any(|partition| partition.spec == spec) { - existed.push(spec); - } else { - registered_partitions.push(partition_from_spec(spec.clone())); - created.push(spec); + registered_partitions.push(partition_from_spec(spec)); } } - inner.partition_page_responses.remove(&key); - let response = CreatePartitionsResponse { created, existed }; + let response = json!({"success": true}); (StatusCode::OK, Json(response)).into_response() } @@ -911,14 +902,17 @@ impl RESTServer { ); return (status, Json(error)).into_response(); } - if let Some(responses) = inner.partition_page_responses.get(&key) { - let responses = responses.clone(); + if inner.partition_page_responses.contains_key(&key) { let request_index = { - let calls = inner.partition_list_calls.entry(key.clone()).or_default(); + let calls = inner + .partition_list_call_counts + .entry(key.clone()) + .or_default(); let request_index = *calls; *calls += 1; request_index }; + let responses = &inner.partition_page_responses[&key]; let expected_token = request_index .checked_sub(1) .and_then(|index| responses.get(index)) @@ -975,18 +969,13 @@ impl RESTServer { return (StatusCode::NOT_FOUND, Json(error)).into_response(); } - let registered_partitions = inner.partitions.entry(key.clone()).or_default(); - let missing = request - .partition_specs - .iter() - .filter(|spec| { - !registered_partitions - .iter() - .any(|partition| partition.spec == **spec) - }) - .cloned() - .collect::>(); - if !missing.is_empty() && !request.ignore_if_not_exists { + let registered_partitions = inner.partitions.entry(key).or_default(); + let has_missing = request.partition_specs.iter().any(|spec| { + !registered_partitions + .iter() + .any(|partition| partition.spec == *spec) + }); + if has_missing && !request.ignore_if_not_exists { let error = ErrorResponse::new( Some("partition".to_string()), Some(table), @@ -996,60 +985,9 @@ impl RESTServer { return (StatusCode::NOT_FOUND, Json(error)).into_response(); } - let mut dropped = Vec::new(); - registered_partitions.retain(|partition| { - if request.partition_specs.contains(&partition.spec) { - dropped.push(partition.spec.clone()); - false - } else { - true - } - }); - inner.partition_page_responses.remove(&key); - let response = DropPartitionsResponse { dropped, missing }; - (StatusCode::OK, Json(response)).into_response() - } - - /// Handle POST /databases/:db/tables/:table/partitions/list-by-names. - pub async fn list_partitions_by_names( - Path((db, table)): Path<(String, String)>, - Extension(state): Extension>, - Json(request): Json, - ) -> impl IntoResponse { - let mut inner = state.inner.lock().unwrap(); - inner - .list_partitions_by_names_calls - .push((db.clone(), table.clone(), request.clone())); - if let Some(status) = inner.list_partitions_by_names_error_status { - let error = ErrorResponse::new( - Some("partition".to_string()), - Some(table.clone()), - Some("Invalid partition request".to_string()), - Some(status.as_u16() as i32), - ); - return (status, Json(error)).into_response(); - } - - let key = format!("{db}.{table}"); - if !inner.tables.contains_key(&key) { - let error = ErrorResponse::new( - Some("table".to_string()), - Some(table), - Some("Not Found".to_string()), - Some(404), - ); - return (StatusCode::NOT_FOUND, Json(error)).into_response(); - } - - let partitions = inner - .partitions - .get(&key) - .into_iter() - .flatten() - .filter(|partition| request.partition_specs.contains(&partition.spec)) - .cloned() - .collect(); - let response = ListPartitionsResponse::new(Some(partitions), None); + registered_partitions + .retain(|partition| !request.partition_specs.contains(&partition.spec)); + let response = json!({"success": true}); (StatusCode::OK, Json(response)).into_response() } @@ -1109,17 +1047,6 @@ impl RESTServer { if s.no_permission_tables.remove(&source_key) { s.no_permission_tables.insert(dest_key.clone()); } - if let Some(partitions) = s.partitions.remove(&source_key) { - s.partitions.insert(dest_key.clone(), partitions); - } - if let Some(responses) = s.partition_page_responses.remove(&source_key) { - s.partition_page_responses - .insert(dest_key.clone(), responses); - } - if let Some(calls) = s.partition_list_calls.remove(&source_key) { - s.partition_list_calls.insert(dest_key, calls); - } - (StatusCode::OK, Json(serde_json::json!(""))).into_response() } else { let err = ErrorResponse::new( @@ -1232,14 +1159,6 @@ impl RESTServer { self.inner.lock().unwrap().list_partitions_error_status = status; } - /// Make the list-partitions-by-names endpoint return the given status. - pub fn set_list_partitions_by_names_error_status(&self, status: Option) { - self.inner - .lock() - .unwrap() - .list_partitions_by_names_error_status = status; - } - /// Add a table with schema and path to the server state. /// /// This is needed for `RESTCatalog::get_table` which requires @@ -1313,7 +1232,7 @@ impl RESTServer { ); inner.partitions.insert(key.clone(), partitions); inner.partition_page_responses.remove(&key); - inner.partition_list_calls.remove(&key); + inner.partition_list_call_counts.remove(&key); } /// Set explicit list-partitions pages and response tokens in request order. @@ -1336,15 +1255,15 @@ impl RESTServer { inner .partition_page_responses .insert(key.clone(), responses); - inner.partition_list_calls.remove(&key); + inner.partition_list_call_counts.remove(&key); } /// Return how many paged partition-list requests the table received. - pub fn table_partition_list_calls(&self, database: &str, table: &str) -> usize { + pub fn table_partition_list_call_count(&self, database: &str, table: &str) -> usize { self.inner .lock() .unwrap() - .partition_list_calls + .partition_list_call_counts .get(&format!("{database}.{table}")) .copied() .unwrap_or_default() @@ -1360,17 +1279,6 @@ impl RESTServer { self.inner.lock().unwrap().drop_partitions_calls.clone() } - /// Return all list-partitions-by-names calls received by the server. - pub fn list_partitions_by_names_calls( - &self, - ) -> Vec<(String, String, ListPartitionsByNamesRequest)> { - self.inner - .lock() - .unwrap() - .list_partitions_by_names_calls - .clone() - } - /// Get the server URL. pub fn url(&self) -> Option { self.addr.map(|a| format!("http://{a}")) @@ -1496,10 +1404,6 @@ pub async fn start_mock_server( &format!("{prefix}/databases/:db/tables/:table/partitions/drop"), post(RESTServer::drop_partitions), ) - .route( - &format!("{prefix}/databases/:db/tables/:table/partitions/list-by-names"), - post(RESTServer::list_partitions_by_names), - ) .route( &format!("{prefix}/databases/:db/views"), get(RESTServer::list_views).post(RESTServer::create_view), diff --git a/crates/paimon/tests/rest_api_test.rs b/crates/paimon/tests/rest_api_test.rs index 8ab6d5da..fc879119 100644 --- a/crates/paimon/tests/rest_api_test.rs +++ b/crates/paimon/tests/rest_api_test.rs @@ -24,9 +24,7 @@ use std::collections::HashMap; use paimon::api::auth::{DLFECSTokenLoader, DLFToken, DLFTokenLoader}; use paimon::api::rest_api::RESTApi; -use paimon::api::{ - ConfigResponse, CreatePartitionsRequest, DropPartitionsRequest, ListPartitionsByNamesRequest, -}; +use paimon::api::{ConfigResponse, CreatePartitionsRequest, DropPartitionsRequest}; use paimon::catalog::{Function, FunctionDefinition, Identifier, ViewSchema}; use paimon::common::Options; use paimon::spec::DataField; @@ -606,113 +604,39 @@ async fn test_drop_table_no_permission() { // ==================== Partition Tests ==================== #[tokio::test] -async fn test_list_partitions_by_names_rejects_more_than_1000_specs() { - let (ctx, identifier) = setup_partition_api().await; - let partition_specs = (0..=1000) - .map(|day| HashMap::from([("day".to_string(), day.to_string())])) - .collect::>(); - - let error = ctx - .api - .list_partitions_by_names(&identifier, partition_specs) - .await - .unwrap_err(); - - let paimon::Error::DataInvalid { message, .. } = error else { - panic!("expected a request-size validation error, got: {error}"); - }; - assert_eq!( - message, - "List partitions by names accepts at most 1000 partition specs for table \ - default.managed_table, got 1001" - ); - assert!(ctx.server.list_partitions_by_names_calls().is_empty()); -} - -#[tokio::test] -async fn test_create_partitions_posts_expected_request() { +async fn test_partition_mutations_post_expected_requests() { let (ctx, identifier) = setup_partition_api().await; let partition_specs = vec![HashMap::from([ ("dt".to_string(), "2026-07-22".to_string()), ("hour".to_string(), "10".to_string()), ])]; - let response = ctx - .api + ctx.api .create_partitions(&identifier, partition_specs.clone(), false) .await .unwrap(); - assert_eq!(response.created, partition_specs.clone()); - assert!(response.existed.is_empty()); assert_eq!( - ctx.server.create_partitions_calls().pop(), - Some(( + ctx.server.create_partitions_calls(), + vec![( "default".to_string(), "managed_table".to_string(), - CreatePartitionsRequest::new(partition_specs, false), - )) + CreatePartitionsRequest::new(partition_specs.clone(), false), + )] ); -} - -#[tokio::test] -async fn test_drop_partitions_posts_expected_request() { - let (ctx, identifier) = setup_partition_api().await; - let partition_specs = vec![HashMap::from([( - "dt".to_string(), - "2026-07-22".to_string(), - )])]; - ctx.server - .set_table_partitions("default", "managed_table", partition_specs.clone()); - let response = ctx - .api + ctx.api .drop_partitions(&identifier, partition_specs.clone(), false) .await .unwrap(); - assert_eq!(response.dropped, partition_specs.clone()); - assert!(response.missing.is_empty()); assert_eq!( - ctx.server.drop_partitions_calls().pop(), - Some(( + ctx.server.drop_partitions_calls(), + vec![( "default".to_string(), "managed_table".to_string(), DropPartitionsRequest::new(partition_specs, false), - )) - ); -} - -#[tokio::test] -async fn test_list_partitions_by_names_posts_expected_request() { - let (ctx, identifier) = setup_partition_api().await; - let partition_specs = vec![ - HashMap::from([("dt".to_string(), "2026-07-22".to_string())]), - HashMap::from([("dt".to_string(), "2026-07-21".to_string())]), - ]; - ctx.server - .set_table_partitions("default", "managed_table", partition_specs.clone()); - - let partitions = ctx - .api - .list_partitions_by_names(&identifier, partition_specs.clone()) - .await - .unwrap(); - - assert_eq!( - partitions - .into_iter() - .map(|partition| partition.spec) - .collect::>(), - partition_specs.clone() - ); - assert_eq!( - ctx.server.list_partitions_by_names_calls().pop(), - Some(( - "default".to_string(), - "managed_table".to_string(), - ListPartitionsByNamesRequest::new(partition_specs), - )) + )] ); } @@ -760,7 +684,7 @@ async fn test_list_partitions_stops_on_empty_next_page_token() { ); assert_eq!( ctx.server - .table_partition_list_calls("default", "managed_table"), + .table_partition_list_call_count("default", "managed_table"), 1 ); } @@ -793,7 +717,6 @@ async fn test_list_partitions_rejects_repeated_page_token() { )])], Some("a".to_string()), ), - (Vec::new(), None), ], ); @@ -809,7 +732,7 @@ async fn test_list_partitions_rejects_repeated_page_token() { ); assert_eq!( ctx.server - .table_partition_list_calls("default", "managed_table"), + .table_partition_list_call_count("default", "managed_table"), 3 ); } diff --git a/crates/paimon/tests/rest_catalog_test.rs b/crates/paimon/tests/rest_catalog_test.rs index f42626fc..c4643474 100644 --- a/crates/paimon/tests/rest_catalog_test.rs +++ b/crates/paimon/tests/rest_catalog_test.rs @@ -34,7 +34,7 @@ use paimon::spec::{ BigIntType, BlobType, BlobViewStruct, DataField, DataType, Datum, IntType, PredicateBuilder, Schema, SchemaChange, VarCharType, }; -use paimon::{CatalogOptions, FileSystemCatalog, RESTEnv, Table}; +use paimon::{CatalogOptions, FileSystemCatalog, Table}; mod mock_server; use mock_server::{start_mock_server, RESTServer}; @@ -84,7 +84,7 @@ fn test_schema() -> Schema { .expect("Failed to build schema") } -fn catalog_managed_format_table_schema(options: &[(&str, &str)]) -> Schema { +fn format_table_schema(options: &[(&str, &str)]) -> Schema { let mut builder = Schema::builder() .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) .column("id", DataType::BigInt(BigIntType::new())) @@ -192,7 +192,7 @@ fn collect_blob_rows(batches: &[RecordBatch]) -> Vec<(i32, String, Option>(); + ctx.catalog + .create_partitions(&identifier, Vec::new(), false) + .await + .unwrap(); ctx.catalog .create_partitions(&identifier, partition_specs.clone(), true) .await @@ -288,7 +292,7 @@ async fn test_rest_catalog_maps_invalid_partition_request() { } #[tokio::test] -async fn test_rest_catalog_batches_unregister_partitions() { +async fn test_rest_catalog_skips_empty_and_batches_drop_partitions() { let ctx = setup_catalog(vec!["default"]).await; let identifier = Identifier::new("default", "managed_table"); ctx.server.add_table("default", "managed_table"); @@ -299,7 +303,11 @@ async fn test_rest_catalog_batches_unregister_partitions() { .set_table_partitions("default", "managed_table", partition_specs.clone()); ctx.catalog - .unregister_partitions(&identifier, partition_specs.clone()) + .drop_partitions(&identifier, Vec::new()) + .await + .unwrap(); + ctx.catalog + .drop_partitions(&identifier, partition_specs.clone()) .await .unwrap(); @@ -312,69 +320,6 @@ async fn test_rest_catalog_batches_unregister_partitions() { .all(|(_, _, request)| request.ignore_if_not_exists)); } -#[tokio::test] -async fn test_rest_catalog_batches_list_partitions_by_names() { - let ctx = setup_catalog(vec!["default"]).await; - let identifier = Identifier::new("default", "managed_table"); - ctx.server.add_table("default", "managed_table"); - let partition_specs = (0..1001) - .map(|value| HashMap::from([("dt".to_string(), value.to_string())])) - .collect::>(); - ctx.server - .set_table_partitions("default", "managed_table", partition_specs.clone()); - - let listed = ctx - .catalog - .list_partitions_by_names(&identifier, partition_specs.clone()) - .await - .unwrap(); - - let calls = ctx.server.list_partitions_by_names_calls(); - assert_eq!(calls.len(), 2); - assert_eq!(calls[0].2.partition_specs, partition_specs[..1000]); - assert_eq!(calls[1].2.partition_specs, partition_specs[1000..]); - let listed_specs = listed - .into_iter() - .map(|partition| partition.spec) - .collect::>(); - assert_eq!(listed_specs, partition_specs); -} - -#[tokio::test] -async fn test_rest_catalog_rejects_invalid_or_out_of_range_partition_page_token() { - let ctx = setup_catalog(vec!["default"]).await; - let identifier = Identifier::new("default", "managed_table"); - ctx.server.add_table("default", "managed_table"); - ctx.server.set_table_partition_page_responses( - "default", - "managed_table", - vec![( - vec![HashMap::from([( - "dt".to_string(), - "2026-07-22".to_string(), - )])], - None, - )], - ); - - for token in ["invalid", "1"] { - let error = ctx - .catalog - .list_partitions_paged(&identifier, None, Some(token)) - .await - .unwrap_err(); - assert!( - matches!( - error, - paimon::Error::RestApi { - source: paimon::api::RestError::BadRequest { .. } - } - ), - "expected bad page token error for {token}, got: {error}" - ); - } -} - #[tokio::test] async fn test_rest_catalog_partition_operations_map_missing_table() { let ctx = setup_catalog(vec!["default"]).await; @@ -391,16 +336,10 @@ async fn test_rest_catalog_partition_operations_map_missing_table() { .unwrap_err(); let drop_error = ctx .catalog - .unregister_partitions(&identifier, partition_specs.clone()) - .await - .unwrap_err(); - let list_error = ctx - .catalog - .list_partitions_by_names(&identifier, partition_specs) + .drop_partitions(&identifier, partition_specs.clone()) .await .unwrap_err(); - - for error in [create_error, drop_error, list_error] { + for error in [create_error, drop_error] { assert!(matches!( error, paimon::Error::TableNotExist { full_name } if full_name == "default.missing" @@ -821,9 +760,9 @@ async fn test_rest_catalog_reads_format_table() { } #[tokio::test] -async fn test_rest_catalog_marks_internal_format_table_as_catalog_managed() { +async fn test_rest_catalog_validates_dynamic_managed_partition_options() { let ctx = setup_catalog(vec!["default"]).await; - let schema = catalog_managed_format_table_schema(&[]); + let schema = format_table_schema(&[]); let identifier = Identifier::new("default", "managed_format"); add_internal_table_with_schema( &ctx.server, @@ -835,12 +774,77 @@ async fn test_rest_catalog_marks_internal_format_table_as_catalog_managed() { let table = ctx.catalog.get_table(&identifier).await.unwrap(); assert!(table.has_catalog_managed_partitions()); - assert!(table - .copy_with_options(HashMap::from([( + for (key, value) in [ + ("metastore.partitioned-table", "false"), + ("format-table.partition-path-only-value", "true"), + ("format-table.implementation", "engine"), + ] { + let error = table + .copy_with_time_travel(HashMap::from([(key.to_string(), value.to_string())])) + .await + .unwrap_err(); + assert!( + matches!(error, paimon::Error::DataInvalid { .. }) && error.to_string().contains(key), + "expected {key} validation error, got: {error}" + ); + } + + let copied = table + .copy_with_time_travel(HashMap::from([ + ( + "metastore.partitioned-table".to_string(), + "TRUE".to_string(), + ), + ( + "format-table.partition-path-only-value".to_string(), + "false".to_string(), + ), + ( + "format-table.implementation".to_string(), + "PAIMON".to_string(), + ), + ("type".to_string(), "table".to_string()), + ])) + .await + .unwrap(); + assert!(copied.has_catalog_managed_partitions()); + assert!(copied + .new_read_builder() + .new_scan() + .plan() + .await + .unwrap() + .splits() + .is_empty()); +} + +#[tokio::test] +async fn test_rest_catalog_rejects_enabling_managed_partitions_in_dynamic_options() { + let ctx = setup_catalog(vec!["default"]).await; + let schema = format_table_schema(&[("metastore.partitioned-table", "false")]); + let identifier = Identifier::new("default", "unmanaged_format"); + add_internal_table_with_schema( + &ctx.server, + "unmanaged_format", + schema, + "memory:/unmanaged_format", + ); + + let table = ctx.catalog.get_table(&identifier).await.unwrap(); + assert!(!table.has_catalog_managed_partitions()); + + let error = table + .copy_with_time_travel(HashMap::from([( "metastore.partitioned-table".to_string(), - "false".to_string(), + "true".to_string(), )])) - .has_catalog_managed_partitions()); + .await + .unwrap_err(); + assert!( + matches!(error, paimon::Error::DataInvalid { .. }) + && error.to_string().contains("metastore.partitioned-table"), + "expected partition source validation error, got: {error}" + ); } #[tokio::test] @@ -857,8 +861,13 @@ async fn test_rest_catalog_rejects_invalid_catalog_managed_partition_options() { "format-table.implementation", "unknown", ), + ( + "invalid_partition_layout", + "format-table.partition-path-only-value", + "tru", + ), ] { - let schema = catalog_managed_format_table_schema(&[(option, bad_value)]); + let schema = format_table_schema(&[(option, bad_value)]); add_internal_table_with_schema(&ctx.server, table, schema, &format!("memory:/{table}")); let error = ctx @@ -876,134 +885,10 @@ async fn test_rest_catalog_rejects_invalid_catalog_managed_partition_options() { } } -#[tokio::test] -async fn test_rest_catalog_accepts_supported_partition_options() { - let ctx = setup_catalog(vec!["default"]).await; - for (table, partitioned, implementation, expected_managed) in [ - ("managed_paimon", "true", "paimon", true), - ("unmanaged_paimon", "false", "paimon", false), - ("unmanaged_engine", "false", "engine", false), - ] { - let schema = Schema::builder() - .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) - .partition_keys(["dt"]) - .option("type", "format-table") - .option("file.format", "parquet") - .option("metastore.partitioned-table", partitioned) - .option("format-table.implementation", implementation) - .build() - .unwrap(); - add_internal_table_with_schema(&ctx.server, table, schema, &format!("memory:/{table}")); - - let loaded = ctx - .catalog - .get_table(&Identifier::new("default", table)) - .await - .unwrap(); - - assert_eq!( - loaded.has_catalog_managed_partitions(), - expected_managed, - "{table}" - ); - } -} - -#[tokio::test] -async fn test_rest_env_new_does_not_enable_catalog_managed_partitions() { - let ctx = setup_catalog(vec!["default"]).await; - let schema = catalog_managed_format_table_schema(&[]); - let identifier = Identifier::new("default", "manual_rest_env"); - add_internal_table_with_schema( - &ctx.server, - "manual_rest_env", - schema, - "memory:/manual_rest_env", - ); - - let loaded = ctx.catalog.get_table(&identifier).await.unwrap(); - let plain_rest_env = RESTEnv::new( - identifier.clone(), - "manual-uuid".to_string(), - loaded.rest_env().unwrap().api().clone(), - Options::new(), - false, - ); - let manually_constructed = Table::new( - loaded.file_io().clone(), - identifier, - loaded.location().to_string(), - loaded.schema().clone(), - Some(plain_rest_env), - ); - - assert!(!manually_constructed.has_catalog_managed_partitions()); -} - -#[cfg(not(windows))] -#[tokio::test] -async fn test_managed_format_partition_listing_requires_rest_endpoint() { - let temp_dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(temp_dir.path().join("dt=2026-07-22")).unwrap(); - let table_path = format!("file://{}", temp_dir.path().display()); - let ctx = setup_catalog(vec!["default"]).await; - let schema = catalog_managed_format_table_schema(&[]); - let identifier = Identifier::new("default", "managed_partition_endpoint"); - add_internal_table_with_schema( - &ctx.server, - "managed_partition_endpoint", - schema, - &table_path, - ); - ctx.server - .set_list_partitions_error_status(Some(StatusCode::NOT_IMPLEMENTED)); - ctx.server - .set_list_partitions_by_names_error_status(Some(StatusCode::NOT_IMPLEMENTED)); - - let error = ctx.catalog.list_partitions(&identifier).await.unwrap_err(); - - assert!(matches!( - error, - paimon::Error::RestApi { - source: paimon::api::RestError::NotImplemented { .. } - } - )); - - let paged_error = ctx - .catalog - .list_partitions_paged(&identifier, Some(10), None) - .await - .unwrap_err(); - assert!(matches!( - paged_error, - paimon::Error::RestApi { - source: paimon::api::RestError::NotImplemented { .. } - } - )); - - let by_names_error = ctx - .catalog - .list_partitions_by_names( - &identifier, - vec![HashMap::from([( - "dt".to_string(), - "2026-07-22".to_string(), - )])], - ) - .await - .unwrap_err(); - assert!(matches!( - by_names_error, - paimon::Error::RestApi { - source: paimon::api::RestError::NotImplemented { .. } - } - )); -} - #[tokio::test] async fn test_rest_catalog_rejects_external_catalog_managed_format_table() { let ctx = setup_catalog(vec!["default"]).await; - let schema = catalog_managed_format_table_schema(&[]); + let schema = format_table_schema(&[]); let identifier = Identifier::new("default", "external_managed_format"); ctx.server.add_table_with_schema( "default", @@ -1025,7 +910,7 @@ async fn test_rest_catalog_rejects_external_catalog_managed_format_table() { #[tokio::test] async fn test_rest_catalog_rejects_engine_managed_format_table() { let ctx = setup_catalog(vec!["default"]).await; - let schema = catalog_managed_format_table_schema(&[("format-table.implementation", "engine")]); + let schema = format_table_schema(&[("format-table.implementation", "engine")]); let identifier = Identifier::new("default", "engine_managed_format"); add_internal_table_with_schema( &ctx.server, @@ -1046,7 +931,7 @@ async fn test_rest_catalog_rejects_engine_managed_format_table() { #[cfg(not(windows))] #[tokio::test] -async fn test_managed_format_scan_hides_unregistered_partition() { +async fn test_managed_format_scan_uses_registered_partition_paths_once() { let tmp = tempfile::tempdir().unwrap(); for dt in ["2026-07-21", "2026-07-22"] { let partition_dir = tmp.path().join(format!("dt={dt}")); @@ -1056,16 +941,16 @@ async fn test_managed_format_scan_hides_unregistered_partition() { let table_path = format!("file://{}", tmp.path().display()); let ctx = setup_catalog(vec!["default"]).await; - let schema = catalog_managed_format_table_schema(&[]); + let schema = format_table_schema(&[]); let identifier = Identifier::new("default", "managed_visibility"); add_internal_table_with_schema(&ctx.server, "managed_visibility", schema, &table_path); ctx.server.set_table_partitions( "default", "managed_visibility", - vec![HashMap::from([( - "dt".to_string(), - "2026-07-22".to_string(), - )])], + vec![ + HashMap::from([("dt".to_string(), "2026-07-22".to_string())]), + HashMap::from([("dt".to_string(), "2026-07-22".to_string())]), + ], ); let table = ctx.catalog.get_table(&identifier).await.unwrap(); @@ -1075,97 +960,6 @@ async fn test_managed_format_scan_hides_unregistered_partition() { assert!(plan.splits()[0].bucket_path().ends_with("/dt=2026-07-22")); } -#[tokio::test] -async fn test_rest_format_table_rejects_partition_option_changes_in_dynamic_copy() { - let ctx = setup_catalog(vec!["default"]).await; - let schema = catalog_managed_format_table_schema(&[]); - let identifier = Identifier::new("default", "managed_dynamic_layout"); - add_internal_table_with_schema( - &ctx.server, - "managed_dynamic_layout", - schema, - "memory:/managed_dynamic_layout", - ); - - let table = ctx.catalog.get_table(&identifier).await.unwrap(); - for (key, value) in [ - ("metastore.partitioned-table", "false"), - ("format-table.partition-path-only-value", "true"), - ("format-table.implementation", "engine"), - ] { - let error = table - .copy_with_time_travel(HashMap::from([(key.to_string(), value.to_string())])) - .await - .unwrap_err(); - assert!( - matches!(error, paimon::Error::DataInvalid { .. }) && error.to_string().contains(key), - "expected {key} validation error, got: {error}" - ); - } - - let copied = table - .copy_with_time_travel(HashMap::from([ - ( - "metastore.partitioned-table".to_string(), - "TRUE".to_string(), - ), - ( - "format-table.partition-path-only-value".to_string(), - "false".to_string(), - ), - ( - "format-table.implementation".to_string(), - "PAIMON".to_string(), - ), - ("read.batch-size".to_string(), "2048".to_string()), - ("type".to_string(), "table".to_string()), - ("path".to_string(), "memory:/dynamic-path".to_string()), - ("file.format".to_string(), "orc".to_string()), - ])) - .await - .unwrap(); - assert!(copied.has_catalog_managed_partitions()); - assert_eq!( - copied.schema().options().get("read.batch-size"), - Some(&"2048".to_string()) - ); -} - -#[tokio::test] -async fn test_rest_format_table_rejects_enabling_catalog_partitions_in_dynamic_copy() { - let ctx = setup_catalog(vec!["default"]).await; - let schema = Schema::builder() - .column("dt", DataType::VarChar(VarCharType::new(255).unwrap())) - .partition_keys(["dt"]) - .option("type", "format-table") - .option("file.format", "parquet") - .option("metastore.partitioned-table", "false") - .build() - .unwrap(); - let identifier = Identifier::new("default", "unmanaged_dynamic_source"); - add_internal_table_with_schema( - &ctx.server, - "unmanaged_dynamic_source", - schema, - "memory:/unmanaged_dynamic_source", - ); - - let table = ctx.catalog.get_table(&identifier).await.unwrap(); - let error = table - .copy_with_time_travel(HashMap::from([( - "metastore.partitioned-table".to_string(), - "true".to_string(), - )])) - .await - .unwrap_err(); - - assert!( - matches!(error, paimon::Error::DataInvalid { .. }) - && error.to_string().contains("metastore.partitioned-table"), - "expected partition option validation error, got: {error}" - ); -} - #[cfg(not(windows))] #[tokio::test] async fn test_managed_format_scan_treats_registered_missing_directory_as_empty() { @@ -1173,7 +967,7 @@ async fn test_managed_format_scan_treats_registered_missing_directory_as_empty() let table_path = format!("file://{}", tmp.path().display()); let ctx = setup_catalog(vec!["default"]).await; - let schema = catalog_managed_format_table_schema(&[]); + let schema = format_table_schema(&[]); let identifier = Identifier::new("default", "managed_missing_directory"); add_internal_table_with_schema( &ctx.server, @@ -1200,13 +994,10 @@ async fn test_managed_format_scan_treats_registered_missing_directory_as_empty() #[tokio::test] async fn test_managed_format_scan_propagates_partition_listing_error() { let tmp = tempfile::tempdir().unwrap(); - let partition_dir = tmp.path().join("dt=2026-07-22"); - std::fs::create_dir_all(&partition_dir).unwrap(); - std::fs::write(partition_dir.join("part-0.parquet"), b"data").unwrap(); let table_path = format!("file://{}", tmp.path().display()); let ctx = setup_catalog(vec!["default"]).await; - let schema = catalog_managed_format_table_schema(&[]); + let schema = format_table_schema(&[]); let identifier = Identifier::new("default", "managed_scan_error"); add_internal_table_with_schema(&ctx.server, "managed_scan_error", schema, &table_path); ctx.server @@ -1235,7 +1026,7 @@ async fn test_managed_format_scan_reports_table_for_malformed_partition_metadata let table_path = format!("file://{}", tmp.path().display()); let ctx = setup_catalog(vec!["default"]).await; - let schema = catalog_managed_format_table_schema(&[]); + let schema = format_table_schema(&[]); let identifier = Identifier::new("default", "managed_corrupt_metadata"); add_internal_table_with_schema(&ctx.server, "managed_corrupt_metadata", schema, &table_path); ctx.server.set_table_partitions( diff --git a/docs/src/sql.md b/docs/src/sql.md index 14c11f66..a84a956e 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -40,7 +40,7 @@ Mosaic support is always available and currently read-only. SQL queries can read SQL support has two layers: - DataFusion provides the parser, query planner, optimizer, execution engine, expressions, scalar functions, aggregate functions, and window functions. SQL statements that `SQLContext` does not intercept are delegated to DataFusion. This includes the DataFusion SQL surface for `SELECT` queries, CTEs (including recursive CTEs), subqueries, joins including `LATERAL` joins, SQL lambda functions, grouping, `HAVING`, window clauses, `QUALIFY`, set operations, `ORDER BY`, `LIMIT`/`OFFSET`, `EXPLAIN`, information-schema commands such as `SHOW TABLES`, `DESCRIBE`, `COPY`, and ordinary `INSERT`. -- Paimon-specific table management and row-level writes are implemented by `SQLContext`. This includes Paimon `CREATE TABLE`, `ALTER TABLE`, `DROP TABLE`, `CREATE TEMPORARY TABLE`, `CREATE TEMPORARY VIEW`, REST Catalog persistent `CREATE VIEW`, `DROP VIEW`, and `CREATE FUNCTION`, catalog-managed Format Table partition commands for internal tables loaded from REST Catalog (`SHOW PARTITIONS`, `ADD/DROP PARTITION`, and `MSCK REPAIR TABLE`), `DROP TEMPORARY TABLE` / `VIEW`, `INSERT OVERWRITE ... PARTITION`, `UPDATE`, `DELETE`, `MERGE INTO`, `TRUNCATE TABLE`, `CALL sys.*`, Paimon time travel, and `SET` / `RESET 'paimon.*'`. +- Paimon-specific table management and row-level writes are implemented by `SQLContext`. This includes Paimon `CREATE TABLE`, `ALTER TABLE`, `DROP TABLE`, `CREATE TEMPORARY TABLE`, `CREATE TEMPORARY VIEW`, REST Catalog persistent `CREATE VIEW`, `DROP VIEW`, and `CREATE FUNCTION`, `DROP TEMPORARY TABLE` / `VIEW`, `INSERT OVERWRITE ... PARTITION`, `UPDATE`, `DELETE`, `MERGE INTO`, `TRUNCATE TABLE`, `ALTER TABLE ... DROP PARTITION`, `CALL sys.*`, Paimon time travel, and `SET` / `RESET 'paimon.*'`. Not every DataFusion DDL/DML statement maps to a Paimon table operation. For Paimon catalogs, `CREATE EXTERNAL TABLE`, `LOCATION`, `CREATE MATERIALIZED VIEW`, and persistent `CREATE TABLE AS SELECT` are rejected or not implemented. Persistent `CREATE FUNCTION` is supported only for the REST Catalog SQL scalar form documented below. DataFusion `COPY` can export query results to files; it does not create or commit Paimon table files. @@ -703,80 +703,6 @@ ALTER TABLE paimon.my_db.users SET TBLPROPERTIES('data-evolution.enabled' = 'tru ALTER TABLE IF EXISTS paimon.my_db.users ADD COLUMN age INT; ``` -### Catalog-managed Format Table partitions - -`SQLContext` supports partition commands for internal REST Catalog Format -Tables that are partitioned, set `metastore.partitioned-table=true`, and do -not set `format-table.implementation=engine`. - -REST registrations determine the partitions returned by `SHOW PARTITIONS` and -included in scans. Unregistered directories are ignored. A registered -partition with a missing directory reads as empty; other storage errors are -reported. - -#### SHOW PARTITIONS - -```sql -SHOW PARTITIONS paimon.my_db.events; -SHOW PARTITIONS paimon.my_db.events PARTITION (region = 'us'); -``` - -The filter may contain any subset of partition keys. Results use declared key -order, are sorted lexicographically, and normalize values through their column -types. Default values are shown as `null`. - -#### ADD PARTITION - -```sql -ALTER TABLE paimon.my_db.events ADD PARTITION (dt = '2026-07-22', region = 'us'); - -ALTER TABLE paimon.my_db.events ADD IF NOT EXISTS PARTITION (dt = '2026-07-22', region = 'us'); -``` - -Each specification must include every partition key. Partitions are registered -in REST before their directories are created. `IF NOT EXISTS` ignores existing -registrations; otherwise duplicates are conflicts. - -Supported types are integers, `BOOLEAN`, `CHAR`, `VARCHAR`, and `DATE`. Values -are normalized through the target type. `NULL` and blank character values use -`partition.default-name`; dates honor `partition.legacy-name`. Paths use -Java-compatible escaping. - -#### DROP PARTITION - -```sql -ALTER TABLE paimon.my_db.events DROP PARTITION (dt = '2026-07-22', region = 'us'); - -ALTER TABLE paimon.my_db.events DROP IF EXISTS PARTITION (region = 'us'); -``` - -A complete specification selects its exact raw registration when present; -otherwise typed matching may select an equivalent value such as integer -metadata `01` for SQL value `1`. A partial specification selects every match, -including non-leading keys. Registrations are removed before directories are -deleted. Missing complete specifications fail unless `IF EXISTS` is used. - -#### MSCK REPAIR TABLE - -```sql -MSCK REPAIR TABLE paimon.my_db.events; -REPAIR TABLE paimon.my_db.events ADD PARTITIONS; -MSCK REPAIR TABLE paimon.my_db.events DROP PARTITIONS; -MSCK REPAIR TABLE paimon.my_db.events SYNC PARTITIONS; -``` - -Plain repair and `ADD PARTITIONS` register filesystem partitions missing from -REST. `DROP PARTITIONS` unregisters entries whose directories are absent. -`SYNC PARTITIONS` adds and then drops. Repair only changes REST metadata; it -never creates or deletes directories. - -Filesystem values are registered as written. Paths must use canonical -Java-compatible escaping; malformed or non-canonical escapes fail before REST -metadata is changed. - -`LOCATION`, `DROP ... PURGE`, branch-qualified table names, and these commands -on non-REST or unsupported Format Tables are not supported. - ## DML The table type determines which row-level DML operations are supported: @@ -944,12 +870,6 @@ Multiple partition key-value pairs can be specified: ALTER TABLE paimon.my_db.events DROP PARTITION (dt = '2024-01-01', region = 'us'); ``` -For a catalog-managed Format Table, use the semantics documented in -[Catalog-managed Format Table partitions](#catalog-managed-format-table-partitions). -These commands apply to internal Format Tables loaded from REST Catalog. REST -metadata is unregistered before physical directories are deleted, and multiple -`PARTITION (...)` clauses are supported. - ## Procedures Use `CALL` to invoke built-in procedures. All procedures are under the `sys` namespace. @@ -2050,8 +1970,6 @@ the normal physical format without wrapping the writer. | `'deletion-vectors.enabled' = 'true'` | Enable deletion vectors | | `'cross-partition-update.enabled' = 'true'` | Allow cross-partition updates | | `'changelog-producer' = 'input'` | Changelog producer (PK tables with input mode reject writes) | -| `'metastore.partitioned-table' = 'true'` | Use REST registrations as the authoritative partition set for an internal Format Table | -| `'format-table.partition-path-only-value' = 'true'` | Store Format Table partition directories as value-only path components instead of `key=value` components | ## Full Example