From 1e65d3fc2dfd18d7a83cab16ac6d3e93f8a68718 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 30 Jul 2026 15:08:41 +0200 Subject: [PATCH 01/20] add tests Signed-off-by: Konstantin Morozov --- .../test.py | 24 +++++++++ .../test.py | 49 +++++++++++++++++++ .../test.py | 23 +++++++++ .../test.py | 32 ++++++++++++ 4 files changed, 128 insertions(+) diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index 486adf1f2b17..78a0b94d2414 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -13,6 +13,7 @@ test_export_part_with_year_transform_partition – toYearNumSinceEpoch() partition expression test_export_part_with_bucket_partition – icebergBucket(N, col) partition expression test_export_part_partition_key_mismatch_is_rejected – mismatched partition spec rejected synchronously + test_export_part_same_partition_key_different_column_order – same partition key, different column order """ import logging @@ -462,6 +463,29 @@ def test_export_part_partition_key_mismatch_is_rejected(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") +def test_export_part_same_partition_key_different_column_order(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_reordered_{sfx}" + iceberg = f"iceberg_reordered_{sfx}" + + make_mt(node, mt, "a Int32, b Int32", "a") + make_iceberg_s3(node, iceberg, "b Int32, a Int32", "a") + + node.query(f"INSERT INTO {mt} VALUES (1, 1), (1, 2)") + + part_1 = get_part(node, mt, "1") + export_part(node, mt, part_1, iceberg) + wait_for_export_part(node, mt, part_1) + + result = node.query(f"SELECT a, b FROM {iceberg} ORDER BY a, b").strip() + expected = "1\t1\n1\t2" + assert result == expected, f"Expected:\n{expected}\nGot:\n{result}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + def test_export_part_with_bucket_partition(cluster): """ Export a part from a MergeTree table partitioned by icebergBucket(8, user_id) diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index b8c15c26275f..10bd5959f3e1 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -312,3 +312,52 @@ def test_pending_patch_parts_skip_before_export(cluster): assert "1\n2\n3" in result, "Export should contain original data before patch" node.query(f"DROP TABLE {mt_table}") + + +def test_export_part_same_partition_key_different_column_order(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"reordered_part_mt_table_{postfix}" + s3_table = f"reordered_part_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32) + ENGINE = MergeTree() + PARTITION BY a + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (b Int32, a Int32) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY a + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1), (1, 2)") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + + deadline = time.time() + 30 + while True: + node.query("SYSTEM FLUSH LOGS") + count = node.query( + f"SELECT count() FROM system.part_log WHERE event_type = 'ExportPart' " + f"AND database = currentDatabase() AND table = '{mt_table}' AND part_name = '{part_name}'" + ).strip() + if count != "0": + break + if time.time() > deadline: + raise TimeoutError(f"ExportPart event for part {part_name!r} did not appear within 30s") + time.sleep(0.5) + + dest_result = node.query(f"SELECT a, b FROM {s3_table} ORDER BY a, b") + expected = "1\t1\n1\t2\n" + assert dest_result == expected, f"Expected:\n{expected}\nGot:\n{dest_result}" diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py index ad2deba8de19..844f79e85f7c 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py @@ -1324,6 +1324,29 @@ def test_export_partition_with_renamed_destination_column(cluster): ) +def test_export_partition_same_partition_key_different_column_order(cluster): + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_reordered_{uid}" + iceberg_table = f"iceberg_reordered_{uid}" + + make_rmt(node, mt_table, "a Int32, b Int32", "a", replica_name="replica1") + make_iceberg_s3(node, iceberg_table, "b Int32, a Int32", partition_by="a") + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1), (1, 2)") + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '1' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + wait_for_export_status(node, mt_table, iceberg_table, "1", "COMPLETED") + + result = node.query(f"SELECT a, b FROM {iceberg_table} ORDER BY a, b").strip() + expected = "1\t1\n1\t2" + assert result == expected, f"Expected:\n{expected}\nGot:\n{result}" + + def test_export_partition_with_castable_widening(cluster): """A lossless widening of both a data column (id Int32 -> Int64) and the partition column (year Int32 -> Int64) round-trips.""" diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py index 8d4589292e3c..6a6e63e36bc1 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py @@ -1747,3 +1747,35 @@ def test_export_partition_all_failure_modes(cluster): f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}" f" SETTINGS export_merge_tree_partition_all_on_error = 'skip_conflicts'" ) + + +def test_export_partition_same_partition_key_different_column_order(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"reordered_mt_table_{postfix}" + s3_table = f"reordered_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') + PARTITION BY a + ORDER BY tuple() + """) + + node.query(f""" + CREATE TABLE {s3_table} (b Int32, a Int32) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY a + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1), (1, 2)") + + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '1' TO TABLE {s3_table}") + wait_for_export_status(node, mt_table, s3_table, "1", "COMPLETED") + + dest_result = node.query(f"SELECT * FROM {s3_table} ORDER BY a, b") + + expected = "1\t1\n2\t1\n" + assert dest_result == expected, f"Expected:\n{expected}\nGot:\n{dest_result}" From 3ee4c71aa37563e9d74c82c350fd748f7fc59155 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 30 Jul 2026 15:09:35 +0200 Subject: [PATCH 02/20] refactoring Signed-off-by: Konstantin Morozov --- src/Storages/MergeTree/ExportPartitionUtils.cpp | 13 +++++++++++++ src/Storages/MergeTree/ExportPartitionUtils.h | 4 ++++ src/Storages/MergeTree/MergeTreeData.cpp | 12 +----------- src/Storages/StorageReplicatedMergeTree.cpp | 12 +----------- 4 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 3f85ebc0e1fb..616a3384ba1c 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -634,6 +634,19 @@ namespace ExportPartitionUtils } #endif + void verifyMergeTreePartitionCompatibility( + const StorageMetadataPtr & source_metadata, + const StorageMetadataPtr & destination_metadata) + { + constexpr auto query_to_string = [] (const ASTPtr & ast) + { + return ast ? ast->formatWithSecretsOneLine() : ""; + }; + + if (query_to_string(source_metadata->getPartitionKeyAST()) != query_to_string(destination_metadata->getPartitionKeyAST())) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different partition key"); + } + void verifyExportSchemaCastable( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata, diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index 0bb8acb9bda4..dd1ae4c18094 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -89,6 +89,10 @@ namespace ExportPartitionUtils const std::string & exception_message, const LoggerPtr & log); + void verifyMergeTreePartitionCompatibility( + const StorageMetadataPtr & source_metadata, + const StorageMetadataPtr & destination_metadata); + /// Validates that source columns can be exported into the destination with the /// same positional CAST matching as `INSERT INTO dest SELECT * FROM src`. Lossy /// casts are rejected unless `export_merge_tree_part_allow_lossy_cast` is set. diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 0ed451b59655..ac8ad03c3b63 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6726,11 +6726,6 @@ void MergeTreeData::exportPartToTable( if (!dest_storage->supportsImport(query_context)) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Destination storage {} does not support MergeTree parts or uses unsupported partitioning", dest_storage->getName()); - auto query_to_string = [] (const ASTPtr & ast) - { - return ast ? ast->formatWithSecretsOneLine() : ""; - }; - auto source_metadata_ptr = getInMemoryMetadataPtr(); auto destination_metadata_ptr = dest_storage->getInMemoryMetadataPtr(); @@ -6791,13 +6786,8 @@ void MergeTreeData::exportPartToTable( ExportPartitionUtils::verifyExportSchemaCastable( source_metadata_ptr, destination_metadata_ptr, dest_storage->getStorageID(), query_context); - /// Iceberg partition compatibility is checked above; here we only need the - /// partition-key ASTs to match (partition-column types follow the lossy-cast gate). if (!dest_storage->isDataLake()) - { - if (query_to_string(source_metadata_ptr->getPartitionKeyAST()) != query_to_string(destination_metadata_ptr->getPartitionKeyAST())) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different partition key"); - } + ExportPartitionUtils::verifyMergeTreePartitionCompatibility(source_metadata_ptr, destination_metadata_ptr); auto part = getPartIfExists(part_name, {MergeTreeDataPartState::Active, MergeTreeDataPartState::Outdated}); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 5677a5e111a8..759736dc06be 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -8408,11 +8408,6 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & if (!dest_storage->supportsImport(query_context)) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Destination storage {} does not support MergeTree parts or uses unsupported partitioning", dest_storage->getName()); - auto query_to_string = [] (const ASTPtr & ast) - { - return ast ? ast->formatWithSecretsOneLine() : ""; - }; - auto src_snapshot = getInMemoryMetadataPtr(); auto destination_snapshot = dest_storage->getInMemoryMetadataPtr(); @@ -8420,13 +8415,8 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & ExportPartitionUtils::verifyExportSchemaCastable( src_snapshot, destination_snapshot, dest_storage->getStorageID(), query_context); - /// Iceberg partition compatibility is checked below; here we only need the - /// partition-key ASTs to match (partition-column types follow the lossy-cast gate). if (!dest_storage->isDataLake()) - { - if (query_to_string(src_snapshot->getPartitionKeyAST()) != query_to_string(destination_snapshot->getPartitionKeyAST())) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different partition key"); - } + ExportPartitionUtils::verifyMergeTreePartitionCompatibility(src_snapshot, destination_snapshot); zkutil::ZooKeeperPtr zookeeper = getZooKeeperAndAssertNotReadonly(); From 14a31bf4746186b9391d84b576093d9a917d819d Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 30 Jul 2026 16:55:57 +0200 Subject: [PATCH 03/20] add check names and position Signed-off-by: Konstantin Morozov --- .../MergeTree/ExportPartitionUtils.cpp | 29 ++++++++++++++++--- .../test.py | 15 ++++++---- .../test.py | 24 ++++----------- .../test.py | 10 +++---- .../test.py | 11 ++++--- 5 files changed, 51 insertions(+), 38 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 616a3384ba1c..39256852b002 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -644,7 +645,8 @@ namespace ExportPartitionUtils }; if (query_to_string(source_metadata->getPartitionKeyAST()) != query_to_string(destination_metadata->getPartitionKeyAST())) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different partition key"); + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Cannot export partition: source and destination tables have different `PARTITION BY` expressions"); } void verifyExportSchemaCastable( @@ -670,15 +672,34 @@ namespace ExportPartitionUtils ActionsDAG::MatchColumnsMode::Position, context); - /// Lossy casts may silently change values, so reject them unless the user opts in. - if (context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]) - return; + auto partition_key_columns = source_metadata->getColumnsRequiredForPartitionKey(); + const std::unordered_set partition_key_column_set( + std::make_move_iterator(partition_key_columns.begin()), + std::make_move_iterator(partition_key_columns.end())); + + const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); for (size_t i = 0; i < num_columns; ++i) { const auto & source_column = source_columns[i]; const auto & destination_column = destination_columns[i]; + + if (partition_key_column_set.contains(source_column.name) && source_column.name != destination_column.name) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Cannot export to {}: partition key column '{}' is at position {} in the source " + "table, but the destination's column at that position is named '{}'. EXPORT " + "PART/PARTITION matches columns by position, so partition key columns must be " + "declared at the same position in both tables.", + destination_storage_id.getFullTableName(), + source_column.name, + i, + destination_column.name); + + /// Lossy casts may silently change values, so reject them unless the user opts in. + if (allow_lossy_cast) + continue; + if (!canBeSafelyCast(source_column.type, destination_column.type)) throw Exception(ErrorCodes::INCOMPATIBLE_COLUMNS, "Cannot export to {}: column '{}' requires a lossy cast from {} to {}, " diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index 78a0b94d2414..cefd25cc0796 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -475,12 +475,17 @@ def test_export_part_same_partition_key_different_column_order(cluster): node.query(f"INSERT INTO {mt} VALUES (1, 1), (1, 2)") part_1 = get_part(node, mt, "1") - export_part(node, mt, part_1, iceberg) - wait_for_export_part(node, mt, part_1) - result = node.query(f"SELECT a, b FROM {iceberg} ORDER BY a, b").strip() - expected = "1\t1\n1\t2" - assert result == expected, f"Expected:\n{expected}\nGot:\n{result}" + error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PART '{part_1}' TO TABLE {iceberg} " + f"SETTINGS allow_experimental_export_merge_tree_part = 1, " + f"allow_experimental_insert_into_iceberg = 1" + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error!r}" + + count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" node.query(f"DROP TABLE IF EXISTS {mt} SYNC") node.query(f"DROP TABLE IF EXISTS {iceberg}") diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 10bd5959f3e1..e9b43c0141bb 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -343,21 +343,9 @@ def test_export_part_same_partition_key_different_column_order(cluster): f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" ).strip() - node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - - deadline = time.time() + 30 - while True: - node.query("SYSTEM FLUSH LOGS") - count = node.query( - f"SELECT count() FROM system.part_log WHERE event_type = 'ExportPart' " - f"AND database = currentDatabase() AND table = '{mt_table}' AND part_name = '{part_name}'" - ).strip() - if count != "0": - break - if time.time() > deadline: - raise TimeoutError(f"ExportPart event for part {part_name!r} did not appear within 30s") - time.sleep(0.5) - - dest_result = node.query(f"SELECT a, b FROM {s3_table} ORDER BY a, b") - expected = "1\t1\n1\t2\n" - assert dest_result == expected, f"Expected:\n{expected}\nGot:\n{dest_result}" + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py index 844f79e85f7c..f4fa3f71da90 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py @@ -1336,15 +1336,15 @@ def test_export_partition_same_partition_key_different_column_order(cluster): node.query(f"INSERT INTO {mt_table} VALUES (1, 1), (1, 2)") - node.query( + error = node.query_and_get_error( f"ALTER TABLE {mt_table} EXPORT PARTITION ID '1' TO TABLE {iceberg_table}", settings={"allow_insert_into_iceberg": 1}, ) - wait_for_export_status(node, mt_table, iceberg_table, "1", "COMPLETED") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" - result = node.query(f"SELECT a, b FROM {iceberg_table} ORDER BY a, b").strip() - expected = "1\t1\n1\t2" - assert result == expected, f"Expected:\n{expected}\nGot:\n{result}" + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" def test_export_partition_with_castable_widening(cluster): diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py index 6a6e63e36bc1..7cb9b71202fa 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py @@ -1772,10 +1772,9 @@ def test_export_partition_same_partition_key_different_column_order(cluster): node.query(f"INSERT INTO {mt_table} VALUES (1, 1), (1, 2)") - node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '1' TO TABLE {s3_table}") - wait_for_export_status(node, mt_table, s3_table, "1", "COMPLETED") - - dest_result = node.query(f"SELECT * FROM {s3_table} ORDER BY a, b") + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '1' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" - expected = "1\t1\n2\t1\n" - assert dest_result == expected, f"Expected:\n{expected}\nGot:\n{dest_result}" + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" From 7195b162115a95df8644e916c3d10fe40cd22d07 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 31 Jul 2026 17:10:22 +0200 Subject: [PATCH 04/20] add tests Signed-off-by: Konstantin Morozov --- .../test.py | 183 +++++++++++++- .../test.py | 187 +++++++++++++- .../test.py | 144 ++++++++++- .../test.py | 236 +++++++++++++++++- 4 files changed, 744 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index cefd25cc0796..5494846b4433 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -13,7 +13,12 @@ test_export_part_with_year_transform_partition – toYearNumSinceEpoch() partition expression test_export_part_with_bucket_partition – icebergBucket(N, col) partition expression test_export_part_partition_key_mismatch_is_rejected – mismatched partition spec rejected synchronously - test_export_part_same_partition_key_different_column_order – same partition key, different column order + test_export_part_same_partition_key_different_column_order_single_column – same 1-column partition key, different column order + test_export_part_same_partition_key_different_column_order_multi_column – same 2-column partition key, different column order + test_export_part_multi_column_partition_key_success – composite (a, b) partition key round-trips + test_export_part_multi_column_partition_key_order_mismatch_is_rejected – composite key fields swapped between src/dst + test_export_part_multi_column_partition_key_fewer_in_destination_is_rejected – dst has fewer partition fields than src + test_export_part_multi_column_partition_key_more_in_destination_is_rejected – dst has more partition fields than src """ import logging @@ -463,7 +468,7 @@ def test_export_part_partition_key_mismatch_is_rejected(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -def test_export_part_same_partition_key_different_column_order(cluster): +def test_export_part_same_partition_key_different_column_order_single_column(cluster): node = cluster.instances["node1"] sfx = unique_suffix() mt = f"mt_reordered_{sfx}" @@ -491,6 +496,180 @@ def test_export_part_same_partition_key_different_column_order(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") +def test_export_part_same_partition_key_different_column_order_multi_column(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_multi_reordered_{sfx}" + iceberg = f"iceberg_multi_reordered_{sfx}" + + make_mt(node, mt, "a Int32, b Int32, c Int32, val String", "(a, b, c)") + make_iceberg_s3(node, iceberg, "c Int32, b Int32, a Int32, val String", "(a, b, c)") + + node.query(f"INSERT INTO {mt} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + + pid = first_partition_id(node, mt) + part = get_part(node, mt, pid) + + error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " + f"SETTINGS allow_experimental_export_merge_tree_part = 1, " + f"allow_experimental_insert_into_iceberg = 1" + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error!r}" + + count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_multi_column_partition_key_success(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_multi_pkey_ok_{sfx}" + iceberg = f"iceberg_multi_pkey_ok_{sfx}" + + cols = "a Int32, b Int32, c Int32, val String" + make_mt(node, mt, cols, "(a, b, c)") + make_iceberg_s3(node, iceberg, cols, "(a, b, c)") + + node.query(f"INSERT INTO {mt} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + + pid = first_partition_id(node, mt) + part = get_part(node, mt, pid) + export_part(node, mt, part, iceberg) + wait_for_export_part(node, mt, part) + + count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) + assert count == 2, f"Expected 2 rows in Iceberg table after export, got {count}" + + result = node.query(f"SELECT a, b, c, val FROM {iceberg} ORDER BY val").strip() + assert result == "1\t1\t1\tx\n1\t1\t1\ty", f"Unexpected exported data:\n{result}" + + assert_part_log(node, mt, part) + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_multi_column_partition_key_order_mismatch_is_rejected(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_multi_order_{sfx}" + iceberg = f"iceberg_multi_order_{sfx}" + + cols = "a Int32, b Int32, c Int32, val String" + make_mt(node, mt, cols, "(a, b, c)") + make_iceberg_s3(node, iceberg, cols, "(c, b, a)") + + node.query(f"INSERT INTO {mt} VALUES (1, 2, 3, 'x')") + + pid = first_partition_id(node, mt) + part = get_part(node, mt, pid) + + error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " + f"SETTINGS allow_experimental_export_merge_tree_part = 1, " + f"allow_experimental_insert_into_iceberg = 1" + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" + + count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_multi_column_partition_key_fewer_in_destination_is_rejected(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_multi_fewer_{sfx}" + iceberg = f"iceberg_multi_fewer_{sfx}" + + cols = "a Int32, b Int32, c Int32, val String" + make_mt(node, mt, cols, "(a, b, c)") + make_iceberg_s3(node, iceberg, cols, "(a, b)") + + node.query(f"INSERT INTO {mt} VALUES (1, 2, 3, 'x')") + + pid = first_partition_id(node, mt) + part = get_part(node, mt, pid) + + error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " + f"SETTINGS allow_experimental_export_merge_tree_part = 1, " + f"allow_experimental_insert_into_iceberg = 1" + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" + + count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_multi_column_partition_key_more_in_destination_is_rejected(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_multi_more_{sfx}" + iceberg = f"iceberg_multi_more_{sfx}" + + cols = "a Int32, b Int32, c Int32, val String" + make_mt(node, mt, cols, "(a, b)") + make_iceberg_s3(node, iceberg, cols, "(a, b, c)") + + node.query(f"INSERT INTO {mt} VALUES (1, 2, 3, 'x')") + + pid = first_partition_id(node, mt) + part = get_part(node, mt, pid) + + error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " + f"SETTINGS allow_experimental_export_merge_tree_part = 1, " + f"allow_experimental_insert_into_iceberg = 1" + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" + + count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_transform_partition_key_different_column_order_is_rejected(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_transform_reordered_{sfx}" + iceberg = f"iceberg_transform_reordered_{sfx}" + + make_mt(node, mt, "other_id Int64, user_id Int64", "icebergBucket(8, user_id)") + make_iceberg_s3(node, iceberg, "user_id Int64, other_id Int64", "icebergBucket(8, user_id)") + + node.query(f"INSERT INTO {mt} VALUES (1, 42)") + + pid = first_partition_id(node, mt) + part = get_part(node, mt, pid) + + error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " + f"SETTINGS allow_experimental_export_merge_tree_part = 1, " + f"allow_experimental_insert_into_iceberg = 1" + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error!r}" + + count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + def test_export_part_with_bucket_partition(cluster): """ Export a part from a MergeTree table partitioned by icebergBucket(8, user_id) diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index e9b43c0141bb..132d13ecc303 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -314,7 +314,7 @@ def test_pending_patch_parts_skip_before_export(cluster): node.query(f"DROP TABLE {mt_table}") -def test_export_part_same_partition_key_different_column_order(cluster): +def test_export_part_same_partition_key_different_column_order_single_column(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] @@ -349,3 +349,188 @@ def test_export_part_same_partition_key_different_column_order(cluster): count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_part_same_partition_key_different_column_order_multi_column(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_reordered_part_mt_table_{postfix}" + s3_table = f"multi_reordered_part_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = MergeTree() + PARTITION BY (a, b, c) + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (c Int32, b Int32, a Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b, c) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_part_multi_column_partition_key_success(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_pkey_ok_mt_table_{postfix}" + s3_table = f"multi_pkey_ok_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = MergeTree() + PARTITION BY (a, b, c) + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b, c) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + + time.sleep(5) + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 2, f"Expected 2 rows in destination after export, got {count}" + + result = node.query(f"SELECT a, b, c, val FROM {s3_table} ORDER BY val").strip() + assert result == "1\t1\t1\tx\n1\t1\t1\ty", f"Unexpected exported data:\n{result}" + + +def test_export_part_multi_column_partition_key_order_mismatch_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_order_mt_table_{postfix}" + s3_table = f"multi_order_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = MergeTree() + PARTITION BY (a, b, c) + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (c, b, a) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_part_multi_column_partition_key_fewer_in_destination_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_fewer_mt_table_{postfix}" + s3_table = f"multi_fewer_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = MergeTree() + PARTITION BY (a, b, c) + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_part_multi_column_partition_key_more_in_destination_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_more_mt_table_{postfix}" + s3_table = f"multi_more_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = MergeTree() + PARTITION BY (a, b) + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b, c) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py index f4fa3f71da90..e3aa21baeb33 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py @@ -754,6 +754,7 @@ def check_accepted(mt, iceberg, description): f"ALTER TABLE {mt} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg}", settings={"allow_insert_into_iceberg": 1}, ) + return pid # 1. Compound identity: (year, region) cols = "id Int64, year Int32, region String" @@ -761,7 +762,12 @@ def check_accepted(mt, iceberg, description): make_rmt(node, t, cols, "(year, region)") node.query(f"INSERT INTO {t} VALUES (1, 2023, 'EU')") make_iceberg_s3(node, i, cols, "(year, region)") - check_accepted(t, i, "compound identity (year, region)") + pid = check_accepted(t, i, "compound identity (year, region)") + wait_for_export_status(node, t, i, pid, "COMPLETED") + count = int(node.query(f"SELECT count() FROM {i}").strip()) + assert count == 1, f"[compound identity (year, region)] Expected 1 row in Iceberg table, got {count}" + result = node.query(f"SELECT id, year, region FROM {i}").strip() + assert result == "1\t2023\tEU", f"[compound identity (year, region)] Unexpected exported data:\n{result}" # 2. Year transform cols = "id Int64, event_date Date" @@ -837,6 +843,8 @@ def assert_rejected(mt, iceberg, description): node.query(f"INSERT INTO {t} VALUES (1, 2020, 'EU')") make_iceberg_s3(node, i, cols, "(region, year)") assert_rejected(t, i, "compound field order reversed") + count = int(node.query(f"SELECT count() FROM {i}").strip()) + assert count == 0, f"[compound field order reversed] Expected 0 rows in destination, got {count}" # 2. Transform mismatch: MergeTree year-transform, Iceberg identity on same Date col cols = "id Int64, event_date Date" @@ -869,6 +877,8 @@ def assert_rejected(mt, iceberg, description): node.query(f"INSERT INTO {t} VALUES (1, 2020, 'EU')") make_iceberg_s3(node, i, cols, "year") assert_rejected(t, i, "2-field MergeTree vs 1-field Iceberg") + count = int(node.query(f"SELECT count() FROM {i}").strip()) + assert count == 0, f"[2-field MergeTree vs 1-field Iceberg] Expected 0 rows in destination, got {count}" # 6. Unsupported MergeTree expression: intDiv(year, 100) is not an Iceberg transform cols = "id Int64, year Int32" @@ -1324,7 +1334,7 @@ def test_export_partition_with_renamed_destination_column(cluster): ) -def test_export_partition_same_partition_key_different_column_order(cluster): +def test_export_partition_same_partition_key_different_column_order_single_column(cluster): node = cluster.instances["replica1"] uid = unique_suffix() @@ -1343,6 +1353,136 @@ def test_export_partition_same_partition_key_different_column_order(cluster): assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + error_all = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_partition_same_partition_key_different_column_order_multi_column(cluster): + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_multi_reordered_{uid}" + iceberg_table = f"iceberg_multi_reordered_{uid}" + + cols = "a Int32, b Int32, c Int32, val String" + make_rmt(node, mt_table, cols, "(a, b, c)", replica_name="replica1") + make_iceberg_s3(node, iceberg_table, "c Int32, b Int32, a Int32, val String", partition_by="(a, b, c)") + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + + pid = first_partition_id(node, mt_table) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + + error_all = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_partition_multi_column_partition_key_more_in_destination_is_rejected(cluster): + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_multi_more_{uid}" + iceberg_table = f"iceberg_multi_more_{uid}" + + cols = "a Int32, b Int32, c Int32, val String" + make_rmt(node, mt_table, cols, "(a, b)", replica_name="replica1") + make_iceberg_s3(node, iceberg_table, cols, partition_by="(a, b, c)") + + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + + pid = first_partition_id(node, mt_table) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + + error_all = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_partition_multi_column_partition_key_success_all(cluster): + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_multi_pkey_ok_all_{uid}" + iceberg_table = f"iceberg_multi_pkey_ok_all_{uid}" + + cols = "a Int32, b Int32, c Int32, val String" + make_rmt(node, mt_table, cols, "(a, b, c)", replica_name="replica1") + make_iceberg_s3(node, iceberg_table, cols, partition_by="(a, b, c)") + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (2, 2, 2, 'y')") + + partition_ids = node.query( + f"SELECT DISTINCT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY partition_id" + ).strip().split("\n") + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + + for pid in partition_ids: + wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 2, f"Expected 2 rows in destination after export, got {count}" + + result = node.query(f"SELECT a, b, c, val FROM {iceberg_table} ORDER BY val").strip() + assert result == "1\t1\t1\tx\n2\t2\t2\ty", f"Unexpected exported data:\n{result}" + + +def test_export_partition_transform_partition_key_different_column_order_is_rejected(cluster): + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_transform_reordered_{uid}" + iceberg_table = f"iceberg_transform_reordered_{uid}" + + make_rmt(node, mt_table, "other_id Int64, user_id Int64", "icebergBucket(8, user_id)", replica_name="replica1") + make_iceberg_s3(node, iceberg_table, "user_id Int64, other_id Int64", partition_by="icebergBucket(8, user_id)") + + node.query(f"INSERT INTO {mt_table} VALUES (1, 42)") + + pid = first_partition_id(node, mt_table) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + + error_all = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py index 7cb9b71202fa..cb73cc926f4e 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py @@ -1749,7 +1749,7 @@ def test_export_partition_all_failure_modes(cluster): ) -def test_export_partition_same_partition_key_different_column_order(cluster): +def test_export_partition_same_partition_key_different_column_order_single_column(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["replica1"] @@ -1776,5 +1776,239 @@ def test_export_partition_same_partition_key_different_column_order(cluster): assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_partition_same_partition_key_different_column_order_multi_column(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_reordered_mt_table_{postfix}" + s3_table = f"multi_reordered_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') + PARTITION BY (a, b, c) + ORDER BY tuple() + """) + + node.query(f""" + CREATE TABLE {s3_table} (c Int32, b Int32, a Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b, c) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + + partition_id = node.query( + f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + + error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_partition_multi_column_partition_key_success(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_pkey_ok_mt_table_{postfix}" + s3_table = f"multi_pkey_ok_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') + PARTITION BY (a, b, c) + ORDER BY tuple() + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b, c) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + + partition_id = node.query( + f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") + wait_for_export_status(node, mt_table, s3_table, partition_id, "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 2, f"Expected 2 rows in destination after export, got {count}" + + result = node.query(f"SELECT a, b, c, val FROM {s3_table} ORDER BY val").strip() + assert result == "1\t1\t1\tx\n1\t1\t1\ty", f"Unexpected exported data:\n{result}" + + +def test_export_partition_multi_column_partition_key_order_mismatch_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_order_mt_table_{postfix}" + s3_table = f"multi_order_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') + PARTITION BY (a, b, c) + ORDER BY tuple() + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (c, b, a) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + + partition_id = node.query( + f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + + error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_partition_multi_column_partition_key_fewer_in_destination_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_fewer_mt_table_{postfix}" + s3_table = f"multi_fewer_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') + PARTITION BY (a, b, c) + ORDER BY tuple() + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + + partition_id = node.query( + f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + + error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_partition_multi_column_partition_key_more_in_destination_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_more_mt_table_{postfix}" + s3_table = f"multi_more_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') + PARTITION BY (a, b) + ORDER BY tuple() + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b, c) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + + partition_id = node.query( + f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + + error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_partition_multi_column_partition_key_success_all(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_pkey_ok_all_mt_table_{postfix}" + s3_table = f"multi_pkey_ok_all_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') + PARTITION BY (a, b, c) + ORDER BY tuple() + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b, c) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (2, 2, 2, 'y')") + + partition_ids = node.query( + f"SELECT DISTINCT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY partition_id" + ).strip().split("\n") + + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + + for pid in partition_ids: + wait_for_export_status(node, mt_table, s3_table, pid, "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 2, f"Expected 2 rows in destination after export, got {count}" + + result = node.query(f"SELECT a, b, c, val FROM {s3_table} ORDER BY val").strip() + assert result == "1\t1\t1\tx\n2\t2\t2\ty", f"Unexpected exported data:\n{result}" From 59b845e2900dd11de35a1d72207d46f4ce8de796 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 3 Aug 2026 14:37:30 +0200 Subject: [PATCH 05/20] update tests, add hive part check Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 3 + docs/en/antalya/partition_export.md | 8 + .../MergeTree/ExportPartitionUtils.cpp | 32 +++ .../test.py | 253 +++++++----------- .../test.py | 214 ++++++++------- .../test.py | 168 +++++------- .../test.py | 227 +++++++--------- 7 files changed, 415 insertions(+), 490 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 73d467c5d9b1..2ff4a7301674 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -51,6 +51,9 @@ Source and destination tables must be 100% compatible: 1. **Identical schemas** - same columns, types, and order 2. **Matching partition keys** - partition expressions must be identical +3. **Partition key columns at the same position** - columns are matched by position, similar to `INSERT INTO dest SELECT * FROM src`. It is not enough for the `PARTITION BY` expressions to be textually identical: every column that is part of the source table's partition key must also sit at the same position in the destination table's schema. For example, `CREATE TABLE src (a Int32, b Int32) ... PARTITION BY a` and `CREATE TABLE dst (b Int32, a Int32) ... PARTITION BY a` both have the expression `PARTITION BY a`, but `a` is at position 0 in `src` and position 1 in `dst`, so the export is rejected with `BAD_ARGUMENTS: partition key column 'a' is at position 0 in the source table, but the destination's column at that position is named 'b'`. + + This explicit check only applies to partition key columns. A mismatch in the position of a non-partition-key column is **not** rejected by name - it is only caught if the source and destination types are not castable. If two non-partition-key columns happen to have swapped positions but compatible types, the export succeeds and silently writes values into the wrong destination column, so keep the full column order identical (per point 1) rather than relying on this check alone. In case a table function is used as the destination, the schema can be omitted and it will be inferred from the source table. diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index 687029b9adc6..ccd18914845b 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -43,6 +43,14 @@ TO TABLE [destination_database.]destination_table - **`partition_id`**: The partition identifier to export (e.g., `'2020'`, `'2021'`) - **`destination_table`**: The target table for the export (typically an S3, Azure, or other object storage table) +## Requirements + +`EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/engines/table-engines/mergetree-family/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements, in particular: + +1. **Identical schemas** - same columns, types, and order +2. **Matching partition keys** - partition expressions must be identical +3. **Partition key columns at the same position** - columns are matched by position, so every column that is part of the source table's partition key must also sit at the same position in the destination table's schema, even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/engines/table-engines/mergetree-family/part_export.md#requirements) for a worked example and the exact error message. + ## Settings ### Server Settings diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 39256852b002..d11b74aff13f 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include "Storages/ExportReplicatedMergeTreePartitionManifest.h" #include "Storages/ExportReplicatedMergeTreePartitionTaskEntry.h" #include @@ -13,6 +14,8 @@ #include #include #include +#include +#include #include #include #include @@ -635,6 +638,18 @@ namespace ExportPartitionUtils } #endif + namespace + { + std::optional getDateTimeTimeZoneName(const DataTypePtr & type) + { + if (const auto * datetime_type = typeid_cast(type.get())) + return datetime_type->getTimeZone().getTimeZone(); + if (const auto * datetime64_type = typeid_cast(type.get())) + return datetime64_type->getTimeZone().getTimeZone(); + return {}; + } + } + void verifyMergeTreePartitionCompatibility( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata) @@ -696,6 +711,23 @@ namespace ExportPartitionUtils i, destination_column.name); + if (partition_key_column_set.contains(source_column.name)) + { + const auto source_time_zone = getDateTimeTimeZoneName(source_column.type); + const auto destination_time_zone = getDateTimeTimeZoneName(destination_column.type); + if (source_time_zone && destination_time_zone && *source_time_zone != *destination_time_zone) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Cannot export to {}: partition key column '{}' is {} in the source table " + "but {} in the destination. The destination's hive-style partition path is " + "rendered from the source value without converting the timezone, so this " + "would silently shift the exported value by the timezone offset. Use the " + "same timezone in both tables' partition key column.", + destination_storage_id.getFullTableName(), + destination_column.name, + source_column.type->getName(), + destination_column.type->getName()); + } + /// Lossy casts may silently change values, so reject them unless the user opts in. if (allow_lossy_cast) continue; diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index 5494846b4433..e1dd26ea3820 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -13,16 +13,14 @@ test_export_part_with_year_transform_partition – toYearNumSinceEpoch() partition expression test_export_part_with_bucket_partition – icebergBucket(N, col) partition expression test_export_part_partition_key_mismatch_is_rejected – mismatched partition spec rejected synchronously - test_export_part_same_partition_key_different_column_order_single_column – same 1-column partition key, different column order - test_export_part_same_partition_key_different_column_order_multi_column – same 2-column partition key, different column order - test_export_part_multi_column_partition_key_success – composite (a, b) partition key round-trips - test_export_part_multi_column_partition_key_order_mismatch_is_rejected – composite key fields swapped between src/dst - test_export_part_multi_column_partition_key_fewer_in_destination_is_rejected – dst has fewer partition fields than src - test_export_part_multi_column_partition_key_more_in_destination_is_rejected – dst has more partition fields than src + test_export_part_multi_column_partition_key_success – composite (a, b, c) partition key round-trips + test_export_part_partition_key_mismatch_variants_are_rejected (parametrized) – partition key column reordering, + cardinality mismatches, and transform-expression reordering between src/dst are all rejected synchronously """ import logging import time +from typing import NamedTuple import pytest @@ -468,44 +466,96 @@ def test_export_part_partition_key_mismatch_is_rejected(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -def test_export_part_same_partition_key_different_column_order_single_column(cluster): +class RejectedPartExportCase(NamedTuple): + src_columns: str + src_partition_by: str + dst_columns: str + dst_partition_by: str + insert_values: str + error_substrings: tuple = () + + +REJECTED_PART_EXPORT_CASES = [ + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32", + src_partition_by="a", + dst_columns="b Int32, a Int32", + dst_partition_by="a", + insert_values="(1, 1), (1, 2)", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_single_column", + ), + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="c Int32, b Int32, a Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 1, 1, 'x'), (1, 1, 1, 'y')", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_multi_column", + ), + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(c, b, a)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("partition field 0 mismatch",), + ), + id="multi_column_partition_key_order_mismatch", + ), + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(a, b)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("partition scheme mismatch",), + ), + id="multi_column_partition_key_fewer_in_destination", + ), + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("partition scheme mismatch",), + ), + id="multi_column_partition_key_more_in_destination", + ), + pytest.param( + RejectedPartExportCase( + src_columns="other_id Int64, user_id Int64", + src_partition_by="icebergBucket(8, user_id)", + dst_columns="user_id Int64, other_id Int64", + dst_partition_by="icebergBucket(8, user_id)", + insert_values="(1, 42)", + error_substrings=("partition key column",), + ), + id="transform_partition_key_different_column_order", + ), +] + + +@pytest.mark.parametrize("case", REJECTED_PART_EXPORT_CASES) +def test_export_part_partition_key_mismatch_variants_are_rejected(cluster, case): node = cluster.instances["node1"] sfx = unique_suffix() - mt = f"mt_reordered_{sfx}" - iceberg = f"iceberg_reordered_{sfx}" + mt = f"mt_rejected_{sfx}" + iceberg = f"iceberg_rejected_{sfx}" - make_mt(node, mt, "a Int32, b Int32", "a") - make_iceberg_s3(node, iceberg, "b Int32, a Int32", "a") + make_mt(node, mt, case.src_columns, case.src_partition_by) + make_iceberg_s3(node, iceberg, case.dst_columns, case.dst_partition_by) - node.query(f"INSERT INTO {mt} VALUES (1, 1), (1, 2)") - - part_1 = get_part(node, mt, "1") - - error = node.query_and_get_error( - f"ALTER TABLE {mt} EXPORT PART '{part_1}' TO TABLE {iceberg} " - f"SETTINGS allow_experimental_export_merge_tree_part = 1, " - f"allow_experimental_insert_into_iceberg = 1" - ) - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error!r}" - - count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) - assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" - - node.query(f"DROP TABLE IF EXISTS {mt} SYNC") - node.query(f"DROP TABLE IF EXISTS {iceberg}") - - -def test_export_part_same_partition_key_different_column_order_multi_column(cluster): - node = cluster.instances["node1"] - sfx = unique_suffix() - mt = f"mt_multi_reordered_{sfx}" - iceberg = f"iceberg_multi_reordered_{sfx}" - - make_mt(node, mt, "a Int32, b Int32, c Int32, val String", "(a, b, c)") - make_iceberg_s3(node, iceberg, "c Int32, b Int32, a Int32, val String", "(a, b, c)") - - node.query(f"INSERT INTO {mt} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + node.query(f"INSERT INTO {mt} VALUES {case.insert_values}") pid = first_partition_id(node, mt) part = get_part(node, mt, pid) @@ -516,7 +566,8 @@ def test_export_part_same_partition_key_different_column_order_multi_column(clus f"allow_experimental_insert_into_iceberg = 1" ) assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error!r}" + for substring in case.error_substrings: + assert substring in error, f"Expected {substring!r} in error, got: {error!r}" count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" @@ -535,7 +586,7 @@ def test_export_part_multi_column_partition_key_success(cluster): make_mt(node, mt, cols, "(a, b, c)") make_iceberg_s3(node, iceberg, cols, "(a, b, c)") - node.query(f"INSERT INTO {mt} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + node.query(f"INSERT INTO {mt} VALUES (1, 2, 3, 'x'), (1, 2, 3, 'y')") pid = first_partition_id(node, mt) part = get_part(node, mt, pid) @@ -546,7 +597,7 @@ def test_export_part_multi_column_partition_key_success(cluster): assert count == 2, f"Expected 2 rows in Iceberg table after export, got {count}" result = node.query(f"SELECT a, b, c, val FROM {iceberg} ORDER BY val").strip() - assert result == "1\t1\t1\tx\n1\t1\t1\ty", f"Unexpected exported data:\n{result}" + assert result == "1\t2\t3\tx\n1\t2\t3\ty", f"Unexpected exported data:\n{result}" assert_part_log(node, mt, part) @@ -554,122 +605,6 @@ def test_export_part_multi_column_partition_key_success(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") -def test_export_part_multi_column_partition_key_order_mismatch_is_rejected(cluster): - node = cluster.instances["node1"] - sfx = unique_suffix() - mt = f"mt_multi_order_{sfx}" - iceberg = f"iceberg_multi_order_{sfx}" - - cols = "a Int32, b Int32, c Int32, val String" - make_mt(node, mt, cols, "(a, b, c)") - make_iceberg_s3(node, iceberg, cols, "(c, b, a)") - - node.query(f"INSERT INTO {mt} VALUES (1, 2, 3, 'x')") - - pid = first_partition_id(node, mt) - part = get_part(node, mt, pid) - - error = node.query_and_get_error( - f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " - f"SETTINGS allow_experimental_export_merge_tree_part = 1, " - f"allow_experimental_insert_into_iceberg = 1" - ) - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" - - count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) - assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" - - node.query(f"DROP TABLE IF EXISTS {mt} SYNC") - node.query(f"DROP TABLE IF EXISTS {iceberg}") - - -def test_export_part_multi_column_partition_key_fewer_in_destination_is_rejected(cluster): - node = cluster.instances["node1"] - sfx = unique_suffix() - mt = f"mt_multi_fewer_{sfx}" - iceberg = f"iceberg_multi_fewer_{sfx}" - - cols = "a Int32, b Int32, c Int32, val String" - make_mt(node, mt, cols, "(a, b, c)") - make_iceberg_s3(node, iceberg, cols, "(a, b)") - - node.query(f"INSERT INTO {mt} VALUES (1, 2, 3, 'x')") - - pid = first_partition_id(node, mt) - part = get_part(node, mt, pid) - - error = node.query_and_get_error( - f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " - f"SETTINGS allow_experimental_export_merge_tree_part = 1, " - f"allow_experimental_insert_into_iceberg = 1" - ) - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" - - count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) - assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" - - node.query(f"DROP TABLE IF EXISTS {mt} SYNC") - node.query(f"DROP TABLE IF EXISTS {iceberg}") - - -def test_export_part_multi_column_partition_key_more_in_destination_is_rejected(cluster): - node = cluster.instances["node1"] - sfx = unique_suffix() - mt = f"mt_multi_more_{sfx}" - iceberg = f"iceberg_multi_more_{sfx}" - - cols = "a Int32, b Int32, c Int32, val String" - make_mt(node, mt, cols, "(a, b)") - make_iceberg_s3(node, iceberg, cols, "(a, b, c)") - - node.query(f"INSERT INTO {mt} VALUES (1, 2, 3, 'x')") - - pid = first_partition_id(node, mt) - part = get_part(node, mt, pid) - - error = node.query_and_get_error( - f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " - f"SETTINGS allow_experimental_export_merge_tree_part = 1, " - f"allow_experimental_insert_into_iceberg = 1" - ) - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" - - count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) - assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" - - node.query(f"DROP TABLE IF EXISTS {mt} SYNC") - node.query(f"DROP TABLE IF EXISTS {iceberg}") - - -def test_export_part_transform_partition_key_different_column_order_is_rejected(cluster): - node = cluster.instances["node1"] - sfx = unique_suffix() - mt = f"mt_transform_reordered_{sfx}" - iceberg = f"iceberg_transform_reordered_{sfx}" - - make_mt(node, mt, "other_id Int64, user_id Int64", "icebergBucket(8, user_id)") - make_iceberg_s3(node, iceberg, "user_id Int64, other_id Int64", "icebergBucket(8, user_id)") - - node.query(f"INSERT INTO {mt} VALUES (1, 42)") - - pid = first_partition_id(node, mt) - part = get_part(node, mt, pid) - - error = node.query_and_get_error( - f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " - f"SETTINGS allow_experimental_export_merge_tree_part = 1, " - f"allow_experimental_insert_into_iceberg = 1" - ) - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error!r}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error!r}" - - count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) - assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" - - node.query(f"DROP TABLE IF EXISTS {mt} SYNC") - node.query(f"DROP TABLE IF EXISTS {iceberg}") - - def test_export_part_with_bucket_partition(cluster): """ Export a part from a MergeTree table partitioned by icebergBucket(8, user_id) diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 132d13ecc303..f946b01e726f 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -1,6 +1,7 @@ import logging import time import uuid +from typing import NamedTuple import pytest @@ -314,29 +315,98 @@ def test_pending_patch_parts_skip_before_export(cluster): node.query(f"DROP TABLE {mt_table}") -def test_export_part_same_partition_key_different_column_order_single_column(cluster): +class RejectedPartExportCase(NamedTuple): + src_columns: str + src_partition_by: str + dst_columns: str + dst_partition_by: str + insert_values: str + error_substrings: tuple = () + + +REJECTED_PART_EXPORT_CASES = [ + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32", + src_partition_by="a", + dst_columns="b Int32, a Int32", + dst_partition_by="a", + insert_values="(1, 1), (1, 2)", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_single_column", + ), + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="c Int32, b Int32, a Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 1, 1, 'x'), (1, 1, 1, 'y')", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_multi_column", + ), + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(c, b, a)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("different `PARTITION BY` expressions",), + ), + id="multi_column_partition_key_order_mismatch", + ), + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(a, b)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("different `PARTITION BY` expressions",), + ), + id="multi_column_partition_key_fewer_in_destination", + ), + pytest.param( + RejectedPartExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("different `PARTITION BY` expressions",), + ), + id="multi_column_partition_key_more_in_destination", + ), +] + + +@pytest.mark.parametrize("case", REJECTED_PART_EXPORT_CASES) +def test_export_part_partition_key_mismatch_variants_are_rejected(cluster, case): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"reordered_part_mt_table_{postfix}" - s3_table = f"reordered_part_s3_table_{postfix}" + mt_table = f"rejected_mt_table_{postfix}" + s3_table = f"rejected_s3_table_{postfix}" node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32) + CREATE TABLE {mt_table} ({case.src_columns}) ENGINE = MergeTree() - PARTITION BY a + PARTITION BY {case.src_partition_by} ORDER BY tuple() SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 """) node.query(f""" - CREATE TABLE {s3_table} (b Int32, a Int32) + CREATE TABLE {s3_table} ({case.dst_columns}) ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY a + PARTITION BY {case.dst_partition_by} """) - node.query(f"INSERT INTO {mt_table} VALUES (1, 1), (1, 2)") + node.query(f"INSERT INTO {mt_table} VALUES {case.insert_values}") part_name = node.query( f"SELECT name FROM system.parts WHERE database = currentDatabase() " @@ -345,44 +415,8 @@ def test_export_part_same_partition_key_different_column_order_single_column(clu error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - -def test_export_part_same_partition_key_different_column_order_multi_column(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_reordered_part_mt_table_{postfix}" - s3_table = f"multi_reordered_part_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) - ENGINE = MergeTree() - PARTITION BY (a, b, c) - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (c Int32, b Int32, a Int32, val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY (a, b, c) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + for substring in case.error_substrings: + assert substring in error, f"Expected {substring!r} in error, got: {error}" count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" @@ -410,7 +444,7 @@ def test_export_part_multi_column_partition_key_success(cluster): PARTITION BY (a, b, c) """) - node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x'), (1, 2, 3, 'y')") part_name = node.query( f"SELECT name FROM system.parts WHERE database = currentDatabase() " @@ -425,32 +459,32 @@ def test_export_part_multi_column_partition_key_success(cluster): assert count == 2, f"Expected 2 rows in destination after export, got {count}" result = node.query(f"SELECT a, b, c, val FROM {s3_table} ORDER BY val").strip() - assert result == "1\t1\t1\tx\n1\t1\t1\ty", f"Unexpected exported data:\n{result}" + assert result == "1\t2\t3\tx\n1\t2\t3\ty", f"Unexpected exported data:\n{result}" -def test_export_part_multi_column_partition_key_order_mismatch_is_rejected(cluster): +def test_export_part_partition_key_timezone_mismatch_is_rejected(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_order_mt_table_{postfix}" - s3_table = f"multi_order_s3_table_{postfix}" + mt_table = f"tz_mismatch_mt_table_{postfix}" + s3_table = f"tz_mismatch_s3_table_{postfix}" node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + CREATE TABLE {mt_table} (id Int64, ts DateTime('UTC')) ENGINE = MergeTree() - PARTITION BY (a, b, c) + PARTITION BY ts ORDER BY tuple() SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 """) node.query(f""" - CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + CREATE TABLE {s3_table} (id Int64, ts DateTime('Asia/Tokyo')) ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY (c, b, a) + PARTITION BY ts """) - node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") part_name = node.query( f"SELECT name FROM system.parts WHERE database = currentDatabase() " @@ -459,78 +493,62 @@ def test_export_part_multi_column_partition_key_order_mismatch_is_rejected(clust error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + assert "timezone" in error, f"Expected timezone mismatch message, got: {error}" count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" -def test_export_part_multi_column_partition_key_fewer_in_destination_is_rejected(cluster): +def test_export_part_non_partition_key_timezone_mismatch_is_allowed(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_fewer_mt_table_{postfix}" - s3_table = f"multi_fewer_s3_table_{postfix}" + mt_table = f"tz_ok_mt_table_{postfix}" + s3_table = f"tz_ok_s3_table_{postfix}" node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + CREATE TABLE {mt_table} (id Int64, ts DateTime('UTC')) ENGINE = MergeTree() - PARTITION BY (a, b, c) + PARTITION BY id ORDER BY tuple() SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 """) node.query(f""" - CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + CREATE TABLE {s3_table} (id Int64, ts DateTime('Asia/Tokyo')) ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY (a, b) + PARTITION BY id """) - node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") part_name = node.query( f"SELECT name FROM system.parts WHERE database = currentDatabase() " f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" ).strip() - error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - -def test_export_part_multi_column_partition_key_more_in_destination_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] + node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_more_mt_table_{postfix}" - s3_table = f"multi_more_s3_table_{postfix}" + time.sleep(5) - node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) - ENGINE = MergeTree() - PARTITION BY (a, b) - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 1, f"Expected 1 row in destination after export, got {count}" - node.query(f""" - CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY (a, b, c) - """) + source_ts = node.query(f"SELECT ts FROM {mt_table}").strip() + assert source_ts == "2024-03-05 15:00:00", f"Unexpected source value: {source_ts}" - node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + dest_ts = node.query(f"SELECT ts FROM {s3_table}").strip() + assert dest_ts == "2024-03-06 00:00:00", ( + f"Expected the exported value to be the same instant displayed in the " + f"destination's Asia/Tokyo timezone ('2024-03-06 00:00:00'), got: {dest_ts}" + ) - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() + source_unix_ts = int(node.query(f"SELECT toUnixTimestamp(ts) FROM {mt_table}").strip()) + dest_unix_ts = int(node.query(f"SELECT toUnixTimestamp(ts) FROM {s3_table}").strip()) + assert source_unix_ts == dest_unix_ts, ( + f"Expected exported DateTime value to be preserved regardless of the " + f"destination column's timezone, got source={source_unix_ts}, dest={dest_unix_ts}" + ) - error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py index e3aa21baeb33..53e18c3df66b 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py @@ -3,6 +3,7 @@ import logging import re import time +from typing import NamedTuple import pytest from avro.datafile import DataFileReader @@ -1334,78 +1335,75 @@ def test_export_partition_with_renamed_destination_column(cluster): ) -def test_export_partition_same_partition_key_different_column_order_single_column(cluster): +class RejectedPartitionExportCase(NamedTuple): + src_columns: str + src_partition_by: str + dst_columns: str + dst_partition_by: str + insert_values: str + error_substrings: tuple = () + + +REJECTED_PARTITION_EXPORT_CASES = [ + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32", + src_partition_by="a", + dst_columns="b Int32, a Int32", + dst_partition_by="a", + insert_values="(1, 1), (1, 2)", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_single_column", + ), + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="c Int32, b Int32, a Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 1, 1, 'x'), (1, 1, 1, 'y')", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_multi_column", + ), + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("partition scheme mismatch",), + ), + id="multi_column_partition_key_more_in_destination", + ), + pytest.param( + RejectedPartitionExportCase( + src_columns="other_id Int64, user_id Int64", + src_partition_by="icebergBucket(8, user_id)", + dst_columns="user_id Int64, other_id Int64", + dst_partition_by="icebergBucket(8, user_id)", + insert_values="(1, 42)", + error_substrings=("partition key column",), + ), + id="transform_partition_key_different_column_order", + ), +] + + +@pytest.mark.parametrize("case", REJECTED_PARTITION_EXPORT_CASES) +def test_export_partition_partition_key_mismatch_variants_are_rejected(cluster, case): node = cluster.instances["replica1"] uid = unique_suffix() - mt_table = f"mt_reordered_{uid}" - iceberg_table = f"iceberg_reordered_{uid}" + mt_table = f"mt_rejected_{uid}" + iceberg_table = f"iceberg_rejected_{uid}" - make_rmt(node, mt_table, "a Int32, b Int32", "a", replica_name="replica1") - make_iceberg_s3(node, iceberg_table, "b Int32, a Int32", partition_by="a") + make_rmt(node, mt_table, case.src_columns, case.src_partition_by, replica_name="replica1") + make_iceberg_s3(node, iceberg_table, case.dst_columns, partition_by=case.dst_partition_by) - node.query(f"INSERT INTO {mt_table} VALUES (1, 1), (1, 2)") - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '1' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" - - error_all = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - -def test_export_partition_same_partition_key_different_column_order_multi_column(cluster): - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_multi_reordered_{uid}" - iceberg_table = f"iceberg_multi_reordered_{uid}" - - cols = "a Int32, b Int32, c Int32, val String" - make_rmt(node, mt_table, cols, "(a, b, c)", replica_name="replica1") - make_iceberg_s3(node, iceberg_table, "c Int32, b Int32, a Int32, val String", partition_by="(a, b, c)") - - node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") - - pid = first_partition_id(node, mt_table) - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" - - error_all = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - -def test_export_partition_multi_column_partition_key_more_in_destination_is_rejected(cluster): - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_multi_more_{uid}" - iceberg_table = f"iceberg_multi_more_{uid}" - - cols = "a Int32, b Int32, c Int32, val String" - make_rmt(node, mt_table, cols, "(a, b)", replica_name="replica1") - make_iceberg_s3(node, iceberg_table, cols, partition_by="(a, b, c)") - - node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + node.query(f"INSERT INTO {mt_table} VALUES {case.insert_values}") pid = first_partition_id(node, mt_table) error = node.query_and_get_error( @@ -1413,6 +1411,8 @@ def test_export_partition_multi_column_partition_key_more_in_destination_is_reje settings={"allow_insert_into_iceberg": 1}, ) assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + for substring in case.error_substrings: + assert substring in error, f"Expected {substring!r} in error, got: {error}" error_all = node.query_and_get_error( f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", @@ -1435,7 +1435,7 @@ def test_export_partition_multi_column_partition_key_success_all(cluster): make_rmt(node, mt_table, cols, "(a, b, c)", replica_name="replica1") make_iceberg_s3(node, iceberg_table, cols, partition_by="(a, b, c)") - node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (2, 2, 2, 'y')") + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x'), (4, 5, 6, 'y')") partition_ids = node.query( f"SELECT DISTINCT partition_id FROM system.parts WHERE database = currentDatabase() " @@ -1454,37 +1454,7 @@ def test_export_partition_multi_column_partition_key_success_all(cluster): assert count == 2, f"Expected 2 rows in destination after export, got {count}" result = node.query(f"SELECT a, b, c, val FROM {iceberg_table} ORDER BY val").strip() - assert result == "1\t1\t1\tx\n2\t2\t2\ty", f"Unexpected exported data:\n{result}" - - -def test_export_partition_transform_partition_key_different_column_order_is_rejected(cluster): - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_transform_reordered_{uid}" - iceberg_table = f"iceberg_transform_reordered_{uid}" - - make_rmt(node, mt_table, "other_id Int64, user_id Int64", "icebergBucket(8, user_id)", replica_name="replica1") - make_iceberg_s3(node, iceberg_table, "user_id Int64, other_id Int64", partition_by="icebergBucket(8, user_id)") - - node.query(f"INSERT INTO {mt_table} VALUES (1, 42)") - - pid = first_partition_id(node, mt_table) - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" - - error_all = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + assert result == "1\t2\t3\tx\n4\t5\t6\ty", f"Unexpected exported data:\n{result}" def test_export_partition_with_castable_widening(cluster): diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py index cb73cc926f4e..f73b35e29b1a 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py @@ -1,6 +1,7 @@ import logging import time import uuid +from typing import NamedTuple import pytest @@ -1749,62 +1750,97 @@ def test_export_partition_all_failure_modes(cluster): ) -def test_export_partition_same_partition_key_different_column_order_single_column(cluster): +class RejectedPartitionExportCase(NamedTuple): + src_columns: str + src_partition_by: str + dst_columns: str + dst_partition_by: str + insert_values: str + error_substrings: tuple = () + + +REJECTED_PARTITION_EXPORT_CASES = [ + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32", + src_partition_by="a", + dst_columns="b Int32, a Int32", + dst_partition_by="a", + insert_values="(1, 1), (1, 2)", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_single_column", + ), + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="c Int32, b Int32, a Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 1, 1, 'x'), (1, 1, 1, 'y')", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_multi_column", + ), + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(c, b, a)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("different `PARTITION BY` expressions",), + ), + id="multi_column_partition_key_order_mismatch", + ), + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(a, b)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("different `PARTITION BY` expressions",), + ), + id="multi_column_partition_key_fewer_in_destination", + ), + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 2, 3, 'x')", + error_substrings=("different `PARTITION BY` expressions",), + ), + id="multi_column_partition_key_more_in_destination", + ), +] + + +@pytest.mark.parametrize("case", REJECTED_PARTITION_EXPORT_CASES) +def test_export_partition_partition_key_mismatch_variants_are_rejected(cluster, case): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["replica1"] postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"reordered_mt_table_{postfix}" - s3_table = f"reordered_s3_table_{postfix}" + mt_table = f"rejected_mt_table_{postfix}" + s3_table = f"rejected_s3_table_{postfix}" node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32) + CREATE TABLE {mt_table} ({case.src_columns}) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') - PARTITION BY a + PARTITION BY {case.src_partition_by} ORDER BY tuple() """) node.query(f""" - CREATE TABLE {s3_table} (b Int32, a Int32) + CREATE TABLE {s3_table} ({case.dst_columns}) ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY a + PARTITION BY {case.dst_partition_by} """) - node.query(f"INSERT INTO {mt_table} VALUES (1, 1), (1, 2)") - - error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '1' TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" - - error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - -def test_export_partition_same_partition_key_different_column_order_multi_column(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_reordered_mt_table_{postfix}" - s3_table = f"multi_reordered_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') - PARTITION BY (a, b, c) - ORDER BY tuple() - """) - - node.query(f""" - CREATE TABLE {s3_table} (c Int32, b Int32, a Int32, val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY (a, b, c) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + node.query(f"INSERT INTO {mt_table} VALUES {case.insert_values}") partition_id = node.query( f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " @@ -1813,7 +1849,8 @@ def test_export_partition_same_partition_key_different_column_order_multi_column error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "partition key column" in error, f"Expected partition key column mismatch message, got: {error}" + for substring in case.error_substrings: + assert substring in error, f"Expected {substring!r} in error, got: {error}" error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" @@ -1843,7 +1880,7 @@ def test_export_partition_multi_column_partition_key_success(cluster): PARTITION BY (a, b, c) """) - node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (1, 1, 1, 'y')") + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x'), (1, 2, 3, 'y')") partition_id = node.query( f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " @@ -1857,31 +1894,31 @@ def test_export_partition_multi_column_partition_key_success(cluster): assert count == 2, f"Expected 2 rows in destination after export, got {count}" result = node.query(f"SELECT a, b, c, val FROM {s3_table} ORDER BY val").strip() - assert result == "1\t1\t1\tx\n1\t1\t1\ty", f"Unexpected exported data:\n{result}" + assert result == "1\t2\t3\tx\n1\t2\t3\ty", f"Unexpected exported data:\n{result}" -def test_export_partition_multi_column_partition_key_order_mismatch_is_rejected(cluster): +def test_export_partition_partition_key_timezone_mismatch_is_rejected(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["replica1"] postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_order_mt_table_{postfix}" - s3_table = f"multi_order_s3_table_{postfix}" + mt_table = f"tz_mismatch_mt_table_{postfix}" + s3_table = f"tz_mismatch_s3_table_{postfix}" node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + CREATE TABLE {mt_table} (id Int64, ts DateTime('UTC')) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') - PARTITION BY (a, b, c) + PARTITION BY ts ORDER BY tuple() """) node.query(f""" - CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + CREATE TABLE {s3_table} (id Int64, ts DateTime('Asia/Tokyo')) ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY (c, b, a) + PARTITION BY ts """) - node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") partition_id = node.query( f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " @@ -1890,85 +1927,7 @@ def test_export_partition_multi_column_partition_key_order_mismatch_is_rejected( error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - - error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - -def test_export_partition_multi_column_partition_key_fewer_in_destination_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_fewer_mt_table_{postfix}" - s3_table = f"multi_fewer_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') - PARTITION BY (a, b, c) - ORDER BY tuple() - """) - - node.query(f""" - CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY (a, b) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") - - partition_id = node.query( - f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - - error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - -def test_export_partition_multi_column_partition_key_more_in_destination_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_more_mt_table_{postfix}" - s3_table = f"multi_more_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') - PARTITION BY (a, b) - ORDER BY tuple() - """) - - node.query(f""" - CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY (a, b, c) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x')") - - partition_id = node.query( - f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - - error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + assert "timezone" in error, f"Expected timezone mismatch message, got: {error}" count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" @@ -1995,7 +1954,7 @@ def test_export_partition_multi_column_partition_key_success_all(cluster): PARTITION BY (a, b, c) """) - node.query(f"INSERT INTO {mt_table} VALUES (1, 1, 1, 'x'), (2, 2, 2, 'y')") + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x'), (4, 5, 6, 'y')") partition_ids = node.query( f"SELECT DISTINCT partition_id FROM system.parts WHERE database = currentDatabase() " @@ -2011,4 +1970,4 @@ def test_export_partition_multi_column_partition_key_success_all(cluster): assert count == 2, f"Expected 2 rows in destination after export, got {count}" result = node.query(f"SELECT a, b, c, val FROM {s3_table} ORDER BY val").strip() - assert result == "1\t1\t1\tx\n2\t2\t2\ty", f"Unexpected exported data:\n{result}" + assert result == "1\t2\t3\tx\n4\t5\t6\ty", f"Unexpected exported data:\n{result}" From bf200410111af55668360f77247358465cfbb9e4 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 3 Aug 2026 14:46:38 +0200 Subject: [PATCH 06/20] up tests Signed-off-by: Konstantin Morozov --- .../test.py | 48 +++++-------------- .../test.py | 47 +++++------------- 2 files changed, 22 insertions(+), 73 deletions(-) diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index f946b01e726f..b4a1160313aa 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -380,6 +380,17 @@ class RejectedPartExportCase(NamedTuple): ), id="multi_column_partition_key_more_in_destination", ), + pytest.param( + RejectedPartExportCase( + src_columns="id Int64, ts DateTime('UTC')", + src_partition_by="ts", + dst_columns="id Int64, ts DateTime('Asia/Tokyo')", + dst_partition_by="ts", + insert_values="(1, '2024-03-05 15:00:00')", + error_substrings=("timezone",), + ), + id="partition_key_timezone_mismatch", + ), ] @@ -462,43 +473,6 @@ def test_export_part_multi_column_partition_key_success(cluster): assert result == "1\t2\t3\tx\n1\t2\t3\ty", f"Unexpected exported data:\n{result}" -def test_export_part_partition_key_timezone_mismatch_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_mismatch_mt_table_{postfix}" - s3_table = f"tz_mismatch_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int64, ts DateTime('UTC')) - ENGINE = MergeTree() - PARTITION BY ts - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int64, ts DateTime('Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY ts - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "timezone" in error, f"Expected timezone mismatch message, got: {error}" - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - def test_export_part_non_partition_key_timezone_mismatch_is_allowed(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py index f73b35e29b1a..0b38f97df4a5 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py @@ -1815,6 +1815,17 @@ class RejectedPartitionExportCase(NamedTuple): ), id="multi_column_partition_key_more_in_destination", ), + pytest.param( + RejectedPartitionExportCase( + src_columns="id Int64, ts DateTime('UTC')", + src_partition_by="ts", + dst_columns="id Int64, ts DateTime('Asia/Tokyo')", + dst_partition_by="ts", + insert_values="(1, '2024-03-05 15:00:00')", + error_substrings=("timezone",), + ), + id="partition_key_timezone_mismatch", + ), ] @@ -1897,42 +1908,6 @@ def test_export_partition_multi_column_partition_key_success(cluster): assert result == "1\t2\t3\tx\n1\t2\t3\ty", f"Unexpected exported data:\n{result}" -def test_export_partition_partition_key_timezone_mismatch_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_mismatch_mt_table_{postfix}" - s3_table = f"tz_mismatch_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int64, ts DateTime('UTC')) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') - PARTITION BY ts - ORDER BY tuple() - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int64, ts DateTime('Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY ts - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - partition_id = node.query( - f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - assert "timezone" in error, f"Expected timezone mismatch message, got: {error}" - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - def test_export_partition_multi_column_partition_key_success_all(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["replica1"] From d3b87ca723f82e01c7cbda98e010e750699426c4 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 3 Aug 2026 18:28:26 +0200 Subject: [PATCH 07/20] refactoring Signed-off-by: Konstantin Morozov --- src/Storages/MergeTree/ExportPartitionUtils.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index d11b74aff13f..433c663a445b 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include "Storages/ExportReplicatedMergeTreePartitionManifest.h" #include "Storages/ExportReplicatedMergeTreePartitionTaskEntry.h" #include @@ -17,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -642,9 +642,9 @@ namespace ExportPartitionUtils { std::optional getDateTimeTimeZoneName(const DataTypePtr & type) { - if (const auto * datetime_type = typeid_cast(type.get())) + if (const auto * datetime_type = checkAndGetDataType(type.get())) return datetime_type->getTimeZone().getTimeZone(); - if (const auto * datetime64_type = typeid_cast(type.get())) + if (const auto * datetime64_type = checkAndGetDataType(type.get())) return datetime64_type->getTimeZone().getTimeZone(); return {}; } From 72d988bd2856e31de3e2279ee42f4c7c9832212b Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 6 Aug 2026 15:34:50 +0200 Subject: [PATCH 08/20] verify subcolumns Signed-off-by: Konstantin Morozov --- .../MergeTree/ExportPartitionUtils.cpp | 101 ++- src/Storages/MergeTree/ExportPartitionUtils.h | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- .../test.py | 44 ++ .../test.py | 584 ++++++++++++++++++ 5 files changed, 701 insertions(+), 32 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 433c663a445b..17497b8ad056 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #if USE_AVRO #include @@ -648,9 +649,55 @@ namespace ExportPartitionUtils return datetime64_type->getTimeZone().getTimeZone(); return {}; } + + void verifyPartitionKeyColumn( + const ColumnWithTypeAndName & source_column, + const ColumnWithTypeAndName & destination_column, + size_t position, + const std::vector & required_columns, + const ColumnsDescription & source_columns_description, + const ColumnsDescription & destination_columns_description, + const StorageID & destination_storage_id) + { + if (source_column.name != destination_column.name) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot export to {}: partition key column '{}' is at position {} in the source " + "table, but the destination's column at that position is named '{}'. EXPORT " + "PART/PARTITION matches columns by position, so partition key columns must be " + "declared at the same position in both tables.", + destination_storage_id.getFullTableName(), + source_column.name, + position, + destination_column.name); + + for (const auto & column_name : required_columns) + { + const auto source_resolved = source_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); + const auto destination_resolved + = destination_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); + if (!source_resolved || !destination_resolved) + continue; + + const auto source_time_zone = getDateTimeTimeZoneName(source_resolved->type); + const auto destination_time_zone = getDateTimeTimeZoneName(destination_resolved->type); + if (source_time_zone && destination_time_zone && *source_time_zone != *destination_time_zone) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot export to {}: partition key column '{}' is {} in the source table " + "but {} in the destination. The destination's hive-style partition path is " + "rendered from the source value without converting the timezone, so this " + "would silently shift the exported value by the timezone offset. Use the " + "same timezone in both tables' partition key column.", + destination_storage_id.getFullTableName(), + column_name, + source_resolved->type->getName(), + destination_resolved->type->getName()); + } + } } - void verifyMergeTreePartitionCompatibility( + void assertPartitionKeyASTAreEqual( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata) { @@ -687,10 +734,21 @@ namespace ExportPartitionUtils ActionsDAG::MatchColumnsMode::Position, context); + const auto & source_columns_description = source_metadata->getColumns(); + const auto & destination_columns_description = destination_metadata->getColumns(); auto partition_key_columns = source_metadata->getColumnsRequiredForPartitionKey(); - const std::unordered_set partition_key_column_set( - std::make_move_iterator(partition_key_columns.begin()), - std::make_move_iterator(partition_key_columns.end())); + + /// Owning top-level column name -> required PARTITION BY names it owns. Two cases: + /// - Flat key `PARTITION BY a` -> {"a": ["a"]}: "a" is not a subcolumn, it owns itself. + /// - Composite key `PARTITION BY t.ts` -> {"t": ["t.ts"]}: "t.ts" is a subcolumn of "t". + /// - Multiple subcolumns of one owner, `PARTITION BY (t.a, t.b)` -> {"t": ["t.a", "t.b"]}. + std::unordered_map> owner_to_partition_key_columns; + for (const auto & column_name : partition_key_columns) + { + auto resolved = source_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); + const auto & owner_column_name = resolved ? resolved->getNameInStorage() : column_name; + owner_to_partition_key_columns[owner_column_name].push_back(column_name); + } const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; @@ -700,33 +758,16 @@ namespace ExportPartitionUtils const auto & source_column = source_columns[i]; const auto & destination_column = destination_columns[i]; - if (partition_key_column_set.contains(source_column.name) && source_column.name != destination_column.name) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Cannot export to {}: partition key column '{}' is at position {} in the source " - "table, but the destination's column at that position is named '{}'. EXPORT " - "PART/PARTITION matches columns by position, so partition key columns must be " - "declared at the same position in both tables.", - destination_storage_id.getFullTableName(), - source_column.name, + if (const auto owned_it = owner_to_partition_key_columns.find(source_column.name); + owned_it != owner_to_partition_key_columns.end()) + verifyPartitionKeyColumn( + source_column, + destination_column, i, - destination_column.name); - - if (partition_key_column_set.contains(source_column.name)) - { - const auto source_time_zone = getDateTimeTimeZoneName(source_column.type); - const auto destination_time_zone = getDateTimeTimeZoneName(destination_column.type); - if (source_time_zone && destination_time_zone && *source_time_zone != *destination_time_zone) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Cannot export to {}: partition key column '{}' is {} in the source table " - "but {} in the destination. The destination's hive-style partition path is " - "rendered from the source value without converting the timezone, so this " - "would silently shift the exported value by the timezone offset. Use the " - "same timezone in both tables' partition key column.", - destination_storage_id.getFullTableName(), - destination_column.name, - source_column.type->getName(), - destination_column.type->getName()); - } + owned_it->second, + source_columns_description, + destination_columns_description, + destination_storage_id); /// Lossy casts may silently change values, so reject them unless the user opts in. if (allow_lossy_cast) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index dd1ae4c18094..b58ecad9e7bb 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -89,7 +89,7 @@ namespace ExportPartitionUtils const std::string & exception_message, const LoggerPtr & log); - void verifyMergeTreePartitionCompatibility( + void assertPartitionKeyASTAreEqual( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 759736dc06be..7b7533d26d9b 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -8416,7 +8416,7 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & src_snapshot, destination_snapshot, dest_storage->getStorageID(), query_context); if (!dest_storage->isDataLake()) - ExportPartitionUtils::verifyMergeTreePartitionCompatibility(src_snapshot, destination_snapshot); + ExportPartitionUtils::assertPartitionKeyASTAreEqual(src_snapshot, destination_snapshot); zkutil::ZooKeeperPtr zookeeper = getZooKeeperAndAssertNotReadonly(); diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index e1dd26ea3820..ceafc0a120fd 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -896,3 +896,47 @@ def test_export_part_runtime_cast_failure_propagates_async(cluster): node.query(f"DROP TABLE IF EXISTS {mt} SYNC") node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_tuple_subcolumn_partition_key_iceberg_rejected(cluster): + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_tuple_subcol_{sfx}" + iceberg = f"iceberg_tuple_subcol_{sfx}" + iceberg_partitioned = f"iceberg_tuple_subcol_part_{sfx}" + + create_error = node.query_and_get_error( + f"CREATE TABLE {iceberg_partitioned} (t Tuple(b Int32, a Int32), val String) " + f"ENGINE = IcebergS3('http://minio1:9001/root/data/{iceberg_partitioned}/', 'minio', 'ClickHouse_Minio_P@ssw0rd') " + f"PARTITION BY t.a" + ) + assert "Unknown field to partition" in create_error, ( + f"Expected Iceberg to reject the tuple subcolumn partition key at CREATE time, " + f"got: {create_error!r}" + ) + + make_mt(node, mt, "t Tuple(a Int32, b Int32), val String", "t.a") + make_iceberg_s3(node, iceberg, "t Tuple(b Int32, a Int32), val String", "val") + + node.query(f"INSERT INTO {mt} VALUES ((1, 99), 'x')") + + part = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt}' AND active ORDER BY name LIMIT 1" + ).strip() + + export_error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PART '{part}' TO TABLE {iceberg} " + f"SETTINGS allow_experimental_export_merge_tree_part = 1, " + f"allow_experimental_insert_into_iceberg = 1" + ) + assert "Unknown field to partition" in export_error, ( + f"Expected export validation to reject the tuple subcolumn partition key of {mt}, " + f"got: {export_error!r}" + ) + + count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table after rejected export, got {count}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index b4a1160313aa..a8cd30d50b2c 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -526,3 +526,587 @@ def test_export_part_non_partition_key_timezone_mismatch_is_allowed(cluster): ) +def test_export_part_tuple_subcolumn_partition_key_hive_destination_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + s3_table = f"tuple_subcol_s3_table_{postfix}" + + error = node.query_and_get_error(f""" + CREATE TABLE {s3_table} (t Tuple(b Int32, a Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY t.a + """) + assert "BAD_ARGUMENTS" in error and "part of the storage columns" in error, ( + f"Expected hive partition strategy to reject the tuple subcolumn " + f"partition expression at CREATE time, got: {error!r}" + ) + + +def read_exported_files(node, s3_table): + return node.query( + f"SELECT t.a, t.b, val FROM " + f"s3('http://minio1:9001/root/data/{s3_table}/**', 'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet', " + f"'t Tuple(b Int32, a Int32), val String') " + f"WHERE _file NOT LIKE 'commit%'" + ).strip() + + +def get_export_part_log(node, mt_table): + node.query("SYSTEM FLUSH LOGS") + return node.query( + f"SELECT part_name, error, exception FROM system.part_log " + f"WHERE event_type = 'ExportPart' AND database = currentDatabase() " + f"AND table = '{mt_table}' ORDER BY event_time" + ).strip() + + +def test_export_part_tuple_subcolumn_partition_key_wildcard_destination(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tuple_subcol_wc_mt_table_{postfix}" + s3_table = f"tuple_subcol_wc_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32, b Int32), val String) + ENGINE = MergeTree() + PARTITION BY t.a + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (t Tuple(b Int32, a Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY t.a + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, 99), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + + time.sleep(5) + + part_log = get_export_part_log(node, mt_table) + result = read_exported_files(node, s3_table) + assert result == "1\t99\tx", ( + f"Tuple element values were remapped: source t = (a=1, b=99), " + f"exported files read back (t.a, t.b) as: {result!r}; part_log: {part_log!r}" + ) + + +def test_export_part_subcolumn_partition_key_different_subcolumn_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"subcol_diff_subcol_mt_table_{postfix}" + s3_table = f"subcol_diff_subcol_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Tuple(b Int32, c Int32), val String) + ENGINE = MergeTree() + PARTITION BY a.b + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Tuple(b Int32, c Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY a.c + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, 2), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "different `PARTITION BY` expressions" in error, ( + f"Both tables declare `a` as the same Tuple(b Int32, c Int32) (so the column-cast " + f"check passes and the owner-name-only partition_key_column_set = {{'a'}} in " + f"verifyExportSchemaCastable cannot distinguish `a.b` from `a.c`), but the source " + f"partitions by `a.b` and the destination by `a.c` — a genuinely different " + f"partition key that must be caught by the `PARTITION BY` AST comparison; " + f"got: {error!r}" + ) + + +def test_export_part_tuple_subcolumn_partition_key_owner_column_reordered_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tuple_subcol_owner_mt_table_{postfix}" + s3_table = f"tuple_subcol_owner_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32, b Int32), decoy Tuple(a Int32, b Int32), val String) + ENGINE = MergeTree() + PARTITION BY t.a + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (decoy Tuple(a Int32, b Int32), t Tuple(a Int32, b Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY t.a + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, 100), (2, 200), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "partition key column" in error, ( + f"Expected export to reject `t` and `decoy` swapping positions around the " + f"partition key column `t.a`, the same way a plain (non-tuple) partition key " + f"column position swap is rejected; got: {error!r}" + ) + + +def test_export_part_subcolumn_partition_key_timezone_mismatch_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"subcol_tz_mt_table_{postfix}" + s3_table = f"subcol_tz_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) + ENGINE = MergeTree() + PARTITION BY t.ts + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (t Tuple(a Int32, ts DateTime('Asia/Tokyo')), val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY t.ts + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, '2024-03-05 15:00:00'), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "timezone" in error, ( + f"Expected export to reject the nested partition key column `t.ts` changing " + f"timezone from UTC to Asia/Tokyo between source and destination, the same way " + f"a top-level DateTime partition key column with mismatched timezones is " + f"rejected (see test_export_part_partition_key_mismatch_variants_are_rejected's " + f"'partition_key_timezone_mismatch' case). getDateTimeTimeZoneName() is called " + f"on the owning column's type (Tuple(...)), not on the actual DateTime " + f"subcolumn's type, so it can never recognize a timezone at all here; " + f"got: {error!r}" + ) + + +def test_export_part_multi_level_subcolumn_partition_key_owner_reordered_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"nested_subcol_owner_mt_table_{postfix}" + s3_table = f"nested_subcol_owner_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} ( + t Tuple(x Tuple(a Int32, b Int32), c Int32), + decoy Tuple(x Tuple(a Int32, b Int32), c Int32), + val String + ) + ENGINE = MergeTree() + PARTITION BY t.x.a + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} ( + decoy Tuple(x Tuple(a Int32, b Int32), c Int32), + t Tuple(x Tuple(a Int32, b Int32), c Int32), + val String + ) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY t.x.a + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((((1, 100), 1000)), (((2, 200), 2000)), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "partition key column" in error, ( + f"Expected export to reject `t` and `decoy` swapping positions around the " + f"two-level-deep partition key column `t.x.a`. This only works if " + f"getNameInStorage() resolves all the way to the top-level column `t`, not to " + f"the intermediate level `t.x`; got: {error!r}" + ) + + +def test_export_part_multiple_subcolumn_partition_keys_owner_reordered_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_subcol_key_mt_table_{postfix}" + s3_table = f"multi_subcol_key_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} ( + t Tuple(a Int32, x Int32), + u Tuple(b Int32, y Int32), + decoy Tuple(b Int32, y Int32), + val String + ) + ENGINE = MergeTree() + PARTITION BY (t.a, u.b) + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} ( + t Tuple(a Int32, x Int32), + decoy Tuple(b Int32, y Int32), + u Tuple(b Int32, y Int32), + val String + ) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY (t.a, u.b) + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, 10), (2, 20), (3, 30), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "partition key column 'u'" in error, ( + f"`t` (owner of key part `t.a`) stays at position 0 on both sides, so the guard " + f"must independently catch `u` (owner of key part `u.b`) swapping positions " + f"with `decoy` — a partition key with two subcolumn-owning columns must have " + f"both validated, not just the first one encountered; got: {error!r}" + ) + + +def test_export_part_mixed_flat_and_subcolumn_partition_key_flat_part_reordered_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"mixed_key_mt_table_{postfix}" + s3_table = f"mixed_key_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, t Tuple(b Int32, c Int32), decoy Int32, val String) + ENGINE = MergeTree() + PARTITION BY (a, t.b) + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (decoy Int32, t Tuple(b Int32, c Int32), a Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY (a, t.b) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, (2, 3), 4, 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "partition key column 'a'" in error, ( + f"`t` (owner of key part `t.b`) stays at position 1 on both sides, so the guard " + f"must independently catch the plain, non-tuple key part `a` swapping positions " + f"with `decoy` — the pre-existing flat-column check and the new subcolumn-owner " + f"resolution must both keep working when combined in one `PARTITION BY` " + f"expression; got: {error!r}" + ) + + +def test_export_part_subcolumn_partition_key_owner_reordered_rejected_even_with_allow_lossy_cast(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"lossy_owner_mt_table_{postfix}" + s3_table = f"lossy_owner_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32, b Int32), decoy Tuple(a Int32, b Int32), val String) + ENGINE = MergeTree() + PARTITION BY t.a + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (decoy Tuple(a Int32, b Int32), t Tuple(a Int32, b Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY t.a + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, 100), (2, 200), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table} " + f"SETTINGS export_merge_tree_part_allow_lossy_cast = 1" + ) + assert "BAD_ARGUMENTS" in error and "partition key column" in error, ( + f"The partition-key position/name guard is checked before the " + f"`allow_lossy_cast` early-continue in verifyExportSchemaCastable, so setting " + f"`export_merge_tree_part_allow_lossy_cast = 1` must not suppress the rejection " + f"of `t`/`decoy` swapping positions around the partition key column `t.a`; " + f"got: {error!r}" + ) + + +def test_export_part_tuple_column_real_narrowing_same_order_is_rejected_diagnostic(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tuple_narrow_same_order_mt_table_{postfix}" + s3_table = f"tuple_narrow_same_order_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32, b Int64), val String) + ENGINE = MergeTree() + PARTITION BY val + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (t Tuple(a Int32, b Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY val + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((100, 5000000000), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + print(f"DIAGNOSTIC SAME-ORDER-NARROWING ERROR: {error!r}") + assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( + f"Expected the genuinely narrowing `b` field (Int64 -> Int32, same field order " + f"on both sides) to be rejected as a lossy cast; got: {error!r}" + ) + + +def test_export_part_tuple_column_fewer_fields_in_destination_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tuple_fewer_fields_mt_table_{postfix}" + s3_table = f"tuple_fewer_fields_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) + ENGINE = MergeTree() + PARTITION BY val + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (t Tuple(a Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY val + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, '2024-03-05 15:00:00'), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( + f"`t` is not a partition key column here. The destination's `t` (Tuple(a Int32)) " + f"has fewer fields than the source's `t` (Tuple(a Int32, ts DateTime('UTC'))), " + f"which canBeSafelyCast's tuple arity check (lhs_type_elements_size != " + f"to_tuple_type_elements.size()) must reject; got: {error!r}" + ) + + +def test_export_part_subcolumn_partition_key_tuple_fewer_fields_in_destination_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"subcol_key_fewer_fields_mt_table_{postfix}" + s3_table = f"subcol_key_fewer_fields_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) + ENGINE = MergeTree() + PARTITION BY t.ts + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (t Tuple(a Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY t.a + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, '2024-03-05 15:00:00'), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( + f"The partition key `t.ts` requires a field the destination's `t` doesn't have " + f"at all, so verifyPartitionKeyColumn's timezone lookup finds " + f"destination_resolved == nullopt for 't.ts' and defers (continue); the arity " + f"mismatch must still be caught right after by canBeSafelyCast on the whole " + f"`t` column; got: {error!r}" + ) + + +def test_export_part_tuple_column_fewer_fields_in_source_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tuple_fewer_fields_src_mt_table_{postfix}" + s3_table = f"tuple_fewer_fields_src_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32), val String) + ENGINE = MergeTree() + PARTITION BY val + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY val + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1,), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( + f"Reverse direction of the fewer-fields case: the source's `t` (Tuple(a Int32)) " + f"has fewer fields than the destination's `t` (Tuple(a Int32, ts " + f"DateTime('UTC'))). canBeSafelyCast's arity check is a plain size " + f"inequality, so it must reject this direction too; got: {error!r}" + ) + + +def test_export_part_subcolumn_partition_key_tuple_fewer_fields_in_source_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"subcol_key_fewer_fields_src_mt_table_{postfix}" + s3_table = f"subcol_key_fewer_fields_src_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(a Int32), val String) + ENGINE = MergeTree() + PARTITION BY t.a + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY t.a + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1,), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( + f"Here `t.a` (the partition key) exists and resolves fine on both sides, so " + f"verifyPartitionKeyColumn's timezone check passes without throwing; the extra " + f"`ts` field the destination has beyond the source's `t` must still be caught " + f"by canBeSafelyCast's arity check, independent of the partition key guard " + f"succeeding; got: {error!r}" + ) + + From 462614981e6382f235ecd0d26b8e1095ac73ceb9 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 6 Aug 2026 16:12:28 +0200 Subject: [PATCH 09/20] lowcardinality and nullable Signed-off-by: Konstantin Morozov --- .../MergeTree/ExportPartitionUtils.cpp | 7 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../test.py | 81 +++++++++++++++++++ 3 files changed, 87 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 17497b8ad056..ca954e022beb 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include #include @@ -643,9 +645,10 @@ namespace ExportPartitionUtils { std::optional getDateTimeTimeZoneName(const DataTypePtr & type) { - if (const auto * datetime_type = checkAndGetDataType(type.get())) + const auto unwrapped_type = removeNullable(removeLowCardinality(type)); + if (const auto * datetime_type = checkAndGetDataType(unwrapped_type.get())) return datetime_type->getTimeZone().getTimeZone(); - if (const auto * datetime64_type = checkAndGetDataType(type.get())) + if (const auto * datetime64_type = checkAndGetDataType(unwrapped_type.get())) return datetime64_type->getTimeZone().getTimeZone(); return {}; } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index ac8ad03c3b63..0519730a5ee8 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6787,7 +6787,7 @@ void MergeTreeData::exportPartToTable( source_metadata_ptr, destination_metadata_ptr, dest_storage->getStorageID(), query_context); if (!dest_storage->isDataLake()) - ExportPartitionUtils::verifyMergeTreePartitionCompatibility(source_metadata_ptr, destination_metadata_ptr); + ExportPartitionUtils::assertPartitionKeyASTAreEqual(source_metadata_ptr, destination_metadata_ptr); auto part = getPartIfExists(part_name, {MergeTreeDataPartState::Active, MergeTreeDataPartState::Outdated}); diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index a8cd30d50b2c..4bab7fceba98 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -1110,3 +1110,84 @@ def test_export_part_subcolumn_partition_key_tuple_fewer_fields_in_source_is_rej ) +def test_export_part_nullable_datetime_partition_key_timezone_mismatch_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"nullable_tz_mt_table_{postfix}" + s3_table = f"nullable_tz_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts Nullable(DateTime('UTC'))) + ENGINE = MergeTree() + PARTITION BY ts + ORDER BY id + SETTINGS allow_nullable_key = 1, enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts Nullable(DateTime('Asia/Tokyo'))) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY ts + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "timezone" in error, ( + f"getDateTimeTimeZoneName() only recognizes bare DateTime/DateTime64, so wrapping " + f"the partition key in Nullable(...) hides the timezone from it entirely; " + f"canBeSafelyCast() provides no fallback here either, since " + f"DataTypeDateTime::equals() treats any two DateTime types as equal regardless " + f"of timezone by design, so this needs its own dedicated rejection - even " + f"without export_merge_tree_part_allow_lossy_cast being set; got: {error!r}" + ) + + +def test_export_part_lowcardinality_datetime_partition_key_timezone_mismatch_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"lc_tz_mt_table_{postfix}" + s3_table = f"lc_tz_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts LowCardinality(DateTime('UTC'))) + ENGINE = MergeTree() + PARTITION BY ts + ORDER BY id + SETTINGS allow_suspicious_low_cardinality_types = 1, enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts LowCardinality(DateTime('Asia/Tokyo'))) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY ts + SETTINGS allow_suspicious_low_cardinality_types = 1 + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "timezone" in error, ( + f"Same gap as the Nullable case, but for LowCardinality(DateTime(...)); " + f"got: {error!r}" + ) + + From 70a5ff8cd9f35e52b8cacee73756faa63daf8c02 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 6 Aug 2026 16:51:05 +0200 Subject: [PATCH 10/20] sensetive column Signed-off-by: Konstantin Morozov --- .../MergeTree/ExportPartitionUtils.cpp | 47 ++++++++++ .../test.py | 86 +++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index ca954e022beb..f7591b628e2a 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #if USE_AVRO @@ -653,11 +655,49 @@ namespace ExportPartitionUtils return {}; } + /// toUnixTimestamp-family functions return the same value regardless of the argument's + /// declared timezone, so a column used only inside them doesn't need the check below. + void collectColumnsRequiringTimezoneCheck( + const ASTPtr & node, + bool inside_timezone_invariant_function, + std::unordered_set & columns_requiring_timezone_check) + { + if (!node) + return; + + if (const auto * identifier = node->as()) + { + if (!inside_timezone_invariant_function) + columns_requiring_timezone_check.insert(identifier->name()); + return; + } + + if (const auto * function = node->as()) + { + static const std::unordered_set timezone_invariant_functions = { + "toUnixTimestamp", + "toUnixTimestamp64Milli", + "toUnixTimestamp64Micro", + "toUnixTimestamp64Nano", + }; + const bool wraps_in_invariant_function + = inside_timezone_invariant_function || timezone_invariant_functions.contains(function->name); + if (function->arguments) + for (const auto & argument : function->arguments->children) + collectColumnsRequiringTimezoneCheck(argument, wraps_in_invariant_function, columns_requiring_timezone_check); + return; + } + + for (const auto & child : node->children) + collectColumnsRequiringTimezoneCheck(child, inside_timezone_invariant_function, columns_requiring_timezone_check); + } + void verifyPartitionKeyColumn( const ColumnWithTypeAndName & source_column, const ColumnWithTypeAndName & destination_column, size_t position, const std::vector & required_columns, + const std::unordered_set & columns_requiring_timezone_check, const ColumnsDescription & source_columns_description, const ColumnsDescription & destination_columns_description, const StorageID & destination_storage_id) @@ -676,6 +716,9 @@ namespace ExportPartitionUtils for (const auto & column_name : required_columns) { + if (!columns_requiring_timezone_check.contains(column_name)) + continue; + const auto source_resolved = source_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); const auto destination_resolved = destination_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); @@ -753,6 +796,9 @@ namespace ExportPartitionUtils owner_to_partition_key_columns[owner_column_name].push_back(column_name); } + std::unordered_set columns_requiring_timezone_check; + collectColumnsRequiringTimezoneCheck(source_metadata->getPartitionKeyAST(), false, columns_requiring_timezone_check); + const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); @@ -768,6 +814,7 @@ namespace ExportPartitionUtils destination_column, i, owned_it->second, + columns_requiring_timezone_check, source_columns_description, destination_columns_description, destination_storage_id); diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 4bab7fceba98..59aaa66eaf91 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -1191,3 +1191,89 @@ def test_export_part_lowcardinality_datetime_partition_key_timezone_mismatch_is_ ) +def test_export_part_timezone_invariant_expression_partition_key_is_allowed(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tz_invariant_mt_table_{postfix}" + s3_table = f"tz_invariant_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) + ENGINE = MergeTree() + PARTITION BY toUnixTimestamp(ts) + ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY toUnixTimestamp(ts) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + + time.sleep(3) + result = node.query( + f"SELECT count() FROM s3('http://minio1:9001/root/data/{s3_table}/**', " + f"'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet', " + f"'id Int32, ts DateTime(\\'Asia/Tokyo\\')') " + f"WHERE _file NOT LIKE 'commit%'" + ).strip() + assert result == "1", ( + f"Expected the export to succeed and produce 1 row: toUnixTimestamp(ts) does " + f"not depend on ts's declared timezone, so a mismatched timezone between " + f"source and destination must not block it; got count={result!r}" + ) + + +def test_export_part_timezone_sensitive_expression_partition_key_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tz_sensitive_mt_table_{postfix}" + s3_table = f"tz_sensitive_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) + ENGINE = MergeTree() + PARTITION BY toDate(ts) + ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY toDate(ts) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "timezone" in error, ( + f"Unlike toUnixTimestamp(ts), toDate(ts) genuinely depends on ts's declared " + f"timezone (day boundaries differ between UTC and Asia/Tokyo), so it must " + f"remain protected by the timezone guard - the allowlist in " + f"collectTimezoneSensitiveColumns() must not be so broad that it lets this " + f"through too; got: {error!r}" + ) + + From e8eddd1519d246585d5115146cd6578261707a23 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 6 Aug 2026 17:38:37 +0200 Subject: [PATCH 11/20] resolve some cases Signed-off-by: Konstantin Morozov --- .../MergeTree/ExportPartitionUtils.cpp | 29 ++++++------ .../test.py | 44 ++++++++++++++++++- 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index f7591b628e2a..47c3bb095e93 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -655,41 +655,40 @@ namespace ExportPartitionUtils return {}; } - /// toUnixTimestamp-family functions return the same value regardless of the argument's - /// declared timezone, so a column used only inside them doesn't need the check below. void collectColumnsRequiringTimezoneCheck( const ASTPtr & node, - bool inside_timezone_invariant_function, + const ASTFunction * immediate_parent_function, std::unordered_set & columns_requiring_timezone_check) { if (!node) return; if (const auto * identifier = node->as()) - { - if (!inside_timezone_invariant_function) - columns_requiring_timezone_check.insert(identifier->name()); - return; - } - - if (const auto * function = node->as()) { static const std::unordered_set timezone_invariant_functions = { "toUnixTimestamp", + "toUnixTimestamp64Second", "toUnixTimestamp64Milli", "toUnixTimestamp64Micro", "toUnixTimestamp64Nano", }; - const bool wraps_in_invariant_function - = inside_timezone_invariant_function || timezone_invariant_functions.contains(function->name); + const bool wrapped_by_invariant_function + = immediate_parent_function && timezone_invariant_functions.contains(immediate_parent_function->name); + if (!wrapped_by_invariant_function) + columns_requiring_timezone_check.insert(identifier->name()); + return; + } + + if (const auto * function = node->as()) + { if (function->arguments) for (const auto & argument : function->arguments->children) - collectColumnsRequiringTimezoneCheck(argument, wraps_in_invariant_function, columns_requiring_timezone_check); + collectColumnsRequiringTimezoneCheck(argument, function, columns_requiring_timezone_check); return; } for (const auto & child : node->children) - collectColumnsRequiringTimezoneCheck(child, inside_timezone_invariant_function, columns_requiring_timezone_check); + collectColumnsRequiringTimezoneCheck(child, nullptr, columns_requiring_timezone_check); } void verifyPartitionKeyColumn( @@ -797,7 +796,7 @@ namespace ExportPartitionUtils } std::unordered_set columns_requiring_timezone_check; - collectColumnsRequiringTimezoneCheck(source_metadata->getPartitionKeyAST(), false, columns_requiring_timezone_check); + collectColumnsRequiringTimezoneCheck(source_metadata->getPartitionKeyAST(), nullptr, columns_requiring_timezone_check); const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 59aaa66eaf91..58849e43864c 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -1272,8 +1272,48 @@ def test_export_part_timezone_sensitive_expression_partition_key_is_rejected(clu f"Unlike toUnixTimestamp(ts), toDate(ts) genuinely depends on ts's declared " f"timezone (day boundaries differ between UTC and Asia/Tokyo), so it must " f"remain protected by the timezone guard - the allowlist in " - f"collectTimezoneSensitiveColumns() must not be so broad that it lets this " - f"through too; got: {error!r}" + f"collectColumnsRequiringTimezoneCheck() must not be so broad that it lets " + f"this through too; got: {error!r}" + ) + + +def test_export_part_timezone_sensitive_function_nested_in_invariant_function_is_rejected(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tz_nested_mt_table_{postfix}" + s3_table = f"tz_nested_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) + ENGINE = MergeTree() + PARTITION BY toUnixTimestamp(toDate(ts)) + ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY toUnixTimestamp(toDate(ts)) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "timezone" in error, ( + f"toDate(ts) inside toUnixTimestamp(toDate(ts)) already produces a " + f"timezone-dependent value before toUnixTimestamp ever runs, so wrapping it " + f"in an outer invariant function must not exempt ts from the check - only " + f"ts's *immediate* parent function determines that; got: {error!r}" ) From c3569c422e890cb2373b74a05717e6689e4c5908 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 6 Aug 2026 18:13:17 +0200 Subject: [PATCH 12/20] fix problems with func Signed-off-by: Konstantin Morozov --- .../MergeTree/ExportPartitionUtils.cpp | 13 +- .../test.py | 134 ++++++++++++++++++ 2 files changed, 144 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 47c3bb095e93..cec3fe0289d5 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -657,7 +657,7 @@ namespace ExportPartitionUtils void collectColumnsRequiringTimezoneCheck( const ASTPtr & node, - const ASTFunction * immediate_parent_function, + const ASTFunction * effective_parent_function, std::unordered_set & columns_requiring_timezone_check) { if (!node) @@ -673,7 +673,7 @@ namespace ExportPartitionUtils "toUnixTimestamp64Nano", }; const bool wrapped_by_invariant_function - = immediate_parent_function && timezone_invariant_functions.contains(immediate_parent_function->name); + = effective_parent_function && timezone_invariant_functions.contains(effective_parent_function->name); if (!wrapped_by_invariant_function) columns_requiring_timezone_check.insert(identifier->name()); return; @@ -681,9 +681,16 @@ namespace ExportPartitionUtils if (const auto * function = node->as()) { + static const std::unordered_set value_preserving_functions = { + "identity", + "assumeNotNull", + "materialize", + }; + const ASTFunction * next_parent_function + = value_preserving_functions.contains(function->name) ? effective_parent_function : function; if (function->arguments) for (const auto & argument : function->arguments->children) - collectColumnsRequiringTimezoneCheck(argument, function, columns_requiring_timezone_check); + collectColumnsRequiringTimezoneCheck(argument, next_parent_function, columns_requiring_timezone_check); return; } diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 58849e43864c..aff4d13bb15f 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -1317,3 +1317,137 @@ def test_export_part_timezone_sensitive_function_nested_in_invariant_function_is ) +def test_export_part_unix_timestamp64_second_partition_key_is_allowed(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tz_u64s_mt_table_{postfix}" + s3_table = f"tz_u64s_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts DateTime64(3, 'UTC')) + ENGINE = MergeTree() + PARTITION BY toUnixTimestamp64Second(ts) + ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts DateTime64(3, 'Asia/Tokyo')) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY toUnixTimestamp64Second(ts) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00.000')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + + time.sleep(3) + result = node.query( + f"SELECT count() FROM s3('http://minio1:9001/root/data/{s3_table}/**', " + f"'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet', " + f"'id Int32, ts DateTime64(3, \\'Asia/Tokyo\\')') " + f"WHERE _file NOT LIKE 'commit%'" + ).strip() + assert result == "1", ( + f"toUnixTimestamp64Second(ts) is documented as being relative to UTC, not the " + f"declared timezone of ts (see toUnixTimestamp64Second.cpp's own " + f"documentation), so a mismatched timezone between source and destination " + f"must not block this export; got count={result!r}" + ) + + +def test_export_part_value_preserving_function_wrapped_in_invariant_function_is_allowed(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tz_passthrough_mt_table_{postfix}" + s3_table = f"tz_passthrough_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) + ENGINE = MergeTree() + PARTITION BY toUnixTimestamp(assumeNotNull(ts)) + ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY toUnixTimestamp(assumeNotNull(ts)) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + + time.sleep(3) + result = node.query( + f"SELECT count() FROM s3('http://minio1:9001/root/data/{s3_table}/**', " + f"'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet', " + f"'id Int32, ts DateTime(\\'Asia/Tokyo\\')') " + f"WHERE _file NOT LIKE 'commit%'" + ).strip() + assert result == "1", ( + f"assumeNotNull(ts) is identity on the underlying value, so it must not break " + f"the chain between ts and the enclosing toUnixTimestamp - the check must look " + f"past value-preserving wrapper functions, not just the immediate parent; " + f"got count={result!r}" + ) + + +def test_export_part_timezone_sensitive_function_behind_value_preserving_wrapper_is_rejected( + cluster, +): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tz_nested_passthrough_mt_table_{postfix}" + s3_table = f"tz_nested_passthrough_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) + ENGINE = MergeTree() + PARTITION BY toUnixTimestamp(assumeNotNull(toDate(ts))) + ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY toUnixTimestamp(assumeNotNull(toDate(ts))) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "timezone" in error, ( + f"toDate(ts) is still timezone-sensitive even when reached through the " + f"value-preserving assumeNotNull() and wrapped by the invariant " + f"toUnixTimestamp() two levels up - skipping past passthrough functions must " + f"not also skip past genuinely timezone-sensitive ones; got: {error!r}" + ) + + From 3163636bac50f137f4cc9948a9e5e562ef931c42 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Thu, 6 Aug 2026 18:36:22 +0200 Subject: [PATCH 13/20] issue with func with tz Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 4 ++ docs/en/antalya/partition_export.md | 8 +-- .../test.py | 61 +++++++++++++++++++ 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 2ff4a7301674..4eb9ca299ab9 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -55,6 +55,10 @@ Source and destination tables must be 100% compatible: This explicit check only applies to partition key columns. A mismatch in the position of a non-partition-key column is **not** rejected by name - it is only caught if the source and destination types are not castable. If two non-partition-key columns happen to have swapped positions but compatible types, the export succeeds and silently writes values into the wrong destination column, so keep the full column order identical (per point 1) rather than relying on this check alone. +4. **Matching timezones for `DateTime`/`DateTime64` partition key columns** - if a partition key column has type `DateTime`/`DateTime64` and its declared timezone differs between the source and destination tables, the export is rejected with `BAD_ARGUMENTS`, because the partition value the source computed at insert time may not match what the destination would compute for the same physical instant. This check is skipped only when the partition key wraps the column in a function that is known to return a timezone-independent result, such as `toUnixTimestamp`, `toUnixTimestamp64Second`, `toUnixTimestamp64Milli`, `toUnixTimestamp64Micro`, or `toUnixTimestamp64Nano`, optionally passed through value-preserving wrappers like `assumeNotNull`, `materialize`, or `identity` (e.g. `PARTITION BY toUnixTimestamp(assumeNotNull(ts))`). + + This is a fixed allowlist, not a general proof of timezone-independence, so some genuinely timezone-independent expressions are currently rejected too. For example, `PARTITION BY toUInt32(ts)` is also timezone-independent because converting `DateTime` to a numeric type uses the stored Unix timestamp and does not perform a calendar-time computation. However, `toUInt32` is not on the allowlist, so the export is rejected even though it would be safe. + In case a table function is used as the destination, the schema can be omitted and it will be inferred from the source table. ## Settings diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index ccd18914845b..13e859a03241 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -45,11 +45,12 @@ TO TABLE [destination_database.]destination_table ## Requirements -`EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/engines/table-engines/mergetree-family/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements, in particular: +`EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/antalya/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements, in particular: 1. **Identical schemas** - same columns, types, and order 2. **Matching partition keys** - partition expressions must be identical -3. **Partition key columns at the same position** - columns are matched by position, so every column that is part of the source table's partition key must also sit at the same position in the destination table's schema, even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/engines/table-engines/mergetree-family/part_export.md#requirements) for a worked example and the exact error message. +3. **Partition key columns at the same position** - columns are matched by position, so every column that is part of the source table's partition key must also sit at the same position in the destination table's schema, even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the exact error message. +4. **Matching timezones for `DateTime`/`DateTime64` partition key columns** - a mismatched declared timezone between source and destination is rejected unless the partition key column is wrapped in a known timezone-independent function such as `toUnixTimestamp`. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for the full list and its known limitations. ## Settings @@ -259,5 +260,4 @@ WHERE source_table = 'rmt_table' AND destination_table = 's3_table'; ## Related Features -- [ALTER TABLE EXPORT PART](/docs/en/engines/table-engines/mergetree-family/part_export.md) - Export individual parts (non-replicated) - +- [ALTER TABLE EXPORT PART](/docs/en/antalya/part_export.md) - Export individual parts (non-replicated) diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index aff4d13bb15f..86cc1c0da8c1 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -1451,3 +1451,64 @@ def test_export_part_timezone_sensitive_function_behind_value_preserving_wrapper ) +# `timezone_invariant_functions` in `collectColumnsRequiringTimezoneCheck` is a +# fixed allowlist, not a proof of timezone-independence. This test documents the +# conservative rejection of safe expressions that the allowlist does not recognize. +@pytest.mark.parametrize( + "function_name, source_type, destination_type, value", + [ + pytest.param( + "toUInt32", + "DateTime('UTC')", + "DateTime('Asia/Tokyo')", + "2024-03-05 15:00:00", + id="toUInt32-DateTime", + ), + pytest.param( + "toInt64", + "DateTime64(3, 'UTC')", + "DateTime64(3, 'Asia/Tokyo')", + "2024-03-05 15:00:00.000", + id="toInt64-DateTime64", + ), + ], +) +def test_export_part_unrecognized_timezone_invariant_function_is_rejected( + cluster, function_name, source_type, destination_type, value +): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tz_unrecognized_mt_table_{postfix}" + s3_table = f"tz_unrecognized_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (id Int32, ts {source_type}) + ENGINE = MergeTree() + PARTITION BY {function_name}(ts) + ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (id Int32, ts {destination_type}) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY {function_name}(ts) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, '{value}')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "timezone" in error, ( + f"`{function_name}(ts)` uses the stored epoch value and does not perform a " + f"calendar-time conversion, but `{function_name}` is not on the allowlist. " + f"The export must be conservatively rejected; got: {error!r}" + ) From 01052197776c7e2af2886f9acb01504a6f21f349 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 7 Aug 2026 11:53:21 +0200 Subject: [PATCH 14/20] refuse type checking Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 14 +- docs/en/antalya/partition_export.md | 7 +- .../MergeTree/ExportPartitionUtils.cpp | 160 +--- .../test.py | 805 ++---------------- .../test.py | 23 +- 5 files changed, 132 insertions(+), 877 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 4eb9ca299ab9..63ffdc8d561f 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -47,17 +47,17 @@ SETTINGS allow_experimental_export_merge_tree_part = 1 ## Requirements -Source and destination tables must be 100% compatible: +Source and destination tables must support positional schema conversion: -1. **Identical schemas** - same columns, types, and order -2. **Matching partition keys** - partition expressions must be identical -3. **Partition key columns at the same position** - columns are matched by position, similar to `INSERT INTO dest SELECT * FROM src`. It is not enough for the `PARTITION BY` expressions to be textually identical: every column that is part of the source table's partition key must also sit at the same position in the destination table's schema. For example, `CREATE TABLE src (a Int32, b Int32) ... PARTITION BY a` and `CREATE TABLE dst (b Int32, a Int32) ... PARTITION BY a` both have the expression `PARTITION BY a`, but `a` is at position 0 in `src` and position 1 in `dst`, so the export is rejected with `BAD_ARGUMENTS: partition key column 'a' is at position 0 in the source table, but the destination's column at that position is named 'b'`. +1. **Positionally compatible schemas** - source columns are matched to destination columns by position, similar to `INSERT INTO dest SELECT * FROM src`. Corresponding types must be safely castable by default. Set `export_merge_tree_part_allow_lossy_cast = 1` to permit lossy casts. +2. **Compatible partitioning** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must be representable as an Iceberg partition spec and must match the destination partition fields and transforms. +3. **Partition key columns at the same position** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. For example, `CREATE TABLE src (a Int32, b Int32) ... PARTITION BY a` and `CREATE TABLE dst (b Int32, a Int32) ... PARTITION BY a` both have the expression `PARTITION BY a`, but `a` is at position 0 in `src` and position 1 in `dst`. The export is rejected with a `BAD_ARGUMENTS` exception whose message includes `Cannot export to : partition key column 'a' is at position 0 in the source table, but the destination's column at that position is named 'b'`. - This explicit check only applies to partition key columns. A mismatch in the position of a non-partition-key column is **not** rejected by name - it is only caught if the source and destination types are not castable. If two non-partition-key columns happen to have swapped positions but compatible types, the export succeeds and silently writes values into the wrong destination column, so keep the full column order identical (per point 1) rather than relying on this check alone. + This explicit name check only applies to partition key columns. A mismatch in the position of a non-partition-key column is **not** rejected by name - it is only caught if the source and destination types are not castable. If two non-partition-key columns happen to have swapped positions but compatible types, the export succeeds and silently writes values into the wrong destination column, so keep the intended column order rather than relying on type compatibility alone. -4. **Matching timezones for `DateTime`/`DateTime64` partition key columns** - if a partition key column has type `DateTime`/`DateTime64` and its declared timezone differs between the source and destination tables, the export is rejected with `BAD_ARGUMENTS`, because the partition value the source computed at insert time may not match what the destination would compute for the same physical instant. This check is skipped only when the partition key wraps the column in a function that is known to return a timezone-independent result, such as `toUnixTimestamp`, `toUnixTimestamp64Second`, `toUnixTimestamp64Milli`, `toUnixTimestamp64Micro`, or `toUnixTimestamp64Nano`, optionally passed through value-preserving wrappers like `assumeNotNull`, `materialize`, or `identity` (e.g. `PARTITION BY toUnixTimestamp(assumeNotNull(ts))`). + For `PARTITION BY t.a`, this rule applies to the top-level owning column `t`. The position of `a` inside a named `Tuple` does not have to match because casts between named tuples match their elements by name. For example, exporting from `t Tuple(a Int32, b Int32)` to `t Tuple(b Int32, a Int32)` is allowed when both tables use `PARTITION BY t.a`. - This is a fixed allowlist, not a general proof of timezone-independence, so some genuinely timezone-independent expressions are currently rejected too. For example, `PARTITION BY toUInt32(ts)` is also timezone-independent because converting `DateTime` to a numeric type uses the stored Unix timestamp and does not perform a calendar-time computation. However, `toUInt32` is not on the allowlist, so the export is rejected even though it would be safe. + For partition expressions containing functions, the check applies to their input columns. For example, `PARTITION BY (toYYYYMM(ts), category)` requires both `ts` and `category` to have the same names at the same top-level positions in both tables. In case a table function is used as the destination, the schema can be omitted and it will be inferred from the source table. diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index 13e859a03241..1dc1b34df61e 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -47,10 +47,9 @@ TO TABLE [destination_database.]destination_table `EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/antalya/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements, in particular: -1. **Identical schemas** - same columns, types, and order -2. **Matching partition keys** - partition expressions must be identical -3. **Partition key columns at the same position** - columns are matched by position, so every column that is part of the source table's partition key must also sit at the same position in the destination table's schema, even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the exact error message. -4. **Matching timezones for `DateTime`/`DateTime64` partition key columns** - a mismatched declared timezone between source and destination is rejected unless the partition key column is wrapped in a known timezone-independent function such as `toUnixTimestamp`. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for the full list and its known limitations. +1. **Positionally compatible schemas** - source columns are matched to destination columns by position. Corresponding types must be safely castable unless `export_merge_tree_part_allow_lossy_cast = 1` is set. +2. **Compatible partitioning** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must match the destination partition fields and transforms. +3. **Partition key columns at the same position** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. ## Settings diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index cec3fe0289d5..f302090a4905 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -13,17 +13,10 @@ #include #include #include -#include -#include -#include -#include #include -#include #include #include #include -#include -#include #include #if USE_AVRO @@ -643,112 +636,6 @@ namespace ExportPartitionUtils } #endif - namespace - { - std::optional getDateTimeTimeZoneName(const DataTypePtr & type) - { - const auto unwrapped_type = removeNullable(removeLowCardinality(type)); - if (const auto * datetime_type = checkAndGetDataType(unwrapped_type.get())) - return datetime_type->getTimeZone().getTimeZone(); - if (const auto * datetime64_type = checkAndGetDataType(unwrapped_type.get())) - return datetime64_type->getTimeZone().getTimeZone(); - return {}; - } - - void collectColumnsRequiringTimezoneCheck( - const ASTPtr & node, - const ASTFunction * effective_parent_function, - std::unordered_set & columns_requiring_timezone_check) - { - if (!node) - return; - - if (const auto * identifier = node->as()) - { - static const std::unordered_set timezone_invariant_functions = { - "toUnixTimestamp", - "toUnixTimestamp64Second", - "toUnixTimestamp64Milli", - "toUnixTimestamp64Micro", - "toUnixTimestamp64Nano", - }; - const bool wrapped_by_invariant_function - = effective_parent_function && timezone_invariant_functions.contains(effective_parent_function->name); - if (!wrapped_by_invariant_function) - columns_requiring_timezone_check.insert(identifier->name()); - return; - } - - if (const auto * function = node->as()) - { - static const std::unordered_set value_preserving_functions = { - "identity", - "assumeNotNull", - "materialize", - }; - const ASTFunction * next_parent_function - = value_preserving_functions.contains(function->name) ? effective_parent_function : function; - if (function->arguments) - for (const auto & argument : function->arguments->children) - collectColumnsRequiringTimezoneCheck(argument, next_parent_function, columns_requiring_timezone_check); - return; - } - - for (const auto & child : node->children) - collectColumnsRequiringTimezoneCheck(child, nullptr, columns_requiring_timezone_check); - } - - void verifyPartitionKeyColumn( - const ColumnWithTypeAndName & source_column, - const ColumnWithTypeAndName & destination_column, - size_t position, - const std::vector & required_columns, - const std::unordered_set & columns_requiring_timezone_check, - const ColumnsDescription & source_columns_description, - const ColumnsDescription & destination_columns_description, - const StorageID & destination_storage_id) - { - if (source_column.name != destination_column.name) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Cannot export to {}: partition key column '{}' is at position {} in the source " - "table, but the destination's column at that position is named '{}'. EXPORT " - "PART/PARTITION matches columns by position, so partition key columns must be " - "declared at the same position in both tables.", - destination_storage_id.getFullTableName(), - source_column.name, - position, - destination_column.name); - - for (const auto & column_name : required_columns) - { - if (!columns_requiring_timezone_check.contains(column_name)) - continue; - - const auto source_resolved = source_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); - const auto destination_resolved - = destination_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); - if (!source_resolved || !destination_resolved) - continue; - - const auto source_time_zone = getDateTimeTimeZoneName(source_resolved->type); - const auto destination_time_zone = getDateTimeTimeZoneName(destination_resolved->type); - if (source_time_zone && destination_time_zone && *source_time_zone != *destination_time_zone) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Cannot export to {}: partition key column '{}' is {} in the source table " - "but {} in the destination. The destination's hive-style partition path is " - "rendered from the source value without converting the timezone, so this " - "would silently shift the exported value by the timezone offset. Use the " - "same timezone in both tables' partition key column.", - destination_storage_id.getFullTableName(), - column_name, - source_resolved->type->getName(), - destination_resolved->type->getName()); - } - } - } - void assertPartitionKeyASTAreEqual( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata) @@ -787,24 +674,17 @@ namespace ExportPartitionUtils context); const auto & source_columns_description = source_metadata->getColumns(); - const auto & destination_columns_description = destination_metadata->getColumns(); - auto partition_key_columns = source_metadata->getColumnsRequiredForPartitionKey(); - - /// Owning top-level column name -> required PARTITION BY names it owns. Two cases: - /// - Flat key `PARTITION BY a` -> {"a": ["a"]}: "a" is not a subcolumn, it owns itself. - /// - Composite key `PARTITION BY t.ts` -> {"t": ["t.ts"]}: "t.ts" is a subcolumn of "t". - /// - Multiple subcolumns of one owner, `PARTITION BY (t.a, t.b)` -> {"t": ["t.a", "t.b"]}. - std::unordered_map> owner_to_partition_key_columns; - for (const auto & column_name : partition_key_columns) - { - auto resolved = source_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); - const auto & owner_column_name = resolved ? resolved->getNameInStorage() : column_name; - owner_to_partition_key_columns[owner_column_name].push_back(column_name); + /// Collect the top-level columns that own columns or subcolumns required by `PARTITION BY`. + /// For example, both `PARTITION BY t.a` and `PARTITION BY (t.a, t.b)` add `t`. + std::unordered_set partition_key_owner_columns; + for (const auto & column_or_subcolumn_name : source_metadata->getColumnsRequiredForPartitionKey()) + { + auto resolved = source_columns_description.tryGetColumnOrSubcolumn( + GetColumnsOptions::All, column_or_subcolumn_name); + const auto & column_name = resolved ? resolved->getNameInStorage() : column_or_subcolumn_name; + partition_key_owner_columns.insert(column_name); } - std::unordered_set columns_requiring_timezone_check; - collectColumnsRequiringTimezoneCheck(source_metadata->getPartitionKeyAST(), nullptr, columns_requiring_timezone_check); - const bool allow_lossy_cast = context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; const size_t num_columns = std::min(source_columns.size(), destination_columns.size()); @@ -813,17 +693,19 @@ namespace ExportPartitionUtils const auto & source_column = source_columns[i]; const auto & destination_column = destination_columns[i]; - if (const auto owned_it = owner_to_partition_key_columns.find(source_column.name); - owned_it != owner_to_partition_key_columns.end()) - verifyPartitionKeyColumn( - source_column, - destination_column, + if (partition_key_owner_columns.contains(source_column.name) && source_column.name != destination_column.name) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot export to {}: partition key column '{}' is at position {} in the source " + "table, but the destination's column at that position is named '{}'. EXPORT " + "PART/PARTITION matches columns by position, so partition key columns must be " + "declared at the same position in both tables.", + destination_storage_id.getFullTableName(), + source_column.name, i, - owned_it->second, - columns_requiring_timezone_check, - source_columns_description, - destination_columns_description, - destination_storage_id); + destination_column.name); + } /// Lossy casts may silently change values, so reject them unless the user opts in. if (allow_lossy_cast) diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 86cc1c0da8c1..b77c2f8159cb 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -322,6 +322,7 @@ class RejectedPartExportCase(NamedTuple): dst_partition_by: str insert_values: str error_substrings: tuple = () + partition_strategy: str = "hive" REJECTED_PART_EXPORT_CASES = [ @@ -332,7 +333,9 @@ class RejectedPartExportCase(NamedTuple): dst_columns="b Int32, a Int32", dst_partition_by="a", insert_values="(1, 1), (1, 2)", - error_substrings=("partition key column",), + error_substrings=( + "partition key column 'a' is at position 0 in the source table", + ), ), id="same_partition_key_different_column_order_single_column", ), @@ -343,7 +346,9 @@ class RejectedPartExportCase(NamedTuple): dst_columns="c Int32, b Int32, a Int32, val String", dst_partition_by="(a, b, c)", insert_values="(1, 1, 1, 'x'), (1, 1, 1, 'y')", - error_substrings=("partition key column",), + error_substrings=( + "partition key column 'a' is at position 0 in the source table", + ), ), id="same_partition_key_different_column_order_multi_column", ), @@ -354,7 +359,9 @@ class RejectedPartExportCase(NamedTuple): dst_columns="a Int32, b Int32, c Int32, val String", dst_partition_by="(c, b, a)", insert_values="(1, 2, 3, 'x')", - error_substrings=("different `PARTITION BY` expressions",), + error_substrings=( + "source and destination tables have different `PARTITION BY` expressions", + ), ), id="multi_column_partition_key_order_mismatch", ), @@ -365,7 +372,9 @@ class RejectedPartExportCase(NamedTuple): dst_columns="a Int32, b Int32, c Int32, val String", dst_partition_by="(a, b)", insert_values="(1, 2, 3, 'x')", - error_substrings=("different `PARTITION BY` expressions",), + error_substrings=( + "source and destination tables have different `PARTITION BY` expressions", + ), ), id="multi_column_partition_key_fewer_in_destination", ), @@ -376,20 +385,51 @@ class RejectedPartExportCase(NamedTuple): dst_columns="a Int32, b Int32, c Int32, val String", dst_partition_by="(a, b, c)", insert_values="(1, 2, 3, 'x')", - error_substrings=("different `PARTITION BY` expressions",), + error_substrings=( + "source and destination tables have different `PARTITION BY` expressions", + ), ), id="multi_column_partition_key_more_in_destination", ), pytest.param( RejectedPartExportCase( - src_columns="id Int64, ts DateTime('UTC')", - src_partition_by="ts", - dst_columns="id Int64, ts DateTime('Asia/Tokyo')", - dst_partition_by="ts", - insert_values="(1, '2024-03-05 15:00:00')", - error_substrings=("timezone",), + src_columns="ts DateTime, category String, decoy DateTime, val String", + src_partition_by="(toYYYYMM(ts), category)", + dst_columns="decoy DateTime, category String, ts DateTime, val String", + dst_partition_by="(toYYYYMM(ts), category)", + insert_values=( + "('2024-03-05 15:00:00', 'category', " + "'2024-03-06 15:00:00', 'x')" + ), + error_substrings=( + "partition key column 'ts' is at position 0 in the source table", + ), + partition_strategy="wildcard", ), - id="partition_key_timezone_mismatch", + id="function_and_column_partition_key_owner_reordered", + ), + pytest.param( + RejectedPartExportCase( + src_columns=( + "t Tuple(ts DateTime, value Int32), category String, " + "decoy Tuple(ts DateTime, value Int32), val String" + ), + src_partition_by="(toYYYYMM(t.ts), category)", + dst_columns=( + "decoy Tuple(ts DateTime, value Int32), category String, " + "t Tuple(ts DateTime, value Int32), val String" + ), + dst_partition_by="(toYYYYMM(t.ts), category)", + insert_values=( + "(('2024-03-05 15:00:00', 1), 'category', " + "('2024-03-06 15:00:00', 2), 'x')" + ), + error_substrings=( + "partition key column 't' is at position 0 in the source table", + ), + partition_strategy="wildcard", + ), + id="function_over_subcolumn_partition_key_owner_reordered", ), ] @@ -411,9 +451,14 @@ def test_export_part_partition_key_mismatch_variants_are_rejected(cluster, case) SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 """) + filename = ( + f"{s3_table}/{{_partition_id}}/{{_file}}" + if case.partition_strategy == "wildcard" + else s3_table + ) node.query(f""" CREATE TABLE {s3_table} ({case.dst_columns}) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + ENGINE = S3(s3_conn, filename='{filename}', format=Parquet, partition_strategy='{case.partition_strategy}') PARTITION BY {case.dst_partition_by} """) @@ -429,8 +474,11 @@ def test_export_part_partition_key_mismatch_variants_are_rejected(cluster, case) for substring in case.error_substrings: assert substring in error, f"Expected {substring!r} in error, got: {error}" - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + if case.partition_strategy == "hive": + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, ( + f"Expected 0 rows in destination after rejected export, got {count}" + ) def test_export_part_multi_column_partition_key_success(cluster): @@ -473,77 +521,6 @@ def test_export_part_multi_column_partition_key_success(cluster): assert result == "1\t2\t3\tx\n1\t2\t3\ty", f"Unexpected exported data:\n{result}" -def test_export_part_non_partition_key_timezone_mismatch_is_allowed(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_ok_mt_table_{postfix}" - s3_table = f"tz_ok_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int64, ts DateTime('UTC')) - ENGINE = MergeTree() - PARTITION BY id - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int64, ts DateTime('Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY id - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - - time.sleep(5) - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 1, f"Expected 1 row in destination after export, got {count}" - - source_ts = node.query(f"SELECT ts FROM {mt_table}").strip() - assert source_ts == "2024-03-05 15:00:00", f"Unexpected source value: {source_ts}" - - dest_ts = node.query(f"SELECT ts FROM {s3_table}").strip() - assert dest_ts == "2024-03-06 00:00:00", ( - f"Expected the exported value to be the same instant displayed in the " - f"destination's Asia/Tokyo timezone ('2024-03-06 00:00:00'), got: {dest_ts}" - ) - - source_unix_ts = int(node.query(f"SELECT toUnixTimestamp(ts) FROM {mt_table}").strip()) - dest_unix_ts = int(node.query(f"SELECT toUnixTimestamp(ts) FROM {s3_table}").strip()) - assert source_unix_ts == dest_unix_ts, ( - f"Expected exported DateTime value to be preserved regardless of the " - f"destination column's timezone, got source={source_unix_ts}, dest={dest_unix_ts}" - ) - - -def test_export_part_tuple_subcolumn_partition_key_hive_destination_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - s3_table = f"tuple_subcol_s3_table_{postfix}" - - error = node.query_and_get_error(f""" - CREATE TABLE {s3_table} (t Tuple(b Int32, a Int32), val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY t.a - """) - assert "BAD_ARGUMENTS" in error and "part of the storage columns" in error, ( - f"Expected hive partition strategy to reject the tuple subcolumn " - f"partition expression at CREATE time, got: {error!r}" - ) - - def read_exported_files(node, s3_table): return node.query( f"SELECT t.a, t.b, val FROM " @@ -562,7 +539,7 @@ def get_export_part_log(node, mt_table): ).strip() -def test_export_part_tuple_subcolumn_partition_key_wildcard_destination(cluster): +def test_export_part_named_tuple_fields_reordered_with_subcolumn_partition_key_is_allowed(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] @@ -635,10 +612,15 @@ def test_export_part_subcolumn_partition_key_different_subcolumn_is_rejected(clu error = node.query_and_get_error( f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" ) - assert "BAD_ARGUMENTS" in error and "different `PARTITION BY` expressions" in error, ( + assert ( + "BAD_ARGUMENTS" in error + and "source and destination tables have different `PARTITION BY` expressions" + in error + ), ( f"Both tables declare `a` as the same Tuple(b Int32, c Int32) (so the column-cast " - f"check passes and the owner-name-only partition_key_column_set = {{'a'}} in " - f"verifyExportSchemaCastable cannot distinguish `a.b` from `a.c`), but the source " + f"check passes and the owner-name-only `partition_key_owner_columns` contains " + f"only `a`, so `verifyExportSchemaCastable` cannot distinguish `a.b` from " + f"`a.c`), but the source " f"partitions by `a.b` and the destination by `a.c` — a genuinely different " f"partition key that must be caught by the `PARTITION BY` AST comparison; " f"got: {error!r}" @@ -684,29 +666,37 @@ def test_export_part_tuple_subcolumn_partition_key_owner_column_reordered_is_rej ) -def test_export_part_subcolumn_partition_key_timezone_mismatch_is_rejected(cluster): +def test_export_part_multiple_partition_key_subcolumns_with_same_owner_reordered_is_rejected(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"subcol_tz_mt_table_{postfix}" - s3_table = f"subcol_tz_s3_table_{postfix}" + mt_table = f"same_owner_subcolumns_mt_table_{postfix}" + s3_table = f"same_owner_subcolumns_s3_table_{postfix}" node.query(f""" - CREATE TABLE {mt_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) + CREATE TABLE {mt_table} ( + t Tuple(a Int32, b Int32), + decoy Tuple(a Int32, b Int32), + val String + ) ENGINE = MergeTree() - PARTITION BY t.ts + PARTITION BY (t.a, t.b) ORDER BY tuple() SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 """) node.query(f""" - CREATE TABLE {s3_table} (t Tuple(a Int32, ts DateTime('Asia/Tokyo')), val String) + CREATE TABLE {s3_table} ( + decoy Tuple(a Int32, b Int32), + t Tuple(a Int32, b Int32), + val String + ) ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY t.ts + PARTITION BY (t.a, t.b) """) - node.query(f"INSERT INTO {mt_table} VALUES ((1, '2024-03-05 15:00:00'), 'x')") + node.query(f"INSERT INTO {mt_table} VALUES ((1, 10), (2, 20), 'x')") part_name = node.query( f"SELECT name FROM system.parts WHERE database = currentDatabase() " @@ -716,15 +706,9 @@ def test_export_part_subcolumn_partition_key_timezone_mismatch_is_rejected(clust error = node.query_and_get_error( f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" ) - assert "BAD_ARGUMENTS" in error and "timezone" in error, ( - f"Expected export to reject the nested partition key column `t.ts` changing " - f"timezone from UTC to Asia/Tokyo between source and destination, the same way " - f"a top-level DateTime partition key column with mismatched timezones is " - f"rejected (see test_export_part_partition_key_mismatch_variants_are_rejected's " - f"'partition_key_timezone_mismatch' case). getDateTimeTimeZoneName() is called " - f"on the owning column's type (Tuple(...)), not on the actual DateTime " - f"subcolumn's type, so it can never recognize a timezone at all here; " - f"got: {error!r}" + assert "BAD_ARGUMENTS" in error and "partition key column 't'" in error, ( + f"Expected both `t.a` and `t.b` to resolve to the same top-level owner `t` " + f"and reject swapping `t` with `decoy`; got: {error!r}" ) @@ -771,7 +755,7 @@ def test_export_part_multi_level_subcolumn_partition_key_owner_reordered_is_reje assert "BAD_ARGUMENTS" in error and "partition key column" in error, ( f"Expected export to reject `t` and `decoy` swapping positions around the " f"two-level-deep partition key column `t.x.a`. This only works if " - f"getNameInStorage() resolves all the way to the top-level column `t`, not to " + f"`getNameInStorage` resolves all the way to the top-level column `t`, not to " f"the intermediate level `t.x`; got: {error!r}" ) @@ -907,608 +891,3 @@ def test_export_part_subcolumn_partition_key_owner_reordered_rejected_even_with_ f"of `t`/`decoy` swapping positions around the partition key column `t.a`; " f"got: {error!r}" ) - - -def test_export_part_tuple_column_real_narrowing_same_order_is_rejected_diagnostic(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tuple_narrow_same_order_mt_table_{postfix}" - s3_table = f"tuple_narrow_same_order_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (t Tuple(a Int32, b Int64), val String) - ENGINE = MergeTree() - PARTITION BY val - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (t Tuple(a Int32, b Int32), val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY val - """) - - node.query(f"INSERT INTO {mt_table} VALUES ((100, 5000000000), 'x')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - print(f"DIAGNOSTIC SAME-ORDER-NARROWING ERROR: {error!r}") - assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( - f"Expected the genuinely narrowing `b` field (Int64 -> Int32, same field order " - f"on both sides) to be rejected as a lossy cast; got: {error!r}" - ) - - -def test_export_part_tuple_column_fewer_fields_in_destination_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tuple_fewer_fields_mt_table_{postfix}" - s3_table = f"tuple_fewer_fields_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) - ENGINE = MergeTree() - PARTITION BY val - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (t Tuple(a Int32), val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY val - """) - - node.query(f"INSERT INTO {mt_table} VALUES ((1, '2024-03-05 15:00:00'), 'x')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( - f"`t` is not a partition key column here. The destination's `t` (Tuple(a Int32)) " - f"has fewer fields than the source's `t` (Tuple(a Int32, ts DateTime('UTC'))), " - f"which canBeSafelyCast's tuple arity check (lhs_type_elements_size != " - f"to_tuple_type_elements.size()) must reject; got: {error!r}" - ) - - -def test_export_part_subcolumn_partition_key_tuple_fewer_fields_in_destination_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"subcol_key_fewer_fields_mt_table_{postfix}" - s3_table = f"subcol_key_fewer_fields_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) - ENGINE = MergeTree() - PARTITION BY t.ts - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (t Tuple(a Int32), val String) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY t.a - """) - - node.query(f"INSERT INTO {mt_table} VALUES ((1, '2024-03-05 15:00:00'), 'x')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( - f"The partition key `t.ts` requires a field the destination's `t` doesn't have " - f"at all, so verifyPartitionKeyColumn's timezone lookup finds " - f"destination_resolved == nullopt for 't.ts' and defers (continue); the arity " - f"mismatch must still be caught right after by canBeSafelyCast on the whole " - f"`t` column; got: {error!r}" - ) - - -def test_export_part_tuple_column_fewer_fields_in_source_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tuple_fewer_fields_src_mt_table_{postfix}" - s3_table = f"tuple_fewer_fields_src_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (t Tuple(a Int32), val String) - ENGINE = MergeTree() - PARTITION BY val - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY val - """) - - node.query(f"INSERT INTO {mt_table} VALUES ((1,), 'x')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( - f"Reverse direction of the fewer-fields case: the source's `t` (Tuple(a Int32)) " - f"has fewer fields than the destination's `t` (Tuple(a Int32, ts " - f"DateTime('UTC'))). canBeSafelyCast's arity check is a plain size " - f"inequality, so it must reject this direction too; got: {error!r}" - ) - - -def test_export_part_subcolumn_partition_key_tuple_fewer_fields_in_source_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"subcol_key_fewer_fields_src_mt_table_{postfix}" - s3_table = f"subcol_key_fewer_fields_src_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (t Tuple(a Int32), val String) - ENGINE = MergeTree() - PARTITION BY t.a - ORDER BY tuple() - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (t Tuple(a Int32, ts DateTime('UTC')), val String) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY t.a - """) - - node.query(f"INSERT INTO {mt_table} VALUES ((1,), 'x')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "INCOMPATIBLE_COLUMNS" in error and "lossy cast" in error, ( - f"Here `t.a` (the partition key) exists and resolves fine on both sides, so " - f"verifyPartitionKeyColumn's timezone check passes without throwing; the extra " - f"`ts` field the destination has beyond the source's `t` must still be caught " - f"by canBeSafelyCast's arity check, independent of the partition key guard " - f"succeeding; got: {error!r}" - ) - - -def test_export_part_nullable_datetime_partition_key_timezone_mismatch_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"nullable_tz_mt_table_{postfix}" - s3_table = f"nullable_tz_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts Nullable(DateTime('UTC'))) - ENGINE = MergeTree() - PARTITION BY ts - ORDER BY id - SETTINGS allow_nullable_key = 1, enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts Nullable(DateTime('Asia/Tokyo'))) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY ts - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "BAD_ARGUMENTS" in error and "timezone" in error, ( - f"getDateTimeTimeZoneName() only recognizes bare DateTime/DateTime64, so wrapping " - f"the partition key in Nullable(...) hides the timezone from it entirely; " - f"canBeSafelyCast() provides no fallback here either, since " - f"DataTypeDateTime::equals() treats any two DateTime types as equal regardless " - f"of timezone by design, so this needs its own dedicated rejection - even " - f"without export_merge_tree_part_allow_lossy_cast being set; got: {error!r}" - ) - - -def test_export_part_lowcardinality_datetime_partition_key_timezone_mismatch_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"lc_tz_mt_table_{postfix}" - s3_table = f"lc_tz_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts LowCardinality(DateTime('UTC'))) - ENGINE = MergeTree() - PARTITION BY ts - ORDER BY id - SETTINGS allow_suspicious_low_cardinality_types = 1, enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts LowCardinality(DateTime('Asia/Tokyo'))) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY ts - SETTINGS allow_suspicious_low_cardinality_types = 1 - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "BAD_ARGUMENTS" in error and "timezone" in error, ( - f"Same gap as the Nullable case, but for LowCardinality(DateTime(...)); " - f"got: {error!r}" - ) - - -def test_export_part_timezone_invariant_expression_partition_key_is_allowed(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_invariant_mt_table_{postfix}" - s3_table = f"tz_invariant_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) - ENGINE = MergeTree() - PARTITION BY toUnixTimestamp(ts) - ORDER BY id - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY toUnixTimestamp(ts) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - - time.sleep(3) - result = node.query( - f"SELECT count() FROM s3('http://minio1:9001/root/data/{s3_table}/**', " - f"'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet', " - f"'id Int32, ts DateTime(\\'Asia/Tokyo\\')') " - f"WHERE _file NOT LIKE 'commit%'" - ).strip() - assert result == "1", ( - f"Expected the export to succeed and produce 1 row: toUnixTimestamp(ts) does " - f"not depend on ts's declared timezone, so a mismatched timezone between " - f"source and destination must not block it; got count={result!r}" - ) - - -def test_export_part_timezone_sensitive_expression_partition_key_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_sensitive_mt_table_{postfix}" - s3_table = f"tz_sensitive_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) - ENGINE = MergeTree() - PARTITION BY toDate(ts) - ORDER BY id - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY toDate(ts) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "BAD_ARGUMENTS" in error and "timezone" in error, ( - f"Unlike toUnixTimestamp(ts), toDate(ts) genuinely depends on ts's declared " - f"timezone (day boundaries differ between UTC and Asia/Tokyo), so it must " - f"remain protected by the timezone guard - the allowlist in " - f"collectColumnsRequiringTimezoneCheck() must not be so broad that it lets " - f"this through too; got: {error!r}" - ) - - -def test_export_part_timezone_sensitive_function_nested_in_invariant_function_is_rejected(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_nested_mt_table_{postfix}" - s3_table = f"tz_nested_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) - ENGINE = MergeTree() - PARTITION BY toUnixTimestamp(toDate(ts)) - ORDER BY id - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY toUnixTimestamp(toDate(ts)) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "BAD_ARGUMENTS" in error and "timezone" in error, ( - f"toDate(ts) inside toUnixTimestamp(toDate(ts)) already produces a " - f"timezone-dependent value before toUnixTimestamp ever runs, so wrapping it " - f"in an outer invariant function must not exempt ts from the check - only " - f"ts's *immediate* parent function determines that; got: {error!r}" - ) - - -def test_export_part_unix_timestamp64_second_partition_key_is_allowed(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_u64s_mt_table_{postfix}" - s3_table = f"tz_u64s_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts DateTime64(3, 'UTC')) - ENGINE = MergeTree() - PARTITION BY toUnixTimestamp64Second(ts) - ORDER BY id - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts DateTime64(3, 'Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY toUnixTimestamp64Second(ts) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00.000')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - - time.sleep(3) - result = node.query( - f"SELECT count() FROM s3('http://minio1:9001/root/data/{s3_table}/**', " - f"'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet', " - f"'id Int32, ts DateTime64(3, \\'Asia/Tokyo\\')') " - f"WHERE _file NOT LIKE 'commit%'" - ).strip() - assert result == "1", ( - f"toUnixTimestamp64Second(ts) is documented as being relative to UTC, not the " - f"declared timezone of ts (see toUnixTimestamp64Second.cpp's own " - f"documentation), so a mismatched timezone between source and destination " - f"must not block this export; got count={result!r}" - ) - - -def test_export_part_value_preserving_function_wrapped_in_invariant_function_is_allowed(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_passthrough_mt_table_{postfix}" - s3_table = f"tz_passthrough_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) - ENGINE = MergeTree() - PARTITION BY toUnixTimestamp(assumeNotNull(ts)) - ORDER BY id - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY toUnixTimestamp(assumeNotNull(ts)) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - - time.sleep(3) - result = node.query( - f"SELECT count() FROM s3('http://minio1:9001/root/data/{s3_table}/**', " - f"'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet', " - f"'id Int32, ts DateTime(\\'Asia/Tokyo\\')') " - f"WHERE _file NOT LIKE 'commit%'" - ).strip() - assert result == "1", ( - f"assumeNotNull(ts) is identity on the underlying value, so it must not break " - f"the chain between ts and the enclosing toUnixTimestamp - the check must look " - f"past value-preserving wrapper functions, not just the immediate parent; " - f"got count={result!r}" - ) - - -def test_export_part_timezone_sensitive_function_behind_value_preserving_wrapper_is_rejected( - cluster, -): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_nested_passthrough_mt_table_{postfix}" - s3_table = f"tz_nested_passthrough_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts DateTime('UTC')) - ENGINE = MergeTree() - PARTITION BY toUnixTimestamp(assumeNotNull(toDate(ts))) - ORDER BY id - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts DateTime('Asia/Tokyo')) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY toUnixTimestamp(assumeNotNull(toDate(ts))) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "BAD_ARGUMENTS" in error and "timezone" in error, ( - f"toDate(ts) is still timezone-sensitive even when reached through the " - f"value-preserving assumeNotNull() and wrapped by the invariant " - f"toUnixTimestamp() two levels up - skipping past passthrough functions must " - f"not also skip past genuinely timezone-sensitive ones; got: {error!r}" - ) - - -# `timezone_invariant_functions` in `collectColumnsRequiringTimezoneCheck` is a -# fixed allowlist, not a proof of timezone-independence. This test documents the -# conservative rejection of safe expressions that the allowlist does not recognize. -@pytest.mark.parametrize( - "function_name, source_type, destination_type, value", - [ - pytest.param( - "toUInt32", - "DateTime('UTC')", - "DateTime('Asia/Tokyo')", - "2024-03-05 15:00:00", - id="toUInt32-DateTime", - ), - pytest.param( - "toInt64", - "DateTime64(3, 'UTC')", - "DateTime64(3, 'Asia/Tokyo')", - "2024-03-05 15:00:00.000", - id="toInt64-DateTime64", - ), - ], -) -def test_export_part_unrecognized_timezone_invariant_function_is_rejected( - cluster, function_name, source_type, destination_type, value -): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["node1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_unrecognized_mt_table_{postfix}" - s3_table = f"tz_unrecognized_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (id Int32, ts {source_type}) - ENGINE = MergeTree() - PARTITION BY {function_name}(ts) - ORDER BY id - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 - """) - - node.query(f""" - CREATE TABLE {s3_table} (id Int32, ts {destination_type}) - ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY {function_name}(ts) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, '{value}')") - - part_name = node.query( - f"SELECT name FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" - ) - assert "BAD_ARGUMENTS" in error and "timezone" in error, ( - f"`{function_name}(ts)` uses the stored epoch value and does not perform a " - f"calendar-time conversion, but `{function_name}` is not on the allowlist. " - f"The export must be conservatively rejected; got: {error!r}" - ) diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py index 0b38f97df4a5..aeb738496b86 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py @@ -1789,7 +1789,9 @@ class RejectedPartitionExportCase(NamedTuple): dst_columns="a Int32, b Int32, c Int32, val String", dst_partition_by="(c, b, a)", insert_values="(1, 2, 3, 'x')", - error_substrings=("different `PARTITION BY` expressions",), + error_substrings=( + "source and destination tables have different `PARTITION BY` expressions", + ), ), id="multi_column_partition_key_order_mismatch", ), @@ -1800,7 +1802,9 @@ class RejectedPartitionExportCase(NamedTuple): dst_columns="a Int32, b Int32, c Int32, val String", dst_partition_by="(a, b)", insert_values="(1, 2, 3, 'x')", - error_substrings=("different `PARTITION BY` expressions",), + error_substrings=( + "source and destination tables have different `PARTITION BY` expressions", + ), ), id="multi_column_partition_key_fewer_in_destination", ), @@ -1811,21 +1815,12 @@ class RejectedPartitionExportCase(NamedTuple): dst_columns="a Int32, b Int32, c Int32, val String", dst_partition_by="(a, b, c)", insert_values="(1, 2, 3, 'x')", - error_substrings=("different `PARTITION BY` expressions",), + error_substrings=( + "source and destination tables have different `PARTITION BY` expressions", + ), ), id="multi_column_partition_key_more_in_destination", ), - pytest.param( - RejectedPartitionExportCase( - src_columns="id Int64, ts DateTime('UTC')", - src_partition_by="ts", - dst_columns="id Int64, ts DateTime('Asia/Tokyo')", - dst_partition_by="ts", - insert_values="(1, '2024-03-05 15:00:00')", - error_substrings=("timezone",), - ), - id="partition_key_timezone_mismatch", - ), ] From 96ed6ff3b43ce38868d2269da3c2a0e528315998 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 7 Aug 2026 12:25:54 +0200 Subject: [PATCH 15/20] fix tupleElement Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 6 +- docs/en/antalya/partition_export.md | 2 +- .../MergeTree/ExportPartitionUtils.cpp | 74 +++++++++++++++---- .../test.py | 52 +++++-------- 4 files changed, 86 insertions(+), 48 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 63ffdc8d561f..24ce21db5b02 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -51,11 +51,13 @@ Source and destination tables must support positional schema conversion: 1. **Positionally compatible schemas** - source columns are matched to destination columns by position, similar to `INSERT INTO dest SELECT * FROM src`. Corresponding types must be safely castable by default. Set `export_merge_tree_part_allow_lossy_cast = 1` to permit lossy casts. 2. **Compatible partitioning** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must be representable as an Iceberg partition spec and must match the destination partition fields and transforms. -3. **Partition key columns at the same position** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. For example, `CREATE TABLE src (a Int32, b Int32) ... PARTITION BY a` and `CREATE TABLE dst (b Int32, a Int32) ... PARTITION BY a` both have the expression `PARTITION BY a`, but `a` is at position 0 in `src` and position 1 in `dst`. The export is rejected with a `BAD_ARGUMENTS` exception whose message includes `Cannot export to : partition key column 'a' is at position 0 in the source table, but the destination's column at that position is named 'b'`. +3. **Matching partition key column positions and layouts** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. If such a column has a named `Tuple` type, its element names must also be declared in the same order, recursively for nested tuples. For example, `CREATE TABLE src (a Int32, b Int32) ... PARTITION BY a` and `CREATE TABLE dst (b Int32, a Int32) ... PARTITION BY a` both have the expression `PARTITION BY a`, but `a` is at position 0 in `src` and position 1 in `dst`. The export is rejected with a `BAD_ARGUMENTS` exception whose message includes `Cannot export to : partition key column 'a' is at position 0 in the source table, but the destination's column at that position is named 'b'`. This explicit name check only applies to partition key columns. A mismatch in the position of a non-partition-key column is **not** rejected by name - it is only caught if the source and destination types are not castable. If two non-partition-key columns happen to have swapped positions but compatible types, the export succeeds and silently writes values into the wrong destination column, so keep the intended column order rather than relying on type compatibility alone. - For `PARTITION BY t.a`, this rule applies to the top-level owning column `t`. The position of `a` inside a named `Tuple` does not have to match because casts between named tuples match their elements by name. For example, exporting from `t Tuple(a Int32, b Int32)` to `t Tuple(b Int32, a Int32)` is allowed when both tables use `PARTITION BY t.a`. + For `PARTITION BY t.a`, this rule applies to the top-level owning column `t`. Exporting from `t Tuple(a Int32, b Int32)` to `t Tuple(b Int32, a Int32)` is rejected, even though `a` is accessed by name. Requiring a stable layout for every partition-key owner also protects positional expressions such as `tupleElement(t, 1)` from changing their meaning after conversion. + + In this case, the export throws a `BAD_ARGUMENTS` exception whose message includes `partition key column 't' has a different Tuple element layout in the source (Tuple(a Int32, b Int32)) and destination (Tuple(b Int32, a Int32)). Tuple element names must be declared in the same order in both tables`. For partition expressions containing functions, the check applies to their input columns. For example, `PARTITION BY (toYYYYMM(ts), category)` requires both `ts` and `category` to have the same names at the same top-level positions in both tables. diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index 1dc1b34df61e..90b979690c95 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -49,7 +49,7 @@ TO TABLE [destination_database.]destination_table 1. **Positionally compatible schemas** - source columns are matched to destination columns by position. Corresponding types must be safely castable unless `export_merge_tree_part_allow_lossy_cast = 1` is set. 2. **Compatible partitioning** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must match the destination partition fields and transforms. -3. **Partition key columns at the same position** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. +3. **Matching partition key column positions and layouts** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. Named `Tuple` elements within such a column must also be declared in the same order. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. ## Settings diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index f302090a4905..467dfb4394fa 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -13,7 +13,11 @@ #include #include #include +#include +#include +#include #include +#include #include #include #include @@ -636,6 +640,61 @@ namespace ExportPartitionUtils } #endif + namespace + { + bool haveSameTupleElementLayout(const DataTypePtr & source_type, const DataTypePtr & destination_type) + { + const auto source_type_unwrapped = removeNullable(removeLowCardinality(source_type)); + const auto destination_type_unwrapped = removeNullable(removeLowCardinality(destination_type)); + + const auto * source_tuple = checkAndGetDataType(source_type_unwrapped.get()); + const auto * destination_tuple = checkAndGetDataType(destination_type_unwrapped.get()); + if (!source_tuple || !destination_tuple) + return source_tuple == destination_tuple; + + if (source_tuple->getElementNames() != destination_tuple->getElementNames()) + return false; + + const auto & source_elements = source_tuple->getElements(); + const auto & destination_elements = destination_tuple->getElements(); + for (size_t i = 0; i < source_elements.size(); ++i) + if (!haveSameTupleElementLayout(source_elements[i], destination_elements[i])) + return false; + + return true; + } + + void verifyPartitionKeyColumn( + const ColumnWithTypeAndName & source_column, + const ColumnWithTypeAndName & destination_column, + size_t position, + const StorageID & destination_storage_id) + { + if (source_column.name != destination_column.name) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot export to {}: partition key column '{}' is at position {} in the source " + "table, but the destination's column at that position is named '{}'. EXPORT " + "PART/PARTITION matches columns by position, so partition key columns must be " + "declared at the same position in both tables.", + destination_storage_id.getFullTableName(), + source_column.name, + position, + destination_column.name); + + if (!haveSameTupleElementLayout(source_column.type, destination_column.type)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot export to {}: partition key column '{}' has a different Tuple element " + "layout in the source ({}) and destination ({}). Tuple element names must be " + "declared in the same order in both tables.", + destination_storage_id.getFullTableName(), + source_column.name, + source_column.type->getName(), + destination_column.type->getName()); + } + } + void assertPartitionKeyASTAreEqual( const StorageMetadataPtr & source_metadata, const StorageMetadataPtr & destination_metadata) @@ -693,19 +752,8 @@ namespace ExportPartitionUtils const auto & source_column = source_columns[i]; const auto & destination_column = destination_columns[i]; - if (partition_key_owner_columns.contains(source_column.name) && source_column.name != destination_column.name) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Cannot export to {}: partition key column '{}' is at position {} in the source " - "table, but the destination's column at that position is named '{}'. EXPORT " - "PART/PARTITION matches columns by position, so partition key columns must be " - "declared at the same position in both tables.", - destination_storage_id.getFullTableName(), - source_column.name, - i, - destination_column.name); - } + if (partition_key_owner_columns.contains(source_column.name)) + verifyPartitionKeyColumn(source_column, destination_column, i, destination_storage_id); /// Lossy casts may silently change values, so reject them unless the user opts in. if (allow_lossy_cast) diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index b77c2f8159cb..241c689c259c 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -521,36 +521,27 @@ def test_export_part_multi_column_partition_key_success(cluster): assert result == "1\t2\t3\tx\n1\t2\t3\ty", f"Unexpected exported data:\n{result}" -def read_exported_files(node, s3_table): - return node.query( - f"SELECT t.a, t.b, val FROM " - f"s3('http://minio1:9001/root/data/{s3_table}/**', 'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet', " - f"'t Tuple(b Int32, a Int32), val String') " - f"WHERE _file NOT LIKE 'commit%'" - ).strip() - - -def get_export_part_log(node, mt_table): - node.query("SYSTEM FLUSH LOGS") - return node.query( - f"SELECT part_name, error, exception FROM system.part_log " - f"WHERE event_type = 'ExportPart' AND database = currentDatabase() " - f"AND table = '{mt_table}' ORDER BY event_time" - ).strip() - - -def test_export_part_named_tuple_fields_reordered_with_subcolumn_partition_key_is_allowed(cluster): +@pytest.mark.parametrize( + "partition_by", + [ + pytest.param("t.a", id="named_subcolumn"), + pytest.param("tupleElement(t, 1)", id="positional_tuple_element"), + ], +) +def test_export_part_named_tuple_fields_reordered_for_partition_key_is_rejected( + cluster, partition_by +): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tuple_subcol_wc_mt_table_{postfix}" - s3_table = f"tuple_subcol_wc_s3_table_{postfix}" + mt_table = f"reordered_tuple_mt_table_{postfix}" + s3_table = f"reordered_tuple_s3_table_{postfix}" node.query(f""" CREATE TABLE {mt_table} (t Tuple(a Int32, b Int32), val String) ENGINE = MergeTree() - PARTITION BY t.a + PARTITION BY {partition_by} ORDER BY tuple() SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 """) @@ -558,7 +549,7 @@ def test_export_part_named_tuple_fields_reordered_with_subcolumn_partition_key_i node.query(f""" CREATE TABLE {s3_table} (t Tuple(b Int32, a Int32), val String) ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') - PARTITION BY t.a + PARTITION BY {partition_by} """) node.query(f"INSERT INTO {mt_table} VALUES ((1, 99), 'x')") @@ -568,15 +559,12 @@ def test_export_part_named_tuple_fields_reordered_with_subcolumn_partition_key_i f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" ).strip() - node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") - - time.sleep(5) - - part_log = get_export_part_log(node, mt_table) - result = read_exported_files(node, s3_table) - assert result == "1\t99\tx", ( - f"Tuple element values were remapped: source t = (a=1, b=99), " - f"exported files read back (t.a, t.b) as: {result!r}; part_log: {part_log!r}" + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error and "different Tuple element layout" in error, ( + f"Expected export to reject reordered named `Tuple` fields used by " + f"`PARTITION BY {partition_by}`, got: {error!r}" ) From 30f8d6f9bab376f56ec541297a86ea08c70b799c Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 7 Aug 2026 12:45:41 +0200 Subject: [PATCH 16/20] tuple in array/map Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 4 +- docs/en/antalya/partition_export.md | 2 +- .../MergeTree/ExportPartitionUtils.cpp | 44 +++++++++++++--- .../test.py | 51 ++++++++++++++++--- 4 files changed, 83 insertions(+), 18 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 24ce21db5b02..4ec36db5ea6f 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -51,12 +51,14 @@ Source and destination tables must support positional schema conversion: 1. **Positionally compatible schemas** - source columns are matched to destination columns by position, similar to `INSERT INTO dest SELECT * FROM src`. Corresponding types must be safely castable by default. Set `export_merge_tree_part_allow_lossy_cast = 1` to permit lossy casts. 2. **Compatible partitioning** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must be representable as an Iceberg partition spec and must match the destination partition fields and transforms. -3. **Matching partition key column positions and layouts** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. If such a column has a named `Tuple` type, its element names must also be declared in the same order, recursively for nested tuples. For example, `CREATE TABLE src (a Int32, b Int32) ... PARTITION BY a` and `CREATE TABLE dst (b Int32, a Int32) ... PARTITION BY a` both have the expression `PARTITION BY a`, but `a` is at position 0 in `src` and position 1 in `dst`. The export is rejected with a `BAD_ARGUMENTS` exception whose message includes `Cannot export to : partition key column 'a' is at position 0 in the source table, but the destination's column at that position is named 'b'`. +3. **Matching partition key column positions and layouts** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. If such a column contains a named `Tuple`, its element names must also be declared in the same order. This comparison is recursive through nested tuples and through container types such as `Array` and `Map`. For example, `CREATE TABLE src (a Int32, b Int32) ... PARTITION BY a` and `CREATE TABLE dst (b Int32, a Int32) ... PARTITION BY a` both have the expression `PARTITION BY a`, but `a` is at position 0 in `src` and position 1 in `dst`. The export is rejected with a `BAD_ARGUMENTS` exception whose message includes `Cannot export to : partition key column 'a' is at position 0 in the source table, but the destination's column at that position is named 'b'`. This explicit name check only applies to partition key columns. A mismatch in the position of a non-partition-key column is **not** rejected by name - it is only caught if the source and destination types are not castable. If two non-partition-key columns happen to have swapped positions but compatible types, the export succeeds and silently writes values into the wrong destination column, so keep the intended column order rather than relying on type compatibility alone. For `PARTITION BY t.a`, this rule applies to the top-level owning column `t`. Exporting from `t Tuple(a Int32, b Int32)` to `t Tuple(b Int32, a Int32)` is rejected, even though `a` is accessed by name. Requiring a stable layout for every partition-key owner also protects positional expressions such as `tupleElement(t, 1)` from changing their meaning after conversion. + The same rule applies when the named tuple is nested inside a container. For example, `arr Array(Tuple(a Int32, b Int32))` and `arr Array(Tuple(b Int32, a Int32))` are incompatible when `arr` provides an input to the partition key. Likewise, tuple layouts in both the key and value types of `Map` are checked recursively. + In this case, the export throws a `BAD_ARGUMENTS` exception whose message includes `partition key column 't' has a different Tuple element layout in the source (Tuple(a Int32, b Int32)) and destination (Tuple(b Int32, a Int32)). Tuple element names must be declared in the same order in both tables`. For partition expressions containing functions, the check applies to their input columns. For example, `PARTITION BY (toYYYYMM(ts), category)` requires both `ts` and `category` to have the same names at the same top-level positions in both tables. diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index 90b979690c95..7a9773ed293d 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -49,7 +49,7 @@ TO TABLE [destination_database.]destination_table 1. **Positionally compatible schemas** - source columns are matched to destination columns by position. Corresponding types must be safely castable unless `export_merge_tree_part_allow_lossy_cast = 1` is set. 2. **Compatible partitioning** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must match the destination partition fields and transforms. -3. **Matching partition key column positions and layouts** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. Named `Tuple` elements within such a column must also be declared in the same order. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. +3. **Matching partition key column positions and layouts** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. Named `Tuple` elements within such a column must also be declared in the same order, including tuples nested inside `Array` or `Map`. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. ## Settings diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 467dfb4394fa..1140ae21d3f3 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -13,7 +13,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -649,18 +651,44 @@ namespace ExportPartitionUtils const auto * source_tuple = checkAndGetDataType(source_type_unwrapped.get()); const auto * destination_tuple = checkAndGetDataType(destination_type_unwrapped.get()); - if (!source_tuple || !destination_tuple) - return source_tuple == destination_tuple; + if (source_tuple || destination_tuple) + { + if (!source_tuple || !destination_tuple) + return false; + + if (source_tuple->getElementNames() != destination_tuple->getElementNames()) + return false; + + const auto & source_elements = source_tuple->getElements(); + const auto & destination_elements = destination_tuple->getElements(); + for (size_t i = 0; i < source_elements.size(); ++i) + if (!haveSameTupleElementLayout(source_elements[i], destination_elements[i])) + return false; + + return true; + } - if (source_tuple->getElementNames() != destination_tuple->getElementNames()) - return false; + const auto * source_array = checkAndGetDataType(source_type_unwrapped.get()); + const auto * destination_array = checkAndGetDataType(destination_type_unwrapped.get()); + if (source_array || destination_array) + { + if (!source_array || !destination_array) + return false; + + return haveSameTupleElementLayout(source_array->getNestedType(), destination_array->getNestedType()); + } - const auto & source_elements = source_tuple->getElements(); - const auto & destination_elements = destination_tuple->getElements(); - for (size_t i = 0; i < source_elements.size(); ++i) - if (!haveSameTupleElementLayout(source_elements[i], destination_elements[i])) + const auto * source_map = checkAndGetDataType(source_type_unwrapped.get()); + const auto * destination_map = checkAndGetDataType(destination_type_unwrapped.get()); + if (source_map || destination_map) + { + if (!source_map || !destination_map) return false; + return haveSameTupleElementLayout(source_map->getKeyType(), destination_map->getKeyType()) + && haveSameTupleElementLayout(source_map->getValueType(), destination_map->getValueType()); + } + return true; } diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 241c689c259c..ba806b8efc03 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -522,14 +522,49 @@ def test_export_part_multi_column_partition_key_success(cluster): @pytest.mark.parametrize( - "partition_by", + "owner_name, source_type, destination_type, partition_by, insert_value", [ - pytest.param("t.a", id="named_subcolumn"), - pytest.param("tupleElement(t, 1)", id="positional_tuple_element"), + pytest.param( + "t", + "Tuple(a Int32, b Int32)", + "Tuple(b Int32, a Int32)", + "t.a", + "(1, 99)", + id="named_subcolumn", + ), + pytest.param( + "t", + "Tuple(a Int32, b Int32)", + "Tuple(b Int32, a Int32)", + "tupleElement(t, 1)", + "(1, 99)", + id="positional_tuple_element", + ), + pytest.param( + "arr", + "Array(Tuple(a Int32, b Int32))", + "Array(Tuple(b Int32, a Int32))", + "tupleElement(arr[1], 'a')", + "[(1, 99)]", + id="tuple_nested_in_array", + ), + pytest.param( + "m", + "Map(String, Tuple(a Int32, b Int32))", + "Map(String, Tuple(b Int32, a Int32))", + "tupleElement(m['key'], 'a')", + "map('key', (1, 99))", + id="tuple_nested_in_map_value", + ), ], ) -def test_export_part_named_tuple_fields_reordered_for_partition_key_is_rejected( - cluster, partition_by +def test_export_part_tuple_fields_reordered_for_partition_key_is_rejected( + cluster, + owner_name, + source_type, + destination_type, + partition_by, + insert_value, ): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] @@ -539,7 +574,7 @@ def test_export_part_named_tuple_fields_reordered_for_partition_key_is_rejected( s3_table = f"reordered_tuple_s3_table_{postfix}" node.query(f""" - CREATE TABLE {mt_table} (t Tuple(a Int32, b Int32), val String) + CREATE TABLE {mt_table} ({owner_name} {source_type}, val String) ENGINE = MergeTree() PARTITION BY {partition_by} ORDER BY tuple() @@ -547,12 +582,12 @@ def test_export_part_named_tuple_fields_reordered_for_partition_key_is_rejected( """) node.query(f""" - CREATE TABLE {s3_table} (t Tuple(b Int32, a Int32), val String) + CREATE TABLE {s3_table} ({owner_name} {destination_type}, val String) ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') PARTITION BY {partition_by} """) - node.query(f"INSERT INTO {mt_table} VALUES ((1, 99), 'x')") + node.query(f"INSERT INTO {mt_table} VALUES ({insert_value}, 'x')") part_name = node.query( f"SELECT name FROM system.parts WHERE database = currentDatabase() " From 4a69876bd3ac39bd5e31e3dc8f18879a21a515b4 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 7 Aug 2026 14:59:50 +0200 Subject: [PATCH 17/20] support unnamed tuple Signed-off-by: Konstantin Morozov --- .../MergeTree/ExportPartitionUtils.cpp | 7 +++- .../test.py | 32 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index 1140ae21d3f3..f3c4aadfb31f 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -656,7 +656,12 @@ namespace ExportPartitionUtils if (!source_tuple || !destination_tuple) return false; - if (source_tuple->getElementNames() != destination_tuple->getElementNames()) + if (source_tuple->hasExplicitNames() && destination_tuple->hasExplicitNames()) + { + if (source_tuple->getElementNames() != destination_tuple->getElementNames()) + return false; + } + else if (source_tuple->getElements().size() != destination_tuple->getElements().size()) return false; const auto & source_elements = source_tuple->getElements(); diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index ba806b8efc03..107fc755c269 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -603,6 +603,38 @@ def test_export_part_tuple_fields_reordered_for_partition_key_is_rejected( ) +def test_export_part_unnamed_tuple_partition_key_owner_matching_named_destination_is_allowed(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"unnamed_tuple_ok_mt_table_{postfix}" + s3_table = f"unnamed_tuple_ok_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (t Tuple(Int32, Int32), val String) + ENGINE = MergeTree() + PARTITION BY tupleElement(t, 1) + ORDER BY tuple() + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1 + """) + + node.query(f""" + CREATE TABLE {s3_table} (t Tuple(x Int32, y Int32), val String) + ENGINE = S3(s3_conn, filename='{s3_table}/{{_partition_id}}/{{_file}}', format=Parquet, partition_strategy='wildcard') + PARTITION BY tupleElement(t, 1) + """) + + node.query(f"INSERT INTO {mt_table} VALUES ((1, 99), 'x')") + + part_name = node.query( + f"SELECT name FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + node.query(f"ALTER TABLE {mt_table} EXPORT PART '{part_name}' TO TABLE {s3_table}") + + def test_export_part_subcolumn_partition_key_different_subcolumn_is_rejected(cluster): skip_if_remote_database_disk_enabled(cluster) node = cluster.instances["node1"] From 7a1e3bd4dd3d642933c69d35d620faf7df908699 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 7 Aug 2026 15:00:37 +0200 Subject: [PATCH 18/20] up doc Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 4ec36db5ea6f..35a6368b5ceb 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -57,6 +57,8 @@ Source and destination tables must support positional schema conversion: For `PARTITION BY t.a`, this rule applies to the top-level owning column `t`. Exporting from `t Tuple(a Int32, b Int32)` to `t Tuple(b Int32, a Int32)` is rejected, even though `a` is accessed by name. Requiring a stable layout for every partition-key owner also protects positional expressions such as `tupleElement(t, 1)` from changing their meaning after conversion. + The element-name check only applies when both the source and destination `Tuple` declare explicit names; an unnamed `Tuple` (e.g. `Tuple(Int32, Int32)`) is compared to the destination by element position and type only. For example, exporting from `t Tuple(Int32, Int32)` to `t Tuple(x Int32, y Int32)` is allowed as long as element types match positionally. + The same rule applies when the named tuple is nested inside a container. For example, `arr Array(Tuple(a Int32, b Int32))` and `arr Array(Tuple(b Int32, a Int32))` are incompatible when `arr` provides an input to the partition key. Likewise, tuple layouts in both the key and value types of `Map` are checked recursively. In this case, the export throws a `BAD_ARGUMENTS` exception whose message includes `partition key column 't' has a different Tuple element layout in the source (Tuple(a Int32, b Int32)) and destination (Tuple(b Int32, a Int32)). Tuple element names must be declared in the same order in both tables`. From 34c64b49087ecda6eb6690fd1bd4181195a59b47 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 10 Aug 2026 12:33:24 +0200 Subject: [PATCH 19/20] restore message Signed-off-by: Konstantin Morozov --- src/Storages/MergeTree/ExportPartitionUtils.cpp | 3 +-- .../test_export_merge_tree_part_to_object_storage/test.py | 8 ++++---- .../test.py | 6 +++--- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index f3c4aadfb31f..906a71ef5737 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -738,8 +738,7 @@ namespace ExportPartitionUtils }; if (query_to_string(source_metadata->getPartitionKeyAST()) != query_to_string(destination_metadata->getPartitionKeyAST())) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Cannot export partition: source and destination tables have different `PARTITION BY` expressions"); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different partition key"); } void verifyExportSchemaCastable( diff --git a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py index 107fc755c269..f4147c4bd412 100644 --- a/tests/integration/test_export_merge_tree_part_to_object_storage/test.py +++ b/tests/integration/test_export_merge_tree_part_to_object_storage/test.py @@ -360,7 +360,7 @@ class RejectedPartExportCase(NamedTuple): dst_partition_by="(c, b, a)", insert_values="(1, 2, 3, 'x')", error_substrings=( - "source and destination tables have different `PARTITION BY` expressions", + "Tables have different partition key", ), ), id="multi_column_partition_key_order_mismatch", @@ -373,7 +373,7 @@ class RejectedPartExportCase(NamedTuple): dst_partition_by="(a, b)", insert_values="(1, 2, 3, 'x')", error_substrings=( - "source and destination tables have different `PARTITION BY` expressions", + "Tables have different partition key", ), ), id="multi_column_partition_key_fewer_in_destination", @@ -386,7 +386,7 @@ class RejectedPartExportCase(NamedTuple): dst_partition_by="(a, b, c)", insert_values="(1, 2, 3, 'x')", error_substrings=( - "source and destination tables have different `PARTITION BY` expressions", + "Tables have different partition key", ), ), id="multi_column_partition_key_more_in_destination", @@ -669,7 +669,7 @@ def test_export_part_subcolumn_partition_key_different_subcolumn_is_rejected(clu ) assert ( "BAD_ARGUMENTS" in error - and "source and destination tables have different `PARTITION BY` expressions" + and "Tables have different partition key" in error ), ( f"Both tables declare `a` as the same Tuple(b Int32, c Int32) (so the column-cast " diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py index aeb738496b86..4dd57f2202b9 100644 --- a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py +++ b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py @@ -1790,7 +1790,7 @@ class RejectedPartitionExportCase(NamedTuple): dst_partition_by="(c, b, a)", insert_values="(1, 2, 3, 'x')", error_substrings=( - "source and destination tables have different `PARTITION BY` expressions", + "Tables have different partition key", ), ), id="multi_column_partition_key_order_mismatch", @@ -1803,7 +1803,7 @@ class RejectedPartitionExportCase(NamedTuple): dst_partition_by="(a, b)", insert_values="(1, 2, 3, 'x')", error_substrings=( - "source and destination tables have different `PARTITION BY` expressions", + "Tables have different partition key", ), ), id="multi_column_partition_key_fewer_in_destination", @@ -1816,7 +1816,7 @@ class RejectedPartitionExportCase(NamedTuple): dst_partition_by="(a, b, c)", insert_values="(1, 2, 3, 'x')", error_substrings=( - "source and destination tables have different `PARTITION BY` expressions", + "Tables have different partition key", ), ), id="multi_column_partition_key_more_in_destination", From f4d7a87d5f32ca6ed69c3f4642c0e1192c03e862 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Tue, 11 Aug 2026 10:03:24 +0200 Subject: [PATCH 20/20] update doc Signed-off-by: Konstantin Morozov --- docs/en/antalya/part_export.md | 20 +++++++++++++------- docs/en/antalya/partition_export.md | 8 ++++---- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/docs/en/antalya/part_export.md b/docs/en/antalya/part_export.md index 35a6368b5ceb..9e0cd7051414 100644 --- a/docs/en/antalya/part_export.md +++ b/docs/en/antalya/part_export.md @@ -47,17 +47,23 @@ SETTINGS allow_experimental_export_merge_tree_part = 1 ## Requirements -Source and destination tables must support positional schema conversion: +Source and destination tables must support positional schema conversion. The following differences between the two schemas are allowed: -1. **Positionally compatible schemas** - source columns are matched to destination columns by position, similar to `INSERT INTO dest SELECT * FROM src`. Corresponding types must be safely castable by default. Set `export_merge_tree_part_allow_lossy_cast = 1` to permit lossy casts. -2. **Compatible partitioning** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must be representable as an Iceberg partition spec and must match the destination partition fields and transforms. -3. **Matching partition key column positions and layouts** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. If such a column contains a named `Tuple`, its element names must also be declared in the same order. This comparison is recursive through nested tuples and through container types such as `Array` and `Map`. For example, `CREATE TABLE src (a Int32, b Int32) ... PARTITION BY a` and `CREATE TABLE dst (b Int32, a Int32) ... PARTITION BY a` both have the expression `PARTITION BY a`, but `a` is at position 0 in `src` and position 1 in `dst`. The export is rejected with a `BAD_ARGUMENTS` exception whose message includes `Cannot export to : partition key column 'a' is at position 0 in the source table, but the destination's column at that position is named 'b'`. +- **Column names** may differ between source and destination for non-partition-key columns - columns are matched by position, similar to `INSERT INTO dest SELECT * FROM src`, not by name. +- **Column types** may differ, as long as the source type is safely castable to the destination type. Set `export_merge_tree_part_allow_lossy_cast = 1` to also permit lossy casts. +- **`Tuple` element names** may differ if either the source or destination declares the tuple without named elements: an unnamed `Tuple` (e.g. `Tuple(Int32, Int32)`) is matched against the destination by element position and type only, not by name. For example, exporting from `t Tuple(Int32, Int32)` to `t Tuple(x Int32, y Int32)` is allowed as long as element types match positionally. - This explicit name check only applies to partition key columns. A mismatch in the position of a non-partition-key column is **not** rejected by name - it is only caught if the source and destination types are not castable. If two non-partition-key columns happen to have swapped positions but compatible types, the export succeeds and silently writes values into the wrong destination column, so keep the intended column order rather than relying on type compatibility alone. +The following must match between source and destination: - For `PARTITION BY t.a`, this rule applies to the top-level owning column `t`. Exporting from `t Tuple(a Int32, b Int32)` to `t Tuple(b Int32, a Int32)` is rejected, even though `a` is accessed by name. Requiring a stable layout for every partition-key owner also protects positional expressions such as `tupleElement(t, 1)` from changing their meaning after conversion. +1. **Column count** - source and destination must have the same number of columns. +2. **`PARTITION BY` expressions** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must be representable as an Iceberg partition spec and must match the destination partition fields and transforms. +3. **The position of every column backing the partition key** - it is not enough for the `PARTITION BY` expressions to be textually identical: every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. If such a column contains a named `Tuple`, its element names must also be declared in the same order (an unnamed `Tuple` on either side is exempt from this, per the allowance above). This comparison is recursive through nested tuples and through container types such as `Array` and `Map`. + + For example, `CREATE TABLE src (a Int32, b Int32) ... PARTITION BY a` and `CREATE TABLE dst (b Int32, a Int32) ... PARTITION BY a` both have the expression `PARTITION BY a`, but `a` is at position 0 in `src` and position 1 in `dst`. The export is rejected with a `BAD_ARGUMENTS` exception whose message includes `Cannot export to : partition key column 'a' is at position 0 in the source table, but the destination's column at that position is named 'b'`. - The element-name check only applies when both the source and destination `Tuple` declare explicit names; an unnamed `Tuple` (e.g. `Tuple(Int32, Int32)`) is compared to the destination by element position and type only. For example, exporting from `t Tuple(Int32, Int32)` to `t Tuple(x Int32, y Int32)` is allowed as long as element types match positionally. + This position check applies only to partition-key columns. A mismatch in the position of a non-partition-key column is allowed by name (see above) and is only rejected if the resulting types aren't castable. If two non-partition-key columns happen to have swapped positions but compatible types, the export succeeds and silently writes values into the wrong destination column, so keep the intended column order rather than relying on type compatibility alone. + + For `PARTITION BY t.a`, this rule applies to the top-level owning column `t`. Exporting from `t Tuple(a Int32, b Int32)` to `t Tuple(b Int32, a Int32)` is rejected, even though `a` is accessed by name. Requiring a stable layout for every partition-key owner also protects positional expressions such as `tupleElement(t, 1)` from changing their meaning after conversion. The same rule applies when the named tuple is nested inside a container. For example, `arr Array(Tuple(a Int32, b Int32))` and `arr Array(Tuple(b Int32, a Int32))` are incompatible when `arr` provides an input to the partition key. Likewise, tuple layouts in both the key and value types of `Map` are checked recursively. diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index 7a9773ed293d..a7b67a2e384b 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -45,11 +45,11 @@ TO TABLE [destination_database.]destination_table ## Requirements -`EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/antalya/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements, in particular: +`EXPORT PARTITION` exports each part via the same mechanism as [`EXPORT PART`](/docs/en/antalya/part_export.md#requirements), so the source and destination tables must satisfy the same compatibility requirements. Column names may differ (columns are matched by position, not by name), and column types may differ as long as they are safely castable (or `export_merge_tree_part_allow_lossy_cast = 1` is set). Beyond that, the following must match: -1. **Positionally compatible schemas** - source columns are matched to destination columns by position. Corresponding types must be safely castable unless `export_merge_tree_part_allow_lossy_cast = 1` is set. -2. **Compatible partitioning** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must match the destination partition fields and transforms. -3. **Matching partition key column positions and layouts** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. Named `Tuple` elements within such a column must also be declared in the same order, including tuples nested inside `Array` or `Map`. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. +1. **Column count** - source and destination must have the same number of columns. +2. **`PARTITION BY` expressions** - for destinations other than data lakes, the source and destination `PARTITION BY` expressions must be identical. For Apache Iceberg destinations, the source partition key must match the destination partition fields and transforms. +3. **Partition key column positions and layouts** - every top-level column that provides a column or subcolumn used by the source table's partition key must have the same name at the same position in the destination table's schema. Named `Tuple` elements within such a column must also be declared in the same order, including tuples nested inside `Array` or `Map`. This applies even if both tables' `PARTITION BY` expressions are textually identical. See [`EXPORT PART` requirements](/docs/en/antalya/part_export.md#requirements) for a worked example and the corresponding exception message. ## Settings