From de10aa504f3d150cf8e6cc0fd2e0159e377b04f9 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 22 Jul 2026 23:31:10 -0700 Subject: [PATCH 01/17] Support schema-evolution added fields inside list/map in PruneDataType PruneDataType rejected any nested difference inside a list/map as a partial projection. A field added inside a list (schema evolution) made the read type a superset of the file type and hit the same fail-fast, so data-evolution reads of old files errored with 'partial projection inside list'. Reconcile the repeated item by field id: added read fields are allowed (null-filled downstream), dropped file fields still fail. Adds unit tests for the list/map added-field and drop-and-add cases. --- .../core/utils/nested_projection_utils.cpp | 95 ++++++++++++++++--- .../utils/nested_projection_utils_test.cpp | 45 +++++++++ 2 files changed, 129 insertions(+), 11 deletions(-) diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp index 5f82a22ab..c34efdb90 100644 --- a/src/paimon/core/utils/nested_projection_utils.cpp +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -181,6 +181,75 @@ bool IsVariantAccessSubstitution(const std::shared_ptr& read_ty return true; } +// Reconcile a LIST/MAP item type. Read may ADD fields (schema evolution; +// null-filled downstream) but must not DROP a file field -- format readers +// cannot partially project inside a repeated group. Returns the file-readable +// item type. `container` is "list" or "map" for the error message. +Result> PruneRepeatedItemType( + const std::shared_ptr& read_type, + const std::shared_ptr& data_type, const char* container) { + if (read_type->Equals(data_type)) { + return data_type; + } + if (IsVariantAccessSubstitution(read_type, data_type)) { + return read_type; + } + if (read_type->id() != data_type->id()) { + return Status::Invalid( + fmt::format("PruneDataType nested item type mismatch inside {}: read {} vs data {}", + container, read_type->ToString(), data_type->ToString())); + } + switch (data_type->id()) { + case arrow::Type::STRUCT: { + arrow::FieldVector item_fields; + for (const auto& data_child : data_type->fields()) { + PAIMON_ASSIGN_OR_RAISE(int32_t data_child_id, + NestedProjectionUtils::GetPaimonFieldId(data_child)); + std::shared_ptr read_child; + for (const auto& candidate : read_type->fields()) { + PAIMON_ASSIGN_OR_RAISE(int32_t candidate_id, + NestedProjectionUtils::GetPaimonFieldId(candidate)); + if (candidate_id == data_child_id) { + read_child = candidate; + break; + } + } + if (!read_child) { + // A file field is dropped -- a real partial projection. + return Status::Invalid(fmt::format( + "PruneDataType does not support partial projection inside {}: src {} vs " + "target {}", + container, data_type->ToString(), read_type->ToString())); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr item_child_type, + PruneRepeatedItemType(read_child->type(), data_child->type(), container)); + item_fields.push_back(data_child->WithType(item_child_type)); + } + return arrow::struct_(item_fields); + } + case arrow::Type::LIST: { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr item, + PruneRepeatedItemType(read_type->field(0)->type(), + data_type->field(0)->type(), container)); + return arrow::list(data_type->field(0)->WithType(item)); + } + case arrow::Type::MAP: { + auto read_map = std::static_pointer_cast(read_type); + auto data_map = std::static_pointer_cast(data_type); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr key, + PruneRepeatedItemType(read_map->key_type(), data_map->key_type(), container)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr item, + PruneRepeatedItemType(read_map->item_type(), data_map->item_type(), container)); + return arrow::map(key, item); + } + default: + return data_type; + } +} + } // namespace Result>> NestedProjectionUtils::PruneDataType( @@ -235,22 +304,26 @@ Result>> NestedProjectionUtils::P if (IsVariantAccessSubstitution(read_type, data_type)) { return std::optional>(read_type); } - // Keep behavior aligned with format readers: partial projection inside - // LIST is unsupported and must fail fast. - return Status::Invalid( - fmt::format("PruneDataType does not support partial projection inside list: src {} " - "vs target {}", - data_type->ToString(), read_type->ToString())); + // Added fields (schema evolution) are allowed; dropped fields still fail. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr item, + PruneRepeatedItemType(read_type->field(0)->type(), + data_type->field(0)->type(), "list")); + return std::optional>( + arrow::list(data_type->field(0)->WithType(item))); } case arrow::Type::MAP: { if (IsVariantAccessSubstitution(read_type, data_type)) { return std::optional>(read_type); } - // Keep behavior aligned with format readers: partial projection inside - // MAP is unsupported and must fail fast. - return Status::Invalid(fmt::format( - "PruneDataType does not support partial projection inside map: src {} vs target {}", - data_type->ToString(), read_type->ToString())); + auto read_map = std::static_pointer_cast(read_type); + auto data_map = std::static_pointer_cast(data_type); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr key, + PruneRepeatedItemType(read_map->key_type(), data_map->key_type(), "map")); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr item, + PruneRepeatedItemType(read_map->item_type(), data_map->item_type(), "map")); + return std::optional>(arrow::map(key, item)); } default: // Atomic type: return data_type as-is (type evolution is handled diff --git a/src/paimon/core/utils/nested_projection_utils_test.cpp b/src/paimon/core/utils/nested_projection_utils_test.cpp index 1fa8f6f72..a791b1b96 100644 --- a/src/paimon/core/utils/nested_projection_utils_test.cpp +++ b/src/paimon/core/utils/nested_projection_utils_test.cpp @@ -222,6 +222,51 @@ TEST(NestedProjectionUtilsTest, PruneDataTypeListDroppingSiblingOfVariantStillFa "partial projection inside list"); } +TEST(NestedProjectionUtilsTest, PruneDataTypeListStructSchemaEvolutionAddedField) { + // Schema evolution: a field (id=12) was added inside the list's struct, so + // read is a superset of the file (nothing dropped). Should read the file's + // struct and return it; the added field is null-filled downstream. + auto data_inner = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)}); + auto read_inner = arrow::struct_({MakeField("a", arrow::int32(), 10), + MakeField("b", arrow::utf8(), 11), + MakeField("c", arrow::int32(), 12)}); + auto data_type = arrow::list(arrow::field("item", data_inner)); + auto read_type = arrow::list(arrow::field("item", read_inner)); + + ASSERT_OK_AND_ASSIGN(auto result, NestedProjectionUtils::PruneDataType(read_type, data_type)); + ASSERT_TRUE(result.has_value()); + ASSERT_TRUE(result.value()->Equals(*data_type)) << result.value()->ToString(); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeMapStructSchemaEvolutionAddedField) { + // Added field inside a MAP value. + auto data_inner = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)}); + auto read_inner = arrow::struct_({MakeField("a", arrow::int32(), 10), + MakeField("b", arrow::utf8(), 11), + MakeField("c", arrow::int32(), 12)}); + auto data_type = arrow::map(arrow::utf8(), data_inner); + auto read_type = arrow::map(arrow::utf8(), read_inner); + + ASSERT_OK_AND_ASSIGN(auto result, NestedProjectionUtils::PruneDataType(read_type, data_type)); + ASSERT_TRUE(result.has_value()); + ASSERT_TRUE(result.value()->Equals(*data_type)) << result.value()->ToString(); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeListStructDropAndAddStillFails) { + // Dropping a file field (b) is a real partial projection -- keep failing. + auto data_inner = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)}); + auto read_inner = arrow::struct_( + {MakeField("a", arrow::int32(), 10), MakeField("c", arrow::int32(), 12)}); + auto data_type = arrow::list(arrow::field("item", data_inner)); + auto read_type = arrow::list(arrow::field("item", read_inner)); + + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::PruneDataType(read_type, data_type), + "partial projection inside list"); +} + TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionNoProjection) { auto file_schema = arrow::schema({ MakeField("f0", arrow::int32(), 1), From 0d00149690ae949513984255cd2cd4ace4a42695 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 22 Jul 2026 23:44:14 -0700 Subject: [PATCH 02/17] Null-fill nested added fields in data-evolution reader assembly After PruneDataType allows a schema-evolution added field inside a list/map, the sub-reader yields the file's structure (e.g. list>) while the output must match the read schema (list>). Reshape each exist sub-array to the read type, null-filling the added nested field (recursive over struct/list/map, preserving offsets and validity). NOTE: not yet compiled/tested; pending local build of paimon-cpp. --- .../reader/data_evolution_file_reader.cpp | 92 ++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/src/paimon/common/reader/data_evolution_file_reader.cpp b/src/paimon/common/reader/data_evolution_file_reader.cpp index eefe900cc..1235341bf 100644 --- a/src/paimon/common/reader/data_evolution_file_reader.cpp +++ b/src/paimon/common/reader/data_evolution_file_reader.cpp @@ -16,6 +16,8 @@ #include "paimon/common/reader/data_evolution_file_reader.h" +#include "arrow/array/array_nested.h" +#include "arrow/array/util.h" #include "arrow/c/abi.h" #include "arrow/c/bridge.h" #include "fmt/format.h" @@ -23,8 +25,91 @@ #include "paimon/common/reader/reader_utils.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/core/utils/nested_projection_utils.h" namespace paimon { + +namespace { + +// Reshape an exist-field array (read as the file's structure) to `read_type`, +// null-filling nested fields added by schema evolution. No-op when the types +// already match. Structs match children by paimon field id; list/map recurse +// into their items positionally, preserving offsets and validity. +Result> AlignArrayToReadType( + const std::shared_ptr& array, + const std::shared_ptr& read_type, arrow::MemoryPool* pool) { + if (array->type()->Equals(*read_type)) { + return array; + } + const auto& data = array->data(); + switch (read_type->id()) { + case arrow::Type::STRUCT: { + const auto& array_type = array->type(); + std::vector> children; + children.reserve(read_type->num_fields()); + for (const auto& read_field : read_type->fields()) { + PAIMON_ASSIGN_OR_RAISE(int32_t read_id, + NestedProjectionUtils::GetPaimonFieldId(read_field)); + int32_t match = -1; + for (int32_t j = 0; j < array_type->num_fields(); j++) { + PAIMON_ASSIGN_OR_RAISE( + int32_t data_id, + NestedProjectionUtils::GetPaimonFieldId(array_type->field(j))); + if (data_id == read_id) { + match = j; + break; + } + } + if (match >= 0) { + auto child = arrow::MakeArray(data->child_data[match]); + PAIMON_ASSIGN_OR_RAISE(child, + AlignArrayToReadType(child, read_field->type(), pool)); + children.push_back(child->data()); + } else { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr null_child, + arrow::MakeArrayOfNull(read_field->type(), data->offset + data->length, + pool)); + children.push_back(null_child->data()); + } + } + auto new_data = data->Copy(); + new_data->type = read_type; + new_data->child_data = std::move(children); + return arrow::MakeArray(new_data); + } + case arrow::Type::LIST: { + auto read_list = std::static_pointer_cast(read_type); + auto values = arrow::MakeArray(data->child_data[0]); + PAIMON_ASSIGN_OR_RAISE(values, + AlignArrayToReadType(values, read_list->value_type(), pool)); + auto new_data = data->Copy(); + new_data->type = read_type; + new_data->child_data = {values->data()}; + return arrow::MakeArray(new_data); + } + case arrow::Type::MAP: { + auto read_map = std::static_pointer_cast(read_type); + const auto& entries_data = data->child_data[0]; + auto key = arrow::MakeArray(entries_data->child_data[0]); + auto value = arrow::MakeArray(entries_data->child_data[1]); + PAIMON_ASSIGN_OR_RAISE(key, AlignArrayToReadType(key, read_map->key_type(), pool)); + PAIMON_ASSIGN_OR_RAISE(value, AlignArrayToReadType(value, read_map->item_type(), pool)); + auto new_entries = entries_data->Copy(); + new_entries->type = arrow::struct_({read_map->key_field(), read_map->item_field()}); + new_entries->child_data = {key->data(), value->data()}; + auto new_data = data->Copy(); + new_data->type = read_type; + new_data->child_data = {new_entries}; + return arrow::MakeArray(new_data); + } + default: + return array; + } +} + +} // namespace + Result> DataEvolutionFileReader::Create( std::vector>&& readers, const std::shared_ptr& read_schema, int32_t read_batch_size, @@ -82,7 +167,12 @@ Result DataEvolutionFileReader::NextBatchWithB } const auto& sub_array = array_for_each_reader[reader_offsets_[i]]; assert(sub_array->num_fields() > field_offsets_[i]); - target_sub_array_vec.push_back(sub_array->field(field_offsets_[i])); + // Null-fill nested fields added by schema evolution (no-op otherwise). + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr aligned, + AlignArrayToReadType(sub_array->field(field_offsets_[i]), + read_schema_->field(i)->type(), arrow_pool_.get())); + target_sub_array_vec.push_back(aligned); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( std::shared_ptr target_array, From d0ed34772e4c8a69f2df506c5bc049c7f1a62894 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 22 Jul 2026 23:55:44 -0700 Subject: [PATCH 03/17] Reshape exist arrays for nested evolution in single-file reader path Factor AlignArrayToReadType into NestedProjectionUtils (shared by the data-evolution and field-mapping readers). In the single-file path: skip the scalar cast for differing LIST/MAP types (like STRUCT), and reshape each exist array to the read type so an evolution-added nested field is null-filled. NOTE: not yet compiled/tested; pending local build. --- .../reader/data_evolution_file_reader.cpp | 86 +------------------ src/paimon/core/io/field_mapping_reader.cpp | 28 ++++-- src/paimon/core/utils/field_mapping.cpp | 10 ++- .../core/utils/nested_projection_utils.cpp | 71 +++++++++++++++ .../core/utils/nested_projection_utils.h | 8 ++ 5 files changed, 108 insertions(+), 95 deletions(-) diff --git a/src/paimon/common/reader/data_evolution_file_reader.cpp b/src/paimon/common/reader/data_evolution_file_reader.cpp index 1235341bf..dbdfec41e 100644 --- a/src/paimon/common/reader/data_evolution_file_reader.cpp +++ b/src/paimon/common/reader/data_evolution_file_reader.cpp @@ -29,87 +29,6 @@ namespace paimon { -namespace { - -// Reshape an exist-field array (read as the file's structure) to `read_type`, -// null-filling nested fields added by schema evolution. No-op when the types -// already match. Structs match children by paimon field id; list/map recurse -// into their items positionally, preserving offsets and validity. -Result> AlignArrayToReadType( - const std::shared_ptr& array, - const std::shared_ptr& read_type, arrow::MemoryPool* pool) { - if (array->type()->Equals(*read_type)) { - return array; - } - const auto& data = array->data(); - switch (read_type->id()) { - case arrow::Type::STRUCT: { - const auto& array_type = array->type(); - std::vector> children; - children.reserve(read_type->num_fields()); - for (const auto& read_field : read_type->fields()) { - PAIMON_ASSIGN_OR_RAISE(int32_t read_id, - NestedProjectionUtils::GetPaimonFieldId(read_field)); - int32_t match = -1; - for (int32_t j = 0; j < array_type->num_fields(); j++) { - PAIMON_ASSIGN_OR_RAISE( - int32_t data_id, - NestedProjectionUtils::GetPaimonFieldId(array_type->field(j))); - if (data_id == read_id) { - match = j; - break; - } - } - if (match >= 0) { - auto child = arrow::MakeArray(data->child_data[match]); - PAIMON_ASSIGN_OR_RAISE(child, - AlignArrayToReadType(child, read_field->type(), pool)); - children.push_back(child->data()); - } else { - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - std::shared_ptr null_child, - arrow::MakeArrayOfNull(read_field->type(), data->offset + data->length, - pool)); - children.push_back(null_child->data()); - } - } - auto new_data = data->Copy(); - new_data->type = read_type; - new_data->child_data = std::move(children); - return arrow::MakeArray(new_data); - } - case arrow::Type::LIST: { - auto read_list = std::static_pointer_cast(read_type); - auto values = arrow::MakeArray(data->child_data[0]); - PAIMON_ASSIGN_OR_RAISE(values, - AlignArrayToReadType(values, read_list->value_type(), pool)); - auto new_data = data->Copy(); - new_data->type = read_type; - new_data->child_data = {values->data()}; - return arrow::MakeArray(new_data); - } - case arrow::Type::MAP: { - auto read_map = std::static_pointer_cast(read_type); - const auto& entries_data = data->child_data[0]; - auto key = arrow::MakeArray(entries_data->child_data[0]); - auto value = arrow::MakeArray(entries_data->child_data[1]); - PAIMON_ASSIGN_OR_RAISE(key, AlignArrayToReadType(key, read_map->key_type(), pool)); - PAIMON_ASSIGN_OR_RAISE(value, AlignArrayToReadType(value, read_map->item_type(), pool)); - auto new_entries = entries_data->Copy(); - new_entries->type = arrow::struct_({read_map->key_field(), read_map->item_field()}); - new_entries->child_data = {key->data(), value->data()}; - auto new_data = data->Copy(); - new_data->type = read_type; - new_data->child_data = {new_entries}; - return arrow::MakeArray(new_data); - } - default: - return array; - } -} - -} // namespace - Result> DataEvolutionFileReader::Create( std::vector>&& readers, const std::shared_ptr& read_schema, int32_t read_batch_size, @@ -170,8 +89,9 @@ Result DataEvolutionFileReader::NextBatchWithB // Null-fill nested fields added by schema evolution (no-op otherwise). PAIMON_ASSIGN_OR_RAISE( std::shared_ptr aligned, - AlignArrayToReadType(sub_array->field(field_offsets_[i]), - read_schema_->field(i)->type(), arrow_pool_.get())); + NestedProjectionUtils::AlignArrayToReadType(sub_array->field(field_offsets_[i]), + read_schema_->field(i)->type(), + arrow_pool_.get())); target_sub_array_vec.push_back(aligned); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( diff --git a/src/paimon/core/io/field_mapping_reader.cpp b/src/paimon/core/io/field_mapping_reader.cpp index 3c225e7d6..3fe076c30 100644 --- a/src/paimon/core/io/field_mapping_reader.cpp +++ b/src/paimon/core/io/field_mapping_reader.cpp @@ -35,6 +35,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/core/casting/cast_executor.h" #include "paimon/core/casting/casting_utils.h" +#include "paimon/core/utils/nested_projection_utils.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/nested_projection_utils.h" #include "paimon/memory/bytes.h" @@ -149,6 +150,12 @@ Result> FieldMappingReader::Create( if (mapping_reader->non_partition_info_.cast_executors[i] != nullptr) { mapping_reader->need_casting_ = true; } + // A nested type that differs by a schema-evolution added field must be + // reshaped (null-filled) to the read type in CastNonPartitionArrayIfNeed. + if (!mapping_reader->non_partition_info_.non_partition_data_schema[i].Type()->Equals( + *mapping_reader->non_partition_info_.non_partition_read_schema[i].Type())) { + mapping_reader->need_casting_ = true; + } // Field name change (RENAME COLUMN) also requires mapping: data schema // carries the file's physical name while read schema carries the // post-rename logical name. If we skipped mapping, the inner reader's @@ -201,6 +208,7 @@ Result> FieldMappingReader::CastNonPartitionArrayI casted_array.reserve(field_count); casted_field_names.reserve(field_count); for (int32_t i = 0; i < field_count; i++) { + std::shared_ptr column; if (non_partition_info_.cast_executors[i] != nullptr) { auto single_column_array = struct_array->field(i); // if src array is dict, cast to string first @@ -213,18 +221,22 @@ Result> FieldMappingReader::CastNonPartitionArrayI arrow::compute::CastOptions::Safe(), arrow_pool_.get())); } PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr casted, - non_partition_info_.cast_executors[i]->Cast( - single_column_array, non_partition_info_.non_partition_read_schema[i].Type(), - arrow_pool_.get())); - casted_array.push_back(casted); - casted_field_names.push_back(non_partition_info_.non_partition_data_schema[i].Name()); + column, non_partition_info_.cast_executors[i]->Cast( + single_column_array, + non_partition_info_.non_partition_read_schema[i].Type(), + arrow_pool_.get())); } else { // read and data type may both be string type, but after adapter transform, type may be // dictionary, need reconstruct struct type - casted_array.push_back(struct_array->field(i)); - casted_field_names.push_back(non_partition_info_.non_partition_data_schema[i].Name()); + column = struct_array->field(i); } + // Null-fill nested fields added by schema evolution (no-op otherwise). + PAIMON_ASSIGN_OR_RAISE( + column, NestedProjectionUtils::AlignArrayToReadType( + column, non_partition_info_.non_partition_read_schema[i].Type(), + arrow_pool_.get())); + casted_array.push_back(column); + casted_field_names.push_back(non_partition_info_.non_partition_data_schema[i].Name()); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, arrow::StructArray::Make(casted_array, casted_field_names)); diff --git a/src/paimon/core/utils/field_mapping.cpp b/src/paimon/core/utils/field_mapping.cpp index 0c90ae1b2..02cafa692 100644 --- a/src/paimon/core/utils/field_mapping.cpp +++ b/src/paimon/core/utils/field_mapping.cpp @@ -172,10 +172,12 @@ Result>> FieldMappingBuilder::CreateDa FieldTypeUtils::ConvertToFieldType(data_fields[i].Type()->id())); if (!read_fields[i].Type()->Equals(data_fields[i].Type())) { - if (read_type == FieldType::STRUCT) { - // STRUCT may still differ by nested pruning shape. No cast is - // needed — type pruning is handled by PruneDataType during - // field mapping construction. + auto read_type_id = read_fields[i].Type()->id(); + if (read_type_id == arrow::Type::STRUCT || read_type_id == arrow::Type::LIST || + read_type_id == arrow::Type::MAP) { + // Nested types differ only by pruning shape or a schema-evolution + // added field. No scalar cast -- handled by PruneDataType and the + // AlignArrayToReadType reshape in the reader. cast_executors.push_back(nullptr); continue; } diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp index c34efdb90..4c96c3368 100644 --- a/src/paimon/core/utils/nested_projection_utils.cpp +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -26,6 +26,7 @@ #include "arrow/array/array_primitive.h" #include "arrow/array/builder_primitive.h" #include "arrow/array/concatenate.h" +#include "arrow/array/util.h" #include "arrow/type.h" #include "fmt/format.h" #include "paimon/common/data/variant/variant_access_utils.h" @@ -525,4 +526,74 @@ Result> NestedProjectionUtils::FilterMapArrayBySel return result_map; } +Result> NestedProjectionUtils::AlignArrayToReadType( + const std::shared_ptr& array, + const std::shared_ptr& read_type, arrow::MemoryPool* pool) { + if (array->type()->Equals(*read_type)) { + return array; + } + const auto& data = array->data(); + switch (read_type->id()) { + case arrow::Type::STRUCT: { + const auto& array_type = array->type(); + std::vector> children; + children.reserve(read_type->num_fields()); + for (const auto& read_field : read_type->fields()) { + PAIMON_ASSIGN_OR_RAISE(int32_t read_id, GetPaimonFieldId(read_field)); + int32_t match = -1; + for (int32_t j = 0; j < array_type->num_fields(); j++) { + PAIMON_ASSIGN_OR_RAISE(int32_t data_id, GetPaimonFieldId(array_type->field(j))); + if (data_id == read_id) { + match = j; + break; + } + } + if (match >= 0) { + auto child = arrow::MakeArray(data->child_data[match]); + PAIMON_ASSIGN_OR_RAISE(child, + AlignArrayToReadType(child, read_field->type(), pool)); + children.push_back(child->data()); + } else { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr null_child, + arrow::MakeArrayOfNull(read_field->type(), data->offset + data->length, + pool)); + children.push_back(null_child->data()); + } + } + auto new_data = data->Copy(); + new_data->type = read_type; + new_data->child_data = std::move(children); + return arrow::MakeArray(new_data); + } + case arrow::Type::LIST: { + auto read_list = std::static_pointer_cast(read_type); + auto values = arrow::MakeArray(data->child_data[0]); + PAIMON_ASSIGN_OR_RAISE(values, + AlignArrayToReadType(values, read_list->value_type(), pool)); + auto new_data = data->Copy(); + new_data->type = read_type; + new_data->child_data = {values->data()}; + return arrow::MakeArray(new_data); + } + case arrow::Type::MAP: { + auto read_map = std::static_pointer_cast(read_type); + const auto& entries_data = data->child_data[0]; + auto key = arrow::MakeArray(entries_data->child_data[0]); + auto value = arrow::MakeArray(entries_data->child_data[1]); + PAIMON_ASSIGN_OR_RAISE(key, AlignArrayToReadType(key, read_map->key_type(), pool)); + PAIMON_ASSIGN_OR_RAISE(value, AlignArrayToReadType(value, read_map->item_type(), pool)); + auto new_entries = entries_data->Copy(); + new_entries->type = arrow::struct_({read_map->key_field(), read_map->item_field()}); + new_entries->child_data = {key->data(), value->data()}; + auto new_data = data->Copy(); + new_data->type = read_type; + new_data->child_data = {new_entries}; + return arrow::MakeArray(new_data); + } + default: + return array; + } +} + } // namespace paimon diff --git a/src/paimon/core/utils/nested_projection_utils.h b/src/paimon/core/utils/nested_projection_utils.h index 760b72176..388b1c6a9 100644 --- a/src/paimon/core/utils/nested_projection_utils.h +++ b/src/paimon/core/utils/nested_projection_utils.h @@ -85,6 +85,14 @@ class PAIMON_EXPORT NestedProjectionUtils { const std::shared_ptr& map_array, const std::vector& selected_keys, arrow::MemoryPool* pool); + /// Reshape `array` (read as the file's structure) to `read_type`, null-filling + /// nested fields added by schema evolution (a read struct that is a superset of + /// the file struct). No-op when the types already match. STRUCT children match + /// by paimon field id; LIST/MAP recurse into items, preserving offsets and validity. + static Result> AlignArrayToReadType( + const std::shared_ptr& array, + const std::shared_ptr& read_type, arrow::MemoryPool* pool); + private: static Result HasNestedSubfieldProjectionType( const std::shared_ptr& file_type, From 9d50b65b765077e85bb905444cfa4690b62e80bf Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 23 Jul 2026 02:33:06 -0700 Subject: [PATCH 04/17] Add array-level test for AlignArrayToReadType nested null-fill Verifies the reshape actually null-fills a schema-evolution added field inside list and preserves existing values. --- .../utils/nested_projection_utils_test.cpp | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/paimon/core/utils/nested_projection_utils_test.cpp b/src/paimon/core/utils/nested_projection_utils_test.cpp index a791b1b96..7c275bc0f 100644 --- a/src/paimon/core/utils/nested_projection_utils_test.cpp +++ b/src/paimon/core/utils/nested_projection_utils_test.cpp @@ -267,6 +267,47 @@ TEST(NestedProjectionUtilsTest, PruneDataTypeListStructDropAndAddStillFails) { "partial projection inside list"); } +TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeNullFillsAddedListStructField) { + auto* pool = arrow::default_memory_pool(); + // file array: list> = [ [{1,"x"},{2,"y"}], [{3,"z"}] ] + arrow::Int32Builder ab(pool); + ASSERT_TRUE(ab.AppendValues({1, 2, 3}).ok()); + std::shared_ptr a; + ASSERT_TRUE(ab.Finish(&a).ok()); + arrow::StringBuilder bb(pool); + ASSERT_TRUE(bb.AppendValues({"x", "y", "z"}).ok()); + std::shared_ptr b; + ASSERT_TRUE(bb.Finish(&b).ok()); + auto data_struct_type = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)}); + std::shared_ptr struct_arr = + arrow::StructArray::Make({a, b}, data_struct_type->fields()).ValueOrDie(); + arrow::Int32Builder offb(pool); + ASSERT_TRUE(offb.AppendValues({0, 2, 3}).ok()); + std::shared_ptr offsets; + ASSERT_TRUE(offb.Finish(&offsets).ok()); + std::shared_ptr list_arr = + arrow::ListArray::FromArrays(*offsets, *struct_arr, pool).ValueOrDie(); + + // read type adds c:int(12) inside the struct. + auto read_struct = arrow::struct_({MakeField("a", arrow::int32(), 10), + MakeField("b", arrow::utf8(), 11), + MakeField("c", arrow::int32(), 12)}); + auto read_type = arrow::list(arrow::field("item", read_struct)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr aligned, + NestedProjectionUtils::AlignArrayToReadType(list_arr, read_type, pool)); + ASSERT_TRUE(aligned->type()->Equals(*read_type)) << aligned->type()->ToString(); + auto out_struct = std::static_pointer_cast( + std::static_pointer_cast(aligned)->values()); + auto c_col = out_struct->GetFieldByName("c"); + ASSERT_NE(c_col, nullptr); + ASSERT_EQ(c_col->null_count(), c_col->length()); // added field is all null + auto a_col = std::static_pointer_cast(out_struct->GetFieldByName("a")); + ASSERT_EQ(a_col->Value(0), 1); + ASSERT_EQ(a_col->Value(2), 3); +} + TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionNoProjection) { auto file_schema = arrow::schema({ MakeField("f0", arrow::int32(), 1), From bf772a8731c1c6f7df7168c6dc764aafdf9bd8c7 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 23 Jul 2026 02:45:55 -0700 Subject: [PATCH 05/17] Add end-to-end test: read list file with an evolution-added inner field Drives FieldMappingReader over a list> batch against a read schema whose struct gained c(id=12); asserts the output is list> with c null-filled. --- .../core/io/field_mapping_reader_test.cpp | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/paimon/core/io/field_mapping_reader_test.cpp b/src/paimon/core/io/field_mapping_reader_test.cpp index be9cc35a3..8d49152fa 100644 --- a/src/paimon/core/io/field_mapping_reader_test.cpp +++ b/src/paimon/core/io/field_mapping_reader_test.cpp @@ -448,6 +448,59 @@ TEST_F(FieldMappingReaderTest, TestDictionaryTypeWithSchemaEvolution) { partition, expected_array); } +TEST_F(FieldMappingReaderTest, TestSchemaEvolutionAddedFieldInsideList) { + // A field `c`(id=12) was added inside the list's struct after the file was + // written. Reading the old file with the new schema must null-fill `c`. + auto id_field = [](const std::string& name, const std::shared_ptr& type, + int32_t id) { + return DataField::ConvertDataFieldToArrowField(DataField(id, arrow::field(name, type))); + }; + auto data_struct = + arrow::struct_({id_field("a", arrow::int32(), 10), id_field("b", arrow::utf8(), 11)}); + auto read_struct = arrow::struct_({id_field("a", arrow::int32(), 10), + id_field("b", arrow::utf8(), 11), + id_field("c", arrow::int32(), 12)}); + std::vector data_fields = { + DataField(100, arrow::field("data_contents", arrow::list(arrow::field("item", data_struct))))}; + std::vector read_fields = { + DataField(100, arrow::field("data_contents", arrow::list(arrow::field("item", read_struct))))}; + auto data_schema = DataField::ConvertDataFieldsToArrowSchema(data_fields); + auto read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields); + + auto data_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(data_schema->fields()), R"([ + [[[1, "x"], [2, "y"]]], + [[[3, "z"]]] + ])") + .ValueOrDie()); + + ASSERT_OK_AND_ASSIGN(auto mapping_builder, + FieldMappingBuilder::Create(read_schema, /*partition_keys=*/{}, + /*predicate=*/nullptr)); + ASSERT_OK_AND_ASSIGN(auto mapping, mapping_builder->CreateFieldMapping(data_fields)); + auto mock = std::make_unique( + data_array, arrow::struct_(data_schema->fields()), /*read_batch_size=*/8); + ASSERT_OK_AND_ASSIGN( + auto reader, FieldMappingReader::Create(read_schema->num_fields(), std::move(mock), + BinaryRow::EmptyRow(), std::move(mapping), + /*skip_map_selected_keys_filter_field_ids=*/{}, + pool_)); + ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(reader.get())); + + auto expect_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(read_schema->fields()), R"([ + [[[1, "x", null], [2, "y", null]]], + [[[3, "z", null]]] + ])") + .ValueOrDie(); + auto expected_chunk = + std::make_shared(arrow::ArrayVector({expect_array})); + ASSERT_TRUE(result_array->type()->Equals(expected_chunk->type())) + << result_array->type()->ToString() << " vs " << expected_chunk->type()->ToString(); + ASSERT_TRUE(result_array->Equals(expected_chunk)) + << result_array->ToString() << " vs " << expected_chunk->ToString(); +} + TEST_F(FieldMappingReaderTest, TestSchemaEvolutionWithModifyType) { std::vector data_fields = {DataField(0, arrow::field("f0", arrow::utf8())), DataField(1, arrow::field("f1", arrow::float32())), From 83ee0e2ff84170a4eb7fdf9f999e62573e7dd87e Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 23 Jul 2026 02:54:35 -0700 Subject: [PATCH 06/17] Trim comments --- src/paimon/core/io/field_mapping_reader.cpp | 3 +-- src/paimon/core/utils/field_mapping.cpp | 5 ++--- src/paimon/core/utils/nested_projection_utils.cpp | 7 +++---- src/paimon/core/utils/nested_projection_utils.h | 7 +++---- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/paimon/core/io/field_mapping_reader.cpp b/src/paimon/core/io/field_mapping_reader.cpp index 3fe076c30..bd1de4987 100644 --- a/src/paimon/core/io/field_mapping_reader.cpp +++ b/src/paimon/core/io/field_mapping_reader.cpp @@ -150,8 +150,7 @@ Result> FieldMappingReader::Create( if (mapping_reader->non_partition_info_.cast_executors[i] != nullptr) { mapping_reader->need_casting_ = true; } - // A nested type that differs by a schema-evolution added field must be - // reshaped (null-filled) to the read type in CastNonPartitionArrayIfNeed. + // A differing nested type needs the AlignArrayToReadType reshape below. if (!mapping_reader->non_partition_info_.non_partition_data_schema[i].Type()->Equals( *mapping_reader->non_partition_info_.non_partition_read_schema[i].Type())) { mapping_reader->need_casting_ = true; diff --git a/src/paimon/core/utils/field_mapping.cpp b/src/paimon/core/utils/field_mapping.cpp index 02cafa692..493d350c3 100644 --- a/src/paimon/core/utils/field_mapping.cpp +++ b/src/paimon/core/utils/field_mapping.cpp @@ -175,9 +175,8 @@ Result>> FieldMappingBuilder::CreateDa auto read_type_id = read_fields[i].Type()->id(); if (read_type_id == arrow::Type::STRUCT || read_type_id == arrow::Type::LIST || read_type_id == arrow::Type::MAP) { - // Nested types differ only by pruning shape or a schema-evolution - // added field. No scalar cast -- handled by PruneDataType and the - // AlignArrayToReadType reshape in the reader. + // Nested type differs by pruning/evolution; the reader's reshape + // handles it, no scalar cast. cast_executors.push_back(nullptr); continue; } diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp index 4c96c3368..61c5e539c 100644 --- a/src/paimon/core/utils/nested_projection_utils.cpp +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -182,10 +182,9 @@ bool IsVariantAccessSubstitution(const std::shared_ptr& read_ty return true; } -// Reconcile a LIST/MAP item type. Read may ADD fields (schema evolution; -// null-filled downstream) but must not DROP a file field -- format readers -// cannot partially project inside a repeated group. Returns the file-readable -// item type. `container` is "list" or "map" for the error message. +// Reconcile a LIST/MAP item: read may ADD fields (evolution, null-filled +// downstream) but must not DROP one. Returns the file-readable item type; +// `container` names the container ("list"/"map") for the error message. Result> PruneRepeatedItemType( const std::shared_ptr& read_type, const std::shared_ptr& data_type, const char* container) { diff --git a/src/paimon/core/utils/nested_projection_utils.h b/src/paimon/core/utils/nested_projection_utils.h index 388b1c6a9..2b4fd2838 100644 --- a/src/paimon/core/utils/nested_projection_utils.h +++ b/src/paimon/core/utils/nested_projection_utils.h @@ -85,10 +85,9 @@ class PAIMON_EXPORT NestedProjectionUtils { const std::shared_ptr& map_array, const std::vector& selected_keys, arrow::MemoryPool* pool); - /// Reshape `array` (read as the file's structure) to `read_type`, null-filling - /// nested fields added by schema evolution (a read struct that is a superset of - /// the file struct). No-op when the types already match. STRUCT children match - /// by paimon field id; LIST/MAP recurse into items, preserving offsets and validity. + /// Reshape `array` to `read_type`, null-filling nested fields added by schema + /// evolution. No-op when types match. STRUCT matches children by paimon field id; + /// LIST/MAP recurse into items, preserving offsets and validity. static Result> AlignArrayToReadType( const std::shared_ptr& array, const std::shared_ptr& read_type, arrow::MemoryPool* pool); From 48c30b9cca7c05ef2534455509da8c1d1562595d Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 23 Jul 2026 03:09:14 -0700 Subject: [PATCH 07/17] Address review comments on PR #446 - Remove duplicate nested_projection_utils.h include. - AlignArrayToReadType: fail fast on a kind/atomic-type mismatch instead of building an invalid ArrayData; only reshape a field whose data and read types differ (skips dictionary-encoded arrays of unchanged type). - Preserve MapType keys_sorted and key/value field metadata when pruning map item types. --- src/paimon/core/io/field_mapping_reader.cpp | 16 ++++++++++------ .../core/utils/nested_projection_utils.cpp | 17 ++++++++++++++--- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/paimon/core/io/field_mapping_reader.cpp b/src/paimon/core/io/field_mapping_reader.cpp index bd1de4987..bffba1601 100644 --- a/src/paimon/core/io/field_mapping_reader.cpp +++ b/src/paimon/core/io/field_mapping_reader.cpp @@ -35,7 +35,6 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/core/casting/cast_executor.h" #include "paimon/core/casting/casting_utils.h" -#include "paimon/core/utils/nested_projection_utils.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/nested_projection_utils.h" #include "paimon/memory/bytes.h" @@ -229,11 +228,16 @@ Result> FieldMappingReader::CastNonPartitionArrayI // dictionary, need reconstruct struct type column = struct_array->field(i); } - // Null-fill nested fields added by schema evolution (no-op otherwise). - PAIMON_ASSIGN_OR_RAISE( - column, NestedProjectionUtils::AlignArrayToReadType( - column, non_partition_info_.non_partition_read_schema[i].Type(), - arrow_pool_.get())); + // Null-fill nested fields added by schema evolution. Only when the data and + // read types differ -- the reader may hand back a dictionary-encoded array + // for an unchanged type, which is not a reshape target. + if (!non_partition_info_.non_partition_data_schema[i].Type()->Equals( + *non_partition_info_.non_partition_read_schema[i].Type())) { + PAIMON_ASSIGN_OR_RAISE( + column, NestedProjectionUtils::AlignArrayToReadType( + column, non_partition_info_.non_partition_read_schema[i].Type(), + arrow_pool_.get())); + } casted_array.push_back(column); casted_field_names.push_back(non_partition_info_.non_partition_data_schema[i].Name()); } diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp index 61c5e539c..c2bd05664 100644 --- a/src/paimon/core/utils/nested_projection_utils.cpp +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -243,7 +243,9 @@ Result> PruneRepeatedItemType( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr item, PruneRepeatedItemType(read_map->item_type(), data_map->item_type(), container)); - return arrow::map(key, item); + return std::static_pointer_cast(std::make_shared( + data_map->key_field()->WithType(key), data_map->item_field()->WithType(item), + data_map->keys_sorted())); } default: return data_type; @@ -323,7 +325,9 @@ Result>> NestedProjectionUtils::P PAIMON_ASSIGN_OR_RAISE( std::shared_ptr item, PruneRepeatedItemType(read_map->item_type(), data_map->item_type(), "map")); - return std::optional>(arrow::map(key, item)); + return std::optional>(std::make_shared( + data_map->key_field()->WithType(key), data_map->item_field()->WithType(item), + data_map->keys_sorted())); } default: // Atomic type: return data_type as-is (type evolution is handled @@ -531,6 +535,12 @@ Result> NestedProjectionUtils::AlignArrayToReadTyp if (array->type()->Equals(*read_type)) { return array; } + // Only same-kind structural differences (added nested fields) are handled; a + // differing kind or atomic type (nested type evolution) is not. + if (array->type()->id() != read_type->id()) { + return Status::Invalid(fmt::format("AlignArrayToReadType cannot reconcile {} to {}", + array->type()->ToString(), read_type->ToString())); + } const auto& data = array->data(); switch (read_type->id()) { case arrow::Type::STRUCT: { @@ -591,7 +601,8 @@ Result> NestedProjectionUtils::AlignArrayToReadTyp return arrow::MakeArray(new_data); } default: - return array; + return Status::Invalid(fmt::format("AlignArrayToReadType cannot reconcile {} to {}", + array->type()->ToString(), read_type->ToString())); } } From 113adb1b0b1a0fddf2b52390bfbbcea5ee7c4ec7 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 23 Jul 2026 03:36:13 -0700 Subject: [PATCH 08/17] Handle dictionary-encoded nested leaves from ORC lazy decoding ORC lazy decoding returns nested strings (including map keys) as Arrow dictionary arrays. AlignArrayToReadType now keeps a leaf's actual encoding (a dictionary of the read value type is kept as-is) and builds the reshaped STRUCT/LIST/MAP output type from the children's actual types, instead of forcing read_type and rejecting dictionary vs string. Add a dictionary-leaf test. --- .../core/io/field_mapping_reader_test.cpp | 4 +- .../core/utils/nested_projection_utils.cpp | 48 +++++++++++++++---- .../utils/nested_projection_utils_test.cpp | 31 ++++++++++++ 3 files changed, 71 insertions(+), 12 deletions(-) diff --git a/src/paimon/core/io/field_mapping_reader_test.cpp b/src/paimon/core/io/field_mapping_reader_test.cpp index 8d49152fa..ce776e852 100644 --- a/src/paimon/core/io/field_mapping_reader_test.cpp +++ b/src/paimon/core/io/field_mapping_reader_test.cpp @@ -461,9 +461,9 @@ TEST_F(FieldMappingReaderTest, TestSchemaEvolutionAddedFieldInsideList) { id_field("b", arrow::utf8(), 11), id_field("c", arrow::int32(), 12)}); std::vector data_fields = { - DataField(100, arrow::field("data_contents", arrow::list(arrow::field("item", data_struct))))}; + DataField(100, arrow::field("items", arrow::list(arrow::field("item", data_struct))))}; std::vector read_fields = { - DataField(100, arrow::field("data_contents", arrow::list(arrow::field("item", read_struct))))}; + DataField(100, arrow::field("items", arrow::list(arrow::field("item", read_struct))))}; auto data_schema = DataField::ConvertDataFieldsToArrowSchema(data_fields); auto read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields); diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp index c2bd05664..a5a568098 100644 --- a/src/paimon/core/utils/nested_projection_utils.cpp +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -535,17 +535,20 @@ Result> NestedProjectionUtils::AlignArrayToReadTyp if (array->type()->Equals(*read_type)) { return array; } - // Only same-kind structural differences (added nested fields) are handled; a - // differing kind or atomic type (nested type evolution) is not. - if (array->type()->id() != read_type->id()) { - return Status::Invalid(fmt::format("AlignArrayToReadType cannot reconcile {} to {}", - array->type()->ToString(), read_type->ToString())); - } + // Leaves keep their encoding (e.g. ORC dictionary); only STRUCT/LIST/MAP are + // reshaped to null-fill added fields, with the output type built from actual children. const auto& data = array->data(); switch (read_type->id()) { case arrow::Type::STRUCT: { + if (array->type()->id() != arrow::Type::STRUCT) { + return Status::Invalid(fmt::format("AlignArrayToReadType cannot reconcile {} to {}", + array->type()->ToString(), + read_type->ToString())); + } const auto& array_type = array->type(); + arrow::FieldVector out_fields; std::vector> children; + out_fields.reserve(read_type->num_fields()); children.reserve(read_type->num_fields()); for (const auto& read_field : read_type->fields()) { PAIMON_ASSIGN_OR_RAISE(int32_t read_id, GetPaimonFieldId(read_field)); @@ -562,45 +565,70 @@ Result> NestedProjectionUtils::AlignArrayToReadTyp PAIMON_ASSIGN_OR_RAISE(child, AlignArrayToReadType(child, read_field->type(), pool)); children.push_back(child->data()); + out_fields.push_back(read_field->WithType(child->type())); } else { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( std::shared_ptr null_child, arrow::MakeArrayOfNull(read_field->type(), data->offset + data->length, pool)); children.push_back(null_child->data()); + out_fields.push_back(read_field); } } auto new_data = data->Copy(); - new_data->type = read_type; + new_data->type = arrow::struct_(out_fields); new_data->child_data = std::move(children); return arrow::MakeArray(new_data); } case arrow::Type::LIST: { + if (array->type()->id() != arrow::Type::LIST) { + return Status::Invalid(fmt::format("AlignArrayToReadType cannot reconcile {} to {}", + array->type()->ToString(), + read_type->ToString())); + } auto read_list = std::static_pointer_cast(read_type); + auto data_list = std::static_pointer_cast(array->type()); auto values = arrow::MakeArray(data->child_data[0]); PAIMON_ASSIGN_OR_RAISE(values, AlignArrayToReadType(values, read_list->value_type(), pool)); auto new_data = data->Copy(); - new_data->type = read_type; + new_data->type = arrow::list(data_list->value_field()->WithType(values->type())); new_data->child_data = {values->data()}; return arrow::MakeArray(new_data); } case arrow::Type::MAP: { + if (array->type()->id() != arrow::Type::MAP) { + return Status::Invalid(fmt::format("AlignArrayToReadType cannot reconcile {} to {}", + array->type()->ToString(), + read_type->ToString())); + } auto read_map = std::static_pointer_cast(read_type); + auto data_map = std::static_pointer_cast(array->type()); const auto& entries_data = data->child_data[0]; auto key = arrow::MakeArray(entries_data->child_data[0]); auto value = arrow::MakeArray(entries_data->child_data[1]); PAIMON_ASSIGN_OR_RAISE(key, AlignArrayToReadType(key, read_map->key_type(), pool)); PAIMON_ASSIGN_OR_RAISE(value, AlignArrayToReadType(value, read_map->item_type(), pool)); + auto key_field = data_map->key_field()->WithType(key->type()); + auto item_field = data_map->item_field()->WithType(value->type()); auto new_entries = entries_data->Copy(); - new_entries->type = arrow::struct_({read_map->key_field(), read_map->item_field()}); + new_entries->type = arrow::struct_({key_field, item_field}); new_entries->child_data = {key->data(), value->data()}; auto new_data = data->Copy(); - new_data->type = read_type; + new_data->type = + std::make_shared(key_field, item_field, data_map->keys_sorted()); new_data->child_data = {new_entries}; return arrow::MakeArray(new_data); } default: + // A dictionary of the read value type (ORC lazy decoding) is kept; + // any other leaf mismatch is unsupported nested type evolution. + if (array->type()->id() == arrow::Type::DICTIONARY && + std::static_pointer_cast(array->type()) + ->value_type() + ->Equals(*read_type)) { + return array; + } return Status::Invalid(fmt::format("AlignArrayToReadType cannot reconcile {} to {}", array->type()->ToString(), read_type->ToString())); } diff --git a/src/paimon/core/utils/nested_projection_utils_test.cpp b/src/paimon/core/utils/nested_projection_utils_test.cpp index 7c275bc0f..d79e9510b 100644 --- a/src/paimon/core/utils/nested_projection_utils_test.cpp +++ b/src/paimon/core/utils/nested_projection_utils_test.cpp @@ -16,6 +16,7 @@ #include "paimon/core/utils/nested_projection_utils.h" +#include "arrow/array/array_dict.h" #include "arrow/array/array_nested.h" #include "arrow/array/builder_binary.h" #include "arrow/array/builder_dict.h" @@ -308,6 +309,36 @@ TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeNullFillsAddedListStructFiel ASSERT_EQ(a_col->Value(2), 3); } +TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeKeepsDictionaryLeafAndNullFills) { + // ORC lazy decoding returns nested strings as dictionary; a dict-encoded `a` + // must be kept (not rejected) while the added `b` is null-filled. + auto* pool = arrow::default_memory_pool(); + arrow::StringBuilder vb(pool); + ASSERT_TRUE(vb.AppendValues({"x", "y"}).ok()); + std::shared_ptr dict_values; + ASSERT_TRUE(vb.Finish(&dict_values).ok()); + arrow::Int32Builder ib(pool); + ASSERT_TRUE(ib.AppendValues({0, 1, 0}).ok()); + std::shared_ptr dict_indices; + ASSERT_TRUE(ib.Finish(&dict_indices).ok()); + auto dict_type = arrow::dictionary(arrow::int32(), arrow::utf8()); + auto a_dict = + arrow::DictionaryArray::FromArrays(dict_type, dict_indices, dict_values).ValueOrDie(); + auto data_struct = arrow::struct_({MakeField("a", dict_type, 10)}); + auto struct_arr = arrow::StructArray::Make({a_dict}, data_struct->fields()).ValueOrDie(); + + auto read_type = + arrow::struct_({MakeField("a", arrow::utf8(), 10), MakeField("b", arrow::int32(), 11)}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr aligned, + NestedProjectionUtils::AlignArrayToReadType(struct_arr, read_type, pool)); + auto out = std::static_pointer_cast(aligned); + ASSERT_EQ(out->num_fields(), 2); + ASSERT_EQ(out->field(0)->type_id(), arrow::Type::DICTIONARY); // `a` kept dict-encoded + auto b = out->GetFieldByName("b"); + ASSERT_NE(b, nullptr); + ASSERT_EQ(b->null_count(), b->length()); // added `b` all null +} + TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionNoProjection) { auto file_schema = arrow::schema({ MakeField("f0", arrow::int32(), 1), From 9dc15e3bba0bb04e7e559526291befc6a103074b Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 23 Jul 2026 04:18:58 -0700 Subject: [PATCH 09/17] Fix field-ID-blind equality and nested rename; clang-format - Compare paimon field IDs in all no-op/substitution checks (EqualWithFieldIds), so a drop+add of a same-name/same-type field (new ID) is not silently exposed as the new field's value. - Reject a rename inside a list/map struct (matching the top-level PruneDataType behaviour). - Add field-ID-change and ORC round-trip regression tests. - clang-format 20 the touched files. --- .../reader/data_evolution_file_reader.cpp | 9 +- src/paimon/core/io/field_mapping_reader.cpp | 8 +- .../core/io/field_mapping_reader_test.cpp | 58 +++++++++--- .../core/utils/nested_projection_utils.cpp | 93 +++++++++++++++---- .../utils/nested_projection_utils_test.cpp | 47 ++++++---- 5 files changed, 158 insertions(+), 57 deletions(-) diff --git a/src/paimon/common/reader/data_evolution_file_reader.cpp b/src/paimon/common/reader/data_evolution_file_reader.cpp index dbdfec41e..adc7cf7ba 100644 --- a/src/paimon/common/reader/data_evolution_file_reader.cpp +++ b/src/paimon/common/reader/data_evolution_file_reader.cpp @@ -87,11 +87,10 @@ Result DataEvolutionFileReader::NextBatchWithB const auto& sub_array = array_for_each_reader[reader_offsets_[i]]; assert(sub_array->num_fields() > field_offsets_[i]); // Null-fill nested fields added by schema evolution (no-op otherwise). - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr aligned, - NestedProjectionUtils::AlignArrayToReadType(sub_array->field(field_offsets_[i]), - read_schema_->field(i)->type(), - arrow_pool_.get())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr aligned, + NestedProjectionUtils::AlignArrayToReadType( + sub_array->field(field_offsets_[i]), + read_schema_->field(i)->type(), arrow_pool_.get())); target_sub_array_vec.push_back(aligned); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( diff --git a/src/paimon/core/io/field_mapping_reader.cpp b/src/paimon/core/io/field_mapping_reader.cpp index bffba1601..efea20720 100644 --- a/src/paimon/core/io/field_mapping_reader.cpp +++ b/src/paimon/core/io/field_mapping_reader.cpp @@ -219,10 +219,10 @@ Result> FieldMappingReader::CastNonPartitionArrayI arrow::compute::CastOptions::Safe(), arrow_pool_.get())); } PAIMON_ASSIGN_OR_RAISE( - column, non_partition_info_.cast_executors[i]->Cast( - single_column_array, - non_partition_info_.non_partition_read_schema[i].Type(), - arrow_pool_.get())); + column, + non_partition_info_.cast_executors[i]->Cast( + single_column_array, non_partition_info_.non_partition_read_schema[i].Type(), + arrow_pool_.get())); } else { // read and data type may both be string type, but after adapter transform, type may be // dictionary, need reconstruct struct type diff --git a/src/paimon/core/io/field_mapping_reader_test.cpp b/src/paimon/core/io/field_mapping_reader_test.cpp index ce776e852..37f2be4e1 100644 --- a/src/paimon/core/io/field_mapping_reader_test.cpp +++ b/src/paimon/core/io/field_mapping_reader_test.cpp @@ -457,9 +457,9 @@ TEST_F(FieldMappingReaderTest, TestSchemaEvolutionAddedFieldInsideList) { }; auto data_struct = arrow::struct_({id_field("a", arrow::int32(), 10), id_field("b", arrow::utf8(), 11)}); - auto read_struct = arrow::struct_({id_field("a", arrow::int32(), 10), - id_field("b", arrow::utf8(), 11), - id_field("c", arrow::int32(), 12)}); + auto read_struct = + arrow::struct_({id_field("a", arrow::int32(), 10), id_field("b", arrow::utf8(), 11), + id_field("c", arrow::int32(), 12)}); std::vector data_fields = { DataField(100, arrow::field("items", arrow::list(arrow::field("item", data_struct))))}; std::vector read_fields = { @@ -480,27 +480,59 @@ TEST_F(FieldMappingReaderTest, TestSchemaEvolutionAddedFieldInsideList) { ASSERT_OK_AND_ASSIGN(auto mapping, mapping_builder->CreateFieldMapping(data_fields)); auto mock = std::make_unique( data_array, arrow::struct_(data_schema->fields()), /*read_batch_size=*/8); - ASSERT_OK_AND_ASSIGN( - auto reader, FieldMappingReader::Create(read_schema->num_fields(), std::move(mock), - BinaryRow::EmptyRow(), std::move(mapping), - /*skip_map_selected_keys_filter_field_ids=*/{}, - pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, FieldMappingReader::Create( + read_schema->num_fields(), std::move(mock), + BinaryRow::EmptyRow(), std::move(mapping), + /*skip_map_selected_keys_filter_field_ids=*/{}, pool_)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(reader.get())); - auto expect_array = arrow::ipc::internal::json::ArrayFromJSON( - arrow::struct_(read_schema->fields()), R"([ + auto expect_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(read_schema->fields()), R"([ [[[1, "x", null], [2, "y", null]]], [[[3, "z", null]]] ])") - .ValueOrDie(); - auto expected_chunk = - std::make_shared(arrow::ArrayVector({expect_array})); + .ValueOrDie(); + auto expected_chunk = std::make_shared(arrow::ArrayVector({expect_array})); ASSERT_TRUE(result_array->type()->Equals(expected_chunk->type())) << result_array->type()->ToString() << " vs " << expected_chunk->type()->ToString(); ASSERT_TRUE(result_array->Equals(expected_chunk)) << result_array->ToString() << " vs " << expected_chunk->ToString(); } +TEST_F(FieldMappingReaderTest, TestSchemaEvolutionAddedFieldInsideListOrc) { + // ORC round-trip: added field inside a list's struct is null-filled. + auto id_field = [](const std::string& name, const std::shared_ptr& type, + int32_t id) { + return DataField::ConvertDataFieldToArrowField(DataField(id, arrow::field(name, type))); + }; + auto data_struct = + arrow::struct_({id_field("a", arrow::int32(), 10), id_field("b", arrow::int32(), 11)}); + auto read_struct = + arrow::struct_({id_field("a", arrow::int32(), 10), id_field("b", arrow::int32(), 11), + id_field("c", arrow::int32(), 12)}); + std::vector data_fields = { + DataField(100, arrow::field("items", arrow::list(arrow::field("item", data_struct))))}; + std::vector read_fields = { + DataField(100, arrow::field("items", arrow::list(arrow::field("item", read_struct))))}; + auto data_schema = DataField::ConvertDataFieldsToArrowSchema(data_fields); + auto read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields); + + auto data_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(data_schema->fields()), R"([ + [[[1, 2], [3, 4]]], + [[[5, 6]]] + ])") + .ValueOrDie()); + auto expect_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(read_schema->fields()), R"([ + [[[1, 2, null], [3, 4, null]]], + [[[5, 6, null]]] + ])") + .ValueOrDie(); + CheckResult(data_schema, data_array, read_schema, /*predicate=*/nullptr, /*partition_keys=*/{}, + BinaryRow::EmptyRow(), expect_array); +} + TEST_F(FieldMappingReaderTest, TestSchemaEvolutionWithModifyType) { std::vector data_fields = {DataField(0, arrow::field("f0", arrow::utf8())), DataField(1, arrow::field("f1", arrow::float32())), diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp index a5a568098..efdf24782 100644 --- a/src/paimon/core/utils/nested_projection_utils.cpp +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -151,12 +151,44 @@ Result NestedProjectionUtils::HasNestedSubfieldProjectionType( namespace { +// Structural equality that also compares paimon field IDs on STRUCT children, so a +// drop+add of a same-name/same-type field (new ID) is not treated as a no-op. +Result EqualWithFieldIds(const std::shared_ptr& a, + const std::shared_ptr& b) { + if (a->id() != b->id() || a->num_fields() != b->num_fields()) { + return false; + } + if (a->num_fields() == 0) { + return a->Equals(*b); + } + for (int32_t i = 0; i < a->num_fields(); ++i) { + const auto& fa = a->field(i); + const auto& fb = b->field(i); + if (a->id() == arrow::Type::STRUCT) { + if (fa->name() != fb->name()) { + return false; + } + // Compare IDs only when present (a map entry's key/value carry none). + auto id_a = NestedProjectionUtils::GetPaimonFieldId(fa); + auto id_b = NestedProjectionUtils::GetPaimonFieldId(fb); + if (id_a.ok() && id_b.ok() && id_a.value() != id_b.value()) { + return false; + } + } + PAIMON_ASSIGN_OR_RAISE(bool child_equal, EqualWithFieldIds(fa->type(), fb->type())); + if (!child_equal) { + return false; + } + } + return true; +} + /// Whether `read_type` is `data_type` with variant columns replaced by their variant-access -/// projections and nothing else changed. Such a read drops no field, so it is not a partial -/// projection of an enclosing repeated group and may pass through where a real one must fail. -bool IsVariantAccessSubstitution(const std::shared_ptr& read_type, - const std::shared_ptr& data_type) { - if (read_type->Equals(data_type)) { +/// projections and nothing else changed (matching paimon field IDs). +Result IsVariantAccessSubstitution(const std::shared_ptr& read_type, + const std::shared_ptr& data_type) { + PAIMON_ASSIGN_OR_RAISE(bool equal, EqualWithFieldIds(read_type, data_type)); + if (equal) { return true; } if (VariantAccessUtils::IsVariantAccessType(read_type) && @@ -170,12 +202,21 @@ bool IsVariantAccessSubstitution(const std::shared_ptr& read_ty for (int32_t i = 0; i < read_type->num_fields(); ++i) { const std::shared_ptr& read_child = read_type->field(i); const std::shared_ptr& data_child = data_type->field(i); - // LIST and MAP name their children by format convention, so only STRUCT is matched - // by name. - if (read_type->id() == arrow::Type::STRUCT && read_child->name() != data_child->name()) { - return false; + // LIST and MAP name their children by format convention, so only STRUCT is + // matched by name and field ID. + if (read_type->id() == arrow::Type::STRUCT) { + if (read_child->name() != data_child->name()) { + return false; + } + auto id_r = NestedProjectionUtils::GetPaimonFieldId(read_child); + auto id_d = NestedProjectionUtils::GetPaimonFieldId(data_child); + if (id_r.ok() && id_d.ok() && id_r.value() != id_d.value()) { + return false; + } } - if (!IsVariantAccessSubstitution(read_child->type(), data_child->type())) { + PAIMON_ASSIGN_OR_RAISE(bool sub, + IsVariantAccessSubstitution(read_child->type(), data_child->type())); + if (!sub) { return false; } } @@ -188,10 +229,12 @@ bool IsVariantAccessSubstitution(const std::shared_ptr& read_ty Result> PruneRepeatedItemType( const std::shared_ptr& read_type, const std::shared_ptr& data_type, const char* container) { - if (read_type->Equals(data_type)) { + PAIMON_ASSIGN_OR_RAISE(bool same, EqualWithFieldIds(read_type, data_type)); + if (same) { return data_type; } - if (IsVariantAccessSubstitution(read_type, data_type)) { + PAIMON_ASSIGN_OR_RAISE(bool substitution, IsVariantAccessSubstitution(read_type, data_type)); + if (substitution) { return read_type; } if (read_type->id() != data_type->id()) { @@ -221,6 +264,12 @@ Result> PruneRepeatedItemType( "target {}", container, data_type->ToString(), read_type->ToString())); } + if (read_child->name() != data_child->name()) { + return Status::Invalid(fmt::format( + "PruneDataType does not support renaming inside {}: field id {} read '{}' " + "vs data '{}'", + container, data_child_id, read_child->name(), data_child->name())); + } PAIMON_ASSIGN_OR_RAISE( std::shared_ptr item_child_type, PruneRepeatedItemType(read_child->type(), data_child->type(), container)); @@ -257,8 +306,9 @@ Result> PruneRepeatedItemType( Result>> NestedProjectionUtils::PruneDataType( const std::shared_ptr& read_type, const std::shared_ptr& data_type) { - // Identical types need no pruning. - if (read_type->Equals(data_type)) { + // Identical types (including paimon field IDs) need no pruning. + PAIMON_ASSIGN_OR_RAISE(bool same, EqualWithFieldIds(read_type, data_type)); + if (same) { return std::optional>(data_type); } @@ -303,7 +353,9 @@ Result>> NestedProjectionUtils::P return std::optional>(arrow::struct_(pruned_fields)); } case arrow::Type::LIST: { - if (IsVariantAccessSubstitution(read_type, data_type)) { + PAIMON_ASSIGN_OR_RAISE(bool list_substitution, + IsVariantAccessSubstitution(read_type, data_type)); + if (list_substitution) { return std::optional>(read_type); } // Added fields (schema evolution) are allowed; dropped fields still fail. @@ -314,7 +366,9 @@ Result>> NestedProjectionUtils::P arrow::list(data_type->field(0)->WithType(item))); } case arrow::Type::MAP: { - if (IsVariantAccessSubstitution(read_type, data_type)) { + PAIMON_ASSIGN_OR_RAISE(bool map_substitution, + IsVariantAccessSubstitution(read_type, data_type)); + if (map_substitution) { return std::optional>(read_type); } auto read_map = std::static_pointer_cast(read_type); @@ -530,9 +584,10 @@ Result> NestedProjectionUtils::FilterMapArrayBySel } Result> NestedProjectionUtils::AlignArrayToReadType( - const std::shared_ptr& array, - const std::shared_ptr& read_type, arrow::MemoryPool* pool) { - if (array->type()->Equals(*read_type)) { + const std::shared_ptr& array, const std::shared_ptr& read_type, + arrow::MemoryPool* pool) { + PAIMON_ASSIGN_OR_RAISE(bool same, EqualWithFieldIds(array->type(), read_type)); + if (same) { return array; } // Leaves keep their encoding (e.g. ORC dictionary); only STRUCT/LIST/MAP are diff --git a/src/paimon/core/utils/nested_projection_utils_test.cpp b/src/paimon/core/utils/nested_projection_utils_test.cpp index d79e9510b..d4ad3932c 100644 --- a/src/paimon/core/utils/nested_projection_utils_test.cpp +++ b/src/paimon/core/utils/nested_projection_utils_test.cpp @@ -224,14 +224,12 @@ TEST(NestedProjectionUtilsTest, PruneDataTypeListDroppingSiblingOfVariantStillFa } TEST(NestedProjectionUtilsTest, PruneDataTypeListStructSchemaEvolutionAddedField) { - // Schema evolution: a field (id=12) was added inside the list's struct, so - // read is a superset of the file (nothing dropped). Should read the file's - // struct and return it; the added field is null-filled downstream. + // Added field (id=12) inside the list's struct: return the file struct. auto data_inner = arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)}); - auto read_inner = arrow::struct_({MakeField("a", arrow::int32(), 10), - MakeField("b", arrow::utf8(), 11), - MakeField("c", arrow::int32(), 12)}); + auto read_inner = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11), + MakeField("c", arrow::int32(), 12)}); auto data_type = arrow::list(arrow::field("item", data_inner)); auto read_type = arrow::list(arrow::field("item", read_inner)); @@ -244,9 +242,9 @@ TEST(NestedProjectionUtilsTest, PruneDataTypeMapStructSchemaEvolutionAddedField) // Added field inside a MAP value. auto data_inner = arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)}); - auto read_inner = arrow::struct_({MakeField("a", arrow::int32(), 10), - MakeField("b", arrow::utf8(), 11), - MakeField("c", arrow::int32(), 12)}); + auto read_inner = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11), + MakeField("c", arrow::int32(), 12)}); auto data_type = arrow::map(arrow::utf8(), data_inner); auto read_type = arrow::map(arrow::utf8(), read_inner); @@ -259,8 +257,8 @@ TEST(NestedProjectionUtilsTest, PruneDataTypeListStructDropAndAddStillFails) { // Dropping a file field (b) is a real partial projection -- keep failing. auto data_inner = arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)}); - auto read_inner = arrow::struct_( - {MakeField("a", arrow::int32(), 10), MakeField("c", arrow::int32(), 12)}); + auto read_inner = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("c", arrow::int32(), 12)}); auto data_type = arrow::list(arrow::field("item", data_inner)); auto read_type = arrow::list(arrow::field("item", read_inner)); @@ -291,9 +289,9 @@ TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeNullFillsAddedListStructFiel arrow::ListArray::FromArrays(*offsets, *struct_arr, pool).ValueOrDie(); // read type adds c:int(12) inside the struct. - auto read_struct = arrow::struct_({MakeField("a", arrow::int32(), 10), - MakeField("b", arrow::utf8(), 11), - MakeField("c", arrow::int32(), 12)}); + auto read_struct = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11), + MakeField("c", arrow::int32(), 12)}); auto read_type = arrow::list(arrow::field("item", read_struct)); ASSERT_OK_AND_ASSIGN(std::shared_ptr aligned, @@ -310,8 +308,7 @@ TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeNullFillsAddedListStructFiel } TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeKeepsDictionaryLeafAndNullFills) { - // ORC lazy decoding returns nested strings as dictionary; a dict-encoded `a` - // must be kept (not rejected) while the added `b` is null-filled. + // ORC lazy decoding returns nested strings as dictionary: keep it, null-fill b. auto* pool = arrow::default_memory_pool(); arrow::StringBuilder vb(pool); ASSERT_TRUE(vb.AppendValues({"x", "y"}).ok()); @@ -339,6 +336,24 @@ TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeKeepsDictionaryLeafAndNullFi ASSERT_EQ(b->null_count(), b->length()); // added `b` all null } +TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeFieldIdChangeNullFillsNotLeak) { + // a(id=10) replaced by a(id=11), same name/type: new field must read null, not leak. + auto* pool = arrow::default_memory_pool(); + arrow::Int32Builder ab(pool); + ASSERT_TRUE(ab.AppendValues({42}).ok()); + std::shared_ptr a_arr; + ASSERT_TRUE(ab.Finish(&a_arr).ok()); + auto data_struct = arrow::struct_({MakeField("a", arrow::int32(), 10)}); + auto struct_arr = arrow::StructArray::Make({a_arr}, data_struct->fields()).ValueOrDie(); + auto read_type = arrow::struct_({MakeField("a", arrow::int32(), 11)}); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr aligned, + NestedProjectionUtils::AlignArrayToReadType(struct_arr, read_type, pool)); + auto a_out = std::static_pointer_cast(aligned)->GetFieldByName("a"); + ASSERT_NE(a_out, nullptr); + ASSERT_EQ(a_out->null_count(), a_out->length()); +} + TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionNoProjection) { auto file_schema = arrow::schema({ MakeField("f0", arrow::int32(), 1), From 94920117e0121bd6e66f3ce0b2562cbb2c5f1e5a Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 23 Jul 2026 04:31:23 -0700 Subject: [PATCH 10/17] Address review: drop redundant DE-reader alignment; JSON dict test Each column-split file is already aligned to its read schema by its FieldMappingReader, so the data-evolution reader assembly no longer re-aligns. Build the dictionary-leaf test array from JSON. --- .../common/reader/data_evolution_file_reader.cpp | 9 ++------- .../core/utils/nested_projection_utils_test.cpp | 11 ++--------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/src/paimon/common/reader/data_evolution_file_reader.cpp b/src/paimon/common/reader/data_evolution_file_reader.cpp index adc7cf7ba..a4e49ecbb 100644 --- a/src/paimon/common/reader/data_evolution_file_reader.cpp +++ b/src/paimon/common/reader/data_evolution_file_reader.cpp @@ -25,7 +25,6 @@ #include "paimon/common/reader/reader_utils.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" -#include "paimon/core/utils/nested_projection_utils.h" namespace paimon { @@ -86,12 +85,8 @@ Result DataEvolutionFileReader::NextBatchWithB } const auto& sub_array = array_for_each_reader[reader_offsets_[i]]; assert(sub_array->num_fields() > field_offsets_[i]); - // Null-fill nested fields added by schema evolution (no-op otherwise). - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr aligned, - NestedProjectionUtils::AlignArrayToReadType( - sub_array->field(field_offsets_[i]), - read_schema_->field(i)->type(), arrow_pool_.get())); - target_sub_array_vec.push_back(aligned); + // Each file is already aligned to its read schema by its FieldMappingReader. + target_sub_array_vec.push_back(sub_array->field(field_offsets_[i])); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( std::shared_ptr target_array, diff --git a/src/paimon/core/utils/nested_projection_utils_test.cpp b/src/paimon/core/utils/nested_projection_utils_test.cpp index d4ad3932c..ff2faa055 100644 --- a/src/paimon/core/utils/nested_projection_utils_test.cpp +++ b/src/paimon/core/utils/nested_projection_utils_test.cpp @@ -22,6 +22,7 @@ #include "arrow/array/builder_dict.h" #include "arrow/array/builder_nested.h" #include "arrow/array/builder_primitive.h" +#include "arrow/ipc/json_simple.h" #include "arrow/memory_pool.h" #include "arrow/type.h" #include "gtest/gtest.h" @@ -310,17 +311,9 @@ TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeNullFillsAddedListStructFiel TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeKeepsDictionaryLeafAndNullFills) { // ORC lazy decoding returns nested strings as dictionary: keep it, null-fill b. auto* pool = arrow::default_memory_pool(); - arrow::StringBuilder vb(pool); - ASSERT_TRUE(vb.AppendValues({"x", "y"}).ok()); - std::shared_ptr dict_values; - ASSERT_TRUE(vb.Finish(&dict_values).ok()); - arrow::Int32Builder ib(pool); - ASSERT_TRUE(ib.AppendValues({0, 1, 0}).ok()); - std::shared_ptr dict_indices; - ASSERT_TRUE(ib.Finish(&dict_indices).ok()); auto dict_type = arrow::dictionary(arrow::int32(), arrow::utf8()); auto a_dict = - arrow::DictionaryArray::FromArrays(dict_type, dict_indices, dict_values).ValueOrDie(); + arrow::ipc::internal::json::ArrayFromJSON(dict_type, R"(["x", "y", "x"])").ValueOrDie(); auto data_struct = arrow::struct_({MakeField("a", dict_type, 10)}); auto struct_arr = arrow::StructArray::Make({a_dict}, data_struct->fields()).ValueOrDie(); From 1740dd52848d347db6aa72197234b1996fef2433 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 23 Jul 2026 05:24:49 -0700 Subject: [PATCH 11/17] Add list/map schema-evolution inte test; match nested fields by name The integration test surfaced that the parquet reader drops nested field-id metadata, so AlignArrayToReadType's id-based child matching errored. Match nested struct children by name (preserved by all readers), requiring agreement only when both sides carry an ID, so a drop+add of a same-name field is still not silently mapped. Add TestSchemaEvolutionAddFieldInsideListAndMap covering an added field inside a list and a map end-to-end on parquet and orc. --- .../core/utils/nested_projection_utils.cpp | 17 ++-- test/inte/write_and_read_inte_test.cpp | 85 +++++++++++++++++++ 2 files changed, 97 insertions(+), 5 deletions(-) diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp index efdf24782..f5e506d6d 100644 --- a/src/paimon/core/utils/nested_projection_utils.cpp +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -606,14 +606,21 @@ Result> NestedProjectionUtils::AlignArrayToReadTyp out_fields.reserve(read_type->num_fields()); children.reserve(read_type->num_fields()); for (const auto& read_field : read_type->fields()) { - PAIMON_ASSIGN_OR_RAISE(int32_t read_id, GetPaimonFieldId(read_field)); + // Match by name (parquet drops nested field-id metadata); if both + // carry IDs they must agree, so a drop+add same-name field won't match. + auto read_id = GetPaimonFieldId(read_field); int32_t match = -1; for (int32_t j = 0; j < array_type->num_fields(); j++) { - PAIMON_ASSIGN_OR_RAISE(int32_t data_id, GetPaimonFieldId(array_type->field(j))); - if (data_id == read_id) { - match = j; - break; + const auto& array_field = array_type->field(j); + if (array_field->name() != read_field->name()) { + continue; + } + auto data_id = GetPaimonFieldId(array_field); + if (read_id.ok() && data_id.ok() && read_id.value() != data_id.value()) { + continue; } + match = j; + break; } if (match >= 0) { auto child = arrow::MakeArray(data->child_data[match]); diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 80b9d099c..3c6b8ed0c 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -422,6 +422,91 @@ TEST_P(WriteAndReadInteTest, TestNestedType) { ASSERT_TRUE(success); } +TEST_P(WriteAndReadInteTest, TestSchemaEvolutionAddFieldInsideListAndMap) { + auto [file_format, file_system] = GetParam(); + if (file_format == "lance" || file_format == "avro") { + return; + } + auto list_struct = + arrow::struct_({arrow::field("a", arrow::int32()), arrow::field("b", arrow::utf8())}); + auto map_value = arrow::struct_({arrow::field("m1", arrow::int32())}); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("items", arrow::list(arrow::field("item", list_struct))), + arrow::field("props", arrow::map(arrow::utf8(), map_value)), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, + options, /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch, TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [[10, "x"], [20, "y"]], [["k1", [100]]]], + [2, [[30, "z"]], [["k2", [200]]]] + ])", + /*partition_map=*/{}, + /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + SchemaManager schema_manager(dir_->GetFileSystem(), table_path); + ASSERT_OK_AND_ASSIGN(auto schema_v0, schema_manager.ReadSchema(0)); + std::vector fields_v0 = schema_v0->Fields(); + int32_t next_id = schema_v0->HighestFieldId(); + + auto items_field = fields_v0[1].ArrowField(); + auto items_list = arrow::internal::checked_pointer_cast(items_field->type()); + auto items_struct = + arrow::internal::checked_pointer_cast(items_list->value_type()); + auto c_field = DataField::ConvertDataFieldToArrowField( + DataField(++next_id, arrow::field("c", arrow::int32()))); + auto new_items_type = arrow::list(items_list->value_field()->WithType( + arrow::struct_({items_struct->field(0), items_struct->field(1), c_field}))); + + auto props_field = fields_v0[2].ArrowField(); + auto props_map = arrow::internal::checked_pointer_cast(props_field->type()); + auto props_value = + arrow::internal::checked_pointer_cast(props_map->item_type()); + auto m2_field = DataField::ConvertDataFieldToArrowField( + DataField(++next_id, arrow::field("m2", arrow::int32()))); + auto new_props_type = arrow::map( + props_map->key_type(), + props_map->item_field()->WithType(arrow::struct_({props_value->field(0), m2_field}))); + + std::vector fields_v1 = fields_v0; + fields_v1[1] = DataField(fields_v0[1].Id(), items_field->WithType(new_items_type)); + fields_v1[2] = DataField(fields_v0[2].Id(), props_field->WithType(new_props_type)); + ASSERT_OK(WriteNextSchema(fields_v1, next_id, options)); + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("items", arrow::list(arrow::field( + "item", arrow::struct_({arrow::field("a", arrow::int32()), + arrow::field("b", arrow::utf8()), + arrow::field("c", arrow::int32())})))), + arrow::field("props", arrow::map(arrow::utf8(), + arrow::struct_({arrow::field("m1", arrow::int32()), + arrow::field("m2", arrow::int32())}))), + }); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(expected_type, splits, + R"([ + [0, 1, [[10, "x", null], [20, "y", null]], [["k1", [100, null]]]], + [0, 2, [[30, "z", null]], [["k2", [200, null]]]] + ])")); + ASSERT_TRUE(success); +} + TEST_P(WriteAndReadInteTest, TestAppendExternalPath) { arrow::FieldVector fields = { arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), From 55b47dcd213b0fee4a8bfe37b414a66f46c760a4 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 23 Jul 2026 06:06:16 -0700 Subject: [PATCH 12/17] Cast leaves to the read type and compare nested nullability AlignArrayToReadType now produces exactly the read type: STRUCT/LIST/MAP are rebuilt with read-side types and nullability, and a leaf is cast to the read type. This fixes two ORC lazy-decoding failures -- a dictionary that no longer equals logical string, and a nested NOT NULL drop that left old/new files with mismatched types (arrow::ChunkedArray::Make failed). EqualWithFieldIds now compares Field::nullable(). Cover dictionary, read-side nullability, and enable ORC lazy decoding in the inte test. --- .../core/utils/nested_projection_utils.cpp | 40 +++++++------------ .../utils/nested_projection_utils_test.cpp | 30 ++++++++++---- test/inte/write_and_read_inte_test.cpp | 8 +++- 3 files changed, 44 insertions(+), 34 deletions(-) diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp index f5e506d6d..55e55b769 100644 --- a/src/paimon/core/utils/nested_projection_utils.cpp +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -27,11 +27,13 @@ #include "arrow/array/builder_primitive.h" #include "arrow/array/concatenate.h" #include "arrow/array/util.h" +#include "arrow/compute/cast.h" #include "arrow/type.h" #include "fmt/format.h" #include "paimon/common/data/variant/variant_access_utils.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/core/casting/casting_utils.h" #include "paimon/status.h" namespace paimon { @@ -164,6 +166,9 @@ Result EqualWithFieldIds(const std::shared_ptr& a, for (int32_t i = 0; i < a->num_fields(); ++i) { const auto& fa = a->field(i); const auto& fb = b->field(i); + if (fa->nullable() != fb->nullable()) { + return false; + } if (a->id() == arrow::Type::STRUCT) { if (fa->name() != fb->name()) { return false; @@ -590,8 +595,9 @@ Result> NestedProjectionUtils::AlignArrayToReadTyp if (same) { return array; } - // Leaves keep their encoding (e.g. ORC dictionary); only STRUCT/LIST/MAP are - // reshaped to null-fill added fields, with the output type built from actual children. + // Produce exactly `read_type` so every file yields the same output type: reshape + // STRUCT/LIST/MAP to null-fill added fields with read-side types/nullability, and + // cast a leaf (which decodes an ORC dictionary and widens e.g. large_string). const auto& data = array->data(); switch (read_type->id()) { case arrow::Type::STRUCT: { @@ -601,9 +607,7 @@ Result> NestedProjectionUtils::AlignArrayToReadTyp read_type->ToString())); } const auto& array_type = array->type(); - arrow::FieldVector out_fields; std::vector> children; - out_fields.reserve(read_type->num_fields()); children.reserve(read_type->num_fields()); for (const auto& read_field : read_type->fields()) { // Match by name (parquet drops nested field-id metadata); if both @@ -627,18 +631,16 @@ Result> NestedProjectionUtils::AlignArrayToReadTyp PAIMON_ASSIGN_OR_RAISE(child, AlignArrayToReadType(child, read_field->type(), pool)); children.push_back(child->data()); - out_fields.push_back(read_field->WithType(child->type())); } else { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( std::shared_ptr null_child, arrow::MakeArrayOfNull(read_field->type(), data->offset + data->length, pool)); children.push_back(null_child->data()); - out_fields.push_back(read_field); } } auto new_data = data->Copy(); - new_data->type = arrow::struct_(out_fields); + new_data->type = read_type; new_data->child_data = std::move(children); return arrow::MakeArray(new_data); } @@ -649,12 +651,11 @@ Result> NestedProjectionUtils::AlignArrayToReadTyp read_type->ToString())); } auto read_list = std::static_pointer_cast(read_type); - auto data_list = std::static_pointer_cast(array->type()); auto values = arrow::MakeArray(data->child_data[0]); PAIMON_ASSIGN_OR_RAISE(values, AlignArrayToReadType(values, read_list->value_type(), pool)); auto new_data = data->Copy(); - new_data->type = arrow::list(data_list->value_field()->WithType(values->type())); + new_data->type = read_type; new_data->child_data = {values->data()}; return arrow::MakeArray(new_data); } @@ -665,34 +666,23 @@ Result> NestedProjectionUtils::AlignArrayToReadTyp read_type->ToString())); } auto read_map = std::static_pointer_cast(read_type); - auto data_map = std::static_pointer_cast(array->type()); const auto& entries_data = data->child_data[0]; auto key = arrow::MakeArray(entries_data->child_data[0]); auto value = arrow::MakeArray(entries_data->child_data[1]); PAIMON_ASSIGN_OR_RAISE(key, AlignArrayToReadType(key, read_map->key_type(), pool)); PAIMON_ASSIGN_OR_RAISE(value, AlignArrayToReadType(value, read_map->item_type(), pool)); - auto key_field = data_map->key_field()->WithType(key->type()); - auto item_field = data_map->item_field()->WithType(value->type()); auto new_entries = entries_data->Copy(); - new_entries->type = arrow::struct_({key_field, item_field}); + new_entries->type = arrow::struct_({read_map->key_field(), read_map->item_field()}); new_entries->child_data = {key->data(), value->data()}; auto new_data = data->Copy(); - new_data->type = - std::make_shared(key_field, item_field, data_map->keys_sorted()); + new_data->type = read_type; new_data->child_data = {new_entries}; return arrow::MakeArray(new_data); } default: - // A dictionary of the read value type (ORC lazy decoding) is kept; - // any other leaf mismatch is unsupported nested type evolution. - if (array->type()->id() == arrow::Type::DICTIONARY && - std::static_pointer_cast(array->type()) - ->value_type() - ->Equals(*read_type)) { - return array; - } - return Status::Invalid(fmt::format("AlignArrayToReadType cannot reconcile {} to {}", - array->type()->ToString(), read_type->ToString())); + // Leaf: cast to the read type (decodes an ORC dictionary, widens + // large_string to string, etc.). + return CastingUtils::Cast(array, read_type, arrow::compute::CastOptions::Safe(), pool); } } diff --git a/src/paimon/core/utils/nested_projection_utils_test.cpp b/src/paimon/core/utils/nested_projection_utils_test.cpp index ff2faa055..0d2731eea 100644 --- a/src/paimon/core/utils/nested_projection_utils_test.cpp +++ b/src/paimon/core/utils/nested_projection_utils_test.cpp @@ -308,10 +308,10 @@ TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeNullFillsAddedListStructFiel ASSERT_EQ(a_col->Value(2), 3); } -TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeKeepsDictionaryLeafAndNullFills) { - // ORC lazy decoding returns nested strings as dictionary: keep it, null-fill b. +TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeDecodesDictionaryLeafAndNullFills) { + // ORC lazy decoding returns dictionary; decode/cast to string. auto* pool = arrow::default_memory_pool(); - auto dict_type = arrow::dictionary(arrow::int32(), arrow::utf8()); + auto dict_type = arrow::dictionary(arrow::int64(), arrow::large_utf8()); auto a_dict = arrow::ipc::internal::json::ArrayFromJSON(dict_type, R"(["x", "y", "x"])").ValueOrDie(); auto data_struct = arrow::struct_({MakeField("a", dict_type, 10)}); @@ -321,12 +321,28 @@ TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeKeepsDictionaryLeafAndNullFi arrow::struct_({MakeField("a", arrow::utf8(), 10), MakeField("b", arrow::int32(), 11)}); ASSERT_OK_AND_ASSIGN(std::shared_ptr aligned, NestedProjectionUtils::AlignArrayToReadType(struct_arr, read_type, pool)); + ASSERT_TRUE(aligned->type()->Equals(*read_type)) << aligned->type()->ToString(); auto out = std::static_pointer_cast(aligned); - ASSERT_EQ(out->num_fields(), 2); - ASSERT_EQ(out->field(0)->type_id(), arrow::Type::DICTIONARY); // `a` kept dict-encoded + ASSERT_EQ(std::static_pointer_cast(out->GetFieldByName("a"))->GetString(0), + "x"); auto b = out->GetFieldByName("b"); - ASSERT_NE(b, nullptr); - ASSERT_EQ(b->null_count(), b->length()); // added `b` all null + ASSERT_EQ(b->null_count(), b->length()); +} + +TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeAppliesReadNullability) { + auto* pool = arrow::default_memory_pool(); + arrow::Int32Builder ab(pool); + ASSERT_TRUE(ab.AppendValues({1, 2}).ok()); + std::shared_ptr a_arr; + ASSERT_TRUE(ab.Finish(&a_arr).ok()); + auto data_field = DataField::ConvertDataFieldToArrowField( + DataField(10, arrow::field("a", arrow::int32(), /*nullable=*/false))); + auto struct_arr = arrow::StructArray::Make({a_arr}, {data_field}).ValueOrDie(); + auto read_type = arrow::struct_({MakeField("a", arrow::int32(), 10)}); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr aligned, + NestedProjectionUtils::AlignArrayToReadType(struct_arr, read_type, pool)); + ASSERT_TRUE(aligned->type()->field(0)->nullable()); } TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeFieldIdChangeNullFillsNotLeak) { diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 3c6b8ed0c..83fa39f43 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -436,9 +436,13 @@ TEST_P(WriteAndReadInteTest, TestSchemaEvolutionAddFieldInsideListAndMap) { arrow::field("props", arrow::map(arrow::utf8(), map_value)), }; std::map options = { - {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, - {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, file_system}, + // Exercise ORC lazy decoding, which returns nested strings as dictionaries. + {"orc.read.enable-lazy-decoding", "true"}, }; if (file_system == "jindo") { options = AddOptionsForJindo(options); From ee14e41d363483d5d4d8fd3edfdbcb44b21d7790 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 23 Jul 2026 06:08:45 -0700 Subject: [PATCH 13/17] Trim comment --- src/paimon/core/utils/nested_projection_utils.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp index 55e55b769..7317ffc89 100644 --- a/src/paimon/core/utils/nested_projection_utils.cpp +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -595,9 +595,8 @@ Result> NestedProjectionUtils::AlignArrayToReadTyp if (same) { return array; } - // Produce exactly `read_type` so every file yields the same output type: reshape - // STRUCT/LIST/MAP to null-fill added fields with read-side types/nullability, and - // cast a leaf (which decodes an ORC dictionary and widens e.g. large_string). + // Produce exactly `read_type` so every file yields the same output type: rebuild + // STRUCT/LIST/MAP with read-side types/nullability and cast a leaf (decodes dict). const auto& data = array->data(); switch (read_type->id()) { case arrow::Type::STRUCT: { From b8ad6793825c0d46f24abe6a395db06c6c5afd93 Mon Sep 17 00:00:00 2001 From: "xiaohongbo.xhb" Date: Thu, 23 Jul 2026 07:12:23 -0700 Subject: [PATCH 14/17] Update expected message for map value type-change rejection --- test/inte/write_and_read_inte_test.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 83fa39f43..4bbb98450 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -2549,10 +2549,9 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingStructValueSchemaEvolutionRea fields_with_changed_tag_value[1] = DataField(fields_v0[1].Id(), tag_field->WithType(changed_tag_value_type)); ASSERT_OK(WriteNextSchema(fields_with_changed_tag_value, schema_v0->HighestFieldId(), options)); - ASSERT_NOK_WITH_MSG(read_fields({"tags"}), - "PruneDataType does not support partial projection inside map: src " - "map> vs target " - "map>"); + ASSERT_NOK_WITH_MSG( + read_fields({"tags"}), + "PruneDataType nested item type mismatch inside map: read string vs data int64"); auto profile_field = fields_v0[2].ArrowField(); auto profile_struct = From 00131a8f4222a4491c5f3499cb25250ad38b359b Mon Sep 17 00:00:00 2001 From: "xiaohongbo.xhb" Date: Thu, 23 Jul 2026 07:25:39 -0700 Subject: [PATCH 15/17] Restrict AlignArrayToReadType leaf cast to representation-only changes --- .../core/utils/nested_projection_utils.cpp | 37 +++++++++++++++++-- .../utils/nested_projection_utils_test.cpp | 14 +++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp index 7317ffc89..e4488c50f 100644 --- a/src/paimon/core/utils/nested_projection_utils.cpp +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -588,6 +588,27 @@ Result> NestedProjectionUtils::FilterMapArrayBySel return result_map; } +namespace { +// Strips physical-only differences from a leaf type: an ORC lazy-decoding +// dictionary wrapper and the 32/64-bit offset width of string/binary. Two leaves +// with equal normalized types hold the same logical values. +std::shared_ptr NormalizeLeafRepresentation( + const std::shared_ptr& type) { + auto t = type; + if (t->id() == arrow::Type::DICTIONARY) { + t = std::static_pointer_cast(t)->value_type(); + } + switch (t->id()) { + case arrow::Type::LARGE_STRING: + return arrow::utf8(); + case arrow::Type::LARGE_BINARY: + return arrow::binary(); + default: + return t; + } +} +} // namespace + Result> NestedProjectionUtils::AlignArrayToReadType( const std::shared_ptr& array, const std::shared_ptr& read_type, arrow::MemoryPool* pool) { @@ -678,10 +699,20 @@ Result> NestedProjectionUtils::AlignArrayToReadTyp new_data->child_data = {new_entries}; return arrow::MakeArray(new_data); } - default: - // Leaf: cast to the read type (decodes an ORC dictionary, widens - // large_string to string, etc.). + default: { + // Leaf: only physical-representation differences are valid here (ORC + // dictionary encoding, string/binary offset width). Genuine type + // evolution is handled by FieldMappingReader's cast executors and + // rejected upstream in PruneDataType, so fail anything else. + if (!NormalizeLeafRepresentation(array->type()) + ->Equals(*NormalizeLeafRepresentation(read_type))) { + return Status::Invalid( + fmt::format("AlignArrayToReadType unsupported leaf type change: data {} vs " + "read {}", + array->type()->ToString(), read_type->ToString())); + } return CastingUtils::Cast(array, read_type, arrow::compute::CastOptions::Safe(), pool); + } } } diff --git a/src/paimon/core/utils/nested_projection_utils_test.cpp b/src/paimon/core/utils/nested_projection_utils_test.cpp index 0d2731eea..87211250d 100644 --- a/src/paimon/core/utils/nested_projection_utils_test.cpp +++ b/src/paimon/core/utils/nested_projection_utils_test.cpp @@ -363,6 +363,20 @@ TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeFieldIdChangeNullFillsNotLea ASSERT_EQ(a_out->null_count(), a_out->length()); } +TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeRejectsLeafTypeChange) { + auto* pool = arrow::default_memory_pool(); + arrow::Int32Builder ab(pool); + ASSERT_TRUE(ab.AppendValues({1}).ok()); + std::shared_ptr a_arr; + ASSERT_TRUE(ab.Finish(&a_arr).ok()); + auto data_struct = arrow::struct_({MakeField("a", arrow::int32(), 10)}); + auto struct_arr = arrow::StructArray::Make({a_arr}, data_struct->fields()).ValueOrDie(); + auto read_type = arrow::struct_({MakeField("a", arrow::int64(), 10)}); + + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::AlignArrayToReadType(struct_arr, read_type, pool), + "AlignArrayToReadType unsupported leaf type change"); +} + TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionNoProjection) { auto file_schema = arrow::schema({ MakeField("f0", arrow::int32(), 1), From 0ffc4766f9141560138565b00e983dcce3efc071 Mon Sep 17 00:00:00 2001 From: "xiaohongbo.xhb" Date: Thu, 23 Jul 2026 19:10:59 -0700 Subject: [PATCH 16/17] Only normalize large_string in leaf representation check, not large_binary --- .../core/utils/nested_projection_utils.cpp | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp index e4488c50f..0114cc701 100644 --- a/src/paimon/core/utils/nested_projection_utils.cpp +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -589,23 +589,21 @@ Result> NestedProjectionUtils::FilterMapArrayBySel } namespace { -// Strips physical-only differences from a leaf type: an ORC lazy-decoding -// dictionary wrapper and the 32/64-bit offset width of string/binary. Two leaves -// with equal normalized types hold the same logical values. +// Strips physical-only differences from a leaf type: ORC lazy decoding wraps +// strings in a dictionary and may widen them to large_string. binary is not +// dictionary-encoded and large_binary is blob's real type, so neither is +// normalized. Two leaves with equal normalized types hold the same logical +// values. std::shared_ptr NormalizeLeafRepresentation( const std::shared_ptr& type) { auto t = type; if (t->id() == arrow::Type::DICTIONARY) { t = std::static_pointer_cast(t)->value_type(); } - switch (t->id()) { - case arrow::Type::LARGE_STRING: - return arrow::utf8(); - case arrow::Type::LARGE_BINARY: - return arrow::binary(); - default: - return t; + if (t->id() == arrow::Type::LARGE_STRING) { + return arrow::utf8(); } + return t; } } // namespace From 2e70b7d38475d7628536e117487037e122076382 Mon Sep 17 00:00:00 2001 From: "xiaohongbo.xhb" Date: Thu, 23 Jul 2026 21:14:10 -0700 Subject: [PATCH 17/17] Add nested large_binary blob alignment test --- .../utils/nested_projection_utils_test.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/paimon/core/utils/nested_projection_utils_test.cpp b/src/paimon/core/utils/nested_projection_utils_test.cpp index 87211250d..979bd9293 100644 --- a/src/paimon/core/utils/nested_projection_utils_test.cpp +++ b/src/paimon/core/utils/nested_projection_utils_test.cpp @@ -377,6 +377,24 @@ TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeRejectsLeafTypeChange) { "AlignArrayToReadType unsupported leaf type change"); } +TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeKeepsNestedLargeBinaryBlob) { + auto* pool = arrow::default_memory_pool(); + arrow::LargeBinaryBuilder blobb(pool); + ASSERT_TRUE(blobb.AppendValues({"a", "b"}).ok()); + std::shared_ptr blob; + ASSERT_TRUE(blobb.Finish(&blob).ok()); + auto data_struct = arrow::struct_({MakeField("blob", arrow::large_binary(), 10)}); + auto struct_arr = arrow::StructArray::Make({blob}, data_struct->fields()).ValueOrDie(); + auto read_type = arrow::struct_( + {MakeField("blob", arrow::large_binary(), 10), MakeField("c", arrow::int32(), 11)}); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr aligned, + NestedProjectionUtils::AlignArrayToReadType(struct_arr, read_type, pool)); + auto blob_out = std::static_pointer_cast(aligned)->GetFieldByName("blob"); + ASSERT_EQ(blob_out->type_id(), arrow::Type::LARGE_BINARY); + ASSERT_TRUE(blob_out->Equals(*blob)); +} + TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionNoProjection) { auto file_schema = arrow::schema({ MakeField("f0", arrow::int32(), 1),