From 8944a254dd2f2cd70e490245f764e1c1fc32950d Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Wed, 8 Jul 2026 10:58:01 +0800 Subject: [PATCH 01/23] Fix interfaces.h hunk: correct count + trailing context --- cmake_modules/arrow.diff | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake_modules/arrow.diff b/cmake_modules/arrow.diff index ae7752097..70d33c7b5 100644 --- a/cmake_modules/arrow.diff +++ b/cmake_modules/arrow.diff @@ -431,9 +431,10 @@ diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.c diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h --- a/cpp/src/arrow/io/interfaces.h +++ b/cpp/src/arrow/io/interfaces.h -@@ -211,7 +211,7 @@ +@@ -210,5 +210,5 @@ /// \brief Advance or skip stream indicated number of bytes /// \param[in] nbytes the number to move forward /// \return Status - Status Advance(int64_t nbytes); + virtual Status Advance(int64_t nbytes); + \ No newline at end of file From e0a7b12581a25c3e237dd431a021349f218c9e6e Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Wed, 8 Jul 2026 11:27:57 +0800 Subject: [PATCH 02/23] feat: support page index filter for nested column type --- .../format/parquet/file_reader_wrapper.cpp | 29 ++-- .../page_filtered_row_group_reader.cpp | 164 ++++++++++-------- .../parquet/page_filtered_row_group_reader.h | 31 ++-- .../page_filtered_row_group_reader_test.cpp | 16 +- .../parquet/parquet_file_batch_reader.cpp | 9 +- 5 files changed, 132 insertions(+), 117 deletions(-) diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp index e7d6bf606..66e2e11e4 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper.cpp @@ -29,6 +29,7 @@ #include "paimon/format/parquet/parquet_format_defs.h" #include "paimon/macros.h" #include "parquet/arrow/reader.h" +#include "parquet/arrow/schema.h" #include "parquet/file_reader.h" #include "parquet/metadata.h" #include "parquet/page_index.h" @@ -234,8 +235,9 @@ Result> FileReaderWrapper::NextPageFiltered( PAIMON_ASSIGN_OR_RAISE( current_page_filtered_reader_, PageFilteredRowGroupReader::ReadFilteredRowGroup( - file_reader_->parquet_reader(), target_rg, target_column_indices_, - page_filtered_read_schema_, file_reader_->properties().cache_options(), + file_reader_.get(), file_reader_->parquet_reader(), target_rg, + target_column_indices_, page_filtered_read_schema_, + file_reader_->properties().cache_options(), pre_buffered, page_ranges, max_chunksize, pool_)); current_filtered_row_ranges_ = target_rg.row_ranges; current_filtered_rg_start_ = all_row_group_ranges_[rg_id].first; @@ -343,20 +345,17 @@ Status FileReaderWrapper::BuildPageFilteredSchema(const std::vector& co if (page_filtered_read_schema_) { return Status::OK(); } - std::shared_ptr schema; - PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetSchema(&schema)); - auto parquet_schema = file_reader_->parquet_reader()->metadata()->schema(); + // Use SchemaManifest to build schema with proper nested field structure. + // GetFieldIndices maps leaf column indices to top-level field indices, + // correctly handling nested types (List, Struct, Map). + const auto& manifest = file_reader_->manifest(); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::vector field_indices, + manifest.GetFieldIndices( + std::vector(column_indices.begin(), column_indices.end()))); std::vector> fields; - for (int32_t col_idx : column_indices) { - const std::string& col_name = parquet_schema->Column(col_idx)->name(); - auto field = schema->GetFieldByName(col_name); - if (!field) { - return Status::Invalid(fmt::format( - "PrepareForReading: Parquet column {} ('{}') has no matching Arrow field in " - "file schema", - col_idx, col_name)); - } - fields.push_back(field); + for (int field_idx : field_indices) { + fields.push_back(manifest.schema_fields[field_idx].field); } page_filtered_read_schema_ = arrow::schema(fields); return Status::OK(); diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index 9c87438b8..9d9e41981 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -27,7 +27,9 @@ #include "arrow/util/future.h" #include "fmt/format.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "parquet/arrow/reader.h" #include "parquet/arrow/reader_internal.h" +#include "parquet/arrow/schema.h" #include "parquet/metadata.h" #include "parquet/schema.h" @@ -123,87 +125,37 @@ std::pair PageFilteredRowGroupReader::ComputeCompressedRowRa } Status PageFilteredRowGroupReader::ExecuteSkipReadPattern( - const std::shared_ptr<::parquet::internal::RecordReader>& record_reader, - const RowRanges& ranges, int64_t total_row_count, int32_t row_group_index, - int32_t column_index) { + ::parquet::arrow::ColumnReader* column_reader, const RowRanges& ranges, + int64_t total_row_count, int32_t row_group_index, int32_t field_index) { int64_t current_row = 0; for (const auto& range : ranges.GetRanges()) { if (range.from > current_row) { int64_t to_skip = range.from - current_row; - int64_t skipped = record_reader->SkipRecords(to_skip); + int64_t skipped = column_reader->SkipRecords(to_skip); if (skipped != to_skip) { return Status::Invalid(fmt::format( "PageFilteredRowGroupReader: expected to skip {} records but skipped {} " - "(row_group={}, column={})", - to_skip, skipped, row_group_index, column_index)); + "(row_group={}, field={})", + to_skip, skipped, row_group_index, field_index)); } current_row = range.from; } int64_t to_read = range.Count(); - int64_t read = record_reader->ReadRecords(to_read); + int64_t read = column_reader->ReadRecords(to_read); if (read != to_read) { - return Status::Invalid( - fmt::format("PageFilteredRowGroupReader: expected to read {} records but read {} " - "(row_group={}, column={}, range=[{},{}])", - to_read, read, row_group_index, column_index, range.from, range.to)); + return Status::Invalid(fmt::format( + "PageFilteredRowGroupReader: expected to read {} records but read {} " + "(row_group={}, field={}, range=[{},{}])", + to_read, read, row_group_index, field_index, range.from, range.to)); } current_row += to_read; } if (current_row < total_row_count) { - record_reader->SkipRecords(total_row_count - current_row); + column_reader->SkipRecords(total_row_count - current_row); } return Status::OK(); } -Result> PageFilteredRowGroupReader::ReadFilteredColumn( - const std::shared_ptr<::parquet::RowGroupReader>& row_group_reader, - ::parquet::ParquetFileReader* parquet_reader, - const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, - int32_t row_group_index, int32_t column_index, const RowRanges& row_ranges, - const std::shared_ptr& field, int64_t row_group_row_count, - std::shared_ptr<::arrow::MemoryPool> pool) { - auto file_metadata = parquet_reader->metadata(); - const auto* col_descriptor = file_metadata->schema()->Column(column_index); - - // Try to get OffsetIndex for I/O-level page skipping - RowRanges effective_ranges = row_ranges; - int64_t effective_row_count = row_group_row_count; - - std::shared_ptr<::parquet::OffsetIndex> offset_index; - if (rg_page_index_reader) { - offset_index = rg_page_index_reader->GetOffsetIndex(column_index); - } - - auto page_reader = row_group_reader->GetColumnPageReader(column_index); - - if (offset_index) { - // Set data_page_filter for I/O-level page skipping - page_reader->set_data_page_filter( - MakePageFilter(row_ranges, offset_index, row_group_row_count)); - // Compute compressed RowRanges for the decode-level skip/read pattern - auto [compressed_ranges, compressed_total] = - ComputeCompressedRowRanges(row_ranges, offset_index, row_group_row_count); - effective_ranges = std::move(compressed_ranges); - effective_row_count = compressed_total; - } - - // Create RecordReader - ::parquet::internal::LevelInfo leaf_info = - ::parquet::internal::LevelInfo::ComputeLevelInfo(col_descriptor); - auto record_reader = - ::parquet::internal::RecordReader::Make(col_descriptor, leaf_info, pool.get()); - record_reader->SetPageReader(std::move(page_reader)); - - PAIMON_RETURN_NOT_OK(ExecuteSkipReadPattern( - record_reader, effective_ranges, effective_row_count, row_group_index, column_index)); - - std::shared_ptr chunked_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(::parquet::arrow::TransferColumnData( - record_reader.get(), field, col_descriptor, pool.get(), &chunked_array)); - - return chunked_array; -} - Status PageFilteredRowGroupReader::WaitForPreBuffer( ::parquet::ParquetFileReader* parquet_reader, int32_t row_group_index, const std::vector& column_indices, const ::arrow::io::CacheOptions& cache_options, @@ -228,7 +180,72 @@ Status PageFilteredRowGroupReader::WaitForPreBuffer( return Status::OK(); } +Result> PageFilteredRowGroupReader::ReadFilteredField( + ::parquet::arrow::FileReader* arrow_file_reader, + const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, + int32_t row_group_index, int32_t field_index, + const std::vector& column_indices, const RowRanges& row_ranges, + int64_t row_group_row_count, std::shared_ptr<::arrow::MemoryPool> pool) { + const auto& manifest = arrow_file_reader->manifest(); + const auto& schema_field = manifest.schema_fields[field_index]; + bool is_flat = schema_field.is_leaf(); + + // For flat columns: compute compressed RowRanges if OffsetIndex is available. + // data_page_filter + compressed_ranges enable I/O-level page skipping. + // For nested fields: use original row_ranges, decode-level skipping only. + RowRanges effective_ranges = row_ranges; + int64_t effective_total = row_group_row_count; + if (is_flat && rg_page_index_reader) { + auto offset_index = rg_page_index_reader->GetOffsetIndex(schema_field.column_index); + if (offset_index) { + auto [compressed, total] = + ComputeCompressedRowRanges(row_ranges, offset_index, row_group_row_count); + effective_ranges = std::move(compressed); + effective_total = total; + } + } + + // Factory: set data_page_filter only for flat columns with OffsetIndex + auto factory = [row_group_index, is_flat, &rg_page_index_reader, &row_ranges, + row_group_row_count](int col_idx, ::parquet::ParquetFileReader* reader) + -> ::parquet::arrow::FileColumnIterator* { + auto* iter = new ::parquet::arrow::FileColumnIterator(col_idx, reader, {row_group_index}); + if (is_flat && rg_page_index_reader) { + auto offset_index = rg_page_index_reader->GetOffsetIndex(col_idx); + if (offset_index) { + iter->set_data_page_filter( + MakePageFilter(row_ranges, offset_index, row_group_row_count)); + } + } + return iter; + }; + + // Build reader tree with leaf column filtering + std::unique_ptr<::parquet::arrow::ColumnReader> column_reader; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow_file_reader->GetColumn( + field_index, column_indices, factory, &column_reader)); + + if (!column_reader) { + return std::shared_ptr(); + } + + // Phase 1: Prepare for reading + PAIMON_RETURN_NOT_OK_FROM_ARROW(column_reader->BeginRead(effective_total)); + + // Phase 2: Execute skip/read pattern + PAIMON_RETURN_NOT_OK(ExecuteSkipReadPattern(column_reader.get(), effective_ranges, + effective_total, row_group_index, field_index)); + + // Phase 3: Build the Arrow array (TransferColumnData for leaves + assemble for nested) + std::shared_ptr chunked_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW( + column_reader->BuildArray(effective_total, &chunked_array)); + + return chunked_array; +} + Result> PageFilteredRowGroupReader::ReadFilteredRowGroup( + ::parquet::arrow::FileReader* arrow_file_reader, ::parquet::ParquetFileReader* parquet_reader, const TargetRowGroup& target_row_group, const std::vector& column_indices, const std::shared_ptr& arrow_schema, const ::arrow::io::CacheOptions& cache_options, bool pre_buffered, @@ -248,7 +265,6 @@ Result> PageFilteredRowGroupReader::Re PAIMON_RETURN_NOT_OK(WaitForPreBuffer(parquet_reader, row_group_index, column_indices, cache_options, pre_buffered, page_ranges, pool)); - auto row_group_reader = parquet_reader->RowGroup(row_group_index); auto rg_metadata = parquet_reader->metadata()->RowGroup(row_group_index); int64_t row_group_row_count = rg_metadata->num_rows(); @@ -260,23 +276,27 @@ Result> PageFilteredRowGroupReader::Re rg_page_index_reader = page_index_reader->RowGroup(row_group_index); } - // Read each column with page filtering + // Group leaf column indices by top-level field using SchemaManifest + const auto& manifest = arrow_file_reader->manifest(); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::vector field_indices, + manifest.GetFieldIndices(std::vector(column_indices.begin(), column_indices.end()))); + + // Read each field with page filtering (unified path for flat and nested) std::vector> columns; - columns.reserve(column_indices.size()); + columns.reserve(field_indices.size()); - for (size_t i = 0; i < column_indices.size(); ++i) { + for (int field_idx : field_indices) { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr chunked_array, - ReadFilteredColumn(row_group_reader, parquet_reader, rg_page_index_reader, - row_group_index, column_indices[i], row_ranges, - arrow_schema->field(static_cast(i)), row_group_row_count, - pool)); + ReadFilteredField(arrow_file_reader, rg_page_index_reader, row_group_index, + field_idx, column_indices, row_ranges, row_group_row_count, pool)); - if (chunked_array->length() != expected_rows) { + if (chunked_array && chunked_array->length() != expected_rows) { return Status::Invalid(fmt::format( - "PageFilteredRowGroupReader: column {} produced {} rows but expected {} " + "PageFilteredRowGroupReader: field {} produced {} rows but expected {} " "(row_group={})", - column_indices[i], chunked_array->length(), expected_rows, row_group_index)); + field_idx, chunked_array->length(), expected_rows, row_group_index)); } columns.push_back(std::move(chunked_array)); diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h b/src/paimon/format/parquet/page_filtered_row_group_reader.h index 5092bb5ca..c0ee79731 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.h +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h @@ -28,6 +28,7 @@ #include "arrow/type.h" #include "paimon/format/parquet/row_ranges.h" #include "paimon/result.h" +#include "parquet/arrow/reader.h" #include "parquet/column_reader.h" #include "parquet/file_reader.h" #include "parquet/page_index.h" @@ -44,6 +45,7 @@ class PageFilteredRowGroupReader { ~PageFilteredRowGroupReader() = delete; /// Read a row group with page-level filtering. + /// @param arrow_file_reader The Arrow FileReader for ColumnReader tree creation /// @param parquet_reader The underlying ParquetFileReader /// @param target_row_group Target row group with index and row ranges /// @param column_indices Leaf column indices to read @@ -56,6 +58,7 @@ class PageFilteredRowGroupReader { /// @param max_chunksize Per-batch row cap for the returned reader. /// @return A RecordBatchReader streaming the filtered rows. static Result> ReadFilteredRowGroup( + ::parquet::arrow::FileReader* arrow_file_reader, ::parquet::ParquetFileReader* parquet_reader, const TargetRowGroup& target_row_group, const std::vector& column_indices, const std::shared_ptr& arrow_schema, @@ -86,30 +89,30 @@ class PageFilteredRowGroupReader { const std::vector<::arrow::io::ReadRange>& page_ranges, std::shared_ptr<::arrow::MemoryPool> pool); - /// Execute the skip/read pattern on a RecordReader based on RowRanges. - static Status ExecuteSkipReadPattern( - const std::shared_ptr<::parquet::internal::RecordReader>& record_reader, - const RowRanges& ranges, int64_t total_row_count, int32_t row_group_index, - int32_t column_index); + /// Execute the skip/read pattern on a ColumnReader based on RowRanges. + static Status ExecuteSkipReadPattern(::parquet::arrow::ColumnReader* column_reader, + const RowRanges& ranges, int64_t total_row_count, + int32_t row_group_index, int32_t field_index); /// Create a data_page_filter callback for a column based on RowRanges + OffsetIndex. static std::function MakePageFilter( const RowRanges& row_ranges, const std::shared_ptr<::parquet::OffsetIndex>& offset_index, int64_t row_group_row_count); - /// Read a single column using skip/read pattern driven by RowRanges. - static Result> ReadFilteredColumn( - const std::shared_ptr<::parquet::RowGroupReader>& row_group_reader, - ::parquet::ParquetFileReader* parquet_reader, - const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, - int32_t row_group_index, int32_t column_index, const RowRanges& row_ranges, - const std::shared_ptr& field, int64_t row_group_row_count, - std::shared_ptr<::arrow::MemoryPool> pool); - /// Compute compressed RowRanges after data_page_filter skips non-matching pages. static std::pair ComputeCompressedRowRanges( const RowRanges& original_ranges, const std::shared_ptr<::parquet::OffsetIndex>& offset_index, int64_t row_group_row_count); + + /// Read a field (flat or nested) using ColumnReader tree. + /// For flat columns: sets data_page_filter + uses compressed RowRanges (I/O + decode skipping). + /// For nested fields: decode-level skipping only via SkipRecords/ReadRecords. + static Result> ReadFilteredField( + ::parquet::arrow::FileReader* arrow_file_reader, + const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, + int32_t row_group_index, int32_t field_index, + const std::vector& column_indices, const RowRanges& row_ranges, + int64_t row_group_row_count, std::shared_ptr<::arrow::MemoryPool> pool); }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index d6bb36ceb..44ee8d28e 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -932,10 +932,10 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnRowGroupFilter) { // Should get rows 50-99 = 50 rows ASSERT_TRUE(result); - ASSERT_EQ(50, result->length()); + ASSERT_EQ(30, result->length()); // Build expected result: rows 50-99 from the original data - auto expected = data->Slice(50, 50); + auto expected = data->Slice(70 , 30); ASSERT_TRUE(expected->Equals(result->chunk(0))); } @@ -1052,10 +1052,10 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedListColumnRowGroupFilter) { ReadWithPredicateImpl(file_name, read_schema, predicate, &result); ASSERT_TRUE(result); - ASSERT_EQ(50, result->length()); + ASSERT_EQ(30, result->length()); // Build expected result: rows 50-99 from the original data - auto expected = data->Slice(50, 50); + auto expected = data->Slice(70, 30); ASSERT_TRUE(expected->Equals(result->chunk(0))); } @@ -1080,10 +1080,10 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedMapColumnRowGroupFilter) { ReadWithPredicateImpl(file_name, read_schema, predicate, &result); ASSERT_TRUE(result); - ASSERT_EQ(50, result->length()); + ASSERT_EQ(30, result->length()); // Build expected result: rows 50-99 from the original data - auto expected = data->Slice(50, 50); + auto expected = data->Slice(70, 30); ASSERT_TRUE(expected->Equals(result->chunk(0))); } @@ -1137,10 +1137,10 @@ TEST_F(PageFilteredRowGroupReaderTest, MultipleAdjacentNestedColumns) { ReadWithPredicateImpl(file_name, read_schema, predicate, &result); ASSERT_TRUE(result); - ASSERT_EQ(50, result->length()); + ASSERT_EQ(30, result->length()); // Build expected result: rows 50-99 from the original data - auto expected = data->Slice(50, 50); + auto expected = data->Slice(70, 30); ASSERT_TRUE(expected->Equals(result->chunk(0))); } diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index 43f4c64c8..11edb868d 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -134,13 +134,6 @@ Status ParquetFileBatchReader::SetReadSchema( arrow::ImportSchema(schema)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, reader_->GetSchema()); - bool has_nested_field = false; - for (const auto& field : read_schema->fields()) { - if (ArrowSchemaValidator::IsNestedType(field->type())) { - has_nested_field = true; - break; - } - } // Recursively match read_schema against file_schema by field names. // STRUCT supports sub-field projection; LIST/MAP require exact type match. @@ -178,7 +171,7 @@ Status ParquetFileBatchReader::SetReadSchema( DEFAULT_PARQUET_READ_ENABLE_PAGE_INDEX_FILTER)); // walkaround: page index filter does not support nested fields for now, skip page index // filter if there is any nested field in the schema - if (enable_page_index_filter && !has_nested_field) { + if (enable_page_index_filter) { // Build column name to index map for page-level filtering. // For leaf columns, indices[0] is the correct leaf column index in Parquet. // For nested types (struct/list/map), FlattenSchema produces multiple leaf indices, From 45058ce27a703a3432a888aad759b7bb847f380d Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Wed, 8 Jul 2026 14:11:07 +0800 Subject: [PATCH 03/23] test: add test cases --- .../page_filtered_row_group_reader_test.cpp | 217 ++++++++++-------- 1 file changed, 125 insertions(+), 92 deletions(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index 44ee8d28e..68a6abede 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -44,6 +44,7 @@ #include "paimon/status.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/utils/roaring_bitmap32.h" #include "parquet/arrow/reader.h" #include "parquet/file_reader.h" #include "parquet/properties.h" @@ -873,34 +874,32 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesWithDictionaryEncoding) auto partial_concat = arrow::Concatenate(result_partial->chunks()).ValueOrDie(); ASSERT_TRUE(partial_concat->Equals(expected_struct)); } -/// Helper: build a StructArray with a top-level int32 "id" column and a nested struct column -/// "info" containing two int32 fields: "x" and "y". -/// id[i] = i, info.x[i] = i * 100, info.y[i] = i * 100 + 1, for i in [0, N). -/// -/// Arrow schema: { id: int32, info: struct } -/// Parquet leaf columns: [id (index 0), info.x (index 1), info.y (index 2)] -static std::shared_ptr MakeNestedStructData(int32_t num_rows) { - arrow::Int32Builder id_builder, x_builder, y_builder; +/// Helper: build an Int32Array with sequential values 0..N-1. +static std::shared_ptr MakeIdColumn(int32_t num_rows) { + arrow::Int32Builder id_builder; EXPECT_TRUE(id_builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + id_builder.UnsafeAppend(i); + } + return id_builder.Finish().ValueOrDie(); +} + +/// Helper: build a struct array (without id column). +/// x[i] = i * 100, y[i] = i * 100 + 1, for i in [0, N). +static std::shared_ptr MakeNestedStructData(int32_t num_rows) { + arrow::Int32Builder x_builder, y_builder; EXPECT_TRUE(x_builder.Reserve(num_rows).ok()); EXPECT_TRUE(y_builder.Reserve(num_rows).ok()); for (int32_t i = 0; i < num_rows; ++i) { - id_builder.UnsafeAppend(i); x_builder.UnsafeAppend(i * 100); y_builder.UnsafeAppend(i * 100 + 1); } - auto id_array = id_builder.Finish().ValueOrDie(); auto x_array = x_builder.Finish().ValueOrDie(); auto y_array = y_builder.Finish().ValueOrDie(); auto field_x = arrow::field("x", arrow::int32()); auto field_y = arrow::field("y", arrow::int32()); - auto inner_struct = - arrow::StructArray::Make({x_array, y_array}, {field_x, field_y}).ValueOrDie(); - - auto field_id = arrow::field("id", arrow::int32()); - auto field_info = arrow::field("info", arrow::struct_({field_x, field_y})); - return arrow::StructArray::Make({id_array, inner_struct}, {field_id, field_info}).ValueOrDie(); + return arrow::StructArray::Make({x_array, y_array}, {field_x, field_y}).ValueOrDie(); } /// Test: rowgroup-level filtering on a file with nested struct columns. @@ -916,13 +915,19 @@ static std::shared_ptr MakeNestedStructData(int32_t num_rows /// The read schema requests both "id" and "info" columns. TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnRowGroupFilter) { std::string file_name = dir_->Str() + "/nested_struct_filter.parquet"; - auto data = MakeNestedStructData(100); - WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); auto field_x = arrow::field("x", arrow::int32()); auto field_y = arrow::field("y", arrow::int32()); - auto read_schema = arrow::schema({arrow::field("id", arrow::int32()), - arrow::field("info", arrow::struct_({field_x, field_y}))}); + auto field_id = arrow::field("id", arrow::int32()); + auto field_info = arrow::field("info", arrow::struct_({field_x, field_y})); + + auto id_array = MakeIdColumn(100); + auto info_array = MakeNestedStructData(100); + auto data = arrow::StructArray::Make({id_array, info_array}, {field_id, field_info}) + .ValueOrDie(); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + auto read_schema = arrow::schema({field_id, field_info}); auto predicate = PredicateBuilder::GreaterOrEqual( /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70)); @@ -935,7 +940,7 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnRowGroupFilter) { ASSERT_EQ(30, result->length()); // Build expected result: rows 50-99 from the original data - auto expected = data->Slice(70 , 30); + auto expected = data->Slice(70, 30); ASSERT_TRUE(expected->Equals(result->chunk(0))); } @@ -949,13 +954,18 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnRowGroupFilter) { /// Predicate on "id": id >= 70. TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnOnlyReadIdField) { std::string file_name = dir_->Str() + "/nested_struct_only_nested.parquet"; - auto data = MakeNestedStructData(100); - WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); - auto field_id = arrow::field("id", arrow::int32()); auto field_x = arrow::field("x", arrow::int32()); auto field_y = arrow::field("y", arrow::int32()); + auto field_id = arrow::field("id", arrow::int32()); auto field_info = arrow::field("info", arrow::struct_({field_x, field_y})); + + auto id_array = MakeIdColumn(100); + auto info_array = MakeNestedStructData(100); + auto data = arrow::StructArray::Make({id_array, info_array}, {field_id, field_info}) + .ValueOrDie(); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + // Read "id" column only auto read_schema = arrow::schema({field_id}); @@ -975,19 +985,9 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnOnlyReadIdField) { ASSERT_TRUE(data->field(0)->Slice(70, 30)->Equals(result_struct->field(0))); } -/// Helper: build a StructArray with an int32 "id" column and a list "tags" column. -/// id[i] = i, tags[i] = [i*10, i*10+1], for i in [0, N). -/// -/// Arrow schema: { id: int32, tags: list } -/// Parquet leaf columns: [id (index 0), tags.item (index 1)] -static std::shared_ptr MakeListColumnData(int32_t num_rows) { - arrow::Int32Builder id_builder; - EXPECT_TRUE(id_builder.Reserve(num_rows).ok()); - for (int32_t i = 0; i < num_rows; ++i) { - id_builder.UnsafeAppend(i); - } - auto id_array = id_builder.Finish().ValueOrDie(); - +/// Helper: build a list array (without id column). +/// tags[i] = [i*10, i*10+1], for i in [0, N). +static std::shared_ptr MakeListColumnData(int32_t num_rows) { auto value_builder = std::make_shared(); arrow::ListBuilder list_builder(arrow::default_memory_pool(), value_builder); for (int32_t i = 0; i < num_rows; ++i) { @@ -995,26 +995,12 @@ static std::shared_ptr MakeListColumnData(int32_t num_rows) EXPECT_TRUE(value_builder->Append(i * 10).ok()); EXPECT_TRUE(value_builder->Append(i * 10 + 1).ok()); } - auto list_array = list_builder.Finish().ValueOrDie(); - - auto field_id = arrow::field("id", arrow::int32()); - auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))); - return arrow::StructArray::Make({id_array, list_array}, {field_id, field_tags}).ValueOrDie(); + return list_builder.Finish().ValueOrDie(); } -/// Helper: build a StructArray with an int32 "id" column and a map "props" column. -/// id[i] = i, props[i] = {"k_i": i * 100}, for i in [0, N). -/// -/// Arrow schema: { id: int32, props: map } -/// Parquet leaf columns: [id (index 0), props.key (index 1), props.value (index 2)] -static std::shared_ptr MakeMapColumnData(int32_t num_rows) { - arrow::Int32Builder id_builder; - EXPECT_TRUE(id_builder.Reserve(num_rows).ok()); - for (int32_t i = 0; i < num_rows; ++i) { - id_builder.UnsafeAppend(i); - } - auto id_array = id_builder.Finish().ValueOrDie(); - +/// Helper: build a map array (without id column). +/// props[i] = {"k_i": i * 100}, for i in [0, N). +static std::shared_ptr MakeMapColumnData(int32_t num_rows) { auto key_builder = std::make_shared(); auto value_builder = std::make_shared(); arrow::MapBuilder map_builder(arrow::default_memory_pool(), key_builder, value_builder); @@ -1024,11 +1010,7 @@ static std::shared_ptr MakeMapColumnData(int32_t num_rows) { EXPECT_TRUE(key_builder->Append(key).ok()); EXPECT_TRUE(value_builder->Append(i * 100).ok()); } - auto map_array = map_builder.Finish().ValueOrDie(); - - auto field_id = arrow::field("id", arrow::int32()); - auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32())); - return arrow::StructArray::Make({id_array, map_array}, {field_id, field_props}).ValueOrDie(); + return map_builder.Finish().ValueOrDie(); } /// Test: rowgroup-level filtering on a file with a list column. @@ -1038,12 +1020,17 @@ static std::shared_ptr MakeMapColumnData(int32_t num_rows) { /// Predicate: id >= 70 → row groups 0 skipped, row groups 1 read → 50 rows expected. TEST_F(PageFilteredRowGroupReaderTest, NestedListColumnRowGroupFilter) { std::string file_name = dir_->Str() + "/nested_list_filter.parquet"; - auto data = MakeListColumnData(100); + + auto field_id = arrow::field("id", arrow::int32()); + auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))); + + auto id_array = MakeIdColumn(100); + auto tags_array = MakeListColumnData(100); + auto data = arrow::StructArray::Make({id_array, tags_array}, {field_id, field_tags}) + .ValueOrDie(); WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); - auto read_schema = - arrow::schema({arrow::field("id", arrow::int32()), - arrow::field("tags", arrow::list(arrow::field("item", arrow::int32())))}); + auto read_schema = arrow::schema({field_id, field_tags}); auto predicate = PredicateBuilder::GreaterOrEqual( /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70)); @@ -1066,12 +1053,17 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedListColumnRowGroupFilter) { /// Predicate: id >= 70 → row groups 0 skipped, row groups 1 read → 50 rows expected. TEST_F(PageFilteredRowGroupReaderTest, NestedMapColumnRowGroupFilter) { std::string file_name = dir_->Str() + "/nested_map_filter.parquet"; - auto data = MakeMapColumnData(100); + + auto field_id = arrow::field("id", arrow::int32()); + auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32())); + + auto id_array = MakeIdColumn(100); + auto props_array = MakeMapColumnData(100); + auto data = arrow::StructArray::Make({id_array, props_array}, {field_id, field_props}) + .ValueOrDie(); WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); - auto read_schema = - arrow::schema({arrow::field("id", arrow::int32()), - arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()))}); + auto read_schema = arrow::schema({field_id, field_props}); auto predicate = PredicateBuilder::GreaterOrEqual( /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70)); @@ -1095,35 +1087,16 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedMapColumnRowGroupFilter) { TEST_F(PageFilteredRowGroupReaderTest, MultipleAdjacentNestedColumns) { std::string file_name = dir_->Str() + "/multi_nested.parquet"; - // Build data with id, info (struct), tags (list) - arrow::Int32Builder id_builder, x_builder, y_builder; - ASSERT_TRUE(id_builder.Reserve(100).ok()); - ASSERT_TRUE(x_builder.Reserve(100).ok()); - ASSERT_TRUE(y_builder.Reserve(100).ok()); - auto value_builder = std::make_shared(); - arrow::ListBuilder list_builder(arrow::default_memory_pool(), value_builder); - - for (int32_t i = 0; i < 100; ++i) { - id_builder.UnsafeAppend(i); - x_builder.UnsafeAppend(i * 100); - y_builder.UnsafeAppend(i * 100 + 1); - ASSERT_TRUE(list_builder.Append().ok()); - ASSERT_TRUE(value_builder->Append(i * 10).ok()); - } - auto id_array = id_builder.Finish().ValueOrDie(); - auto x_array = x_builder.Finish().ValueOrDie(); - auto y_array = y_builder.Finish().ValueOrDie(); - auto list_array = list_builder.Finish().ValueOrDie(); - auto field_x = arrow::field("x", arrow::int32()); auto field_y = arrow::field("y", arrow::int32()); - auto inner_struct = - arrow::StructArray::Make({x_array, y_array}, {field_x, field_y}).ValueOrDie(); - auto field_id = arrow::field("id", arrow::int32()); auto field_info = arrow::field("info", arrow::struct_({field_x, field_y})); auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))); - auto data = arrow::StructArray::Make({id_array, inner_struct, list_array}, + + auto id_array = MakeIdColumn(100); + auto info_array = MakeNestedStructData(100); + auto tags_array = MakeListColumnData(100); + auto data = arrow::StructArray::Make({id_array, info_array, tags_array}, {field_id, field_info, field_tags}) .ValueOrDie(); @@ -1144,4 +1117,64 @@ TEST_F(PageFilteredRowGroupReaderTest, MultipleAdjacentNestedColumns) { ASSERT_TRUE(expected->Equals(result->chunk(0))); } +/// Test: predicate pushdown with all nested column types (struct, list, map). +/// +/// Schema: { id: int32, info: struct, +/// tags: list, props: map } +/// 100 rows, 10 rows per page, 50 rows per row group → 2 row groups. +/// Predicate: id in [15, 29] or id in [80, 99] (Between is inclusive). +/// Read schema: full schema (all columns). +/// Page-level filtering (10 rows/page): +/// Between(15, 29) → pages 1-2 (rows 10-29) +/// Between(80, 99) → pages 8-9 (rows 80-99) +/// Total: 40 rows. +TEST_F(PageFilteredRowGroupReaderTest, MultipleNestedColumns) { + std::string file_name = dir_->Str() + "/multi_nested_columns.parquet"; + + auto field_x = arrow::field("x", arrow::int32()); + auto field_y = arrow::field("y", arrow::int32()); + auto field_id = arrow::field("id", arrow::int32()); + auto field_info = arrow::field("info", arrow::struct_({field_x, field_y})); + auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))); + auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32())); + + // Build data with all nested column types using shared helpers + auto id_array = MakeIdColumn(100); + auto info_array = MakeNestedStructData(100); + auto tags_array = MakeListColumnData(100); + auto props_array = MakeMapColumnData(100); + auto data = arrow::StructArray::Make({id_array, info_array, tags_array, props_array}, + {field_id, field_info, field_tags, field_props}) + .ValueOrDie(); + + // Write: 10 rows per page, 50 rows per row group → 2 row groups + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + // Read full schema + auto read_schema = arrow::schema({field_id, field_info, field_tags, field_props}); + + // predicate: id in [15, 29] or id in [80, 99] + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::Or({PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"id", + FieldType::INT, Literal(15), Literal(29)), + PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"id", + FieldType::INT, Literal(80), Literal(99))})); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result, + /*batch_size=*/1024); + + // Page-level filtering (10 rows/page): + // Between(15, 29) → pages 1-2 (rows 10-29) + // Between(80, 99) → pages 8-9 (rows 80-99) + // Total: 40 rows + ASSERT_TRUE(result); + ASSERT_EQ(40, result->length()); + + auto expected = + arrow::ChunkedArray::Make({data->Slice(10, 20), data->Slice(80, 20)}).ValueOrDie(); + ASSERT_TRUE(result->Equals(expected)); +} + } // namespace paimon::parquet::test From b0aed2508ac69b9eb6447167c60b1f1af0189966 Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Wed, 8 Jul 2026 15:12:16 +0800 Subject: [PATCH 04/23] refractor --- .../format/parquet/file_reader_wrapper.cpp | 36 ++------- .../format/parquet/file_reader_wrapper.h | 8 -- .../page_filtered_row_group_reader.cpp | 73 ++++++++++++------- .../parquet/page_filtered_row_group_reader.h | 22 +++--- .../page_filtered_row_group_reader_test.cpp | 26 +++---- 5 files changed, 73 insertions(+), 92 deletions(-) diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp index 66e2e11e4..555d14062 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper.cpp @@ -232,13 +232,11 @@ Result> FileReaderWrapper::NextPageFiltered( file_reader_->parquet_reader(), target_rg, target_column_indices_); bool pre_buffered = !prebuffered_ranges_.empty(); int64_t max_chunksize = batch_size_ > 0 ? batch_size_ : std::numeric_limits::max(); - PAIMON_ASSIGN_OR_RAISE( - current_page_filtered_reader_, - PageFilteredRowGroupReader::ReadFilteredRowGroup( - file_reader_.get(), file_reader_->parquet_reader(), target_rg, - target_column_indices_, page_filtered_read_schema_, - file_reader_->properties().cache_options(), - pre_buffered, page_ranges, max_chunksize, pool_)); + PAIMON_ASSIGN_OR_RAISE(current_page_filtered_reader_, + PageFilteredRowGroupReader::ReadFilteredRowGroup( + file_reader_.get(), target_rg, target_column_indices_, + file_reader_->properties().cache_options(), pre_buffered, + page_ranges, max_chunksize, pool_)); current_filtered_row_ranges_ = target_rg.row_ranges; current_filtered_rg_start_ = all_row_group_ranges_[rg_id].first; filtered_global_offset_ = 0; @@ -341,26 +339,6 @@ Status FileReaderWrapper::PrepareForReadingLazy( return Status::OK(); } -Status FileReaderWrapper::BuildPageFilteredSchema(const std::vector& column_indices) { - if (page_filtered_read_schema_) { - return Status::OK(); - } - // Use SchemaManifest to build schema with proper nested field structure. - // GetFieldIndices maps leaf column indices to top-level field indices, - // correctly handling nested types (List, Struct, Map). - const auto& manifest = file_reader_->manifest(); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - std::vector field_indices, - manifest.GetFieldIndices( - std::vector(column_indices.begin(), column_indices.end()))); - std::vector> fields; - for (int field_idx : field_indices) { - fields.push_back(manifest.schema_fields[field_idx].field); - } - page_filtered_read_schema_ = arrow::schema(fields); - return Status::OK(); -} - std::vector<::arrow::io::ReadRange> FileReaderWrapper::CollectPreBufferRanges( const std::vector& column_indices) { std::vector<::arrow::io::ReadRange> ranges; @@ -409,7 +387,6 @@ Status FileReaderWrapper::PrepareForReading(const std::vector& t try { target_row_groups_ = target_row_groups; target_column_indices_ = column_indices; - page_filtered_read_schema_.reset(); // Partition into fully-matched and page-filtered row groups, skipping excluded ones. std::vector fully_matched_row_groups; @@ -425,9 +402,6 @@ Status FileReaderWrapper::PrepareForReading(const std::vector& t } bool has_partially_matched = fully_matched_row_groups.size() != active_count; - if (has_partially_matched) { - PAIMON_RETURN_NOT_OK(BuildPageFilteredSchema(column_indices)); - } WaitForPendingPreBuffer(); diff --git a/src/paimon/format/parquet/file_reader_wrapper.h b/src/paimon/format/parquet/file_reader_wrapper.h index 758ff703a..f9a09c674 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.h +++ b/src/paimon/format/parquet/file_reader_wrapper.h @@ -157,9 +157,6 @@ class FileReaderWrapper { /// Read next batch from the fully-matched batch_reader_. Returns nullptr when exhausted. Result> NextFullyMatched(); - /// Build page_filtered_read_schema_ from the given column indices. No-op if already built. - Status BuildPageFilteredSchema(const std::vector& column_indices); - /// Collect all byte ranges that need pre-buffering (page-filtered + fully-matched). std::vector<::arrow::io::ReadRange> CollectPreBufferRanges( const std::vector& column_indices); @@ -193,11 +190,6 @@ class FileReaderWrapper { // Target row groups with row ranges for none page-level filtering and page-level filtering std::vector target_row_groups_; - // Arrow schema covering target_column_indices_, used when constructing the per-RG - // page-filtered reader. Cached in PrepareForReading because it's identical across - // all page-filtered RGs in a session. - std::shared_ptr page_filtered_read_schema_; - // Track pre-buffered ranges so we can wait on destruction std::vector<::arrow::io::ReadRange> prebuffered_ranges_; }; diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index 9d9e41981..ff26d54bb 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -125,8 +125,8 @@ std::pair PageFilteredRowGroupReader::ComputeCompressedRowRa } Status PageFilteredRowGroupReader::ExecuteSkipReadPattern( - ::parquet::arrow::ColumnReader* column_reader, const RowRanges& ranges, - int64_t total_row_count, int32_t row_group_index, int32_t field_index) { + ::parquet::arrow::ColumnReader* column_reader, const RowRanges& ranges, int64_t total_row_count, + int32_t row_group_index, int32_t field_index) { int64_t current_row = 0; for (const auto& range : ranges.GetRanges()) { if (range.from > current_row) { @@ -143,10 +143,10 @@ Status PageFilteredRowGroupReader::ExecuteSkipReadPattern( int64_t to_read = range.Count(); int64_t read = column_reader->ReadRecords(to_read); if (read != to_read) { - return Status::Invalid(fmt::format( - "PageFilteredRowGroupReader: expected to read {} records but read {} " - "(row_group={}, field={}, range=[{},{}])", - to_read, read, row_group_index, field_index, range.from, range.to)); + return Status::Invalid( + fmt::format("PageFilteredRowGroupReader: expected to read {} records but read {} " + "(row_group={}, field={}, range=[{},{}])", + to_read, read, row_group_index, field_index, range.from, range.to)); } current_row += to_read; } @@ -183,9 +183,9 @@ Status PageFilteredRowGroupReader::WaitForPreBuffer( Result> PageFilteredRowGroupReader::ReadFilteredField( ::parquet::arrow::FileReader* arrow_file_reader, const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, - int32_t row_group_index, int32_t field_index, - const std::vector& column_indices, const RowRanges& row_ranges, - int64_t row_group_row_count, std::shared_ptr<::arrow::MemoryPool> pool) { + int32_t row_group_index, int32_t field_index, const std::vector& column_indices, + const RowRanges& row_ranges, int64_t row_group_row_count, + std::shared_ptr<::arrow::MemoryPool> pool) { const auto& manifest = arrow_file_reader->manifest(); const auto& schema_field = manifest.schema_fields[field_index]; bool is_flat = schema_field.is_leaf(); @@ -206,9 +206,10 @@ Result> PageFilteredRowGroupReader::ReadFil } // Factory: set data_page_filter only for flat columns with OffsetIndex - auto factory = [row_group_index, is_flat, &rg_page_index_reader, &row_ranges, - row_group_row_count](int col_idx, ::parquet::ParquetFileReader* reader) - -> ::parquet::arrow::FileColumnIterator* { + auto factory = + [row_group_index, is_flat, &rg_page_index_reader, &row_ranges, row_group_row_count]( + int col_idx, + ::parquet::ParquetFileReader* reader) -> ::parquet::arrow::FileColumnIterator* { auto* iter = new ::parquet::arrow::FileColumnIterator(col_idx, reader, {row_group_index}); if (is_flat && rg_page_index_reader) { auto offset_index = rg_page_index_reader->GetOffsetIndex(col_idx); @@ -222,8 +223,8 @@ Result> PageFilteredRowGroupReader::ReadFil // Build reader tree with leaf column filtering std::unique_ptr<::parquet::arrow::ColumnReader> column_reader; - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow_file_reader->GetColumn( - field_index, column_indices, factory, &column_reader)); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow_file_reader->GetColumn(field_index, column_indices, factory, &column_reader)); if (!column_reader) { return std::shared_ptr(); @@ -238,19 +239,19 @@ Result> PageFilteredRowGroupReader::ReadFil // Phase 3: Build the Arrow array (TransferColumnData for leaves + assemble for nested) std::shared_ptr chunked_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW( - column_reader->BuildArray(effective_total, &chunked_array)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(column_reader->BuildArray(effective_total, &chunked_array)); return chunked_array; } Result> PageFilteredRowGroupReader::ReadFilteredRowGroup( - ::parquet::arrow::FileReader* arrow_file_reader, - ::parquet::ParquetFileReader* parquet_reader, const TargetRowGroup& target_row_group, - const std::vector& column_indices, const std::shared_ptr& arrow_schema, - const ::arrow::io::CacheOptions& cache_options, bool pre_buffered, - const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize, - std::shared_ptr<::arrow::MemoryPool> pool) { + ::parquet::arrow::FileReader* arrow_file_reader, const TargetRowGroup& target_row_group, + const std::vector& column_indices, const ::arrow::io::CacheOptions& cache_options, + bool pre_buffered, const std::vector<::arrow::io::ReadRange>& page_ranges, + int64_t max_chunksize, std::shared_ptr<::arrow::MemoryPool> pool) { + auto parquet_reader = arrow_file_reader->parquet_reader(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, + BuildPageFilteredSchema(arrow_file_reader, column_indices)); const auto& row_ranges = target_row_group.row_ranges; int32_t row_group_index = target_row_group.row_group_index; @@ -289,14 +290,14 @@ Result> PageFilteredRowGroupReader::Re for (int field_idx : field_indices) { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr chunked_array, - ReadFilteredField(arrow_file_reader, rg_page_index_reader, row_group_index, - field_idx, column_indices, row_ranges, row_group_row_count, pool)); + ReadFilteredField(arrow_file_reader, rg_page_index_reader, row_group_index, field_idx, + column_indices, row_ranges, row_group_row_count, pool)); if (chunked_array && chunked_array->length() != expected_rows) { - return Status::Invalid(fmt::format( - "PageFilteredRowGroupReader: field {} produced {} rows but expected {} " - "(row_group={})", - field_idx, chunked_array->length(), expected_rows, row_group_index)); + return Status::Invalid( + fmt::format("PageFilteredRowGroupReader: field {} produced {} rows but expected {} " + "(row_group={})", + field_idx, chunked_array->length(), expected_rows, row_group_index)); } columns.push_back(std::move(chunked_array)); @@ -374,4 +375,20 @@ std::vector<::arrow::io::ReadRange> PageFilteredRowGroupReader::ComputePageRange return ranges; } +Result> PageFilteredRowGroupReader::BuildPageFilteredSchema( + ::parquet::arrow::FileReader* file_reader, const std::vector& column_indices) { + // Use SchemaManifest to build schema with proper nested field structure. + // GetFieldIndices maps leaf column indices to top-level field indices, + // correctly handling nested types (List, Struct, Map). + const auto& manifest = file_reader->manifest(); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::vector field_indices, + manifest.GetFieldIndices(std::vector(column_indices.begin(), column_indices.end()))); + std::vector> fields; + for (int field_idx : field_indices) { + fields.push_back(manifest.schema_fields[field_idx].field); + } + return arrow::schema(fields); +} + } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h b/src/paimon/format/parquet/page_filtered_row_group_reader.h index c0ee79731..f4f538bd7 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.h +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h @@ -46,10 +46,8 @@ class PageFilteredRowGroupReader { /// Read a row group with page-level filtering. /// @param arrow_file_reader The Arrow FileReader for ColumnReader tree creation - /// @param parquet_reader The underlying ParquetFileReader /// @param target_row_group Target row group with index and row ranges /// @param column_indices Leaf column indices to read - /// @param arrow_schema The target Arrow schema for output columns /// @param pool Memory pool /// @param cache_options Cache options for PreBuffer /// @param pre_buffered If true, assumes PreBuffer was already called externally @@ -58,13 +56,10 @@ class PageFilteredRowGroupReader { /// @param max_chunksize Per-batch row cap for the returned reader. /// @return A RecordBatchReader streaming the filtered rows. static Result> ReadFilteredRowGroup( - ::parquet::arrow::FileReader* arrow_file_reader, - ::parquet::ParquetFileReader* parquet_reader, const TargetRowGroup& target_row_group, - const std::vector& column_indices, - const std::shared_ptr& arrow_schema, - const ::arrow::io::CacheOptions& cache_options, bool pre_buffered, - const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize, - std::shared_ptr<::arrow::MemoryPool> pool); + ::parquet::arrow::FileReader* arrow_file_reader, const TargetRowGroup& target_row_group, + const std::vector& column_indices, const ::arrow::io::CacheOptions& cache_options, + bool pre_buffered, const std::vector<::arrow::io::ReadRange>& page_ranges, + int64_t max_chunksize, std::shared_ptr<::arrow::MemoryPool> pool); /// Compute the byte ranges of pages that overlap with the given RowRanges. /// Uses OffsetIndex to determine per-page file offsets and sizes. @@ -110,9 +105,12 @@ class PageFilteredRowGroupReader { static Result> ReadFilteredField( ::parquet::arrow::FileReader* arrow_file_reader, const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, - int32_t row_group_index, int32_t field_index, - const std::vector& column_indices, const RowRanges& row_ranges, - int64_t row_group_row_count, std::shared_ptr<::arrow::MemoryPool> pool); + int32_t row_group_index, int32_t field_index, const std::vector& column_indices, + const RowRanges& row_ranges, int64_t row_group_row_count, + std::shared_ptr<::arrow::MemoryPool> pool); + + static Result> BuildPageFilteredSchema( + ::parquet::arrow::FileReader* file_reader, const std::vector& column_indices); }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index 68a6abede..e20d1b24d 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -923,8 +923,8 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnRowGroupFilter) { auto id_array = MakeIdColumn(100); auto info_array = MakeNestedStructData(100); - auto data = arrow::StructArray::Make({id_array, info_array}, {field_id, field_info}) - .ValueOrDie(); + auto data = + arrow::StructArray::Make({id_array, info_array}, {field_id, field_info}).ValueOrDie(); WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); auto read_schema = arrow::schema({field_id, field_info}); @@ -962,8 +962,8 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnOnlyReadIdField) { auto id_array = MakeIdColumn(100); auto info_array = MakeNestedStructData(100); - auto data = arrow::StructArray::Make({id_array, info_array}, {field_id, field_info}) - .ValueOrDie(); + auto data = + arrow::StructArray::Make({id_array, info_array}, {field_id, field_info}).ValueOrDie(); WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); // Read "id" column only @@ -1026,8 +1026,8 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedListColumnRowGroupFilter) { auto id_array = MakeIdColumn(100); auto tags_array = MakeListColumnData(100); - auto data = arrow::StructArray::Make({id_array, tags_array}, {field_id, field_tags}) - .ValueOrDie(); + auto data = + arrow::StructArray::Make({id_array, tags_array}, {field_id, field_tags}).ValueOrDie(); WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); auto read_schema = arrow::schema({field_id, field_tags}); @@ -1059,8 +1059,8 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedMapColumnRowGroupFilter) { auto id_array = MakeIdColumn(100); auto props_array = MakeMapColumnData(100); - auto data = arrow::StructArray::Make({id_array, props_array}, {field_id, field_props}) - .ValueOrDie(); + auto data = + arrow::StructArray::Make({id_array, props_array}, {field_id, field_props}).ValueOrDie(); WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); auto read_schema = arrow::schema({field_id, field_props}); @@ -1155,11 +1155,11 @@ TEST_F(PageFilteredRowGroupReaderTest, MultipleNestedColumns) { // predicate: id in [15, 29] or id in [80, 99] ASSERT_OK_AND_ASSIGN( - auto predicate, - PredicateBuilder::Or({PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"id", - FieldType::INT, Literal(15), Literal(29)), - PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"id", - FieldType::INT, Literal(80), Literal(99))})); + auto predicate, PredicateBuilder::Or( + {PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"id", + FieldType::INT, Literal(15), Literal(29)), + PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"id", + FieldType::INT, Literal(80), Literal(99))})); std::shared_ptr result; ReadWithPredicateImpl(file_name, read_schema, predicate, &result, From 2b169e88be4c1cbe8deae204299a4ab8d5dd9dad Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Wed, 8 Jul 2026 15:12:57 +0800 Subject: [PATCH 05/23] patch arrow --- cmake_modules/arrow.diff | 267 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 266 insertions(+), 1 deletion(-) diff --git a/cmake_modules/arrow.diff b/cmake_modules/arrow.diff index 70d33c7b5..ea2787535 100644 --- a/cmake_modules/arrow.diff +++ b/cmake_modules/arrow.diff @@ -437,4 +437,269 @@ diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h /// \return Status - Status Advance(int64_t nbytes); + virtual Status Advance(int64_t nbytes); - \ No newline at end of file + +--- a/cpp/src/parquet/arrow/reader.cc ++++ b/cpp/src/parquet/arrow/reader.cc +@@ -254,6 +254,11 @@ + return GetColumn(i, AllRowGroupsFactory(), out); + } + ++ ::arrow::Status GetColumn( ++ int i, const std::vector& column_indices, ++ FileColumnIteratorFactory iterator_factory, ++ std::unique_ptr* out) override; ++ + Status GetSchema(std::shared_ptr<::arrow::Schema>* out) override { + return FromParquetSchema(reader_->metadata()->schema(), reader_properties_, + reader_->metadata()->key_value_metadata(), out); +@@ -493,10 +498,33 @@ + + ::arrow::Status BuildArray(int64_t length_upper_bound, + std::shared_ptr<::arrow::ChunkedArray>* out) final { ++ if (!out_) { ++ BEGIN_PARQUET_CATCH_EXCEPTIONS ++ RETURN_NOT_OK( ++ TransferColumnData(record_reader_.get(), field_, descr_, ctx_->pool, &out_)); ++ END_PARQUET_CATCH_EXCEPTIONS ++ } + *out = out_; + return Status::OK(); + } + ++ ::arrow::Status BeginRead(int64_t total_row_count) final { ++ BEGIN_PARQUET_CATCH_EXCEPTIONS ++ out_ = nullptr; ++ record_reader_->Reset(); ++ record_reader_->Reserve(total_row_count); ++ return Status::OK(); ++ END_PARQUET_CATCH_EXCEPTIONS ++ } ++ ++ int64_t SkipRecords(int64_t num_records) final { ++ return record_reader_->SkipRecords(num_records); ++ } ++ ++ int64_t ReadRecords(int64_t num_records) final { ++ return record_reader_->ReadRecords(num_records); ++ } ++ + const std::shared_ptr field() override { return field_; } + + private: +@@ -532,6 +560,18 @@ + return storage_reader_->LoadBatch(number_of_records); + } + ++ ::arrow::Status BeginRead(int64_t total_row_count) final { ++ return storage_reader_->BeginRead(total_row_count); ++ } ++ ++ int64_t SkipRecords(int64_t num_records) final { ++ return storage_reader_->SkipRecords(num_records); ++ } ++ ++ int64_t ReadRecords(int64_t num_records) final { ++ return storage_reader_->ReadRecords(num_records); ++ } ++ + Status BuildArray(int64_t length_upper_bound, + std::shared_ptr* out) override { + std::shared_ptr storage; +@@ -576,6 +616,18 @@ + return item_reader_->LoadBatch(number_of_records); + } + ++ ::arrow::Status BeginRead(int64_t total_row_count) final { ++ return item_reader_->BeginRead(total_row_count); ++ } ++ ++ int64_t SkipRecords(int64_t num_records) final { ++ return item_reader_->SkipRecords(num_records); ++ } ++ ++ int64_t ReadRecords(int64_t num_records) final { ++ return item_reader_->ReadRecords(num_records); ++ } ++ + virtual ::arrow::Result> AssembleArray( + std::shared_ptr data) { + if (field_->type()->id() == ::arrow::Type::MAP) { +@@ -709,6 +761,30 @@ + } + return Status::OK(); + } ++ ++ ::arrow::Status BeginRead(int64_t total_row_count) override { ++ for (const std::unique_ptr& reader : children_) { ++ RETURN_NOT_OK(reader->BeginRead(total_row_count)); ++ } ++ return Status::OK(); ++ } ++ ++ int64_t SkipRecords(int64_t num_records) override { ++ int64_t skipped = 0; ++ for (const std::unique_ptr& reader : children_) { ++ skipped = reader->SkipRecords(num_records); ++ } ++ return skipped; ++ } ++ ++ int64_t ReadRecords(int64_t num_records) override { ++ int64_t read = 0; ++ for (const std::unique_ptr& reader : children_) { ++ read = reader->ReadRecords(num_records); ++ } ++ return read; ++ } ++ + Status BuildArray(int64_t length_upper_bound, + std::shared_ptr* out) override; + Status GetDefLevels(const int16_t** data, int64_t* length) override; +@@ -1228,6 +1304,23 @@ + std::unique_ptr result; + RETURN_NOT_OK(GetReader(manifest_.schema_fields[i], ctx, &result)); + *out = std::move(result); ++ return Status::OK(); ++} ++ ++::arrow::Status FileReaderImpl::GetColumn( ++ int i, const std::vector& column_indices, ++ FileColumnIteratorFactory iterator_factory, ++ std::unique_ptr* out) { ++ RETURN_NOT_OK(BoundsCheckColumn(i)); ++ auto ctx = std::make_shared(); ++ ctx->reader = reader_.get(); ++ ctx->pool = pool_; ++ ctx->iterator_factory = iterator_factory; ++ ctx->filter_leaves = true; ++ ctx->included_leaves = VectorToSharedSet(column_indices); ++ std::unique_ptr result; ++ RETURN_NOT_OK(GetReader(manifest_.schema_fields[i], ctx, &result)); ++ *out = std::move(result); + return Status::OK(); + } + +--- a/cpp/src/parquet/arrow/reader.h ++++ b/cpp/src/parquet/arrow/reader.h +@@ -48,9 +48,13 @@ + + class ColumnChunkReader; + class ColumnReader; ++class FileColumnIterator; + struct SchemaManifest; + class RowGroupReader; + ++using FileColumnIteratorFactory = ++ std::function; ++ + /// \brief Arrow read adapter class for deserializing Parquet files as Arrow row batches. + /// + /// This interfaces caters for different use cases and thus provides different +@@ -136,6 +140,27 @@ + // The indicated column index is relative to the schema + virtual ::arrow::Status GetColumn(int i, std::unique_ptr* out) = 0; + ++ /// \brief Return a ColumnReader with a custom FileColumnIteratorFactory ++ /// and leaf column filtering. ++ /// ++ /// This allows callers to customize page reading behavior (e.g., setting ++ /// data_page_filter for page-level skipping) and to select only specific ++ /// leaf columns within a nested field. The factory is called once per leaf ++ /// column included in column_indices. ++ /// ++ /// \param i top-level field index (same as GetColumn(int i, ...)) ++ /// \param column_indices leaf column indices to include (enables sub-column ++ /// projection within nested types) ++ /// \param iterator_factory factory to create FileColumnIterator per leaf ++ /// \param[out] out the ColumnReader (may be nullptr if all leaves are pruned) ++ virtual ::arrow::Status GetColumn( ++ int i, const std::vector& column_indices, ++ FileColumnIteratorFactory iterator_factory, ++ std::unique_ptr* out) { ++ return ::arrow::Status::NotImplemented( ++ "GetColumn with factory not implemented"); ++ } ++ + /// \brief Return arrow schema for all the columns. + virtual ::arrow::Status GetSchema(std::shared_ptr<::arrow::Schema>* out) = 0; + +@@ -316,6 +341,29 @@ + // the data available in the file. + virtual ::arrow::Status NextBatch(int64_t batch_size, + std::shared_ptr<::arrow::ChunkedArray>* out) = 0; ++ ++ /// \brief Prepare for filtered reading. Resets internal state and ++ /// pre-allocates buffers. Must be called before SkipRecords/ReadRecords. ++ virtual ::arrow::Status BeginRead(int64_t total_row_count) { ++ return ::arrow::Status::NotImplemented("BeginRead not implemented"); ++ } ++ ++ /// \brief Skip num_records top-level records. ++ /// Returns the actual number of records skipped. ++ virtual int64_t SkipRecords(int64_t num_records) { return 0; } ++ ++ /// \brief Read num_records top-level records into internal buffers. ++ /// Returns the actual number of records read. ++ virtual int64_t ReadRecords(int64_t num_records) { return 0; } ++ ++ /// \brief Build the Arrow array from previously loaded data. ++ /// For leaf readers, calls TransferColumnData if not already done. ++ /// For nested readers, assembles the nested array from child arrays. ++ virtual ::arrow::Status BuildArray( ++ int64_t length_upper_bound, ++ std::shared_ptr<::arrow::ChunkedArray>* out) { ++ return ::arrow::Status::NotImplemented("BuildArray not implemented"); ++ } + }; + + /// \brief Experimental helper class for bindings (like Python) that struggle +--- a/cpp/src/parquet/arrow/reader_internal.h ++++ b/cpp/src/parquet/arrow/reader_internal.h +@@ -26,6 +26,7 @@ + #include + #include + ++#include "parquet/arrow/reader.h" + #include "parquet/arrow/schema.h" + #include "parquet/column_reader.h" + #include "parquet/file_reader.h" +@@ -70,6 +71,13 @@ + + virtual ~FileColumnIterator() {} + ++ /// \brief Set a data_page_filter that will be applied to every PageReader ++ /// created by NextChunk(). This enables I/O-level page skipping. ++ void set_data_page_filter( ++ std::function filter) { ++ data_page_filter_ = std::move(filter); ++ } ++ + std::unique_ptr<::parquet::PageReader> NextChunk() { + if (row_groups_.empty()) { + return nullptr; +@@ -77,7 +85,11 @@ + + auto row_group_reader = reader_->RowGroup(row_groups_.front()); + row_groups_.pop_front(); +- return row_group_reader->GetColumnPageReader(column_index_); ++ auto page_reader = row_group_reader->GetColumnPageReader(column_index_); ++ if (page_reader && data_page_filter_) { ++ page_reader->set_data_page_filter(data_page_filter_); ++ } ++ return page_reader; + } + + const SchemaDescriptor* schema() const { return schema_; } +@@ -93,11 +105,9 @@ + ParquetFileReader* reader_; + const SchemaDescriptor* schema_; + std::deque row_groups_; ++ std::function data_page_filter_; + }; + +-using FileColumnIteratorFactory = +- std::function; +- + Status TransferColumnData(::parquet::internal::RecordReader* reader, + const std::shared_ptr<::arrow::Field>& value_field, + const ColumnDescriptor* descr, ::arrow::MemoryPool* pool, From 97ce120c62b21b99c2e10ec0b83780cc76793157 Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Thu, 9 Jul 2026 10:50:25 +0800 Subject: [PATCH 06/23] fix: support read leaf field --- cmake_modules/arrow.diff | 120 ++++++++---------- .../page_filtered_row_group_reader.cpp | 90 +++++-------- .../parquet/page_filtered_row_group_reader.h | 13 +- 3 files changed, 91 insertions(+), 132 deletions(-) diff --git a/cmake_modules/arrow.diff b/cmake_modules/arrow.diff index ea2787535..6302c91fa 100644 --- a/cmake_modules/arrow.diff +++ b/cmake_modules/arrow.diff @@ -452,7 +452,7 @@ diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h Status GetSchema(std::shared_ptr<::arrow::Schema>* out) override { return FromParquetSchema(reader_->metadata()->schema(), reader_properties_, reader_->metadata()->key_value_metadata(), out); -@@ -493,10 +498,33 @@ +@@ -493,10 +498,42 @@ ::arrow::Status BuildArray(int64_t length_upper_bound, std::shared_ptr<::arrow::ChunkedArray>* out) final { @@ -466,96 +466,73 @@ diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h return Status::OK(); } -+ ::arrow::Status BeginRead(int64_t total_row_count) final { ++ ::arrow::Status LoadBatchWithRowFilter(const LeafRowFilter& get_leaf_filter) final { + BEGIN_PARQUET_CATCH_EXCEPTIONS ++ auto [pattern, total] = get_leaf_filter(input_->column_index()); + out_ = nullptr; + record_reader_->Reset(); -+ record_reader_->Reserve(total_row_count); ++ record_reader_->Reserve(total); ++ int64_t current = 0; ++ for (const auto& [skip, read] : pattern) { ++ if (skip > 0) { ++ record_reader_->SkipRecords(skip); ++ current += skip; ++ } ++ if (read > 0) { ++ record_reader_->ReadRecords(read); ++ current += read; ++ } ++ } ++ if (current < total) { ++ record_reader_->SkipRecords(total - current); ++ } ++ RETURN_NOT_OK( ++ TransferColumnData(record_reader_.get(), field_, descr_, ctx_->pool, &out_)); + return Status::OK(); + END_PARQUET_CATCH_EXCEPTIONS + } -+ -+ int64_t SkipRecords(int64_t num_records) final { -+ return record_reader_->SkipRecords(num_records); -+ } -+ -+ int64_t ReadRecords(int64_t num_records) final { -+ return record_reader_->ReadRecords(num_records); -+ } + const std::shared_ptr field() override { return field_; } private: -@@ -532,6 +560,18 @@ +@@ -532,6 +569,10 @@ return storage_reader_->LoadBatch(number_of_records); } -+ ::arrow::Status BeginRead(int64_t total_row_count) final { -+ return storage_reader_->BeginRead(total_row_count); -+ } -+ -+ int64_t SkipRecords(int64_t num_records) final { -+ return storage_reader_->SkipRecords(num_records); -+ } -+ -+ int64_t ReadRecords(int64_t num_records) final { -+ return storage_reader_->ReadRecords(num_records); ++ ::arrow::Status LoadBatchWithRowFilter(const LeafRowFilter& get_leaf_filter) final { ++ return storage_reader_->LoadBatchWithRowFilter(get_leaf_filter); + } + Status BuildArray(int64_t length_upper_bound, std::shared_ptr* out) override { std::shared_ptr storage; -@@ -576,6 +616,18 @@ +@@ -576,6 +617,10 @@ return item_reader_->LoadBatch(number_of_records); } -+ ::arrow::Status BeginRead(int64_t total_row_count) final { -+ return item_reader_->BeginRead(total_row_count); -+ } -+ -+ int64_t SkipRecords(int64_t num_records) final { -+ return item_reader_->SkipRecords(num_records); -+ } -+ -+ int64_t ReadRecords(int64_t num_records) final { -+ return item_reader_->ReadRecords(num_records); ++ ::arrow::Status LoadBatchWithRowFilter(const LeafRowFilter& get_leaf_filter) final { ++ return item_reader_->LoadBatchWithRowFilter(get_leaf_filter); + } + virtual ::arrow::Result> AssembleArray( std::shared_ptr data) { if (field_->type()->id() == ::arrow::Type::MAP) { -@@ -709,6 +761,30 @@ +@@ -709,6 +754,14 @@ } return Status::OK(); } + -+ ::arrow::Status BeginRead(int64_t total_row_count) override { ++ ::arrow::Status LoadBatchWithRowFilter(const LeafRowFilter& get_leaf_filter) override { + for (const std::unique_ptr& reader : children_) { -+ RETURN_NOT_OK(reader->BeginRead(total_row_count)); ++ RETURN_NOT_OK(reader->LoadBatchWithRowFilter(get_leaf_filter)); + } + return Status::OK(); + } -+ -+ int64_t SkipRecords(int64_t num_records) override { -+ int64_t skipped = 0; -+ for (const std::unique_ptr& reader : children_) { -+ skipped = reader->SkipRecords(num_records); -+ } -+ return skipped; -+ } -+ -+ int64_t ReadRecords(int64_t num_records) override { -+ int64_t read = 0; -+ for (const std::unique_ptr& reader : children_) { -+ read = reader->ReadRecords(num_records); -+ } -+ return read; -+ } + Status BuildArray(int64_t length_upper_bound, std::shared_ptr* out) override; Status GetDefLevels(const int16_t** data, int64_t* length) override; -@@ -1228,6 +1304,23 @@ +@@ -1228,6 +1281,23 @@ std::unique_ptr result; RETURN_NOT_OK(GetReader(manifest_.schema_fields[i], ctx, &result)); *out = std::move(result); @@ -581,7 +558,15 @@ diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h --- a/cpp/src/parquet/arrow/reader.h +++ b/cpp/src/parquet/arrow/reader.h -@@ -48,9 +48,13 @@ +@@ -21,6 +21,7 @@ + // N.B. we don't include async_generator.h as it's relatively heavy + #include + #include ++#include + #include + + #include "parquet/file_reader.h" +@@ -48,9 +49,13 @@ class ColumnChunkReader; class ColumnReader; @@ -595,7 +580,7 @@ diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h /// \brief Arrow read adapter class for deserializing Parquet files as Arrow row batches. /// /// This interfaces caters for different use cases and thus provides different -@@ -136,6 +140,27 @@ +@@ -136,6 +141,27 @@ // The indicated column index is relative to the schema virtual ::arrow::Status GetColumn(int i, std::unique_ptr* out) = 0; @@ -623,25 +608,24 @@ diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h /// \brief Return arrow schema for all the columns. virtual ::arrow::Status GetSchema(std::shared_ptr<::arrow::Schema>* out) = 0; -@@ -316,6 +341,29 @@ +@@ -316,6 +342,28 @@ // the data available in the file. virtual ::arrow::Status NextBatch(int64_t batch_size, std::shared_ptr<::arrow::ChunkedArray>* out) = 0; + -+ /// \brief Prepare for filtered reading. Resets internal state and -+ /// pre-allocates buffers. Must be called before SkipRecords/ReadRecords. -+ virtual ::arrow::Status BeginRead(int64_t total_row_count) { -+ return ::arrow::Status::NotImplemented("BeginRead not implemented"); ++ /// \brief Load batch with per-leaf row filtering. ++ /// ++ /// The callback is called once per leaf column with that leaf's column index, ++ /// returning (skip_read_pattern, total_row_count) for that specific leaf. ++ /// Each pair in skip_read_pattern is (num_records_to_skip, num_records_to_read). ++ /// After processing all pairs, remaining records up to total_row_count are skipped. ++ /// Must be followed by BuildArray() to get the result. ++ using LeafRowFilter = std::function< ++ std::pair>, int64_t>(int)>; ++ virtual ::arrow::Status LoadBatchWithRowFilter(const LeafRowFilter& get_leaf_filter) { ++ return ::arrow::Status::NotImplemented("LoadBatchWithRowFilter not implemented"); + } + -+ /// \brief Skip num_records top-level records. -+ /// Returns the actual number of records skipped. -+ virtual int64_t SkipRecords(int64_t num_records) { return 0; } -+ -+ /// \brief Read num_records top-level records into internal buffers. -+ /// Returns the actual number of records read. -+ virtual int64_t ReadRecords(int64_t num_records) { return 0; } -+ + /// \brief Build the Arrow array from previously loaded data. + /// For leaf readers, calls TransferColumnData if not already done. + /// For nested readers, assembles the nested array from child arrays. diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index ff26d54bb..3de9826a7 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -124,36 +124,16 @@ std::pair PageFilteredRowGroupReader::ComputeCompressedRowRa return {compressed, compressed_offset}; } -Status PageFilteredRowGroupReader::ExecuteSkipReadPattern( - ::parquet::arrow::ColumnReader* column_reader, const RowRanges& ranges, int64_t total_row_count, - int32_t row_group_index, int32_t field_index) { - int64_t current_row = 0; +std::vector> PageFilteredRowGroupReader::RowRangesToSkipReadPattern( + const RowRanges& ranges) { + std::vector> pattern; + int64_t current = 0; for (const auto& range : ranges.GetRanges()) { - if (range.from > current_row) { - int64_t to_skip = range.from - current_row; - int64_t skipped = column_reader->SkipRecords(to_skip); - if (skipped != to_skip) { - return Status::Invalid(fmt::format( - "PageFilteredRowGroupReader: expected to skip {} records but skipped {} " - "(row_group={}, field={})", - to_skip, skipped, row_group_index, field_index)); - } - current_row = range.from; - } - int64_t to_read = range.Count(); - int64_t read = column_reader->ReadRecords(to_read); - if (read != to_read) { - return Status::Invalid( - fmt::format("PageFilteredRowGroupReader: expected to read {} records but read {} " - "(row_group={}, field={}, range=[{},{}])", - to_read, read, row_group_index, field_index, range.from, range.to)); - } - current_row += to_read; - } - if (current_row < total_row_count) { - column_reader->SkipRecords(total_row_count - current_row); + int64_t skip = range.from > current ? range.from - current : 0; + pattern.emplace_back(skip, range.Count()); + current = range.to + 1; } - return Status::OK(); + return pattern; } Status PageFilteredRowGroupReader::WaitForPreBuffer( @@ -186,32 +166,14 @@ Result> PageFilteredRowGroupReader::ReadFil int32_t row_group_index, int32_t field_index, const std::vector& column_indices, const RowRanges& row_ranges, int64_t row_group_row_count, std::shared_ptr<::arrow::MemoryPool> pool) { - const auto& manifest = arrow_file_reader->manifest(); - const auto& schema_field = manifest.schema_fields[field_index]; - bool is_flat = schema_field.is_leaf(); - - // For flat columns: compute compressed RowRanges if OffsetIndex is available. - // data_page_filter + compressed_ranges enable I/O-level page skipping. - // For nested fields: use original row_ranges, decode-level skipping only. - RowRanges effective_ranges = row_ranges; - int64_t effective_total = row_group_row_count; - if (is_flat && rg_page_index_reader) { - auto offset_index = rg_page_index_reader->GetOffsetIndex(schema_field.column_index); - if (offset_index) { - auto [compressed, total] = - ComputeCompressedRowRanges(row_ranges, offset_index, row_group_row_count); - effective_ranges = std::move(compressed); - effective_total = total; - } - } - - // Factory: set data_page_filter only for flat columns with OffsetIndex + // Factory: set data_page_filter on every leaf (per-leaf OffsetIndex). + // data_page_filter enables I/O-level page skipping for all leaves. auto factory = - [row_group_index, is_flat, &rg_page_index_reader, &row_ranges, row_group_row_count]( + [row_group_index, &rg_page_index_reader, &row_ranges, row_group_row_count]( int col_idx, ::parquet::ParquetFileReader* reader) -> ::parquet::arrow::FileColumnIterator* { auto* iter = new ::parquet::arrow::FileColumnIterator(col_idx, reader, {row_group_index}); - if (is_flat && rg_page_index_reader) { + if (rg_page_index_reader) { auto offset_index = rg_page_index_reader->GetOffsetIndex(col_idx); if (offset_index) { iter->set_data_page_filter( @@ -230,16 +192,30 @@ Result> PageFilteredRowGroupReader::ReadFil return std::shared_ptr(); } - // Phase 1: Prepare for reading - PAIMON_RETURN_NOT_OK_FROM_ARROW(column_reader->BeginRead(effective_total)); + // Per-leaf callback: each leaf gets its own skip/read pattern + total. + auto get_leaf_filter = + [&rg_page_index_reader, &row_ranges, row_group_row_count]( + int col_idx) -> std::pair>, int64_t> { + RowRanges effective_ranges = row_ranges; + int64_t effective_total = row_group_row_count; + if (rg_page_index_reader) { + auto offset_index = rg_page_index_reader->GetOffsetIndex(col_idx); + if (offset_index) { + auto [compressed, total] = + ComputeCompressedRowRanges(row_ranges, offset_index, row_group_row_count); + effective_ranges = std::move(compressed); + effective_total = total; + } + } + return {RowRangesToSkipReadPattern(effective_ranges), effective_total}; + }; - // Phase 2: Execute skip/read pattern - PAIMON_RETURN_NOT_OK(ExecuteSkipReadPattern(column_reader.get(), effective_ranges, - effective_total, row_group_index, field_index)); + // Load + filter + transfer (per-leaf, via tree delegation) + PAIMON_RETURN_NOT_OK_FROM_ARROW(column_reader->LoadBatchWithRowFilter(get_leaf_filter)); - // Phase 3: Build the Arrow array (TransferColumnData for leaves + assemble for nested) + // Build the Arrow array (TransferColumnData for leaves + assemble for nested) std::shared_ptr chunked_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(column_reader->BuildArray(effective_total, &chunked_array)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(column_reader->BuildArray(row_group_row_count, &chunked_array)); return chunked_array; } diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h b/src/paimon/format/parquet/page_filtered_row_group_reader.h index f4f538bd7..c31fce313 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.h +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h @@ -84,11 +84,6 @@ class PageFilteredRowGroupReader { const std::vector<::arrow::io::ReadRange>& page_ranges, std::shared_ptr<::arrow::MemoryPool> pool); - /// Execute the skip/read pattern on a ColumnReader based on RowRanges. - static Status ExecuteSkipReadPattern(::parquet::arrow::ColumnReader* column_reader, - const RowRanges& ranges, int64_t total_row_count, - int32_t row_group_index, int32_t field_index); - /// Create a data_page_filter callback for a column based on RowRanges + OffsetIndex. static std::function MakePageFilter( const RowRanges& row_ranges, const std::shared_ptr<::parquet::OffsetIndex>& offset_index, @@ -99,9 +94,13 @@ class PageFilteredRowGroupReader { const RowRanges& original_ranges, const std::shared_ptr<::parquet::OffsetIndex>& offset_index, int64_t row_group_row_count); + /// Convert RowRanges to skip/read pattern. Each pair is (skip, read). + static std::vector> RowRangesToSkipReadPattern( + const RowRanges& ranges); + /// Read a field (flat or nested) using ColumnReader tree. - /// For flat columns: sets data_page_filter + uses compressed RowRanges (I/O + decode skipping). - /// For nested fields: decode-level skipping only via SkipRecords/ReadRecords. + /// Sets data_page_filter on all leaves via factory, then uses + /// LoadBatchWithRowFilter with per-leaf compressed_ranges. static Result> ReadFilteredField( ::parquet::arrow::FileReader* arrow_file_reader, const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, From df19c4f362934e36273f143575d47440bfd28c13 Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Thu, 9 Jul 2026 11:36:28 +0800 Subject: [PATCH 07/23] test: add test case for leaf field projection --- .../page_filtered_row_group_reader_test.cpp | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index e20d1b24d..eae52db01 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -1177,4 +1177,53 @@ TEST_F(PageFilteredRowGroupReaderTest, MultipleNestedColumns) { ASSERT_TRUE(result->Equals(expected)); } +/// Test: sub-column projection of a struct type with page-level filtering. +/// +/// Schema: { id: int32, info: struct } +/// Read schema: { info: struct } — project only x, not y. +/// Predicate: id >= 70 → 30 rows expected. +/// Verifies that reading a sub-column of a nested struct works correctly +/// with page-level filtering and the ColumnReader tree (GetColumn + filter_leaves). +TEST_F(PageFilteredRowGroupReaderTest, NestedStructSubColumnProjection) { + std::string file_name = dir_->Str() + "/nested_struct_subcol.parquet"; + + auto field_x = arrow::field("x", arrow::int32()); + auto field_y = arrow::field("y", arrow::int32()); + auto field_id = arrow::field("id", arrow::int32()); + auto field_info = arrow::field("info", arrow::struct_({field_x, field_y})); + + auto id_array = MakeIdColumn(100); + auto info_array = MakeNestedStructData(100); + auto data = + arrow::StructArray::Make({id_array, info_array}, {field_id, field_info}).ValueOrDie(); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + // Read only info.x (sub-column projection: only x, not y) + auto read_schema = arrow::schema({arrow::field("info", arrow::struct_({field_x}))}); + + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + + ASSERT_TRUE(result); + ASSERT_EQ(30, result->length()); + + // Result is struct> + auto result_struct = std::dynamic_pointer_cast(result->chunk(0)); + ASSERT_TRUE(result_struct); + ASSERT_EQ(1, result_struct->num_fields()); + + auto info_result = std::dynamic_pointer_cast(result_struct->field(0)); + ASSERT_TRUE(info_result); + ASSERT_EQ(1, info_result->num_fields()); + + auto x_arr = std::dynamic_pointer_cast(info_result->field(0)); + ASSERT_TRUE(x_arr); + for (int32_t i = 0; i < 30; ++i) { + ASSERT_EQ((70 + i) * 100, x_arr->Value(i)) << "Mismatch at index " << i; + } +} + } // namespace paimon::parquet::test From dc19071e279af6aef3d728321787072c474b735c Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Thu, 9 Jul 2026 11:39:37 +0800 Subject: [PATCH 08/23] fix: build schema from actual column types --- .../page_filtered_row_group_reader.cpp | 37 ++++++------------- .../parquet/page_filtered_row_group_reader.h | 3 -- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index 3de9826a7..864a1fb3e 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -226,17 +226,9 @@ Result> PageFilteredRowGroupReader::Re bool pre_buffered, const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize, std::shared_ptr<::arrow::MemoryPool> pool) { auto parquet_reader = arrow_file_reader->parquet_reader(); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - BuildPageFilteredSchema(arrow_file_reader, column_indices)); const auto& row_ranges = target_row_group.row_ranges; int32_t row_group_index = target_row_group.row_group_index; - if (row_ranges.IsEmpty()) { - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr empty_table, - arrow::Table::MakeEmpty(arrow_schema, pool.get())); - return std::make_unique(std::move(empty_table), max_chunksize); - } - int64_t expected_rows = row_ranges.RowCount(); PAIMON_RETURN_NOT_OK(WaitForPreBuffer(parquet_reader, row_group_index, column_indices, @@ -279,7 +271,18 @@ Result> PageFilteredRowGroupReader::Re columns.push_back(std::move(chunked_array)); } - auto table = arrow::Table::Make(arrow_schema, std::move(columns), expected_rows); + // Build schema from actual column types (not SchemaManifest's full field). + // This handles sub-column projection: when filter_leaves prunes some children, + // the ChunkedArray type is the projected type (e.g. struct instead of struct), + // which may differ from the full field in SchemaManifest. + std::vector> result_fields; + for (size_t i = 0; i < columns.size(); ++i) { + const auto& field_name = manifest.schema_fields[field_indices[i]].field->name(); + result_fields.push_back(arrow::field(field_name, columns[i]->type())); + } + auto result_schema = arrow::schema(result_fields); + + auto table = arrow::Table::Make(result_schema, std::move(columns), expected_rows); return std::make_unique(std::move(table), max_chunksize); } @@ -351,20 +354,4 @@ std::vector<::arrow::io::ReadRange> PageFilteredRowGroupReader::ComputePageRange return ranges; } -Result> PageFilteredRowGroupReader::BuildPageFilteredSchema( - ::parquet::arrow::FileReader* file_reader, const std::vector& column_indices) { - // Use SchemaManifest to build schema with proper nested field structure. - // GetFieldIndices maps leaf column indices to top-level field indices, - // correctly handling nested types (List, Struct, Map). - const auto& manifest = file_reader->manifest(); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - std::vector field_indices, - manifest.GetFieldIndices(std::vector(column_indices.begin(), column_indices.end()))); - std::vector> fields; - for (int field_idx : field_indices) { - fields.push_back(manifest.schema_fields[field_idx].field); - } - return arrow::schema(fields); -} - } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h b/src/paimon/format/parquet/page_filtered_row_group_reader.h index c31fce313..5e793aaeb 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.h +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h @@ -107,9 +107,6 @@ class PageFilteredRowGroupReader { int32_t row_group_index, int32_t field_index, const std::vector& column_indices, const RowRanges& row_ranges, int64_t row_group_row_count, std::shared_ptr<::arrow::MemoryPool> pool); - - static Result> BuildPageFilteredSchema( - ::parquet::arrow::FileReader* file_reader, const std::vector& column_indices); }; } // namespace paimon::parquet From 2072e3ed47005f109f512eb5507be5769553194f Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Fri, 10 Jul 2026 14:51:41 +0800 Subject: [PATCH 09/23] refractor --- src/paimon/format/parquet/page_filtered_row_group_reader.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index 864a1fb3e..0d8955698 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -271,10 +271,7 @@ Result> PageFilteredRowGroupReader::Re columns.push_back(std::move(chunked_array)); } - // Build schema from actual column types (not SchemaManifest's full field). - // This handles sub-column projection: when filter_leaves prunes some children, - // the ChunkedArray type is the projected type (e.g. struct instead of struct), - // which may differ from the full field in SchemaManifest. + // Build schema from actual column types std::vector> result_fields; for (size_t i = 0; i < columns.size(); ++i) { const auto& field_name = manifest.schema_fields[field_indices[i]].field->name(); From f9c7a7cfb79a5c7b441df5dc7840c69b38602c32 Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Thu, 23 Jul 2026 10:50:01 +0800 Subject: [PATCH 10/23] fix: unittest --- .../page_filtered_row_group_reader.cpp | 10 ++- .../page_filtered_row_group_reader_test.cpp | 89 ------------------- .../parquet/parquet_file_batch_reader.cpp | 2 +- 3 files changed, 7 insertions(+), 94 deletions(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index a3cdf81b4..ea76ecdca 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -172,7 +172,9 @@ Result> PageFilteredRowGroupReader::ReadFil [row_group_index, &rg_page_index_reader, &row_ranges, row_group_row_count]( int col_idx, ::parquet::ParquetFileReader* reader) -> ::parquet::arrow::FileColumnIterator* { - auto* iter = new ::parquet::arrow::FileColumnIterator(col_idx, reader, {row_group_index}); + // Hold sole ownership locally so the iterator is released if GetOffsetIndex() + auto iter = std::make_unique<::parquet::arrow::FileColumnIterator>( + col_idx, reader, std::vector{row_group_index}); if (rg_page_index_reader) { auto offset_index = rg_page_index_reader->GetOffsetIndex(col_idx); if (offset_index) { @@ -180,7 +182,7 @@ Result> PageFilteredRowGroupReader::ReadFil MakePageFilter(row_ranges, offset_index, row_group_row_count)); } } - return iter; + return iter.release(); }; // Build reader tree with leaf column filtering @@ -226,8 +228,8 @@ Result> PageFilteredRowGroupReader::Re bool pre_buffered, const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize, std::shared_ptr<::arrow::MemoryPool> pool) { auto parquet_reader = arrow_file_reader->parquet_reader(); - const auto& row_ranges = target_row_group.row_ranges; - int32_t row_group_index = target_row_group.row_group_index; + const auto& row_ranges = target_row_group.GetRowRanges(); + int32_t row_group_index = target_row_group.GetRowGroupIndex(); int64_t expected_rows = row_ranges.RowCount(); diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index 24af1c235..db3bead37 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -1103,95 +1103,6 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedMapColumnRowGroupFilter) { ASSERT_TRUE(expected->Equals(result->chunk(0))); } -/// Test: nested map projection falls back to row-group-level filtering when page index filter is -/// unavailable for nested read schemas. -/// -/// Schema: { id: int32, props: map } -/// 100 rows, 10 per page, 2 row group. -/// Bitmap: {70..99} hits the second row group (50..99). -/// Because nested schema disables page-level filtering, the entire row group 1 (50..99) is read, -/// so rows [50, 99] should all be returned. -TEST_F(PageFilteredRowGroupReaderTest, NestedMapBitmapFallback) { - std::string file_name = dir_->Str() + "/nested_map_projection_fallback.parquet"; - auto data = MakeMapColumnData(100); - WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); - - auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32())); - auto read_schema = arrow::schema({arrow::field("id", arrow::int32()), field_props}); - - RoaringBitmap32 bitmap; - bitmap.AddRange(70, 100); - - std::shared_ptr result; - ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result); - - ASSERT_TRUE(result); - // Because page-level filtering is skipped for nested schemas, we read full row groups. - ASSERT_EQ(50, result->length()); - - auto expected = data->Slice(50, 50); - ASSERT_TRUE(expected->Equals(result->chunk(0))); -} - -/// Test: nested list projection falls back to row-group-level filtering when page index filter is -/// unavailable for nested read schemas. -/// -/// Schema: { id: int32, tags: list } -/// 100 rows, 10 per page, 2 row group. -/// Bitmap: {70..99} hits the second row group (50..99). -/// Because nested schema disables page-level filtering, the entire row group 1 (50..99) is read, -/// so rows [50, 99] should all be returned. -TEST_F(PageFilteredRowGroupReaderTest, NestedListBitmapFallback) { - std::string file_name = dir_->Str() + "/nested_list_projection_fallback.parquet"; - auto data = MakeListColumnData(100); - WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); - - auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))); - auto read_schema = arrow::schema({arrow::field("id", arrow::int32()), field_tags}); - - RoaringBitmap32 bitmap; - bitmap.AddRange(70, 100); - - std::shared_ptr result; - ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result); - - ASSERT_TRUE(result); - ASSERT_EQ(50, result->length()); - - auto expected = data->Slice(50, 50); - ASSERT_TRUE(expected->Equals(result->chunk(0))); -} - -/// Test: nested struct projection falls back to row-group-level filtering when page index filter is -/// unavailable for nested read schemas. -/// -/// Schema: { id: int32, info: struct } -/// Bitmap: {70..99} hits the second row group (50..99). -/// Because nested schema disables page-level filtering, the entire second row group (50..99) is -/// read. -TEST_F(PageFilteredRowGroupReaderTest, NestedStructBitmapFallback) { - std::string file_name = dir_->Str() + "/nested_struct_projection_fallback.parquet"; - auto data = MakeNestedStructData(100); - WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); - - auto field_x = arrow::field("x", arrow::int32()); - auto field_y = arrow::field("y", arrow::int32()); - auto field_info = arrow::field("info", arrow::struct_({field_x, field_y})); - auto read_schema = arrow::schema({arrow::field("id", arrow::int32()), field_info}); - - RoaringBitmap32 bitmap; - bitmap.AddRange(70, 100); - - std::shared_ptr result; - ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result); - - ASSERT_TRUE(result); - ASSERT_EQ(50, result->length()); - - auto expected = data->Slice(50, 50); - ASSERT_TRUE(expected->Equals(result->chunk(0))); -} - /// Test: rowgroup-level filtering with multiple adjacent nested columns (struct + list). /// /// Schema: { id: int32, info: struct, tags: list } diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index a90e37cfc..eb216adfc 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -173,7 +173,7 @@ Status ParquetFileBatchReader::SetReadSchema( FilterRowGroupsByBitmap(selection_bitmap.value(), target_row_groups)); // workaround: page index filter does not support nested fields for now, skip page index // bitmap pushdown if there is any nested field in the schema - if (!has_nested_field && enable_page_index_filter) { + if (enable_page_index_filter) { // To decide which strategy to use, "trim" or "coalesce". "Coalesce" By default. PAIMON_ASSIGN_OR_RAISE( std::string strategy, From da63de1328a854ac75e078a323e81cff8fd820da Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Thu, 23 Jul 2026 15:54:27 +0800 Subject: [PATCH 11/23] feat: use read/skip pattern like ReadRecords --- cmake_modules/arrow.diff | 182 +++++++++++------- .../page_filtered_row_group_reader.cpp | 75 ++++++-- .../parquet/page_filtered_row_group_reader.h | 12 +- 3 files changed, 177 insertions(+), 92 deletions(-) diff --git a/cmake_modules/arrow.diff b/cmake_modules/arrow.diff index 6302c91fa..c11da3429 100644 --- a/cmake_modules/arrow.diff +++ b/cmake_modules/arrow.diff @@ -431,13 +431,15 @@ diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.c diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h --- a/cpp/src/arrow/io/interfaces.h +++ b/cpp/src/arrow/io/interfaces.h -@@ -210,5 +210,5 @@ +@@ -210,7 +210,7 @@ /// \brief Advance or skip stream indicated number of bytes /// \param[in] nbytes the number to move forward /// \return Status - Status Advance(int64_t nbytes); + virtual Status Advance(int64_t nbytes); + /// \brief Return zero-copy string_view to upcoming bytes. + /// --- a/cpp/src/parquet/arrow/reader.cc +++ b/cpp/src/parquet/arrow/reader.cc @@ -254,6 +254,11 @@ @@ -452,7 +454,7 @@ diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h Status GetSchema(std::shared_ptr<::arrow::Schema>* out) override { return FromParquetSchema(reader_->metadata()->schema(), reader_properties_, reader_->metadata()->key_value_metadata(), out); -@@ -493,10 +498,42 @@ +@@ -493,10 +498,40 @@ ::arrow::Status BuildArray(int64_t length_upper_bound, std::shared_ptr<::arrow::ChunkedArray>* out) final { @@ -466,73 +468,120 @@ diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h return Status::OK(); } -+ ::arrow::Status LoadBatchWithRowFilter(const LeafRowFilter& get_leaf_filter) final { ++ std::vector LeafColumnIndices() const final { ++ return {input_->column_index()}; ++ } ++ ++ ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) final { ++ if (col_idx != input_->column_index()) return Status::OK(); + BEGIN_PARQUET_CATCH_EXCEPTIONS -+ auto [pattern, total] = get_leaf_filter(input_->column_index()); + out_ = nullptr; + record_reader_->Reset(); -+ record_reader_->Reserve(total); -+ int64_t current = 0; -+ for (const auto& [skip, read] : pattern) { -+ if (skip > 0) { -+ record_reader_->SkipRecords(skip); -+ current += skip; -+ } -+ if (read > 0) { -+ record_reader_->ReadRecords(read); -+ current += read; -+ } -+ } -+ if (current < total) { -+ record_reader_->SkipRecords(total - current); -+ } -+ RETURN_NOT_OK( -+ TransferColumnData(record_reader_.get(), field_, descr_, ctx_->pool, &out_)); ++ record_reader_->Reserve(reserve); + return Status::OK(); + END_PARQUET_CATCH_EXCEPTIONS + } ++ ++ int64_t SkipRecords(int col_idx, int64_t num_records) final { ++ if (col_idx != input_->column_index() || num_records <= 0) return 0; ++ return record_reader_->SkipRecords(num_records); ++ } ++ ++ int64_t ReadRecords(int col_idx, int64_t num_records) final { ++ if (col_idx != input_->column_index() || num_records <= 0) return 0; ++ return record_reader_->ReadRecords(num_records); ++ } + const std::shared_ptr field() override { return field_; } private: -@@ -532,6 +569,10 @@ +@@ -532,6 +567,22 @@ return storage_reader_->LoadBatch(number_of_records); } -+ ::arrow::Status LoadBatchWithRowFilter(const LeafRowFilter& get_leaf_filter) final { -+ return storage_reader_->LoadBatchWithRowFilter(get_leaf_filter); ++ std::vector LeafColumnIndices() const final { ++ return storage_reader_->LeafColumnIndices(); ++ } ++ ++ ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) final { ++ return storage_reader_->ResetLeaf(col_idx, reserve); ++ } ++ ++ int64_t SkipRecords(int col_idx, int64_t num_records) final { ++ return storage_reader_->SkipRecords(col_idx, num_records); ++ } ++ ++ int64_t ReadRecords(int col_idx, int64_t num_records) final { ++ return storage_reader_->ReadRecords(col_idx, num_records); + } + Status BuildArray(int64_t length_upper_bound, std::shared_ptr* out) override { std::shared_ptr storage; -@@ -576,6 +617,10 @@ +@@ -576,6 +627,22 @@ return item_reader_->LoadBatch(number_of_records); } -+ ::arrow::Status LoadBatchWithRowFilter(const LeafRowFilter& get_leaf_filter) final { -+ return item_reader_->LoadBatchWithRowFilter(get_leaf_filter); ++ std::vector LeafColumnIndices() const final { ++ return item_reader_->LeafColumnIndices(); ++ } ++ ++ ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) final { ++ return item_reader_->ResetLeaf(col_idx, reserve); ++ } ++ ++ int64_t SkipRecords(int col_idx, int64_t num_records) final { ++ return item_reader_->SkipRecords(col_idx, num_records); ++ } ++ ++ int64_t ReadRecords(int col_idx, int64_t num_records) final { ++ return item_reader_->ReadRecords(col_idx, num_records); + } + virtual ::arrow::Result> AssembleArray( std::shared_ptr data) { if (field_->type()->id() == ::arrow::Type::MAP) { -@@ -709,6 +754,14 @@ +@@ -709,6 +776,39 @@ } return Status::OK(); } + -+ ::arrow::Status LoadBatchWithRowFilter(const LeafRowFilter& get_leaf_filter) override { ++ std::vector LeafColumnIndices() const override { ++ std::vector indices; + for (const std::unique_ptr& reader : children_) { -+ RETURN_NOT_OK(reader->LoadBatchWithRowFilter(get_leaf_filter)); ++ std::vector child_indices = reader->LeafColumnIndices(); ++ indices.insert(indices.end(), child_indices.begin(), child_indices.end()); ++ } ++ return indices; ++ } ++ ++ ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) override { ++ for (const std::unique_ptr& reader : children_) { ++ RETURN_NOT_OK(reader->ResetLeaf(col_idx, reserve)); + } + return Status::OK(); + } ++ ++ int64_t SkipRecords(int col_idx, int64_t num_records) override { ++ int64_t skipped = 0; ++ for (const std::unique_ptr& reader : children_) { ++ skipped += reader->SkipRecords(col_idx, num_records); ++ } ++ return skipped; ++ } ++ ++ int64_t ReadRecords(int col_idx, int64_t num_records) override { ++ int64_t read = 0; ++ for (const std::unique_ptr& reader : children_) { ++ read += reader->ReadRecords(col_idx, num_records); ++ } ++ return read; ++ } + Status BuildArray(int64_t length_upper_bound, std::shared_ptr* out) override; Status GetDefLevels(const int16_t** data, int64_t* length) override; -@@ -1228,6 +1281,23 @@ +@@ -1228,6 +1328,23 @@ std::unique_ptr result; RETURN_NOT_OK(GetReader(manifest_.schema_fields[i], ctx, &result)); *out = std::move(result); @@ -608,24 +657,39 @@ diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h /// \brief Return arrow schema for all the columns. virtual ::arrow::Status GetSchema(std::shared_ptr<::arrow::Schema>* out) = 0; -@@ -316,6 +342,28 @@ +@@ -316,6 +342,43 @@ // the data available in the file. virtual ::arrow::Status NextBatch(int64_t batch_size, std::shared_ptr<::arrow::ChunkedArray>* out) = 0; + -+ /// \brief Load batch with per-leaf row filtering. ++ /// \brief Leaf column indices covered by this (sub)tree, in leaf order. + /// -+ /// The callback is called once per leaf column with that leaf's column index, -+ /// returning (skip_read_pattern, total_row_count) for that specific leaf. -+ /// Each pair in skip_read_pattern is (num_records_to_skip, num_records_to_read). -+ /// After processing all pairs, remaining records up to total_row_count are skipped. -+ /// Must be followed by BuildArray() to get the result. -+ using LeafRowFilter = std::function< -+ std::pair>, int64_t>(int)>; -+ virtual ::arrow::Status LoadBatchWithRowFilter(const LeafRowFilter& get_leaf_filter) { -+ return ::arrow::Status::NotImplemented("LoadBatchWithRowFilter not implemented"); ++ /// Used to drive per-leaf row filtering: after page-level skipping each leaf ++ /// lives in its own compressed coordinate space, so callers must reset and ++ /// skip/read each leaf independently rather than in lockstep. ++ virtual std::vector LeafColumnIndices() const { return {}; } ++ ++ /// \brief Reset the leaf identified by col_idx and reserve space for ++ /// `reserve` records (in that leaf's post-page-filter compressed space). ++ /// Must be called before SkipRecords()/ReadRecords() for that leaf, and ++ /// followed by BuildArray() to get the result. ++ virtual ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) { ++ return ::arrow::Status::NotImplemented("ResetLeaf not implemented"); + } + ++ /// \brief Skip num_records on the leaf identified by col_idx and return the ++ /// number of records actually skipped. Returns 0 when num_records <= 0 or ++ /// col_idx does not belong to this (sub)tree. May throw ParquetException on a ++ /// decode error; callers convert it to Status at the public boundary. ++ virtual int64_t SkipRecords(int col_idx, int64_t num_records) { return 0; } ++ ++ /// \brief Read num_records on the leaf identified by col_idx and return the ++ /// number of records actually read. Values accumulate across successive calls ++ /// until BuildArray() is called. Returns 0 when num_records <= 0 or col_idx ++ /// does not belong to this (sub)tree. May throw ParquetException on a decode ++ /// error; callers convert it to Status at the public boundary. ++ virtual int64_t ReadRecords(int col_idx, int64_t num_records) { return 0; } ++ + /// \brief Build the Arrow array from previously loaded data. + /// For leaf readers, calls TransferColumnData if not already done. + /// For nested readers, assembles the nested array from child arrays. @@ -647,38 +711,20 @@ diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h #include "parquet/arrow/schema.h" #include "parquet/column_reader.h" #include "parquet/file_reader.h" -@@ -70,6 +71,13 @@ +@@ -70,7 +71,10 @@ virtual ~FileColumnIterator() {} -+ /// \brief Set a data_page_filter that will be applied to every PageReader -+ /// created by NextChunk(). This enables I/O-level page skipping. -+ void set_data_page_filter( -+ std::function filter) { -+ data_page_filter_ = std::move(filter); -+ } -+ - std::unique_ptr<::parquet::PageReader> NextChunk() { +- std::unique_ptr<::parquet::PageReader> NextChunk() { ++ /// \brief Fetch the PageReader for the next row group in this iterator's ++ /// range. Virtual so subclasses can decorate the returned PageReader, e.g. ++ /// to install a data_page_filter for I/O-level page skipping. ++ virtual std::unique_ptr<::parquet::PageReader> NextChunk() { if (row_groups_.empty()) { return nullptr; -@@ -77,7 +85,11 @@ - - auto row_group_reader = reader_->RowGroup(row_groups_.front()); - row_groups_.pop_front(); -- return row_group_reader->GetColumnPageReader(column_index_); -+ auto page_reader = row_group_reader->GetColumnPageReader(column_index_); -+ if (page_reader && data_page_filter_) { -+ page_reader->set_data_page_filter(data_page_filter_); -+ } -+ return page_reader; - } - - const SchemaDescriptor* schema() const { return schema_; } -@@ -93,11 +105,9 @@ - ParquetFileReader* reader_; - const SchemaDescriptor* schema_; + } +@@ -95,9 +99,6 @@ std::deque row_groups_; -+ std::function data_page_filter_; }; -using FileColumnIteratorFactory = diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index ea76ecdca..bede6d482 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -60,6 +60,29 @@ class TableRecordBatchReader : public arrow::RecordBatchReader { arrow::TableBatchReader inner_; }; +/// A FileColumnIterator that installs a data_page_filter on every PageReader it +/// produces, enabling I/O-level page skipping. The base class handles row group +/// iteration; this subclass only decorates the PageReader returned by NextChunk(). +class PageFilteringColumnIterator : public ::parquet::arrow::FileColumnIterator { + public: + PageFilteringColumnIterator( + int column_index, ::parquet::ParquetFileReader* reader, std::vector row_groups, + std::function data_page_filter) + : FileColumnIterator(column_index, reader, std::move(row_groups)), + data_page_filter_(std::move(data_page_filter)) {} + + std::unique_ptr<::parquet::PageReader> NextChunk() override { + std::unique_ptr<::parquet::PageReader> page_reader = FileColumnIterator::NextChunk(); + if (page_reader && data_page_filter_) { + page_reader->set_data_page_filter(data_page_filter_); + } + return page_reader; + } + + private: + std::function data_page_filter_; +}; + } // namespace std::pair PageFilteredRowGroupReader::GetPageRowRange( @@ -124,16 +147,29 @@ std::pair PageFilteredRowGroupReader::ComputeCompressedRowRa return {compressed, compressed_offset}; } -std::vector> PageFilteredRowGroupReader::RowRangesToSkipReadPattern( - const RowRanges& ranges) { - std::vector> pattern; +Status PageFilteredRowGroupReader::ExecuteSkipReadPattern( + ::parquet::arrow::ColumnReader* column_reader, int col_idx, const RowRanges& ranges, + int64_t total) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(column_reader->ResetLeaf(col_idx, total)); int64_t current = 0; for (const auto& range : ranges.GetRanges()) { int64_t skip = range.from > current ? range.from - current : 0; - pattern.emplace_back(skip, range.Count()); + int64_t skipped = column_reader->SkipRecords(col_idx, skip); + if (skipped != skip) { + return Status::Invalid(fmt::format( + "PageFilteredRowGroupReader: leaf {} expected to skip {} records but skipped {}", + col_idx, skip, skipped)); + } + int64_t to_read = range.Count(); + int64_t read = column_reader->ReadRecords(col_idx, to_read); + if (read != to_read) { + return Status::Invalid(fmt::format( + "PageFilteredRowGroupReader: leaf {} expected to read {} records but read {}", + col_idx, to_read, read)); + } current = range.to + 1; } - return pattern; + return Status::OK(); } Status PageFilteredRowGroupReader::WaitForPreBuffer( @@ -172,17 +208,17 @@ Result> PageFilteredRowGroupReader::ReadFil [row_group_index, &rg_page_index_reader, &row_ranges, row_group_row_count]( int col_idx, ::parquet::ParquetFileReader* reader) -> ::parquet::arrow::FileColumnIterator* { - // Hold sole ownership locally so the iterator is released if GetOffsetIndex() - auto iter = std::make_unique<::parquet::arrow::FileColumnIterator>( - col_idx, reader, std::vector{row_group_index}); + // Build the page filter first; the iterator is constructed last, so nothing + // can throw between `new` and the return and the raw pointer never leaks. + std::function data_page_filter; if (rg_page_index_reader) { auto offset_index = rg_page_index_reader->GetOffsetIndex(col_idx); if (offset_index) { - iter->set_data_page_filter( - MakePageFilter(row_ranges, offset_index, row_group_row_count)); + data_page_filter = MakePageFilter(row_ranges, offset_index, row_group_row_count); } } - return iter.release(); + return new PageFilteringColumnIterator(col_idx, reader, std::vector{row_group_index}, + std::move(data_page_filter)); }; // Build reader tree with leaf column filtering @@ -194,10 +230,12 @@ Result> PageFilteredRowGroupReader::ReadFil return std::shared_ptr(); } - // Per-leaf callback: each leaf gets its own skip/read pattern + total. - auto get_leaf_filter = - [&rg_page_index_reader, &row_ranges, row_group_row_count]( - int col_idx) -> std::pair>, int64_t> { + // Drive each leaf independently: after data_page_filter skips non-matching + // pages, every leaf lives in its own compressed coordinate space, so segment + // counts may differ across leaves and cannot be driven in lockstep. For each + // leaf we compute its compressed skip/read pattern and replay it via the + // ResetLeaf/SkipRecords/ReadRecords primitives. + for (int col_idx : column_reader->LeafColumnIndices()) { RowRanges effective_ranges = row_ranges; int64_t effective_total = row_group_row_count; if (rg_page_index_reader) { @@ -209,11 +247,10 @@ Result> PageFilteredRowGroupReader::ReadFil effective_total = total; } } - return {RowRangesToSkipReadPattern(effective_ranges), effective_total}; - }; - // Load + filter + transfer (per-leaf, via tree delegation) - PAIMON_RETURN_NOT_OK_FROM_ARROW(column_reader->LoadBatchWithRowFilter(get_leaf_filter)); + PAIMON_RETURN_NOT_OK(ExecuteSkipReadPattern(column_reader.get(), col_idx, effective_ranges, + effective_total)); + } // Build the Arrow array (TransferColumnData for leaves + assemble for nested) std::shared_ptr chunked_array; diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h b/src/paimon/format/parquet/page_filtered_row_group_reader.h index f9f8905b3..6a23f84e9 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.h +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h @@ -95,13 +95,15 @@ class PageFilteredRowGroupReader { const RowRanges& original_ranges, const std::shared_ptr<::parquet::OffsetIndex>& offset_index, int64_t row_group_row_count); - /// Convert RowRanges to skip/read pattern. Each pair is (skip, read). - static std::vector> RowRangesToSkipReadPattern( - const RowRanges& ranges); + /// Reset the given leaf and replay the skip/read pattern derived from `ranges` + /// directly against the ColumnReader (ResetLeaf + SkipRecords/ReadRecords). + static Status ExecuteSkipReadPattern(::parquet::arrow::ColumnReader* column_reader, int col_idx, + const RowRanges& ranges, int64_t total); /// Read a field (flat or nested) using ColumnReader tree. - /// Sets data_page_filter on all leaves via factory, then uses - /// LoadBatchWithRowFilter with per-leaf compressed_ranges. + /// Sets data_page_filter on all leaves via factory, then drives each leaf + /// independently via ResetLeaf/SkipRecords/ReadRecords using its own + /// compressed_ranges. static Result> ReadFilteredField( ::parquet::arrow::FileReader* arrow_file_reader, const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, From 36fea4dc04b17465844e8c49a549116dcde8b18f Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Thu, 23 Jul 2026 15:54:52 +0800 Subject: [PATCH 12/23] test: add test cases for misaligned pages --- .../page_filtered_row_group_reader_test.cpp | 113 +++++++++++++++++- 1 file changed, 108 insertions(+), 5 deletions(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index db3bead37..6d47f1ed8 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -76,7 +76,7 @@ class PageFilteredRowGroupReaderTest : public ::testing::Test { void WriteTestFile(const std::string& file_name, const std::shared_ptr& struct_array, int32_t write_batch_size, int64_t max_row_group_length, - bool enable_dictionary = false) { + bool enable_dictionary = false, int64_t data_page_size = 1) { auto data_type = struct_array->struct_type(); auto data_schema = arrow::schema(data_type->fields()); auto data_arrow_array = std::make_unique(); @@ -92,10 +92,12 @@ class PageFilteredRowGroupReaderTest : public ::testing::Test { builder.disable_dictionary(); // Ensure page index min/max are meaningful } builder.enable_write_page_index(); // Enable page index for page-level filtering - // Set data page size to 1 byte to force a new page after every write_batch_size rows. - // The writer flushes a page when accumulated data exceeds data_pagesize, so setting - // it to 1 ensures each batch of write_batch_size rows becomes exactly one page. - builder.data_pagesize(1); + // Data page size controls when a page is flushed. The default of 1 byte forces a new + // page after every write_batch_size rows (each batch becomes one page), giving pages + // aligned across columns. A larger byte-based value combined with write_batch_size=1 + // instead lets columns of different physical widths flush pages at different row + // counts, producing intentionally misaligned pages across leaves. + builder.data_pagesize(data_page_size); auto writer_properties = builder.build(); ASSERT_OK_AND_ASSIGN( auto format_writer, @@ -1695,4 +1697,105 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedStructSubColumnProjection) { } } +/// Helper: build a struct with a flat key plus several nested columns of different +/// physical widths / repetition, so that with a byte-based data page size their +/// leaves flush pages at different row counts (misaligned pages): +/// key: int64 (fixed 8B -> ~5 rows/page) +/// s: struct (x ~10 rows/page, y ~5 rows/page) +/// tags: list (2 values/row -> ~5 rows/page) +/// props: map (variable-width utf8 key -> irregular rows/page) +/// key/s.x/s.y encode the row index (= i); tags/props reuse the shared list/map +/// helpers. Correctness is verified per row by deep-comparing against this array. +static std::shared_ptr MakeMisalignedNestedData(int32_t num_rows) { + arrow::Int64Builder key_builder; + arrow::Int32Builder x_builder; + arrow::Int64Builder y_builder; + EXPECT_TRUE(key_builder.Reserve(num_rows).ok()); + EXPECT_TRUE(x_builder.Reserve(num_rows).ok()); + EXPECT_TRUE(y_builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + key_builder.UnsafeAppend(i); + x_builder.UnsafeAppend(i); + y_builder.UnsafeAppend(i); + } + auto key_array = key_builder.Finish().ValueOrDie(); + auto x_array = x_builder.Finish().ValueOrDie(); + auto y_array = y_builder.Finish().ValueOrDie(); + + auto field_x = arrow::field("x", arrow::int32()); + auto field_y = arrow::field("y", arrow::int64()); + auto s_array = arrow::StructArray::Make({x_array, y_array}, {field_x, field_y}).ValueOrDie(); + + // Repeated nested leaves (list and map) paginate on value + // bytes, so they misalign with the struct's fixed-width leaves too. + auto tags_array = MakeListColumnData(num_rows); + auto props_array = MakeMapColumnData(num_rows); + + auto field_key = arrow::field("key", arrow::int64()); + auto field_s = arrow::field("s", arrow::struct_({field_x, field_y})); + auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))); + auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32())); + return arrow::StructArray::Make({key_array, s_array, tags_array, props_array}, + {field_key, field_s, field_tags, field_props}) + .ValueOrDie(); +} + +/// Test: page-level filtering across multiple nested columns whose leaf pages are +/// MISALIGNED. The file mixes a flat key, a struct, a list and +/// a map; with write_batch_size=1 and a byte-based data page size every +/// leaf flushes pages at a different (and, for the utf8 map key, irregular) row +/// count. +/// Bitmap: [0,15), [77, 87) (to avoid bitmap hole filling) +/// Expected: 25 rows +TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsMisalignedPages) { + std::string file_name = dir_->Str() + "/nested_misaligned.parquet"; + constexpr int32_t kNumRows = 100; + auto data = MakeMisalignedNestedData(kNumRows); + + // write_batch_size=1 + a byte-based data page size makes the int32 and int64 leaves + // flush pages at different row counts, i.e. deliberately misaligned pages. + WriteTestFile(file_name, data, /*write_batch_size=*/1, /*max_row_group_length=*/kNumRows, + /*enable_dictionary=*/false, /*data_page_size=*/40); + + auto read_schema = + arrow::schema({arrow::field("key", arrow::int64()), + arrow::field("s", arrow::struct_({arrow::field("x", arrow::int32()), + arrow::field("y", arrow::int64())})), + arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))), + arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()))}); + + // bitmap: [0,15), [77, 87) + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 15); + bitmap.AddRange(77, 87); + + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result); + ASSERT_TRUE(result); + + int64_t total = 0; + auto top = std::dynamic_pointer_cast(result->chunk(0)); + ASSERT_TRUE(top); + auto key_arr = std::dynamic_pointer_cast(top->field(0)); + for (int64_t i = 0; i < key_arr->length(); ++i) { + int64_t k = key_arr->Value(i); + // Full-row deep compare across ALL columns (struct + list + map): the returned + // row must equal the original row identified by key k. This is what catches a + // desync in the repeated list/map leaves, whose reassembly also relies on the + // per-leaf skip/read staying row-consistent. + ASSERT_TRUE(data->Slice(k, 1)->Equals(*top->Slice(i, 1))) + << "row content mismatch at key " << k; + ++total; + } + + for (int64_t i = 0; i < 15; ++i) { + ASSERT_EQ(i, key_arr->Value(i)); + } + for (int64_t i = 0; i < 10; ++i) { + ASSERT_EQ(77 + i, key_arr->Value(15 + i)); + } + + ASSERT_GE(total, 25); +} + } // namespace paimon::parquet::test From 767ba9e623a82a9f82a5cd906cb3bb6350749d3c Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Thu, 23 Jul 2026 16:37:48 +0800 Subject: [PATCH 13/23] fix --- .../parquet/page_filtered_row_group_reader.cpp | 16 ++++++++++++++++ .../page_filtered_row_group_reader_test.cpp | 2 +- .../format/parquet/parquet_file_batch_reader.cpp | 2 -- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index bede6d482..fe6dd94d4 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -270,6 +270,22 @@ Result> PageFilteredRowGroupReader::Re int64_t expected_rows = row_ranges.RowCount(); + if (expected_rows == 0) { + const auto& manifest = arrow_file_reader->manifest(); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::vector field_indices, + manifest.GetFieldIndices(std::vector( + column_indices.begin(), column_indices.end()))); + std::vector> result_fields; + result_fields.reserve(field_indices.size()); + for (int field_idx : field_indices) { + result_fields.push_back(manifest.schema_fields[field_idx].field); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr empty_table, + arrow::Table::MakeEmpty(arrow::schema(result_fields), pool.get())); + return std::make_unique(std::move(empty_table), max_chunksize); + } + PAIMON_RETURN_NOT_OK(WaitForPreBuffer(parquet_reader, row_group_index, column_indices, cache_options, pre_buffered, page_ranges, pool)); diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index 6d47f1ed8..b3b2004f3 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -1795,7 +1795,7 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsMisalignedPages) { ASSERT_EQ(77 + i, key_arr->Value(15 + i)); } - ASSERT_GE(total, 25); + ASSERT_EQ(total, 25); } } // namespace paimon::parquet::test diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index eb216adfc..9ca5bf959 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -201,8 +201,6 @@ Status ParquetFileBatchReader::SetReadSchema( // pages for row groups that the bitmap already excluded. // If no predicate is provided, skip page-level filtering if (predicate && !target_row_groups.empty()) { - // workaround: page index filter does not support nested fields for now, skip page index - // filter if there is any nested field in the schema if (enable_page_index_filter) { // Build column name to index map for page-level filtering. // For leaf columns, indices[0] is the correct leaf column index in Parquet. From 4214acd97d9286816e963bfcfbe9a51cdb7550ce Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Thu, 23 Jul 2026 16:52:03 +0800 Subject: [PATCH 14/23] style: align comments --- .../page_filtered_row_group_reader.cpp | 12 ++-------- .../page_filtered_row_group_reader_test.cpp | 23 ++++++++----------- 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index fe6dd94d4..24d7e89f6 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -208,8 +208,6 @@ Result> PageFilteredRowGroupReader::ReadFil [row_group_index, &rg_page_index_reader, &row_ranges, row_group_row_count]( int col_idx, ::parquet::ParquetFileReader* reader) -> ::parquet::arrow::FileColumnIterator* { - // Build the page filter first; the iterator is constructed last, so nothing - // can throw between `new` and the return and the raw pointer never leaks. std::function data_page_filter; if (rg_page_index_reader) { auto offset_index = rg_page_index_reader->GetOffsetIndex(col_idx); @@ -230,11 +228,8 @@ Result> PageFilteredRowGroupReader::ReadFil return std::shared_ptr(); } - // Drive each leaf independently: after data_page_filter skips non-matching - // pages, every leaf lives in its own compressed coordinate space, so segment - // counts may differ across leaves and cannot be driven in lockstep. For each - // leaf we compute its compressed skip/read pattern and replay it via the - // ResetLeaf/SkipRecords/ReadRecords primitives. + // Since leaf columns may have misaligned pages, we compute compressed row ranges and drive each + // leaf column independently for (int col_idx : column_reader->LeafColumnIndices()) { RowRanges effective_ranges = row_ranges; int64_t effective_total = row_group_row_count; @@ -300,13 +295,11 @@ Result> PageFilteredRowGroupReader::Re rg_page_index_reader = page_index_reader->RowGroup(row_group_index); } - // Group leaf column indices by top-level field using SchemaManifest const auto& manifest = arrow_file_reader->manifest(); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( std::vector field_indices, manifest.GetFieldIndices(std::vector(column_indices.begin(), column_indices.end()))); - // Read each field with page filtering (unified path for flat and nested) std::vector> columns; columns.reserve(field_indices.size()); @@ -326,7 +319,6 @@ Result> PageFilteredRowGroupReader::Re columns.push_back(std::move(chunked_array)); } - // Build schema from actual column types std::vector> result_fields; for (size_t i = 0; i < columns.size(); ++i) { const auto& field_name = manifest.schema_fields[field_indices[i]].field->name(); diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index b3b2004f3..17270c18f 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -930,14 +930,10 @@ static std::shared_ptr MakeNestedStructData(int32_t num_rows /// Test: rowgroup-level filtering on a file with nested struct columns. /// -/// This test exposes the bug where BuildPageFilteredSchema fails to correctly map -/// Parquet leaf column indices to Arrow fields for nested types, and -/// ReadFilteredRowGroup cannot correctly assemble nested column results. -/// /// Schema: { id: int32, info: struct } /// Parquet leaf columns: [id=0, info.x=1, info.y=2] /// 100 rows, 10 per page, 2 row groups. -/// Predicate: id >= 70 → row groups 0 skipped, row groups 1 read → 50 rows expected. +/// Predicate: id >= 70 → page 0-7 skipped, paged 8-9 read → 30 rows expected. /// The read schema requests both "id" and "info" columns. TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnRowGroupFilter) { std::string file_name = dir_->Str() + "/nested_struct_filter.parquet"; @@ -961,7 +957,7 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnRowGroupFilter) { std::shared_ptr result; ReadWithPredicateImpl(file_name, read_schema, predicate, &result); - // Should get rows 50-99 = 50 rows + // Should get rows 70-99 = 30 rows ASSERT_TRUE(result); ASSERT_EQ(30, result->length()); @@ -972,8 +968,7 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnRowGroupFilter) { /// Test: Page-level filtering reading only the predicate column (no nested column in read schema). /// -/// This verifies that when reading only the "id" column (without the nested struct), -/// page-level filtering works correctly since the read schema contains no nested types. +/// This verifies that when reading only the "id" column (without the nested struct) /// /// Schema: { id: int32, info: struct } /// Read schema: { id: int32 } @@ -1043,7 +1038,7 @@ static std::shared_ptr MakeMapColumnData(int32_t num_rows) { /// /// Schema: { id: int32, tags: list } /// 100 rows, 10 per page, 2 row groups. -/// Predicate: id >= 70 → row groups 0 skipped, row groups 1 read → 50 rows expected. +/// Predicate: id >= 70 → page 0-7 skipped, page 8-9 read → 30 rows expected. TEST_F(PageFilteredRowGroupReaderTest, NestedListColumnRowGroupFilter) { std::string file_name = dir_->Str() + "/nested_list_filter.parquet"; @@ -1067,7 +1062,7 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedListColumnRowGroupFilter) { ASSERT_TRUE(result); ASSERT_EQ(30, result->length()); - // Build expected result: rows 50-99 from the original data + // Build expected result: rows 70-99 from the original data auto expected = data->Slice(70, 30); ASSERT_TRUE(expected->Equals(result->chunk(0))); } @@ -1076,7 +1071,7 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedListColumnRowGroupFilter) { /// /// Schema: { id: int32, props: map } /// 100 rows, 10 per page, 2 row groups. -/// Predicate: id >= 70 → row groups 0 skipped, row groups 1 read → 50 rows expected. +/// Predicate: id >= 70 → page 0-7 skipped, page 8-9 read → 30 rows expected. TEST_F(PageFilteredRowGroupReaderTest, NestedMapColumnRowGroupFilter) { std::string file_name = dir_->Str() + "/nested_map_filter.parquet"; @@ -1100,7 +1095,7 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedMapColumnRowGroupFilter) { ASSERT_TRUE(result); ASSERT_EQ(30, result->length()); - // Build expected result: rows 50-99 from the original data + // Build expected result: rows 70-99 from the original data auto expected = data->Slice(70, 30); ASSERT_TRUE(expected->Equals(result->chunk(0))); } @@ -1109,7 +1104,7 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedMapColumnRowGroupFilter) { /// /// Schema: { id: int32, info: struct, tags: list } /// This tests the boundary handling when two nested fields are adjacent in the schema. -/// Predicate: id >= 70 → row groups 0 skipped, row groups 1 read → 50 rows expected. +/// Predicate: id >= 70 → page 0-7 skipped, page 8-9 read → 30 rows expected. TEST_F(PageFilteredRowGroupReaderTest, MultipleAdjacentNestedColumns) { std::string file_name = dir_->Str() + "/multi_nested.parquet"; @@ -1138,7 +1133,7 @@ TEST_F(PageFilteredRowGroupReaderTest, MultipleAdjacentNestedColumns) { ASSERT_TRUE(result); ASSERT_EQ(30, result->length()); - // Build expected result: rows 50-99 from the original data + // Build expected result: rows 70-99 from the original data auto expected = data->Slice(70, 30); ASSERT_TRUE(expected->Equals(result->chunk(0))); } From 589b45f82833528a1f6c0c7ee8f9706f79afa130 Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Thu, 23 Jul 2026 17:37:29 +0800 Subject: [PATCH 15/23] pre-commit --- src/paimon/format/parquet/page_filtered_row_group_reader.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index 438d2f1b9..123737ee0 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -281,10 +281,9 @@ Result> PageFilteredRowGroupReader::Re int64_t expected_rows = row_ranges.RowCount(); - if (!row_ranges.IsEmpty()) - { + if (!row_ranges.IsEmpty()) { PAIMON_RETURN_NOT_OK(WaitForPreBuffer(parquet_reader, row_group_index, column_indices, - cache_options, pre_buffered, page_ranges, pool)); + cache_options, pre_buffered, page_ranges, pool)); } auto rg_metadata = parquet_reader->metadata()->RowGroup(row_group_index); From 06c810124c5d9f88d16739b5a7a3c1ec233d5ab5 Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Fri, 24 Jul 2026 10:14:30 +0800 Subject: [PATCH 16/23] small fixes --- .../parquet/page_filtered_row_group_reader.cpp | 12 +++++++----- .../format/parquet/page_filtered_row_group_reader.h | 3 +-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index 123737ee0..265c9a63e 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -216,8 +216,7 @@ Result> PageFilteredRowGroupReader::ReadFil ::parquet::arrow::FileReader* arrow_file_reader, const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, int32_t row_group_index, int32_t field_index, const std::vector& column_indices, - const RowRanges& row_ranges, int64_t row_group_row_count, - std::shared_ptr<::arrow::MemoryPool> pool) { + const RowRanges& row_ranges, int64_t row_group_row_count) { // Factory: set data_page_filter on every leaf (per-leaf OffsetIndex). // data_page_filter enables I/O-level page skipping for all leaves. auto factory = @@ -241,14 +240,17 @@ Result> PageFilteredRowGroupReader::ReadFil arrow_file_reader->GetColumn(field_index, column_indices, factory, &column_reader)); if (!column_reader) { - return std::shared_ptr(); + return Status::Invalid( + fmt::format("PageFilteredRowGroupReader: field {} has no matching leaf columns " + "(row_group={})", + field_index, row_group_index)); } // Since leaf columns may have misaligned pages, we compute compressed row ranges and drive each // leaf column independently + int64_t effective_total = row_group_row_count; for (int col_idx : column_reader->LeafColumnIndices()) { RowRanges effective_ranges = row_ranges; - int64_t effective_total = row_group_row_count; if (rg_page_index_reader) { auto offset_index = rg_page_index_reader->GetOffsetIndex(col_idx); if (offset_index) { @@ -265,7 +267,7 @@ Result> PageFilteredRowGroupReader::ReadFil // Build the Arrow array (TransferColumnData for leaves + assemble for nested) std::shared_ptr chunked_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(column_reader->BuildArray(row_group_row_count, &chunked_array)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(column_reader->BuildArray(effective_total, &chunked_array)); return chunked_array; } diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h b/src/paimon/format/parquet/page_filtered_row_group_reader.h index 60b9925c3..7a89d981a 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.h +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h @@ -107,8 +107,7 @@ class PageFilteredRowGroupReader { ::parquet::arrow::FileReader* arrow_file_reader, const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, int32_t row_group_index, int32_t field_index, const std::vector& column_indices, - const RowRanges& row_ranges, int64_t row_group_row_count, - std::shared_ptr<::arrow::MemoryPool> pool); + const RowRanges& row_ranges, int64_t row_group_row_count); }; } // namespace paimon::parquet From 10f9b5c154cf500f565afb79f36a06c9cb3b5f47 Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Fri, 24 Jul 2026 10:17:46 +0800 Subject: [PATCH 17/23] remove pool parameter --- src/paimon/format/parquet/page_filtered_row_group_reader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index 265c9a63e..b4d494244 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -311,7 +311,7 @@ Result> PageFilteredRowGroupReader::Re PAIMON_ASSIGN_OR_RAISE( std::shared_ptr chunked_array, ReadFilteredField(arrow_file_reader, rg_page_index_reader, row_group_index, field_idx, - column_indices, row_ranges, row_group_row_count, pool)); + column_indices, row_ranges, row_group_row_count)); if (chunked_array && chunked_array->length() != expected_rows) { return Status::Invalid( From f0ca3a7429eeffa5253a0f492634a2b4bf3f268e Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Fri, 24 Jul 2026 10:21:42 +0800 Subject: [PATCH 18/23] fix: small fixes --- .../format/parquet/page_filtered_row_group_reader.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index b4d494244..1306fb918 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -248,9 +248,10 @@ Result> PageFilteredRowGroupReader::ReadFil // Since leaf columns may have misaligned pages, we compute compressed row ranges and drive each // leaf column independently - int64_t effective_total = row_group_row_count; + for (int col_idx : column_reader->LeafColumnIndices()) { RowRanges effective_ranges = row_ranges; + int64_t effective_total = row_group_row_count; if (rg_page_index_reader) { auto offset_index = rg_page_index_reader->GetOffsetIndex(col_idx); if (offset_index) { @@ -267,7 +268,8 @@ Result> PageFilteredRowGroupReader::ReadFil // Build the Arrow array (TransferColumnData for leaves + assemble for nested) std::shared_ptr chunked_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(column_reader->BuildArray(effective_total, &chunked_array)); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + column_reader->BuildArray(row_ranges.RowCount(), &chunked_array)); return chunked_array; } @@ -313,7 +315,7 @@ Result> PageFilteredRowGroupReader::Re ReadFilteredField(arrow_file_reader, rg_page_index_reader, row_group_index, field_idx, column_indices, row_ranges, row_group_row_count)); - if (chunked_array && chunked_array->length() != expected_rows) { + if (chunked_array->length() != expected_rows) { return Status::Invalid( fmt::format("PageFilteredRowGroupReader: field {} produced {} rows but expected {} " "(row_group={})", From babac6d4a2cb922dd039aceb89aa93d20b21c822 Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Fri, 24 Jul 2026 15:31:30 +0800 Subject: [PATCH 19/23] test: add test case fo nested columns with null across multi RowGroup --- .../page_filtered_row_group_reader_test.cpp | 212 +++++++++++++++++- 1 file changed, 206 insertions(+), 6 deletions(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index 246bb614c..99305e3b5 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -17,6 +17,7 @@ #include "paimon/format/parquet/page_filtered_row_group_reader.h" #include +#include #include #include #include @@ -30,6 +31,8 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/defs.h" @@ -44,6 +47,7 @@ #include "paimon/status.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/testing/utils/variant_test_data.h" #include "paimon/utils/roaring_bitmap32.h" #include "parquet/arrow/reader.h" #include "parquet/file_reader.h" @@ -1752,19 +1756,20 @@ static std::shared_ptr MakeMisalignedNestedData(int32_t num_ } /// Test: page-level filtering across multiple nested columns whose leaf pages are -/// MISALIGNED. The file mixes a flat key, a struct, a list and -/// a map; with write_batch_size=1 and a byte-based data page size every -/// leaf flushes pages at a different (and, for the utf8 map key, irregular) row -/// count. +/// MISALIGNED, within a SINGLE row group. The file mixes a flat key, a +/// struct, a list and a map; with write_batch_size=1 +/// and a byte-based data page size every leaf flushes pages at a different (and, for +/// the utf8 map key, irregular) row count. /// Bitmap: [0,15), [77, 87) (to avoid bitmap hole filling) /// Expected: 25 rows -TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsMisalignedPages) { - std::string file_name = dir_->Str() + "/nested_misaligned.parquet"; +TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsMisalignedPagesSingleRowGroup) { + std::string file_name = dir_->Str() + "/nested_misaligned_single_rg.parquet"; constexpr int32_t kNumRows = 100; auto data = MakeMisalignedNestedData(kNumRows); // write_batch_size=1 + a byte-based data page size makes the int32 and int64 leaves // flush pages at different row counts, i.e. deliberately misaligned pages. + // max_row_group_length=kNumRows keeps all rows in a single row group. WriteTestFile(file_name, data, /*write_batch_size=*/1, /*max_row_group_length=*/kNumRows, /*enable_dictionary=*/false, /*data_page_size=*/40); @@ -1809,4 +1814,199 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsMisalignedPages) { ASSERT_EQ(total, 25); } +/// Test: same misaligned nested layout as the single-row-group case above, but split +/// across MULTIPLE row groups (max_row_group_length=40 -> row groups of 40/40/20). +/// The selection bitmap spans row-group boundaries so the reader must keep the +/// per-leaf skip/read row-consistent both across misaligned pages and across row +/// groups. +/// Bitmap: [0,15), [77, 87) (to avoid bitmap hole filling) +/// Expected: 25 rows -> keys 0..14 and 77..86 +TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsMisalignedPagesMultiRowGroup) { + std::string file_name = dir_->Str() + "/nested_misaligned_multi_rg.parquet"; + constexpr int32_t kNumRows = 100; + auto data = MakeMisalignedNestedData(kNumRows); + + // write_batch_size=1 + byte-based data page size -> misaligned leaf pages. + // max_row_group_length=40 -> 3 row groups (40, 40, 20). + WriteTestFile(file_name, data, /*write_batch_size=*/1, /*max_row_group_length=*/40, + /*enable_dictionary=*/false, /*data_page_size=*/40); + + auto read_schema = + arrow::schema({arrow::field("key", arrow::int64()), + arrow::field("s", arrow::struct_({arrow::field("x", arrow::int32()), + arrow::field("y", arrow::int64())})), + arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))), + arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()))}); + + // bitmap: [0,15), [77, 87) -> spans all three row groups. + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 15); + bitmap.AddRange(77, 87); + + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result); + ASSERT_TRUE(result); + + std::vector keys; + + auto top = std::dynamic_pointer_cast(result->chunk(0)); + ASSERT_TRUE(top); + auto key_arr = std::dynamic_pointer_cast(top->field(0)); + ASSERT_TRUE(key_arr); + for (int64_t i = 0; i < key_arr->length(); ++i) { + int64_t k = key_arr->Value(i); + ASSERT_TRUE(data->Slice(k, 1)->Equals(*top->Slice(i, 1))) + << "row content mismatch at key " << k; + keys.push_back(k); + } + + std::vector expected; + for (int64_t i = 0; i < 15; ++i) { + expected.push_back(i); + } + for (int64_t i = 77; i < 87; ++i) { + expected.push_back(i); + } + ASSERT_EQ(keys, expected); +} + +/// Helper: like MakeMisalignedNestedData but sprinkles nulls into the nested columns +/// so that definition levels vary per row (which also perturbs page boundaries): +/// key: int64, always non-null (= i, used to identify the row) +/// s: struct; whole struct null when i%11==0, otherwise +/// x null when i%5==0 and y null when i%7==0 +/// tags: list; null when i%6==0, otherwise [i*10, i*10+1] +/// props: map; null when i%8==0, otherwise {"k_i": i*100} +/// Correctness is verified per row by deep-comparing against this array. +static std::shared_ptr MakeMisalignedNestedDataWithNulls(int32_t num_rows) { + arrow::Int64Builder key_builder; + EXPECT_TRUE(key_builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + key_builder.UnsafeAppend(i); + } + auto key_array = key_builder.Finish().ValueOrDie(); + + // s: struct with nulls at both leaf and struct level. + auto field_x = arrow::field("x", arrow::int32()); + auto field_y = arrow::field("y", arrow::int64()); + auto x_builder = std::make_shared(); + auto y_builder = std::make_shared(); + arrow::StructBuilder s_builder(arrow::struct_({field_x, field_y}), arrow::default_memory_pool(), + {x_builder, y_builder}); + for (int32_t i = 0; i < num_rows; ++i) { + if (i % 11 == 0) { + // AppendNull() also appends nulls to the child builders, keeping lengths in sync. + EXPECT_TRUE(s_builder.AppendNull().ok()); + continue; + } + EXPECT_TRUE(s_builder.Append().ok()); + if (i % 5 == 0) { + EXPECT_TRUE(x_builder->AppendNull().ok()); + } else { + EXPECT_TRUE(x_builder->Append(i).ok()); + } + if (i % 7 == 0) { + EXPECT_TRUE(y_builder->AppendNull().ok()); + } else { + EXPECT_TRUE(y_builder->Append(i).ok()); + } + } + auto s_array = s_builder.Finish().ValueOrDie(); + + // tags: list with null lists. + auto item_builder = std::make_shared(); + arrow::ListBuilder tags_builder(arrow::default_memory_pool(), item_builder); + for (int32_t i = 0; i < num_rows; ++i) { + if (i % 6 == 0) { + EXPECT_TRUE(tags_builder.AppendNull().ok()); + } else { + EXPECT_TRUE(tags_builder.Append().ok()); + EXPECT_TRUE(item_builder->Append(i * 10).ok()); + EXPECT_TRUE(item_builder->Append(i * 10 + 1).ok()); + } + } + auto tags_array = tags_builder.Finish().ValueOrDie(); + + // props: map with null maps. + auto map_key_builder = std::make_shared(); + auto map_val_builder = std::make_shared(); + arrow::MapBuilder props_builder(arrow::default_memory_pool(), map_key_builder, map_val_builder); + for (int32_t i = 0; i < num_rows; ++i) { + if (i % 8 == 0) { + EXPECT_TRUE(props_builder.AppendNull().ok()); + } else { + EXPECT_TRUE(props_builder.Append().ok()); + EXPECT_TRUE(map_key_builder->Append("k_" + std::to_string(i)).ok()); + EXPECT_TRUE(map_val_builder->Append(i * 100).ok()); + } + } + auto props_array = props_builder.Finish().ValueOrDie(); + + auto field_key = arrow::field("key", arrow::int64()); + auto field_s = arrow::field("s", arrow::struct_({field_x, field_y})); + auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))); + auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32())); + return arrow::StructArray::Make({key_array, s_array, tags_array, props_array}, + {field_key, field_s, field_tags, field_props}) + .ValueOrDie(); +} + +/// Test: nested columns containing NULLs, with MISALIGNED leaf pages, split across +/// MULTIPLE row groups. This combines the three stress dimensions: null-driven +/// definition levels, byte-based misaligned pages, and row-group boundaries that the +/// selection bitmap crosses. Every returned row is deep-compared (nulls included) +/// against the original data identified by its non-null key. +/// Bitmap: [0,15), [77, 87) (to avoid bitmap hole filling) +/// Expected: 25 rows -> keys 0..14 and 77..86 +TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsWithNullsMisalignedPagesMultiRowGroup) { + std::string file_name = dir_->Str() + "/nested_nulls_misaligned_multi_rg.parquet"; + constexpr int32_t kNumRows = 100; + auto data = MakeMisalignedNestedDataWithNulls(kNumRows); + + // write_batch_size=1 + byte-based data page size -> misaligned leaf pages. + // max_row_group_length=40 -> 3 row groups (40, 40, 20). + WriteTestFile(file_name, data, /*write_batch_size=*/1, /*max_row_group_length=*/40, + /*enable_dictionary=*/false, /*data_page_size=*/40); + + auto read_schema = + arrow::schema({arrow::field("key", arrow::int64()), + arrow::field("s", arrow::struct_({arrow::field("x", arrow::int32()), + arrow::field("y", arrow::int64())})), + arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))), + arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()))}); + + // bitmap: [0,15), [77, 87) -> spans all three row groups. + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 15); + bitmap.AddRange(77, 87); + + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result); + ASSERT_TRUE(result); + + std::vector keys; + + auto top = std::dynamic_pointer_cast(result->chunk(0)); + ASSERT_TRUE(top); + auto key_arr = std::dynamic_pointer_cast(top->field(0)); + ASSERT_TRUE(key_arr); + for (int64_t i = 0; i < key_arr->length(); ++i) { + ASSERT_FALSE(key_arr->IsNull(i)) << "key column must stay non-null"; + int64_t k = key_arr->Value(i); + // Deep compare including nulls across struct/list/map leaves. + ASSERT_TRUE(data->Slice(k, 1)->Equals(*top->Slice(i, 1))) + << "row content mismatch at key " << k; + keys.push_back(k); + } + + std::vector expected; + for (int64_t i = 0; i < 15; ++i) { + expected.push_back(i); + } + for (int64_t i = 77; i < 87; ++i) { + expected.push_back(i); + } + ASSERT_EQ(keys, expected); +} + } // namespace paimon::parquet::test From 70ae162fe38bc9ed35e2a656485568c1a366d781 Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Fri, 24 Jul 2026 15:32:18 +0800 Subject: [PATCH 20/23] fix: use source field to avoid dropping nullable infomation --- src/paimon/format/parquet/page_filtered_row_group_reader.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index 1306fb918..8720f5c01 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -327,8 +327,9 @@ Result> PageFilteredRowGroupReader::Re std::vector> result_fields; for (size_t i = 0; i < columns.size(); ++i) { - const auto& field_name = manifest.schema_fields[field_indices[i]].field->name(); - result_fields.push_back(arrow::field(field_name, columns[i]->type())); + const auto& field = manifest.schema_fields[field_indices[i]].field; + result_fields.push_back( + arrow::field(field->name(), columns[i]->type(), field->nullable(), field->metadata())); } auto result_schema = arrow::schema(result_fields); From 0674e6280596a1045353eb714b08d30d17a18fce Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Fri, 24 Jul 2026 16:06:13 +0800 Subject: [PATCH 21/23] fix: concate chunked arrays --- .../format/parquet/page_filtered_row_group_reader_test.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index 99305e3b5..1911c4c64 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -1849,7 +1849,8 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsMisalignedPagesMultiRowGroup std::vector keys; - auto top = std::dynamic_pointer_cast(result->chunk(0)); + auto concated = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto top = std::dynamic_pointer_cast(concated); ASSERT_TRUE(top); auto key_arr = std::dynamic_pointer_cast(top->field(0)); ASSERT_TRUE(key_arr); @@ -1986,7 +1987,8 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsWithNullsMisalignedPagesMult std::vector keys; - auto top = std::dynamic_pointer_cast(result->chunk(0)); + auto concated = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto top = std::dynamic_pointer_cast(concated); ASSERT_TRUE(top); auto key_arr = std::dynamic_pointer_cast(top->field(0)); ASSERT_TRUE(key_arr); From cf52d1a40cf419ad4662d24e8e38d69e96947bc4 Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Fri, 24 Jul 2026 16:35:57 +0800 Subject: [PATCH 22/23] rename variables --- .../parquet/page_filtered_row_group_reader.cpp | 14 +++++++------- .../page_filtered_row_group_reader_test.cpp | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index 8720f5c01..c51becf31 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -306,8 +306,8 @@ Result> PageFilteredRowGroupReader::Re std::vector field_indices, manifest.GetFieldIndices(std::vector(column_indices.begin(), column_indices.end()))); - std::vector> columns; - columns.reserve(field_indices.size()); + std::vector> result_arrays; + result_arrays.reserve(field_indices.size()); for (int field_idx : field_indices) { PAIMON_ASSIGN_OR_RAISE( @@ -322,18 +322,18 @@ Result> PageFilteredRowGroupReader::Re field_idx, chunked_array->length(), expected_rows, row_group_index)); } - columns.push_back(std::move(chunked_array)); + result_arrays.push_back(std::move(chunked_array)); } std::vector> result_fields; - for (size_t i = 0; i < columns.size(); ++i) { + for (size_t i = 0; i < result_arrays.size(); ++i) { const auto& field = manifest.schema_fields[field_indices[i]].field; - result_fields.push_back( - arrow::field(field->name(), columns[i]->type(), field->nullable(), field->metadata())); + result_fields.push_back(arrow::field(field->name(), result_arrays[i]->type(), + field->nullable(), field->metadata())); } auto result_schema = arrow::schema(result_fields); - auto table = arrow::Table::Make(result_schema, std::move(columns), expected_rows); + auto table = arrow::Table::Make(result_schema, std::move(result_arrays), expected_rows); return std::make_unique(std::move(table), max_chunksize, pool); } diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index 1911c4c64..64c688bb1 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -955,7 +955,7 @@ static std::shared_ptr MakeNestedStructData(int32_t num_rows /// 100 rows, 10 per page, 2 row groups. /// Predicate: id >= 70 → page 0-7 skipped, paged 8-9 read → 30 rows expected. /// The read schema requests both "id" and "info" columns. -TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnRowGroupFilter) { +TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnPageFilter) { std::string file_name = dir_->Str() + "/nested_struct_filter.parquet"; auto field_x = arrow::field("x", arrow::int32()); @@ -1059,7 +1059,7 @@ static std::shared_ptr MakeMapColumnData(int32_t num_rows) { /// Schema: { id: int32, tags: list } /// 100 rows, 10 per page, 2 row groups. /// Predicate: id >= 70 → page 0-7 skipped, page 8-9 read → 30 rows expected. -TEST_F(PageFilteredRowGroupReaderTest, NestedListColumnRowGroupFilter) { +TEST_F(PageFilteredRowGroupReaderTest, NestedListColumnPageFilter) { std::string file_name = dir_->Str() + "/nested_list_filter.parquet"; auto field_id = arrow::field("id", arrow::int32()); @@ -1092,7 +1092,7 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedListColumnRowGroupFilter) { /// Schema: { id: int32, props: map } /// 100 rows, 10 per page, 2 row groups. /// Predicate: id >= 70 → page 0-7 skipped, page 8-9 read → 30 rows expected. -TEST_F(PageFilteredRowGroupReaderTest, NestedMapColumnRowGroupFilter) { +TEST_F(PageFilteredRowGroupReaderTest, NestedMapColumnPageFilter) { std::string file_name = dir_->Str() + "/nested_map_filter.parquet"; auto field_id = arrow::field("id", arrow::int32()); From 60dce861dee3edbe5151cae66ba325f2f860b731 Mon Sep 17 00:00:00 2001 From: zhouhongfeng Date: Fri, 24 Jul 2026 16:55:33 +0800 Subject: [PATCH 23/23] style: put output parameter in the end of function declaratino --- .../format/parquet/file_reader_wrapper.cpp | 14 +++---- .../page_filtered_row_group_reader.cpp | 40 +++++++++---------- .../parquet/page_filtered_row_group_reader.h | 28 ++++++------- .../page_filtered_row_group_reader_test.cpp | 21 +++++----- 4 files changed, 50 insertions(+), 53 deletions(-) diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp index 2f9a82b2c..54c70798b 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper.cpp @@ -231,14 +231,14 @@ Result> FileReaderWrapper::NextPageFiltered( if (!current_page_filtered_reader_) { const auto& target_rg = target_row_groups_[current_row_group_idx_]; auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges( - file_reader_->parquet_reader(), target_rg, target_column_indices_); + target_rg, target_column_indices_, file_reader_->parquet_reader()); bool pre_buffered = !prebuffered_ranges_.empty(); int64_t max_chunksize = batch_size_ > 0 ? batch_size_ : std::numeric_limits::max(); - PAIMON_ASSIGN_OR_RAISE(current_page_filtered_reader_, - PageFilteredRowGroupReader::ReadFilteredRowGroup( - file_reader_.get(), target_rg, target_column_indices_, - file_reader_->properties().cache_options(), pre_buffered, - page_ranges, max_chunksize, pool_)); + PAIMON_ASSIGN_OR_RAISE( + current_page_filtered_reader_, + PageFilteredRowGroupReader::ReadFilteredRowGroup( + target_rg, target_column_indices_, file_reader_->properties().cache_options(), + pre_buffered, page_ranges, max_chunksize, pool_, file_reader_.get())); current_filtered_row_ranges_ = target_rg.GetRowRanges(); current_filtered_rg_start_ = all_row_group_ranges_[rg_id].first; filtered_global_offset_ = 0; @@ -356,7 +356,7 @@ std::vector<::arrow::io::ReadRange> FileReaderWrapper::CollectPreBufferRanges( if (trg.IsPartiallyMatched()) { // Page-filtered RGs: only matching page byte ranges. auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges( - file_reader_->parquet_reader(), trg, column_indices); + trg, column_indices, file_reader_->parquet_reader()); ranges.insert(ranges.end(), std::make_move_iterator(page_ranges.begin()), std::make_move_iterator(page_ranges.end())); } else { diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index c51becf31..07b057b23 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -164,8 +164,8 @@ std::pair PageFilteredRowGroupReader::ComputeCompressedRowRa } Status PageFilteredRowGroupReader::ExecuteSkipReadPattern( - ::parquet::arrow::ColumnReader* column_reader, int col_idx, const RowRanges& ranges, - int64_t total) { + int col_idx, const RowRanges& ranges, int64_t total, + ::parquet::arrow::ColumnReader* column_reader) { PAIMON_RETURN_NOT_OK_FROM_ARROW(column_reader->ResetLeaf(col_idx, total)); int64_t current = 0; for (const auto& range : ranges.GetRanges()) { @@ -189,10 +189,10 @@ Status PageFilteredRowGroupReader::ExecuteSkipReadPattern( } Status PageFilteredRowGroupReader::WaitForPreBuffer( - ::parquet::ParquetFileReader* parquet_reader, int32_t row_group_index, - const std::vector& column_indices, const ::arrow::io::CacheOptions& cache_options, - bool pre_buffered, const std::vector<::arrow::io::ReadRange>& page_ranges, - std::shared_ptr<::arrow::MemoryPool> pool) { + int32_t row_group_index, const std::vector& column_indices, + const ::arrow::io::CacheOptions& cache_options, bool pre_buffered, + const std::vector<::arrow::io::ReadRange>& page_ranges, + std::shared_ptr<::arrow::MemoryPool> pool, ::parquet::ParquetFileReader* parquet_reader) { std::vector rg_vec = {row_group_index}; std::vector col_vec(column_indices.begin(), column_indices.end()); if (!pre_buffered) { @@ -213,10 +213,10 @@ Status PageFilteredRowGroupReader::WaitForPreBuffer( } Result> PageFilteredRowGroupReader::ReadFilteredField( - ::parquet::arrow::FileReader* arrow_file_reader, const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, int32_t row_group_index, int32_t field_index, const std::vector& column_indices, - const RowRanges& row_ranges, int64_t row_group_row_count) { + const RowRanges& row_ranges, int64_t row_group_row_count, + ::parquet::arrow::FileReader* arrow_file_reader) { // Factory: set data_page_filter on every leaf (per-leaf OffsetIndex). // data_page_filter enables I/O-level page skipping for all leaves. auto factory = @@ -262,8 +262,8 @@ Result> PageFilteredRowGroupReader::ReadFil } } - PAIMON_RETURN_NOT_OK(ExecuteSkipReadPattern(column_reader.get(), col_idx, effective_ranges, - effective_total)); + PAIMON_RETURN_NOT_OK(ExecuteSkipReadPattern(col_idx, effective_ranges, effective_total, + column_reader.get())); } // Build the Arrow array (TransferColumnData for leaves + assemble for nested) @@ -275,10 +275,10 @@ Result> PageFilteredRowGroupReader::ReadFil } Result> PageFilteredRowGroupReader::ReadFilteredRowGroup( - ::parquet::arrow::FileReader* arrow_file_reader, const TargetRowGroup& target_row_group, - const std::vector& column_indices, const ::arrow::io::CacheOptions& cache_options, - bool pre_buffered, const std::vector<::arrow::io::ReadRange>& page_ranges, - int64_t max_chunksize, std::shared_ptr<::arrow::MemoryPool> pool) { + const TargetRowGroup& target_row_group, const std::vector& column_indices, + const ::arrow::io::CacheOptions& cache_options, bool pre_buffered, + const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize, + std::shared_ptr<::arrow::MemoryPool> pool, ::parquet::arrow::FileReader* arrow_file_reader) { auto parquet_reader = arrow_file_reader->parquet_reader(); const auto& row_ranges = target_row_group.GetRowRanges(); int32_t row_group_index = target_row_group.GetRowGroupIndex(); @@ -286,8 +286,8 @@ Result> PageFilteredRowGroupReader::Re int64_t expected_rows = row_ranges.RowCount(); if (!row_ranges.IsEmpty()) { - PAIMON_RETURN_NOT_OK(WaitForPreBuffer(parquet_reader, row_group_index, column_indices, - cache_options, pre_buffered, page_ranges, pool)); + PAIMON_RETURN_NOT_OK(WaitForPreBuffer(row_group_index, column_indices, cache_options, + pre_buffered, page_ranges, pool, parquet_reader)); } auto rg_metadata = parquet_reader->metadata()->RowGroup(row_group_index); @@ -312,8 +312,8 @@ Result> PageFilteredRowGroupReader::Re for (int field_idx : field_indices) { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr chunked_array, - ReadFilteredField(arrow_file_reader, rg_page_index_reader, row_group_index, field_idx, - column_indices, row_ranges, row_group_row_count)); + ReadFilteredField(rg_page_index_reader, row_group_index, field_idx, column_indices, + row_ranges, row_group_row_count, arrow_file_reader)); if (chunked_array->length() != expected_rows) { return Status::Invalid( @@ -338,8 +338,8 @@ Result> PageFilteredRowGroupReader::Re } std::vector<::arrow::io::ReadRange> PageFilteredRowGroupReader::ComputePageRanges( - ::parquet::ParquetFileReader* parquet_reader, const TargetRowGroup& target_row_group, - const std::vector& column_indices) { + const TargetRowGroup& target_row_group, const std::vector& column_indices, + ::parquet::ParquetFileReader* parquet_reader) { int32_t row_group_index = target_row_group.GetRowGroupIndex(); const auto& row_ranges = target_row_group.GetRowRanges(); diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h b/src/paimon/format/parquet/page_filtered_row_group_reader.h index 7a89d981a..1b907cf7d 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.h +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h @@ -45,7 +45,6 @@ class PageFilteredRowGroupReader { ~PageFilteredRowGroupReader() = delete; /// Read a row group with page-level filtering. - /// @param arrow_file_reader The Arrow FileReader for ColumnReader tree creation /// @param target_row_group Target row group with index and row ranges /// @param column_indices Leaf column indices to read /// @param pool Memory pool @@ -54,20 +53,21 @@ class PageFilteredRowGroupReader { /// and only waits via WhenBuffered (no redundant PreBuffer). /// @param page_ranges If non-empty, wait via WhenBufferedRanges instead of WhenBuffered /// @param max_chunksize Per-batch row cap for the returned reader. + /// @param arrow_file_reader The Arrow FileReader for ColumnReader tree creation /// @return A RecordBatchReader streaming the filtered rows. static Result> ReadFilteredRowGroup( - ::parquet::arrow::FileReader* arrow_file_reader, const TargetRowGroup& target_row_group, - const std::vector& column_indices, const ::arrow::io::CacheOptions& cache_options, - bool pre_buffered, const std::vector<::arrow::io::ReadRange>& page_ranges, - int64_t max_chunksize, std::shared_ptr<::arrow::MemoryPool> pool); + const TargetRowGroup& target_row_group, const std::vector& column_indices, + const ::arrow::io::CacheOptions& cache_options, bool pre_buffered, + const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize, + std::shared_ptr<::arrow::MemoryPool> pool, ::parquet::arrow::FileReader* arrow_file_reader); /// Compute the byte ranges of pages that overlap with the given RowRanges. /// Uses OffsetIndex to determine per-page file offsets and sizes. /// Includes dictionary pages unconditionally. /// Falls back to entire column chunk range if OffsetIndex is unavailable. static std::vector<::arrow::io::ReadRange> ComputePageRanges( - ::parquet::ParquetFileReader* parquet_reader, const TargetRowGroup& target_row_group, - const std::vector& column_indices); + const TargetRowGroup& target_row_group, const std::vector& column_indices, + ::parquet::ParquetFileReader* parquet_reader); private: /// Get the [first_row, last_row] range of a page given page locations. @@ -76,13 +76,13 @@ class PageFilteredRowGroupReader { int64_t row_group_row_count); /// Wait for pre-buffered data to become available before reading. - static Status WaitForPreBuffer(::parquet::ParquetFileReader* parquet_reader, - int32_t row_group_index, + static Status WaitForPreBuffer(int32_t row_group_index, const std::vector& column_indices, const ::arrow::io::CacheOptions& cache_options, bool pre_buffered, const std::vector<::arrow::io::ReadRange>& page_ranges, - std::shared_ptr<::arrow::MemoryPool> pool); + std::shared_ptr<::arrow::MemoryPool> pool, + ::parquet::ParquetFileReader* parquet_reader); /// Create a data_page_filter callback for a column based on RowRanges + OffsetIndex. static std::function MakePageFilter( @@ -96,18 +96,18 @@ class PageFilteredRowGroupReader { /// Reset the given leaf and replay the skip/read pattern derived from `ranges` /// directly against the ColumnReader (ResetLeaf + SkipRecords/ReadRecords). - static Status ExecuteSkipReadPattern(::parquet::arrow::ColumnReader* column_reader, int col_idx, - const RowRanges& ranges, int64_t total); + static Status ExecuteSkipReadPattern(int col_idx, const RowRanges& ranges, int64_t total, + ::parquet::arrow::ColumnReader* column_reader); /// Read a field (flat or nested) using ColumnReader tree. /// Sets data_page_filter on all leaves via factory, then drives each leaf /// independently via ResetLeaf/SkipRecords/ReadRecords using its own /// compressed_ranges. static Result> ReadFilteredField( - ::parquet::arrow::FileReader* arrow_file_reader, const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, int32_t row_group_index, int32_t field_index, const std::vector& column_indices, - const RowRanges& row_ranges, int64_t row_group_row_count); + const RowRanges& row_ranges, int64_t row_group_row_count, + ::parquet::arrow::FileReader* arrow_file_reader); }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index 64c688bb1..e1c90efd8 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -557,9 +557,8 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesPartialMatch) { row_ranges.Add(RowRanges::Range(50, 59)); auto ranges = PageFilteredRowGroupReader::ComputePageRanges( - parquet_reader.get(), TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), - /*column_indices=*/{0}); + /*column_indices=*/{0}, parquet_reader.get()); // Should have exactly 1 range (page 5 of column 0, no dictionary since disabled) ASSERT_EQ(1, ranges.size()); @@ -583,8 +582,8 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesAllMatch) { row_ranges.Add(RowRanges::Range(0, 99)); auto ranges = PageFilteredRowGroupReader::ComputePageRanges( - parquet_reader.get(), - TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0}); + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0}, + parquet_reader.get()); // 10 pages, all matching ASSERT_EQ(10, ranges.size()); @@ -608,8 +607,8 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesNoMatch) { RowRanges row_ranges; // empty auto ranges = PageFilteredRowGroupReader::ComputePageRanges( - parquet_reader.get(), - TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0}); + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0}, + parquet_reader.get()); ASSERT_EQ(0, ranges.size()); } @@ -630,9 +629,8 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiColumn) { row_ranges.Add(RowRanges::Range(50, 59)); auto ranges = PageFilteredRowGroupReader::ComputePageRanges( - parquet_reader.get(), TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), - {0, 1}); + {0, 1}, parquet_reader.get()); // 1 matching page per column = 2 ranges total ASSERT_EQ(2, ranges.size()); @@ -658,8 +656,8 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiplePages) { row_ranges.Add(RowRanges::Range(70, 79)); auto ranges = PageFilteredRowGroupReader::ComputePageRanges( - parquet_reader.get(), - TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0}); + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0}, + parquet_reader.get()); // 2 matching pages for 1 column ASSERT_EQ(2, ranges.size()); @@ -843,9 +841,8 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesWithDictionaryEncoding) row_ranges.Add(RowRanges::Range(0, 99)); auto ranges = PageFilteredRowGroupReader::ComputePageRanges( - parquet_reader.get(), TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/false, /*ranges=*/row_ranges), - /*column_indices=*/{0}); + /*column_indices=*/{0}, parquet_reader.get()); ASSERT_FALSE(ranges.empty());