From a710efaf59ad91f08711be02a2e25b32d7dc1324 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Mon, 14 Sep 2026 18:07:47 +0800 Subject: [PATCH] refactor(global-index): extract common sorted index components --- src/paimon/CMakeLists.txt | 6 +- src/paimon/common/global_index/CMakeLists.txt | 6 +- .../btree/btree_compatibility_test.cpp | 56 ++--- .../btree_global_index_integration_test.cpp | 10 +- .../btree/btree_global_index_reader.cpp | 33 ++- .../btree/btree_global_index_reader.h | 10 +- .../btree/btree_global_index_writer.cpp | 23 ++- .../btree/btree_global_index_writer.h | 6 +- .../btree/btree_global_indexer.cpp | 2 - .../btree/lazy_filtered_btree_reader.cpp | 30 +-- .../btree/lazy_filtered_btree_reader.h | 10 +- .../{btree => }/key_serializer.cpp | 77 +++++-- .../global_index/{btree => }/key_serializer.h | 36 ++-- .../{btree => }/key_serializer_test.cpp | 194 +++++++++--------- ...ctor.cpp => sorted_file_meta_selector.cpp} | 139 +++++++------ ...selector.h => sorted_file_meta_selector.h} | 36 ++-- ...cpp => sorted_file_meta_selector_test.cpp} | 146 +++++++------ ...ex_meta.cpp => sorted_index_file_meta.cpp} | 49 ++--- ..._index_meta.h => sorted_index_file_meta.h} | 15 +- ...st.cpp => sorted_index_file_meta_test.cpp} | 140 ++++++------- .../primary_key_sorted_index_scan_test.cpp | 26 +-- 21 files changed, 552 insertions(+), 498 deletions(-) rename src/paimon/common/global_index/{btree => }/key_serializer.cpp (87%) rename src/paimon/common/global_index/{btree => }/key_serializer.h (59%) rename src/paimon/common/global_index/{btree => }/key_serializer_test.cpp (53%) rename src/paimon/common/global_index/{btree/btree_file_meta_selector.cpp => sorted_file_meta_selector.cpp} (51%) rename src/paimon/common/global_index/{btree/btree_file_meta_selector.h => sorted_file_meta_selector.h} (71%) rename src/paimon/common/global_index/{btree/btree_file_meta_selector_test.cpp => sorted_file_meta_selector_test.cpp} (62%) rename src/paimon/common/global_index/{btree/btree_index_meta.cpp => sorted_index_file_meta.cpp} (72%) rename src/paimon/common/global_index/{btree/btree_index_meta.h => sorted_index_file_meta.h} (82%) rename src/paimon/common/global_index/{btree/btree_index_meta_test.cpp => sorted_index_file_meta_test.cpp} (67%) diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index a25397b98..ee1895476 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -606,12 +606,12 @@ if(PAIMON_BUILD_TESTS) common/global_index/bitmap_global_index_result_test.cpp common/global_index/bitmap_scored_global_index_result_test.cpp common/global_index/bitmap/bitmap_global_index_test.cpp - common/global_index/btree/btree_index_meta_test.cpp + common/global_index/sorted_index_file_meta_test.cpp common/global_index/btree/btree_file_footer_test.cpp - common/global_index/btree/key_serializer_test.cpp + common/global_index/key_serializer_test.cpp common/global_index/btree/btree_global_index_integration_test.cpp common/global_index/btree/btree_compatibility_test.cpp - common/global_index/btree/btree_file_meta_selector_test.cpp + common/global_index/sorted_file_meta_selector_test.cpp common/global_index/btree/lazy_filtered_btree_reader_test.cpp common/global_index/rangebitmap/range_bitmap_global_index_test.cpp common/global_index/wrap/file_index_reader_wrapper_test.cpp diff --git a/src/paimon/common/global_index/CMakeLists.txt b/src/paimon/common/global_index/CMakeLists.txt index 0bfad32ed..18806d58a 100644 --- a/src/paimon/common/global_index/CMakeLists.txt +++ b/src/paimon/common/global_index/CMakeLists.txt @@ -22,10 +22,10 @@ set(PAIMON_GLOBAL_INDEX_SRC btree/btree_global_indexer.cpp btree/btree_global_index_reader.cpp btree/btree_global_index_writer.cpp - btree/btree_file_meta_selector.cpp - btree/btree_index_meta.cpp + sorted_file_meta_selector.cpp + sorted_index_file_meta.cpp btree/lazy_filtered_btree_reader.cpp - btree/key_serializer.cpp + key_serializer.cpp rangebitmap/range_bitmap_global_index.cpp rangebitmap/range_bitmap_global_index_factory.cpp offset_global_index_reader.cpp diff --git a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp index 84b451758..80c362058 100644 --- a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp +++ b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp @@ -25,8 +25,8 @@ #include "arrow/c/bridge.h" #include "gtest/gtest.h" #include "paimon/common/global_index/btree/btree_global_indexer.h" -#include "paimon/common/global_index/btree/btree_index_meta.h" -#include "paimon/common/global_index/btree/key_serializer.h" +#include "paimon/common/global_index/key_serializer.h" +#include "paimon/common/global_index/sorted_index_file_meta.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/fs/file_system.h" @@ -779,12 +779,14 @@ TEST_F(BTreeCompatibilityTest, DuplicateKeys) { TEST_F(BTreeCompatibilityTest, MetaDeserialization) { // Test int_50 meta { + ASSERT_OK_AND_ASSIGN(std::shared_ptr key_serializer, + KeySerializer::Create(arrow::int32(), pool_)); std::string meta_path = data_dir_ + "/btree_test_int_50.bin.meta"; auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, - BTreeIndexMeta::Deserialize(meta_bytes, pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + SortedIndexFileMeta::Deserialize(meta_bytes, pool_.get())); ASSERT_TRUE(meta); ASSERT_TRUE(meta->HasNulls()); @@ -792,25 +794,25 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) { ASSERT_TRUE(meta->FirstKey()); ASSERT_OK_AND_ASSIGN(auto min_key, - KeySerializer::DeserializeKey(MemorySlice::Wrap(meta->FirstKey()), - arrow::int32(), pool_.get())); + key_serializer->Deserialize(MemorySlice::Wrap(meta->FirstKey()))); ASSERT_EQ(min_key, Literal(3)); ASSERT_TRUE(meta->LastKey()); ASSERT_OK_AND_ASSIGN(auto max_key, - KeySerializer::DeserializeKey(MemorySlice::Wrap(meta->LastKey()), - arrow::int32(), pool_.get())); + key_serializer->Deserialize(MemorySlice::Wrap(meta->LastKey()))); ASSERT_EQ(max_key, Literal(143)); } // Test float_50 meta { + ASSERT_OK_AND_ASSIGN(std::shared_ptr key_serializer, + KeySerializer::Create(arrow::float32(), pool_)); std::string meta_path = data_dir_ + "/btree_test_float_50.bin.meta"; auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, - BTreeIndexMeta::Deserialize(meta_bytes, pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + SortedIndexFileMeta::Deserialize(meta_bytes, pool_.get())); ASSERT_TRUE(meta); ASSERT_TRUE(meta->HasNulls()); @@ -818,14 +820,12 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) { ASSERT_TRUE(meta->FirstKey()); ASSERT_OK_AND_ASSIGN(auto min_key, - KeySerializer::DeserializeKey(MemorySlice::Wrap(meta->FirstKey()), - arrow::float32(), pool_.get())); + key_serializer->Deserialize(MemorySlice::Wrap(meta->FirstKey()))); ASSERT_EQ(min_key, Literal(static_cast(-INFINITY))); ASSERT_TRUE(meta->LastKey()); ASSERT_OK_AND_ASSIGN(auto max_key, - KeySerializer::DeserializeKey(MemorySlice::Wrap(meta->LastKey()), - arrow::float32(), pool_.get())); + key_serializer->Deserialize(MemorySlice::Wrap(meta->LastKey()))); ASSERT_EQ(max_key, Literal(static_cast(std::nan("")))); } @@ -835,8 +835,8 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) { auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, - BTreeIndexMeta::Deserialize(meta_bytes, pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + SortedIndexFileMeta::Deserialize(meta_bytes, pool_.get())); ASSERT_TRUE(meta); ASSERT_TRUE(meta->HasNulls()); @@ -847,35 +847,37 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) { // Test no_nulls meta { + ASSERT_OK_AND_ASSIGN(std::shared_ptr key_serializer, + KeySerializer::Create(arrow::int32(), pool_)); std::string meta_path = data_dir_ + "/btree_test_int_no_nulls.bin.meta"; auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, - BTreeIndexMeta::Deserialize(meta_bytes, pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + SortedIndexFileMeta::Deserialize(meta_bytes, pool_.get())); ASSERT_TRUE(meta); ASSERT_TRUE(meta->FirstKey()); ASSERT_OK_AND_ASSIGN(auto min_key, - KeySerializer::DeserializeKey(MemorySlice::Wrap(meta->FirstKey()), - arrow::int32(), pool_.get())); + key_serializer->Deserialize(MemorySlice::Wrap(meta->FirstKey()))); ASSERT_EQ(min_key, Literal(4)); ASSERT_TRUE(meta->LastKey()); ASSERT_OK_AND_ASSIGN(auto max_key, - KeySerializer::DeserializeKey(MemorySlice::Wrap(meta->LastKey()), - arrow::int32(), pool_.get())); + key_serializer->Deserialize(MemorySlice::Wrap(meta->LastKey()))); ASSERT_EQ(max_key, Literal(158)); } // Test string_50 meta { + ASSERT_OK_AND_ASSIGN(std::shared_ptr key_serializer, + KeySerializer::Create(arrow::utf8(), pool_)); std::string meta_path = data_dir_ + "/btree_test_string_50.bin.meta"; auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, - BTreeIndexMeta::Deserialize(meta_bytes, pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + SortedIndexFileMeta::Deserialize(meta_bytes, pool_.get())); ASSERT_TRUE(meta); ASSERT_TRUE(meta->HasNulls()); @@ -883,15 +885,13 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) { ASSERT_TRUE(meta->FirstKey()); ASSERT_OK_AND_ASSIGN(auto min_key, - KeySerializer::DeserializeKey(MemorySlice::Wrap(meta->FirstKey()), - arrow::utf8(), pool_.get())); + key_serializer->Deserialize(MemorySlice::Wrap(meta->FirstKey()))); std::string min_key_str = "test_00000"; ASSERT_EQ(min_key, Literal(FieldType::STRING, min_key_str.data(), min_key_str.size())); ASSERT_TRUE(meta->LastKey()); ASSERT_OK_AND_ASSIGN(auto max_key, - KeySerializer::DeserializeKey(MemorySlice::Wrap(meta->LastKey()), - arrow::utf8(), pool_.get())); + key_serializer->Deserialize(MemorySlice::Wrap(meta->LastKey()))); std::string max_key_str = "test_00049"; ASSERT_EQ(max_key, Literal(FieldType::STRING, max_key_str.data(), max_key_str.size())); } diff --git a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp index ce25b9b4e..89d98d43a 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp @@ -26,8 +26,8 @@ #include "paimon/common/factories/io_hook.h" #include "paimon/common/global_index/btree/btree_global_index_writer.h" #include "paimon/common/global_index/btree/btree_global_indexer.h" -#include "paimon/common/global_index/btree/btree_index_meta.h" #include "paimon/common/global_index/btree/lazy_filtered_btree_reader.h" +#include "paimon/common/global_index/sorted_index_file_meta.h" #include "paimon/common/options/memory_size.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/data/decimal.h" @@ -490,8 +490,8 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteEmptyStringKeyMetadata) { ASSERT_OK_AND_ASSIGN(auto metas, writer->Finish()); ASSERT_EQ(metas.size(), 1); - ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, - BTreeIndexMeta::Deserialize(metas[0].metadata, pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + SortedIndexFileMeta::Deserialize(metas[0].metadata, pool_.get())); ASSERT_TRUE(meta->FirstKey()); ASSERT_EQ(meta->FirstKey()->size(), 0); ASSERT_TRUE(meta->LastKey()); @@ -1940,7 +1940,7 @@ TEST_P(BTreeGlobalIndexIntegrationTest, TestIOException) { ASSERT_TRUE(run_complete); } -// Multiple files with BTreeFileMetaSelector filtering +// Multiple files with SortedFileMetaSelector filtering TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadMultiFilesWithMetaSelector) { auto file_writer = std::make_shared(fs_, base_path_); auto field = arrow::field("int_field", arrow::int32()); @@ -1982,7 +1982,7 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadMultiFilesWithMetaSelector) ASSERT_EQ(all_metas.size(), 3); // Create reader over all 3 files (internally uses LazyFilteredBTreeReader + - // BTreeFileMetaSelector) + // SortedFileMetaSelector) auto file_reader = std::make_shared(fs_, base_path_); auto c_schema = CreateArrowSchema(field); ASSERT_OK_AND_ASSIGN(auto reader, diff --git a/src/paimon/common/global_index/btree/btree_global_index_reader.cpp b/src/paimon/common/global_index/btree/btree_global_index_reader.cpp index f9a4868ff..b9b7f4cff 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_reader.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_reader.cpp @@ -20,7 +20,7 @@ #include "paimon/common/global_index/btree/btree_global_index_reader.h" #include "fmt/format.h" -#include "paimon/common/global_index/btree/key_serializer.h" +#include "paimon/common/global_index/key_serializer.h" #include "paimon/common/memory/memory_slice.h" #include "paimon/common/memory/memory_slice_input.h" #include "paimon/global_index/bitmap_global_index_result.h" @@ -32,36 +32,36 @@ Result> BTreeGlobalIndexReader::Create( const std::shared_ptr& sst_file_reader, RoaringBitmap64&& null_bitmap, const std::optional& min_key_slice, const std::optional& max_key_slice, - const std::shared_ptr& key_type, const std::shared_ptr& pool) { + const std::shared_ptr& key_serializer) { + if (key_serializer == nullptr) { + return Status::Invalid("Cannot create BTreeGlobalIndexReader without a key serializer."); + } std::optional min_key; std::optional max_key; if (min_key_slice) { - PAIMON_ASSIGN_OR_RAISE( - min_key, KeySerializer::DeserializeKey(min_key_slice.value(), key_type, pool.get())); + PAIMON_ASSIGN_OR_RAISE(min_key, key_serializer->Deserialize(min_key_slice.value())); } if (max_key_slice) { - PAIMON_ASSIGN_OR_RAISE( - max_key, KeySerializer::DeserializeKey(max_key_slice.value(), key_type, pool.get())); + PAIMON_ASSIGN_OR_RAISE(max_key, key_serializer->Deserialize(max_key_slice.value())); } return std::shared_ptr(new BTreeGlobalIndexReader( sst_file_reader, std::move(null_bitmap), std::move(min_key), std::move(max_key), - min_key_slice, max_key_slice, key_type, pool)); + min_key_slice, max_key_slice, key_serializer)); } BTreeGlobalIndexReader::BTreeGlobalIndexReader( const std::shared_ptr& sst_file_reader, RoaringBitmap64&& null_bitmap, std::optional min_key, std::optional max_key, std::optional min_key_slice, std::optional max_key_slice, - const std::shared_ptr& key_type, const std::shared_ptr& pool) - : pool_(pool), - sst_file_reader_(sst_file_reader), + const std::shared_ptr& key_serializer) + : sst_file_reader_(sst_file_reader), + key_serializer_(key_serializer), null_bitmap_(std::move(null_bitmap)), min_key_(std::move(min_key)), max_key_(std::move(max_key)), min_key_slice_(std::move(min_key_slice)), max_key_slice_(std::move(max_key_slice)), - key_type_(key_type), - comparator_(KeySerializer::CreateComparator(key_type, pool)) {} + comparator_(key_serializer_->CreateComparator()) {} Result> BTreeGlobalIndexReader::VisitIsNotNull() { return std::make_shared( @@ -286,13 +286,12 @@ Result BTreeGlobalIndexReader::RangeQuery(const std::optional from_bytes, - KeySerializer::SerializeKey(from.value(), key_type_, pool_.get())); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr to_bytes, - KeySerializer::SerializeKey(to.value(), key_type_, pool_.get())); + key_serializer_->Serialize(from.value())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr to_bytes, key_serializer_->Serialize(to.value())); MemorySlice from_slice = MemorySlice::Wrap(from_bytes); MemorySlice to_slice = MemorySlice::Wrap(to_bytes); - PAIMON_RETURN_NOT_OK(KeySerializer::ValidateSerializedKey(from_slice, key_type_)); - PAIMON_RETURN_NOT_OK(KeySerializer::ValidateSerializedKey(to_slice, key_type_)); + PAIMON_RETURN_NOT_OK(key_serializer_->ValidateSerializedKey(from_slice)); + PAIMON_RETURN_NOT_OK(key_serializer_->ValidateSerializedKey(to_slice)); // Determine if we can skip lower/upper bound checks using cached serialized min/max keys. // When from == min_key_, all entries are >= from, so skip lower bound comparison. diff --git a/src/paimon/common/global_index/btree/btree_global_index_reader.h b/src/paimon/common/global_index/btree/btree_global_index_reader.h index 1bdbe71e8..39bee677e 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_reader.h +++ b/src/paimon/common/global_index/btree/btree_global_index_reader.h @@ -27,7 +27,7 @@ #include "arrow/api.h" #include "paimon/common/global_index/btree/btree_defs.h" -#include "paimon/common/global_index/btree/key_serializer.h" +#include "paimon/common/global_index/key_serializer.h" #include "paimon/common/sst/sst_file_reader.h" #include "paimon/global_index/global_index_io_meta.h" #include "paimon/global_index/global_index_reader.h" @@ -44,7 +44,7 @@ class BTreeGlobalIndexReader : public GlobalIndexReader, const std::shared_ptr& sst_file_reader, RoaringBitmap64&& null_bitmap, const std::optional& min_key_slice, const std::optional& max_key_slice, - const std::shared_ptr& key_type, const std::shared_ptr& pool); + const std::shared_ptr& key_serializer); Result> VisitIsNotNull() override; @@ -95,8 +95,7 @@ class BTreeGlobalIndexReader : public GlobalIndexReader, RoaringBitmap64&& null_bitmap, std::optional min_key, std::optional max_key, std::optional min_key_slice, std::optional max_key_slice, - const std::shared_ptr& key_type, - const std::shared_ptr& pool); + const std::shared_ptr& key_serializer); Result RangeQuery(const std::optional& from, const std::optional& to, bool from_inclusive, @@ -106,8 +105,8 @@ class BTreeGlobalIndexReader : public GlobalIndexReader, Result AllNonNullRows(); - std::shared_ptr pool_; std::shared_ptr sst_file_reader_; + std::shared_ptr key_serializer_; RoaringBitmap64 null_bitmap_; std::optional min_key_; std::optional max_key_; @@ -115,7 +114,6 @@ class BTreeGlobalIndexReader : public GlobalIndexReader, std::optional min_key_slice_; std::optional max_key_slice_; - std::shared_ptr key_type_; MemorySlice::SliceComparator comparator_; }; diff --git a/src/paimon/common/global_index/btree/btree_global_index_writer.cpp b/src/paimon/common/global_index/btree/btree_global_index_writer.cpp index 37b81c419..fd30728d0 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_writer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_writer.cpp @@ -26,8 +26,9 @@ #include "fmt/format.h" #include "paimon/common/compression/block_compression_factory.h" #include "paimon/common/global_index/btree/btree_defs.h" -#include "paimon/common/global_index/btree/key_serializer.h" #include "paimon/common/global_index/global_index_utils.h" +#include "paimon/common/global_index/key_serializer.h" +#include "paimon/common/global_index/sorted_index_file_meta.h" #include "paimon/common/memory/memory_slice_output.h" #include "paimon/common/predicate/literal_converter.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -45,6 +46,8 @@ Result> BTreeGlobalIndexWriter::Create( PAIMON_RETURN_NOT_OK(Preconditions::CheckNotNull( key_field, fmt::format("field {} not in arrow_array when Create BTreeGlobalIndexWriter", field_name))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key_serializer, + KeySerializer::Create(key_field->type(), pool)); PAIMON_ASSIGN_OR_RAISE(std::string index_file_name, file_writer->NewFileName(BtreeDefs::kIdentifier)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr output_stream, @@ -52,19 +55,19 @@ Result> BTreeGlobalIndexWriter::Create( auto sst_file_writer = std::make_unique(output_stream, /*bloom_filter=*/nullptr, block_size, compression_factory, pool); return std::shared_ptr(new BTreeGlobalIndexWriter( - field_name, arrow_type, key_field->type(), file_writer, index_file_name, output_stream, + field_name, arrow_type, key_serializer, file_writer, index_file_name, output_stream, std::move(sst_file_writer), pool)); } BTreeGlobalIndexWriter::BTreeGlobalIndexWriter( const std::string& field_name, const std::shared_ptr& arrow_type, - const std::shared_ptr& key_type, + const std::shared_ptr& key_serializer, const std::shared_ptr& file_writer, const std::string& index_file_name, const std::shared_ptr& output_stream, std::unique_ptr&& sst_writer, const std::shared_ptr& pool) : field_name_(field_name), arrow_type_(arrow_type), - key_type_(key_type), + key_serializer_(key_serializer), pool_(pool), file_writer_(file_writer), index_file_name_(index_file_name), @@ -135,7 +138,7 @@ Status BTreeGlobalIndexWriter::Flush() { current_row_ids_.clear(); assert(last_key_); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key_bytes, - KeySerializer::SerializeKey(last_key_.value(), key_type_, pool_.get())); + key_serializer_->Serialize(last_key_.value())); return sst_writer_->Write(std::move(key_bytes), output.ToSlice().CopyBytes(pool_.get())); } @@ -190,16 +193,14 @@ Result> BTreeGlobalIndexWriter::Finish() { std::shared_ptr first_key_bytes; std::shared_ptr last_key_bytes; if (first_key_) { - PAIMON_ASSIGN_OR_RAISE(first_key_bytes, KeySerializer::SerializeKey( - first_key_.value(), key_type_, pool_.get())); + PAIMON_ASSIGN_OR_RAISE(first_key_bytes, key_serializer_->Serialize(first_key_.value())); } if (last_key_) { - PAIMON_ASSIGN_OR_RAISE( - last_key_bytes, KeySerializer::SerializeKey(last_key_.value(), key_type_, pool_.get())); + PAIMON_ASSIGN_OR_RAISE(last_key_bytes, key_serializer_->Serialize(last_key_.value())); } // Create index meta - auto index_meta = - std::make_shared(first_key_bytes, last_key_bytes, !null_bitmap_.IsEmpty()); + auto index_meta = std::make_shared(first_key_bytes, last_key_bytes, + !null_bitmap_.IsEmpty()); auto meta_bytes = index_meta->Serialize(pool_.get()); // Create GlobalIndexIOMeta diff --git a/src/paimon/common/global_index/btree/btree_global_index_writer.h b/src/paimon/common/global_index/btree/btree_global_index_writer.h index f8c9ff9c4..1078c0b4b 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_writer.h +++ b/src/paimon/common/global_index/btree/btree_global_index_writer.h @@ -24,13 +24,13 @@ #include "arrow/api.h" #include "paimon/common/global_index/btree/btree_file_footer.h" -#include "paimon/common/global_index/btree/btree_index_meta.h" #include "paimon/common/sst/sst_file_writer.h" #include "paimon/global_index/global_index_writer.h" #include "paimon/global_index/io/global_index_file_writer.h" #include "paimon/predicate/literal.h" #include "paimon/utils/roaring_bitmap64.h" namespace paimon { +class KeySerializer; /// Writer for BTree Global Index files. /// This writer builds an SST file where each key maps to a list of row IDs. @@ -76,7 +76,7 @@ class BTreeGlobalIndexWriter : public GlobalIndexWriter { private: BTreeGlobalIndexWriter(const std::string& field_name, const std::shared_ptr& arrow_type, - const std::shared_ptr& key_type, + const std::shared_ptr& key_serializer, const std::shared_ptr& file_writer, const std::string& index_file_name, const std::shared_ptr& output_stream, @@ -90,7 +90,7 @@ class BTreeGlobalIndexWriter : public GlobalIndexWriter { private: std::string field_name_; std::shared_ptr arrow_type_; - std::shared_ptr key_type_; + std::shared_ptr key_serializer_; std::shared_ptr pool_; std::shared_ptr file_writer_; diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.cpp b/src/paimon/common/global_index/btree/btree_global_indexer.cpp index 8996fe59d..09b50858c 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_indexer.cpp @@ -27,8 +27,6 @@ #include "paimon/common/compression/block_compression_factory.h" #include "paimon/common/global_index/btree/btree_file_footer.h" #include "paimon/common/global_index/btree/btree_global_index_writer.h" -#include "paimon/common/global_index/btree/btree_index_meta.h" -#include "paimon/common/global_index/btree/key_serializer.h" #include "paimon/common/global_index/btree/lazy_filtered_btree_reader.h" #include "paimon/common/memory/memory_slice.h" #include "paimon/common/memory/memory_slice_input.h" diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp index 4ea7576b6..ca5508bf7 100644 --- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp +++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp @@ -25,8 +25,8 @@ #include "paimon/common/executor/future.h" #include "paimon/common/global_index/btree/btree_file_footer.h" #include "paimon/common/global_index/btree/btree_global_index_reader.h" -#include "paimon/common/global_index/btree/btree_index_meta.h" -#include "paimon/common/global_index/btree/key_serializer.h" +#include "paimon/common/global_index/key_serializer.h" +#include "paimon/common/global_index/sorted_index_file_meta.h" #include "paimon/common/global_index/union_global_index_reader.h" #include "paimon/common/memory/memory_slice.h" #include "paimon/common/memory/memory_slice_input.h" @@ -44,22 +44,24 @@ Result> LazyFilteredBTreeReader::Create const std::shared_ptr& file_reader, const std::shared_ptr& cache_manager, const std::shared_ptr& pool, const std::shared_ptr& executor) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_selector, - BTreeFileMetaSelector::Create(files, key_type, pool)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key_serializer, + KeySerializer::Create(key_type, pool)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_selector, + SortedFileMetaSelector::Create(files, key_serializer)); return std::shared_ptr( - new LazyFilteredBTreeReader(read_buffer_size, std::move(file_selector), key_type, + new LazyFilteredBTreeReader(read_buffer_size, std::move(file_selector), key_serializer, file_reader, cache_manager, pool, executor)); } LazyFilteredBTreeReader::LazyFilteredBTreeReader( - std::optional read_buffer_size, std::unique_ptr file_selector, - std::shared_ptr key_type, std::shared_ptr file_reader, - std::shared_ptr cache_manager, std::shared_ptr pool, - std::shared_ptr executor) + std::optional read_buffer_size, std::unique_ptr file_selector, + std::shared_ptr key_serializer, + std::shared_ptr file_reader, std::shared_ptr cache_manager, + std::shared_ptr pool, std::shared_ptr executor) : read_buffer_size_(read_buffer_size), pool_(std::move(pool)), file_selector_(std::move(file_selector)), - key_type_(std::move(key_type)), + key_serializer_(std::move(key_serializer)), file_reader_(std::move(file_reader)), cache_manager_(std::move(cache_manager)), executor_(std::move(executor)) {} @@ -224,11 +226,11 @@ Result> LazyFilteredBTreeReader::GetOrCreateR Result> LazyFilteredBTreeReader::CreateSingleReader( const GlobalIndexIOMeta& meta) { // Create comparator based on field type - auto comparator = KeySerializer::CreateComparator(key_type_, pool_); + auto comparator = key_serializer_->CreateComparator(); // Get min/max key slices from meta data (keep as slices; Create() will deserialize) - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_meta, - BTreeIndexMeta::Deserialize(meta.metadata, pool_.get())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_meta, + SortedIndexFileMeta::Deserialize(meta.metadata, pool_.get())); std::optional min_key_slice; std::optional max_key_slice; if (index_meta->FirstKey()) { @@ -270,7 +272,7 @@ Result> LazyFilteredBTreeReader::CreateSingle comparator, block_cache, pool_)); return BTreeGlobalIndexReader::Create(sst_file_reader, std::move(null_bitmap), min_key_slice, - max_key_slice, key_type_, pool_); + max_key_slice, key_serializer_); } Result LazyFilteredBTreeReader::ReadNullBitmap( diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h index 74ed978b8..46681c1ef 100644 --- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h +++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h @@ -28,7 +28,7 @@ #include "arrow/api.h" #include "paimon/common/global_index/btree/btree_defs.h" -#include "paimon/common/global_index/btree/btree_file_meta_selector.h" +#include "paimon/common/global_index/sorted_file_meta_selector.h" #include "paimon/common/io/cache/cache_manager.h" #include "paimon/common/sst/block_cache.h" #include "paimon/common/sst/block_handle.h" @@ -80,8 +80,8 @@ class LazyFilteredBTreeReader : public GlobalIndexReader { private: LazyFilteredBTreeReader(std::optional read_buffer_size, - std::unique_ptr file_selector, - std::shared_ptr key_type, + std::unique_ptr file_selector, + std::shared_ptr key_serializer, std::shared_ptr file_reader, std::shared_ptr cache_manager, std::shared_ptr pool, std::shared_ptr executor); @@ -102,8 +102,8 @@ class LazyFilteredBTreeReader : public GlobalIndexReader { private: std::optional read_buffer_size_; std::shared_ptr pool_; - std::unique_ptr file_selector_; - std::shared_ptr key_type_; + std::unique_ptr file_selector_; + std::shared_ptr key_serializer_; std::shared_ptr file_reader_; std::shared_ptr cache_manager_; std::map> reader_cache_; diff --git a/src/paimon/common/global_index/btree/key_serializer.cpp b/src/paimon/common/global_index/key_serializer.cpp similarity index 87% rename from src/paimon/common/global_index/btree/key_serializer.cpp rename to src/paimon/common/global_index/key_serializer.cpp index 4b66d1d1f..33bcc1c9d 100644 --- a/src/paimon/common/global_index/btree/key_serializer.cpp +++ b/src/paimon/common/global_index/key_serializer.cpp @@ -17,7 +17,13 @@ * under the License. */ -#include "paimon/common/global_index/btree/key_serializer.h" +#include "paimon/common/global_index/key_serializer.h" + +#include +#include +#include +#include +#include #include "fmt/format.h" #include "paimon/common/memory/memory_slice_input.h" @@ -124,8 +130,36 @@ Status ValidateDecimal(const MemorySlice& slice, } // namespace -Result> KeySerializer::SerializeKey( - const Literal& literal, const std::shared_ptr& type, MemoryPool* pool) { +Result> KeySerializer::Create( + const std::shared_ptr& type, const std::shared_ptr& pool) { + if (type == nullptr) { + return Status::Invalid("Cannot create KeySerializer without a key type."); + } + if (pool == nullptr) { + return Status::Invalid("Cannot create KeySerializer without a memory pool."); + } + switch (type->id()) { + case arrow::Type::type::BOOL: + case arrow::Type::type::INT8: + case arrow::Type::type::INT16: + case arrow::Type::type::INT32: + case arrow::Type::type::INT64: + case arrow::Type::type::DATE32: + case arrow::Type::type::FLOAT: + case arrow::Type::type::DOUBLE: + case arrow::Type::type::STRING: + case arrow::Type::type::TIMESTAMP: + case arrow::Type::type::DECIMAL128: + return std::shared_ptr(new KeySerializer(type, pool)); + default: + return Status::Invalid(fmt::format("Data type {} is not supported by global index now.", + type->ToString())); + } +} + +Result> KeySerializer::Serialize(const Literal& literal) const { + const std::shared_ptr& type = type_; + MemoryPool* pool = pool_.get(); if (literal.IsNull()) { return Status::Invalid("cannot serialize null in KeySerializer"); } @@ -182,7 +216,7 @@ Result> KeySerializer::SerializeKey( case FieldType::TIMESTAMP: { if (!type || type->id() != arrow::Type::TIMESTAMP) { return Status::Invalid( - "ts type cannot cast to arrow::TimestampType in BTreeGlobalIndex"); + "timestamp type cannot cast to arrow::TimestampType in KeySerializer"); } auto ts_type = checked_pointer_cast(type); MemorySliceOutput output(8, pool); @@ -199,7 +233,7 @@ Result> KeySerializer::SerializeKey( case FieldType::DECIMAL: { if (!type || type->id() != arrow::Type::DECIMAL128) { return Status::Invalid( - "decimal type cannot cast to arrow::Decimal128Type in BTreeGlobalIndex"); + "decimal type cannot cast to arrow::Decimal128Type in KeySerializer"); } auto decimal_type = checked_pointer_cast(type); @@ -218,15 +252,15 @@ Result> KeySerializer::SerializeKey( } default: return Status::Invalid( - fmt::format("Not support serialize {} type in BTreeGlobalIndex", + fmt::format("Not support serialize {} type in KeySerializer", FieldTypeUtils::FieldTypeToString(literal.GetType()))); } } -Result KeySerializer::DeserializeKey(const MemorySlice& slice, - const std::shared_ptr& type, - MemoryPool* pool) { - PAIMON_RETURN_NOT_OK(ValidateSerializedKey(slice, type)); +Result KeySerializer::Deserialize(const MemorySlice& slice) const { + const std::shared_ptr& type = type_; + MemoryPool* pool = pool_.get(); + PAIMON_RETURN_NOT_OK(ValidateSerializedKey(slice)); switch (type->id()) { case arrow::Type::type::BOOL: return Literal(slice.ReadByte(0) == 1 ? true : false); @@ -279,16 +313,13 @@ Result KeySerializer::DeserializeKey(const MemorySlice& slice, } } default: - return Status::Invalid(fmt::format( - "Not support deserialize {} type in BTreeGlobalIndex", type->ToString())); + return Status::Invalid( + fmt::format("Not support deserialize {} type in KeySerializer", type->ToString())); } } -Status KeySerializer::ValidateSerializedKey(const MemorySlice& slice, - const std::shared_ptr& type) { - if (type == nullptr) { - return Status::Invalid("Cannot validate a serialized BTree key without a key type."); - } +Status KeySerializer::ValidateSerializedKey(const MemorySlice& slice) const { + const std::shared_ptr& type = type_; switch (type->id()) { case arrow::Type::type::BOOL: { PAIMON_RETURN_NOT_OK(ValidateExactLength(slice, type, sizeof(int8_t))); @@ -325,12 +356,13 @@ Status KeySerializer::ValidateSerializedKey(const MemorySlice& slice, } default: return Status::Invalid(fmt::format( - "Not support validate serialized {} type in BTreeGlobalIndex", type->ToString())); + "Not support validate serialized {} type in KeySerializer", type->ToString())); } } -MemorySlice::SliceComparator KeySerializer::CreateComparator( - const std::shared_ptr& type, const std::shared_ptr& pool) { +MemorySlice::SliceComparator KeySerializer::CreateComparator() const { + const std::shared_ptr& type = type_; + const std::shared_ptr& pool = pool_; // Fast paths for integer and string types: direct value comparison without Literal // deserialization, avoiding heap allocations entirely. switch (type->id()) { @@ -393,8 +425,9 @@ MemorySlice::SliceComparator KeySerializer::CreateComparator( // deserialize to Literal and compare. return [pool = pool, type = type](const MemorySlice& a, const MemorySlice& b) -> Result { - PAIMON_ASSIGN_OR_RAISE(Literal la, DeserializeKey(a, type, pool.get())); - PAIMON_ASSIGN_OR_RAISE(Literal lb, DeserializeKey(b, type, pool.get())); + KeySerializer serializer(type, pool); + PAIMON_ASSIGN_OR_RAISE(Literal la, serializer.Deserialize(a)); + PAIMON_ASSIGN_OR_RAISE(Literal lb, serializer.Deserialize(b)); return la.CompareTo(lb); }; } diff --git a/src/paimon/common/global_index/btree/key_serializer.h b/src/paimon/common/global_index/key_serializer.h similarity index 59% rename from src/paimon/common/global_index/btree/key_serializer.h rename to src/paimon/common/global_index/key_serializer.h index d44b5bdc1..09b5298d6 100644 --- a/src/paimon/common/global_index/btree/key_serializer.h +++ b/src/paimon/common/global_index/key_serializer.h @@ -19,29 +19,41 @@ #pragma once +#include + #include "arrow/api.h" #include "paimon/common/memory/memory_slice.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" #include "paimon/predicate/literal.h" namespace paimon { + +/// Provides core methods to serialize, deserialize, and compare global index keys. class KeySerializer { public: - KeySerializer() = delete; - ~KeySerializer() = delete; + ~KeySerializer() = default; - static Result> SerializeKey(const Literal& literal, - const std::shared_ptr& type, - MemoryPool* pool); + static Result> Create( + const std::shared_ptr& type, const std::shared_ptr& pool); - static Result DeserializeKey(const MemorySlice& slice, - const std::shared_ptr& type, - MemoryPool* pool); + Result> Serialize(const Literal& literal) const; - static Status ValidateSerializedKey(const MemorySlice& slice, - const std::shared_ptr& type); + Result Deserialize(const MemorySlice& slice) const; - static MemorySlice::SliceComparator CreateComparator( - const std::shared_ptr& type, const std::shared_ptr& pool); + Status ValidateSerializedKey(const MemorySlice& slice) const; + + MemorySlice::SliceComparator CreateComparator() const; + + MemoryPool* GetMemoryPool() const { + return pool_.get(); + } + + private: + KeySerializer(const std::shared_ptr& type, + const std::shared_ptr& pool) + : type_(type), pool_(pool) {} + + std::shared_ptr type_; + std::shared_ptr pool_; }; } // namespace paimon diff --git a/src/paimon/common/global_index/btree/key_serializer_test.cpp b/src/paimon/common/global_index/key_serializer_test.cpp similarity index 53% rename from src/paimon/common/global_index/btree/key_serializer_test.cpp rename to src/paimon/common/global_index/key_serializer_test.cpp index 61322fde4..8d29c2fc7 100644 --- a/src/paimon/common/global_index/btree/key_serializer_test.cpp +++ b/src/paimon/common/global_index/key_serializer_test.cpp @@ -17,7 +17,7 @@ * under the License. */ -#include "paimon/common/global_index/btree/key_serializer.h" +#include "paimon/common/global_index/key_serializer.h" #include #include @@ -36,6 +36,29 @@ class KeySerializerTest : public ::testing::Test { pool_ = GetDefaultPool(); } + Result> Serialize(const Literal& literal, + const std::shared_ptr& type, + MemoryPool* /*pool*/) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr serializer, + KeySerializer::Create(type, pool_)); + return serializer->Serialize(literal); + } + + Result Deserialize(const MemorySlice& slice, + const std::shared_ptr& type, + MemoryPool* /*pool*/) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr serializer, + KeySerializer::Create(type, pool_)); + return serializer->Deserialize(slice); + } + + Status ValidateSerializedKey(const MemorySlice& slice, + const std::shared_ptr& type) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr serializer, + KeySerializer::Create(type, pool_)); + return serializer->ValidateSerializedKey(slice); + } + std::shared_ptr pool_; }; @@ -43,63 +66,51 @@ TEST_F(KeySerializerTest, SerializeAndDeserializeAllTypes) { // BOOLEAN { Literal literal(true); - ASSERT_OK_AND_ASSIGN(auto bytes, - KeySerializer::SerializeKey(literal, arrow::boolean(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::boolean(), pool_.get())); auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes)); - ASSERT_OK_AND_ASSIGN(auto result, - KeySerializer::DeserializeKey(slice, arrow::boolean(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::boolean(), pool_.get())); ASSERT_EQ(result.GetValue(), true); Literal literal_false(false); - ASSERT_OK_AND_ASSIGN( - bytes, KeySerializer::SerializeKey(literal_false, arrow::boolean(), pool_.get())); + ASSERT_OK_AND_ASSIGN(bytes, Serialize(literal_false, arrow::boolean(), pool_.get())); slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes)); - ASSERT_OK_AND_ASSIGN(result, - KeySerializer::DeserializeKey(slice, arrow::boolean(), pool_.get())); + ASSERT_OK_AND_ASSIGN(result, Deserialize(slice, arrow::boolean(), pool_.get())); ASSERT_EQ(result.GetValue(), false); } // TINYINT (int8) { Literal literal(static_cast(-42)); - ASSERT_OK_AND_ASSIGN(auto bytes, - KeySerializer::SerializeKey(literal, arrow::int8(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::int8(), pool_.get())); auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes)); - ASSERT_OK_AND_ASSIGN(auto result, - KeySerializer::DeserializeKey(slice, arrow::int8(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::int8(), pool_.get())); ASSERT_EQ(result.GetValue(), -42); } // SMALLINT (int16) { Literal literal(static_cast(12345)); - ASSERT_OK_AND_ASSIGN(auto bytes, - KeySerializer::SerializeKey(literal, arrow::int16(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::int16(), pool_.get())); auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes)); - ASSERT_OK_AND_ASSIGN(auto result, - KeySerializer::DeserializeKey(slice, arrow::int16(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::int16(), pool_.get())); ASSERT_EQ(result.GetValue(), 12345); } // INT (int32) { Literal literal(42); - ASSERT_OK_AND_ASSIGN(auto bytes, - KeySerializer::SerializeKey(literal, arrow::int32(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::int32(), pool_.get())); auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes)); - ASSERT_OK_AND_ASSIGN(auto result, - KeySerializer::DeserializeKey(slice, arrow::int32(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::int32(), pool_.get())); ASSERT_EQ(result.GetValue(), 42); } // DATE (stored as int32) { Literal literal(FieldType::DATE, 18000); - ASSERT_OK_AND_ASSIGN(auto bytes, - KeySerializer::SerializeKey(literal, arrow::date32(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::date32(), pool_.get())); auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes)); - ASSERT_OK_AND_ASSIGN(auto result, - KeySerializer::DeserializeKey(slice, arrow::date32(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::date32(), pool_.get())); ASSERT_EQ(result.GetType(), FieldType::DATE); ASSERT_EQ(result.GetValue(), 18000); } @@ -107,44 +118,36 @@ TEST_F(KeySerializerTest, SerializeAndDeserializeAllTypes) { // BIGINT (int64) { Literal literal(static_cast(123456789012345LL)); - ASSERT_OK_AND_ASSIGN(auto bytes, - KeySerializer::SerializeKey(literal, arrow::int64(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::int64(), pool_.get())); auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes)); - ASSERT_OK_AND_ASSIGN(auto result, - KeySerializer::DeserializeKey(slice, arrow::int64(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::int64(), pool_.get())); ASSERT_EQ(result.GetValue(), 123456789012345LL); } // FLOAT { Literal literal(3.14f); - ASSERT_OK_AND_ASSIGN(auto bytes, - KeySerializer::SerializeKey(literal, arrow::float32(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::float32(), pool_.get())); auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes)); - ASSERT_OK_AND_ASSIGN(auto result, - KeySerializer::DeserializeKey(slice, arrow::float32(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::float32(), pool_.get())); ASSERT_FLOAT_EQ(result.GetValue(), 3.14f); } // DOUBLE { Literal literal(2.718281828); - ASSERT_OK_AND_ASSIGN(auto bytes, - KeySerializer::SerializeKey(literal, arrow::float64(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::float64(), pool_.get())); auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes)); - ASSERT_OK_AND_ASSIGN(auto result, - KeySerializer::DeserializeKey(slice, arrow::float64(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::float64(), pool_.get())); ASSERT_DOUBLE_EQ(result.GetValue(), 2.718281828); } // STRING { Literal literal(FieldType::STRING, "hello world", 11); - ASSERT_OK_AND_ASSIGN(auto bytes, - KeySerializer::SerializeKey(literal, arrow::utf8(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, arrow::utf8(), pool_.get())); auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes)); - ASSERT_OK_AND_ASSIGN(auto result, - KeySerializer::DeserializeKey(slice, arrow::utf8(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, arrow::utf8(), pool_.get())); ASSERT_EQ(result.GetValue(), "hello world"); } @@ -152,11 +155,9 @@ TEST_F(KeySerializerTest, SerializeAndDeserializeAllTypes) { { auto ts_type = arrow::timestamp(arrow::TimeUnit::MILLI); Literal literal(Timestamp::FromEpochMillis(1234567890)); - ASSERT_OK_AND_ASSIGN(auto bytes, - KeySerializer::SerializeKey(literal, ts_type, pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, ts_type, pool_.get())); auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes)); - ASSERT_OK_AND_ASSIGN(auto result, - KeySerializer::DeserializeKey(slice, ts_type, pool_.get())); + ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, ts_type, pool_.get())); ASSERT_EQ(result.GetValue().GetMillisecond(), 1234567890); } @@ -164,11 +165,9 @@ TEST_F(KeySerializerTest, SerializeAndDeserializeAllTypes) { { auto ts_type = arrow::timestamp(arrow::TimeUnit::NANO); Literal literal(Timestamp(5000, 123456)); - ASSERT_OK_AND_ASSIGN(auto bytes, - KeySerializer::SerializeKey(literal, ts_type, pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, ts_type, pool_.get())); auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes)); - ASSERT_OK_AND_ASSIGN(auto result, - KeySerializer::DeserializeKey(slice, ts_type, pool_.get())); + ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, ts_type, pool_.get())); ASSERT_EQ(result.GetValue().GetMillisecond(), 5000); ASSERT_EQ(result.GetValue().GetNanoOfMillisecond(), 123456); } @@ -177,11 +176,9 @@ TEST_F(KeySerializerTest, SerializeAndDeserializeAllTypes) { { auto decimal_type = arrow::decimal128(10, 2); Literal literal(Decimal::FromUnscaledLong(12345, 10, 2)); - ASSERT_OK_AND_ASSIGN(auto bytes, - KeySerializer::SerializeKey(literal, decimal_type, pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, decimal_type, pool_.get())); auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes)); - ASSERT_OK_AND_ASSIGN(auto result, - KeySerializer::DeserializeKey(slice, decimal_type, pool_.get())); + ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, decimal_type, pool_.get())); ASSERT_EQ(result.GetValue().ToUnscaledLong(), 12345); } @@ -189,49 +186,49 @@ TEST_F(KeySerializerTest, SerializeAndDeserializeAllTypes) { { auto decimal_type = arrow::decimal128(25, 3); Literal literal(Decimal(25, 3, 9999999)); - ASSERT_OK_AND_ASSIGN(auto bytes, - KeySerializer::SerializeKey(literal, decimal_type, pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes, Serialize(literal, decimal_type, pool_.get())); auto slice = MemorySlice::Wrap(MemorySegment::Wrap(bytes)); - ASSERT_OK_AND_ASSIGN(auto result, - KeySerializer::DeserializeKey(slice, decimal_type, pool_.get())); + ASSERT_OK_AND_ASSIGN(auto result, Deserialize(slice, decimal_type, pool_.get())); ASSERT_EQ(result.GetValue().ToString(), literal.GetValue().ToString()); } // NULL should fail { Literal null_literal(FieldType::INT); - ASSERT_NOK_WITH_MSG(KeySerializer::SerializeKey(null_literal, arrow::int32(), pool_.get()), + ASSERT_NOK_WITH_MSG(Serialize(null_literal, arrow::int32(), pool_.get()), "cannot serialize null in KeySerializer"); } // unsupported type { - Literal literal(FieldType::BINARY, "ab", 2); - ASSERT_NOK_WITH_MSG(KeySerializer::SerializeKey(literal, arrow::binary(), pool_.get()), - "Not support serialize BINARY type in BTreeGlobalIndex"); + ASSERT_NOK_WITH_MSG(KeySerializer::Create(arrow::binary(), pool_), + "is not supported by global index now"); } } +TEST_F(KeySerializerTest, CreateRejectsMissingDependencies) { + ASSERT_NOK_WITH_MSG(KeySerializer::Create(nullptr, pool_), + "Cannot create KeySerializer without a key type"); + ASSERT_NOK_WITH_MSG(KeySerializer::Create(arrow::int32(), nullptr), + "Cannot create KeySerializer without a memory pool"); +} + TEST_F(KeySerializerTest, CanonicalizesFloatingPointNaN) { const auto float_nan = FloatingPointFromBits(0xffc12345U); const auto canonical_float_nan = FloatingPointFromBits(kCanonicalFloatNaNBits); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr float_bytes, - KeySerializer::SerializeKey(Literal(float_nan), arrow::float32(), pool_.get())); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr canonical_float_bytes, - KeySerializer::SerializeKey(Literal(canonical_float_nan), arrow::float32(), pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr float_bytes, + Serialize(Literal(float_nan), arrow::float32(), pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr canonical_float_bytes, + Serialize(Literal(canonical_float_nan), arrow::float32(), pool_.get())); ASSERT_EQ(std::string(float_bytes->data(), float_bytes->size()), std::string(canonical_float_bytes->data(), canonical_float_bytes->size())); const auto double_nan = FloatingPointFromBits(0xfff8123456789abcULL); const auto canonical_double_nan = FloatingPointFromBits(kCanonicalDoubleNaNBits); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr double_bytes, - KeySerializer::SerializeKey(Literal(double_nan), arrow::float64(), pool_.get())); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr canonical_double_bytes, - KeySerializer::SerializeKey(Literal(canonical_double_nan), arrow::float64(), pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr double_bytes, + Serialize(Literal(double_nan), arrow::float64(), pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr canonical_double_bytes, + Serialize(Literal(canonical_double_nan), arrow::float64(), pool_.get())); ASSERT_EQ(std::string(double_bytes->data(), double_bytes->size()), std::string(canonical_double_bytes->data(), canonical_double_bytes->size())); } @@ -241,50 +238,46 @@ TEST_F(KeySerializerTest, RejectsMalformedSerializedKeys) { return MemorySlice::Wrap(std::make_shared(value, pool_.get())); }; - ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(std::string(3, '\0')), arrow::int32())); - ASSERT_NOK( - KeySerializer::DeserializeKey(wrap(std::string(3, '\0')), arrow::int32(), pool_.get())); + ASSERT_NOK(ValidateSerializedKey(wrap(std::string(3, '\0')), arrow::int32())); + ASSERT_NOK(Deserialize(wrap(std::string(3, '\0')), arrow::int32(), pool_.get())); std::string invalid_boolean(1, static_cast(2)); - ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(invalid_boolean), arrow::boolean())); + ASSERT_NOK(ValidateSerializedKey(wrap(invalid_boolean), arrow::boolean())); auto nanos_timestamp = arrow::timestamp(arrow::TimeUnit::NANO); std::string unterminated_timestamp(9, '\0'); unterminated_timestamp[8] = static_cast(0x80); - ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(unterminated_timestamp), nanos_timestamp)); + ASSERT_NOK(ValidateSerializedKey(wrap(unterminated_timestamp), nanos_timestamp)); std::string out_of_range_timestamp(11, '\0'); out_of_range_timestamp[8] = static_cast(0xC0); out_of_range_timestamp[9] = static_cast(0x84); out_of_range_timestamp[10] = static_cast(0x3D); - ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(out_of_range_timestamp), nanos_timestamp)); + ASSERT_NOK(ValidateSerializedKey(wrap(out_of_range_timestamp), nanos_timestamp)); auto non_compact_decimal = arrow::decimal128(25, 3); - ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(""), non_compact_decimal)); - ASSERT_NOK( - KeySerializer::ValidateSerializedKey(wrap(std::string(17, '\0')), non_compact_decimal)); + ASSERT_NOK(ValidateSerializedKey(wrap(""), non_compact_decimal)); + ASSERT_NOK(ValidateSerializedKey(wrap(std::string(17, '\0')), non_compact_decimal)); auto narrow_decimal = arrow::decimal128(1, 0); - ASSERT_OK_AND_ASSIGN(std::shared_ptr out_of_range_decimal, - KeySerializer::SerializeKey(Literal(Decimal::FromUnscaledLong(10, 1, 0)), - narrow_decimal, pool_.get())); - ASSERT_NOK(KeySerializer::ValidateSerializedKey(MemorySlice::Wrap(out_of_range_decimal), - narrow_decimal)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr out_of_range_decimal, + Serialize(Literal(Decimal::FromUnscaledLong(10, 1, 0)), narrow_decimal, pool_.get())); + ASSERT_NOK(ValidateSerializedKey(MemorySlice::Wrap(out_of_range_decimal), narrow_decimal)); } TEST_F(KeySerializerTest, CreateComparator) { // INT comparator { - auto comparator = KeySerializer::CreateComparator(arrow::int32(), pool_); + ASSERT_OK_AND_ASSIGN(std::shared_ptr serializer, + KeySerializer::Create(arrow::int32(), pool_)); + auto comparator = serializer->CreateComparator(); Literal literal_1(1); Literal literal_2(2); Literal literal_3(1); - ASSERT_OK_AND_ASSIGN(auto bytes_1, - KeySerializer::SerializeKey(literal_1, arrow::int32(), pool_.get())); - ASSERT_OK_AND_ASSIGN(auto bytes_2, - KeySerializer::SerializeKey(literal_2, arrow::int32(), pool_.get())); - ASSERT_OK_AND_ASSIGN(auto bytes_3, - KeySerializer::SerializeKey(literal_3, arrow::int32(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes_1, Serialize(literal_1, arrow::int32(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes_2, Serialize(literal_2, arrow::int32(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes_3, Serialize(literal_3, arrow::int32(), pool_.get())); auto slice_1 = MemorySlice::Wrap(MemorySegment::Wrap(bytes_1)); auto slice_2 = MemorySlice::Wrap(MemorySegment::Wrap(bytes_2)); auto slice_3 = MemorySlice::Wrap(MemorySegment::Wrap(bytes_3)); @@ -299,16 +292,15 @@ TEST_F(KeySerializerTest, CreateComparator) { // STRING comparator { - auto comparator = KeySerializer::CreateComparator(arrow::utf8(), pool_); + ASSERT_OK_AND_ASSIGN(std::shared_ptr serializer, + KeySerializer::Create(arrow::utf8(), pool_)); + auto comparator = serializer->CreateComparator(); Literal literal_a(FieldType::STRING, "apple", 5); Literal literal_b(FieldType::STRING, "banana", 6); Literal literal_c(FieldType::STRING, "apple", 5); - ASSERT_OK_AND_ASSIGN(auto bytes_a, - KeySerializer::SerializeKey(literal_a, arrow::utf8(), pool_.get())); - ASSERT_OK_AND_ASSIGN(auto bytes_b, - KeySerializer::SerializeKey(literal_b, arrow::utf8(), pool_.get())); - ASSERT_OK_AND_ASSIGN(auto bytes_c, - KeySerializer::SerializeKey(literal_c, arrow::utf8(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes_a, Serialize(literal_a, arrow::utf8(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes_b, Serialize(literal_b, arrow::utf8(), pool_.get())); + ASSERT_OK_AND_ASSIGN(auto bytes_c, Serialize(literal_c, arrow::utf8(), pool_.get())); auto slice_a = MemorySlice::Wrap(MemorySegment::Wrap(bytes_a)); auto slice_b = MemorySlice::Wrap(MemorySegment::Wrap(bytes_b)); auto slice_c = MemorySlice::Wrap(MemorySegment::Wrap(bytes_c)); diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp b/src/paimon/common/global_index/sorted_file_meta_selector.cpp similarity index 51% rename from src/paimon/common/global_index/btree/btree_file_meta_selector.cpp rename to src/paimon/common/global_index/sorted_file_meta_selector.cpp index aa2f23c71..365a09661 100644 --- a/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp +++ b/src/paimon/common/global_index/sorted_file_meta_selector.cpp @@ -17,47 +17,45 @@ * under the License. */ -#include "paimon/common/global_index/btree/btree_file_meta_selector.h" +#include "paimon/common/global_index/sorted_file_meta_selector.h" #include "fmt/format.h" #include "paimon/common/memory/memory_slice.h" namespace paimon { -Result> BTreeFileMetaSelector::Create( - const std::vector& files, const std::shared_ptr& key_type, - const std::shared_ptr& pool) { - if (key_type == nullptr) { - return Status::Invalid("Cannot create a BTree file metadata selector without a key type."); - } - if (pool == nullptr) { +Result> SortedFileMetaSelector::Create( + const std::vector& files, + const std::shared_ptr& key_serializer) { + if (key_serializer == nullptr) { return Status::Invalid( - "Cannot create a BTree file metadata selector without a memory pool."); + "Cannot create a sorted file metadata selector without a key serializer."); } - std::vector>> decoded_files; + std::vector>> decoded_files; decoded_files.reserve(files.size()); - MemorySlice::SliceComparator comparator = KeySerializer::CreateComparator(key_type, pool); + MemorySlice::SliceComparator comparator = key_serializer->CreateComparator(); for (const auto& file : files) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_meta, - BTreeIndexMeta::Deserialize(file.metadata, pool.get())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr index_meta, + SortedIndexFileMeta::Deserialize(file.metadata, key_serializer->GetMemoryPool())); bool has_first_key = index_meta->FirstKey() != nullptr; bool has_last_key = index_meta->LastKey() != nullptr; if (has_first_key != has_last_key) { return Status::Invalid(fmt::format( - "BTree index metadata for {} must contain both boundary keys or neither.", + "Sorted index file metadata for {} must contain both boundary keys or neither.", file.file_path)); } if (!has_first_key && !index_meta->HasNulls()) { - return Status::Invalid( - fmt::format("BTree index metadata for {} has no boundary keys or null values.", - file.file_path)); + return Status::Invalid(fmt::format( + "Sorted index file metadata for {} has no boundary keys or null values.", + file.file_path)); } if (index_meta->FirstKey() != nullptr) { - PAIMON_RETURN_NOT_OK(KeySerializer::ValidateSerializedKey( - WrapKeySlice(index_meta->FirstKey()), key_type)); + PAIMON_RETURN_NOT_OK( + key_serializer->ValidateSerializedKey(WrapKeySlice(index_meta->FirstKey()))); } if (index_meta->LastKey() != nullptr) { - PAIMON_RETURN_NOT_OK(KeySerializer::ValidateSerializedKey( - WrapKeySlice(index_meta->LastKey()), key_type)); + PAIMON_RETURN_NOT_OK( + key_serializer->ValidateSerializedKey(WrapKeySlice(index_meta->LastKey()))); } if (index_meta->FirstKey() != nullptr && index_meta->LastKey() != nullptr) { PAIMON_ASSIGN_OR_RAISE(int32_t comparison, @@ -65,35 +63,35 @@ Result> BTreeFileMetaSelector::Create( WrapKeySlice(index_meta->LastKey()))); if (comparison > 0) { return Status::Invalid(fmt::format( - "BTree index metadata for {} has a first key greater than its last key.", + "Sorted index file metadata for {} has a first key greater than its last key.", file.file_path)); } } decoded_files.emplace_back(file, std::move(index_meta)); } - return std::unique_ptr( - new BTreeFileMetaSelector(std::move(decoded_files), key_type, pool)); + return std::unique_ptr( + new SortedFileMetaSelector(std::move(decoded_files), key_serializer)); } -BTreeFileMetaSelector::BTreeFileMetaSelector( - std::vector>> files, - std::shared_ptr key_type, std::shared_ptr pool) +SortedFileMetaSelector::SortedFileMetaSelector( + std::vector>> files, + std::shared_ptr key_serializer) : files_(std::move(files)), - key_type_(std::move(key_type)), - pool_(std::move(pool)), - comparator_(KeySerializer::CreateComparator(key_type_, pool_)) {} + key_serializer_(std::move(key_serializer)), + comparator_(key_serializer_->CreateComparator()) {} -Result> BTreeFileMetaSelector::VisitIsNotNull() { - return Filter([](const BTreeIndexMeta& meta) -> Result { return !meta.OnlyNulls(); }); +Result> SortedFileMetaSelector::VisitIsNotNull() { + return Filter( + [](const SortedIndexFileMeta& meta) -> Result { return !meta.OnlyNulls(); }); } -Result> BTreeFileMetaSelector::VisitIsNull() { - return Filter([](const BTreeIndexMeta& meta) -> Result { return meta.HasNulls(); }); +Result> SortedFileMetaSelector::VisitIsNull() { + return Filter([](const SortedIndexFileMeta& meta) -> Result { return meta.HasNulls(); }); } -Result> BTreeFileMetaSelector::VisitEqual(const Literal& literal) { +Result> SortedFileMetaSelector::VisitEqual(const Literal& literal) { PAIMON_ASSIGN_OR_RAISE(MemorySlice literal_slice, SerializeLiteral(literal)); - return Filter([this, &literal_slice](const BTreeIndexMeta& meta) -> Result { + return Filter([this, &literal_slice](const SortedIndexFileMeta& meta) -> Result { if (meta.OnlyNulls()) { return false; } @@ -101,16 +99,16 @@ Result> BTreeFileMetaSelector::VisitEqual(const L }); } -Result> BTreeFileMetaSelector::VisitNotEqual( +Result> SortedFileMetaSelector::VisitNotEqual( const Literal& literal) { - return Filter([](const BTreeIndexMeta& meta) -> Result { return true; }); + return Filter([](const SortedIndexFileMeta& meta) -> Result { return true; }); } -Result> BTreeFileMetaSelector::VisitLessThan( +Result> SortedFileMetaSelector::VisitLessThan( const Literal& literal) { // file.minKey < literal PAIMON_ASSIGN_OR_RAISE(MemorySlice literal_slice, SerializeLiteral(literal)); - return Filter([this, &literal_slice](const BTreeIndexMeta& meta) -> Result { + return Filter([this, &literal_slice](const SortedIndexFileMeta& meta) -> Result { if (meta.OnlyNulls()) { return false; } @@ -119,11 +117,11 @@ Result> BTreeFileMetaSelector::VisitLessThan( }); } -Result> BTreeFileMetaSelector::VisitLessOrEqual( +Result> SortedFileMetaSelector::VisitLessOrEqual( const Literal& literal) { // file.minKey <= literal PAIMON_ASSIGN_OR_RAISE(MemorySlice literal_slice, SerializeLiteral(literal)); - return Filter([this, &literal_slice](const BTreeIndexMeta& meta) -> Result { + return Filter([this, &literal_slice](const SortedIndexFileMeta& meta) -> Result { if (meta.OnlyNulls()) { return false; } @@ -132,11 +130,11 @@ Result> BTreeFileMetaSelector::VisitLessOrEqual( }); } -Result> BTreeFileMetaSelector::VisitGreaterThan( +Result> SortedFileMetaSelector::VisitGreaterThan( const Literal& literal) { // file.maxKey > literal PAIMON_ASSIGN_OR_RAISE(MemorySlice literal_slice, SerializeLiteral(literal)); - return Filter([this, &literal_slice](const BTreeIndexMeta& meta) -> Result { + return Filter([this, &literal_slice](const SortedIndexFileMeta& meta) -> Result { if (meta.OnlyNulls()) { return false; } @@ -145,11 +143,11 @@ Result> BTreeFileMetaSelector::VisitGreaterThan( }); } -Result> BTreeFileMetaSelector::VisitGreaterOrEqual( +Result> SortedFileMetaSelector::VisitGreaterOrEqual( const Literal& literal) { // file.maxKey >= literal PAIMON_ASSIGN_OR_RAISE(MemorySlice literal_slice, SerializeLiteral(literal)); - return Filter([this, &literal_slice](const BTreeIndexMeta& meta) -> Result { + return Filter([this, &literal_slice](const SortedIndexFileMeta& meta) -> Result { if (meta.OnlyNulls()) { return false; } @@ -158,7 +156,7 @@ Result> BTreeFileMetaSelector::VisitGreaterOrEqua }); } -Result> BTreeFileMetaSelector::VisitIn( +Result> SortedFileMetaSelector::VisitIn( const std::vector& literals) { std::vector literal_slices; literal_slices.reserve(literals.size()); @@ -166,7 +164,7 @@ Result> BTreeFileMetaSelector::VisitIn( PAIMON_ASSIGN_OR_RAISE(MemorySlice slice, SerializeLiteral(literal)); literal_slices.push_back(std::move(slice)); } - return Filter([this, &literal_slices](const BTreeIndexMeta& meta) -> Result { + return Filter([this, &literal_slices](const SortedIndexFileMeta& meta) -> Result { if (meta.OnlyNulls()) { return false; } @@ -180,31 +178,32 @@ Result> BTreeFileMetaSelector::VisitIn( }); } -Result> BTreeFileMetaSelector::VisitNotIn( +Result> SortedFileMetaSelector::VisitNotIn( const std::vector& literals) { // Cannot filter any file by NOT IN condition - return Filter([](const BTreeIndexMeta& meta) -> Result { return true; }); + return Filter([](const SortedIndexFileMeta& meta) -> Result { return true; }); } -Result> BTreeFileMetaSelector::VisitStartsWith( +Result> SortedFileMetaSelector::VisitStartsWith( const Literal& prefix) { - return Filter([](const BTreeIndexMeta& meta) -> Result { return true; }); + return Filter([](const SortedIndexFileMeta& meta) -> Result { return true; }); } -Result> BTreeFileMetaSelector::VisitEndsWith(const Literal& suffix) { - return Filter([](const BTreeIndexMeta& meta) -> Result { return true; }); +Result> SortedFileMetaSelector::VisitEndsWith( + const Literal& suffix) { + return Filter([](const SortedIndexFileMeta& meta) -> Result { return true; }); } -Result> BTreeFileMetaSelector::VisitContains( +Result> SortedFileMetaSelector::VisitContains( const Literal& literal) { - return Filter([](const BTreeIndexMeta& meta) -> Result { return true; }); + return Filter([](const SortedIndexFileMeta& meta) -> Result { return true; }); } -Result> BTreeFileMetaSelector::VisitLike(const Literal& literal) { - return Filter([](const BTreeIndexMeta& meta) -> Result { return true; }); +Result> SortedFileMetaSelector::VisitLike(const Literal& literal) { + return Filter([](const SortedIndexFileMeta& meta) -> Result { return true; }); } -Result> BTreeFileMetaSelector::Filter( +Result> SortedFileMetaSelector::Filter( const MetaPredicate& predicate) const { std::vector result; for (const auto& [io_meta, index_meta] : files_) { @@ -216,8 +215,9 @@ Result> BTreeFileMetaSelector::Filter( return result; } -Result BTreeFileMetaSelector::Overlaps(const BTreeIndexMeta& meta, const MemorySlice& from, - const MemorySlice& to) const { +Result SortedFileMetaSelector::Overlaps(const SortedIndexFileMeta& meta, + const MemorySlice& from, + const MemorySlice& to) const { if (meta.FirstKey()) { PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(to, WrapKeySlice(meta.FirstKey()))); if (cmp < 0) { @@ -233,31 +233,30 @@ Result BTreeFileMetaSelector::Overlaps(const BTreeIndexMeta& meta, const M return true; } -Result BTreeFileMetaSelector::CompareFirstKey(const BTreeIndexMeta& meta, - const MemorySlice& literal) const { +Result SortedFileMetaSelector::CompareFirstKey(const SortedIndexFileMeta& meta, + const MemorySlice& literal) const { if (!meta.FirstKey()) { return -1; } return comparator_(WrapKeySlice(meta.FirstKey()), literal); } -Result BTreeFileMetaSelector::CompareLastKey(const BTreeIndexMeta& meta, - const MemorySlice& literal) const { +Result SortedFileMetaSelector::CompareLastKey(const SortedIndexFileMeta& meta, + const MemorySlice& literal) const { if (!meta.LastKey()) { return 1; } return comparator_(WrapKeySlice(meta.LastKey()), literal); } -MemorySlice BTreeFileMetaSelector::WrapKeySlice(const std::shared_ptr& key) { +MemorySlice SortedFileMetaSelector::WrapKeySlice(const std::shared_ptr& key) { return MemorySlice::Wrap(MemorySegment::WrapView(key->data(), key->size())); } -Result BTreeFileMetaSelector::SerializeLiteral(const Literal& literal) const { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bytes, - KeySerializer::SerializeKey(literal, key_type_, pool_.get())); +Result SortedFileMetaSelector::SerializeLiteral(const Literal& literal) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bytes, key_serializer_->Serialize(literal)); MemorySlice slice = MemorySlice::Wrap(bytes); - PAIMON_RETURN_NOT_OK(KeySerializer::ValidateSerializedKey(slice, key_type_)); + PAIMON_RETURN_NOT_OK(key_serializer_->ValidateSerializedKey(slice)); return slice; } diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector.h b/src/paimon/common/global_index/sorted_file_meta_selector.h similarity index 71% rename from src/paimon/common/global_index/btree/btree_file_meta_selector.h rename to src/paimon/common/global_index/sorted_file_meta_selector.h index ad066278e..871bbab33 100644 --- a/src/paimon/common/global_index/btree/btree_file_meta_selector.h +++ b/src/paimon/common/global_index/sorted_file_meta_selector.h @@ -19,24 +19,27 @@ #pragma once +#include #include #include #include -#include "paimon/common/global_index/btree/btree_index_meta.h" -#include "paimon/common/global_index/btree/key_serializer.h" +#include "paimon/common/global_index/key_serializer.h" +#include "paimon/common/global_index/sorted_index_file_meta.h" #include "paimon/common/memory/memory_slice.h" #include "paimon/global_index/global_index_io_meta.h" #include "paimon/predicate/function_visitor.h" namespace paimon { -/// Selects candidate BTree index files based on filter predicates. -class BTreeFileMetaSelector : public FunctionVisitor> { +/// Selects candidate global index files by per-index-file min/max metadata. +/// +/// All files are expected to belong to the same field. +class SortedFileMetaSelector : public FunctionVisitor> { public: - static Result> Create( + static Result> Create( const std::vector& files, - const std::shared_ptr& key_type, const std::shared_ptr& pool); + const std::shared_ptr& key_serializer); Result> VisitIsNotNull() override; Result> VisitIsNull() override; @@ -55,20 +58,22 @@ class BTreeFileMetaSelector : public FunctionVisitor> VisitLike(const Literal& literal) override; private: - BTreeFileMetaSelector( - std::vector>> files, - std::shared_ptr key_type, std::shared_ptr pool); + SortedFileMetaSelector( + std::vector>> files, + std::shared_ptr key_serializer); - using MetaPredicate = std::function(const BTreeIndexMeta&)>; + using MetaPredicate = std::function(const SortedIndexFileMeta&)>; Result> Filter(const MetaPredicate& predicate) const; - Result Overlaps(const BTreeIndexMeta& meta, const MemorySlice& from, + Result Overlaps(const SortedIndexFileMeta& meta, const MemorySlice& from, const MemorySlice& to) const; - Result CompareFirstKey(const BTreeIndexMeta& meta, const MemorySlice& literal) const; + Result CompareFirstKey(const SortedIndexFileMeta& meta, + const MemorySlice& literal) const; - Result CompareLastKey(const BTreeIndexMeta& meta, const MemorySlice& literal) const; + Result CompareLastKey(const SortedIndexFileMeta& meta, + const MemorySlice& literal) const; Result SerializeLiteral(const Literal& literal) const; @@ -76,9 +81,8 @@ class BTreeFileMetaSelector : public FunctionVisitor& key); - std::vector>> files_; - std::shared_ptr key_type_; - std::shared_ptr pool_; + std::vector>> files_; + std::shared_ptr key_serializer_; MemorySlice::SliceComparator comparator_; }; diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp b/src/paimon/common/global_index/sorted_file_meta_selector_test.cpp similarity index 62% rename from src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp rename to src/paimon/common/global_index/sorted_file_meta_selector_test.cpp index bb025c118..eb0a843c3 100644 --- a/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp +++ b/src/paimon/common/global_index/sorted_file_meta_selector_test.cpp @@ -17,7 +17,7 @@ * under the License. */ -#include "paimon/common/global_index/btree/btree_file_meta_selector.h" +#include "paimon/common/global_index/sorted_file_meta_selector.h" #include #include @@ -25,31 +25,37 @@ #include #include "gtest/gtest.h" -#include "paimon/common/global_index/btree/btree_index_meta.h" -#include "paimon/common/global_index/btree/key_serializer.h" +#include "paimon/common/global_index/key_serializer.h" +#include "paimon/common/global_index/sorted_index_file_meta.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { -class BTreeFileMetaSelectorTest : public ::testing::Test { +class SortedFileMetaSelectorTest : public ::testing::Test { public: void SetUp() override { pool_ = GetDefaultPool(); key_type_ = arrow::int32(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr key_serializer, + KeySerializer::Create(key_type_, pool_)); + key_serializer_ = std::move(key_serializer); // file1: keys [1, 10], has_nulls=true // file2: keys [15, 20], has_nulls=false // file3: keys [21, 30], has_nulls=true // file4: keys [1, 5], has_nulls=false // file5: keys [19, 25], has_nulls=true - auto meta1 = std::make_shared(SerializeInt(1), SerializeInt(10), true); - auto meta2 = std::make_shared(SerializeInt(15), SerializeInt(20), false); - auto meta3 = std::make_shared(SerializeInt(21), SerializeInt(30), true); - auto meta4 = std::make_shared(SerializeInt(1), SerializeInt(5), false); - auto meta5 = std::make_shared(SerializeInt(19), SerializeInt(25), true); + auto meta1 = std::make_shared(SerializeInt(1), SerializeInt(10), true); + auto meta2 = + std::make_shared(SerializeInt(15), SerializeInt(20), false); + auto meta3 = + std::make_shared(SerializeInt(21), SerializeInt(30), true); + auto meta4 = std::make_shared(SerializeInt(1), SerializeInt(5), false); + auto meta5 = + std::make_shared(SerializeInt(19), SerializeInt(25), true); // file6: only-nulls file (no keys, has_nulls=true) - auto meta6 = std::make_shared(nullptr, nullptr, true); + auto meta6 = std::make_shared(nullptr, nullptr, true); files_ = { GlobalIndexIOMeta("file1", 1, meta1->Serialize(pool_.get())), GlobalIndexIOMeta("file2", 1, meta2->Serialize(pool_.get())), @@ -62,8 +68,7 @@ class BTreeFileMetaSelectorTest : public ::testing::Test { std::shared_ptr SerializeInt(int32_t value) const { Literal literal(value); - EXPECT_OK_AND_ASSIGN(std::shared_ptr result, - KeySerializer::SerializeKey(literal, key_type_, pool_.get())); + EXPECT_OK_AND_ASSIGN(std::shared_ptr result, key_serializer_->Serialize(literal)); return result; } @@ -83,12 +88,13 @@ class BTreeFileMetaSelectorTest : public ::testing::Test { private: std::shared_ptr pool_; std::shared_ptr key_type_; + std::shared_ptr key_serializer_; std::vector files_; }; -TEST_F(BTreeFileMetaSelectorTest, TestVisitLessThan) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, - BTreeFileMetaSelector::Create(files_, key_type_, pool_)); +TEST_F(SortedFileMetaSelectorTest, TestVisitLessThan) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + SortedFileMetaSelector::Create(files_, key_serializer_)); // minKey < 8: file1(1), file4(1) ASSERT_OK_AND_ASSIGN(auto result, selector->VisitLessThan(Literal(8))); @@ -103,9 +109,9 @@ TEST_F(BTreeFileMetaSelectorTest, TestVisitLessThan) { ASSERT_TRUE(result.empty()); } -TEST_F(BTreeFileMetaSelectorTest, TestVisitLessOrEqual) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, - BTreeFileMetaSelector::Create(files_, key_type_, pool_)); +TEST_F(SortedFileMetaSelectorTest, TestVisitLessOrEqual) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + SortedFileMetaSelector::Create(files_, key_serializer_)); // minKey <= 20: file1(1), file2(15), file4(1), file5(19) ASSERT_OK_AND_ASSIGN(auto result, selector->VisitLessOrEqual(Literal(20))); @@ -116,9 +122,9 @@ TEST_F(BTreeFileMetaSelectorTest, TestVisitLessOrEqual) { CheckResult(result, {"file1", "file2", "file4"}); } -TEST_F(BTreeFileMetaSelectorTest, TestVisitGreaterThan) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, - BTreeFileMetaSelector::Create(files_, key_type_, pool_)); +TEST_F(SortedFileMetaSelectorTest, TestVisitGreaterThan) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + SortedFileMetaSelector::Create(files_, key_serializer_)); // maxKey > 20: file3(30), file5(25) ASSERT_OK_AND_ASSIGN(auto result, selector->VisitGreaterThan(Literal(20))); @@ -129,9 +135,9 @@ TEST_F(BTreeFileMetaSelectorTest, TestVisitGreaterThan) { ASSERT_TRUE(result.empty()); } -TEST_F(BTreeFileMetaSelectorTest, TestVisitGreaterOrEqual) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, - BTreeFileMetaSelector::Create(files_, key_type_, pool_)); +TEST_F(SortedFileMetaSelectorTest, TestVisitGreaterOrEqual) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + SortedFileMetaSelector::Create(files_, key_serializer_)); // maxKey >= 5: all non-null files (file1..file5) ASSERT_OK_AND_ASSIGN(auto result, selector->VisitGreaterOrEqual(Literal(5))); @@ -142,9 +148,9 @@ TEST_F(BTreeFileMetaSelectorTest, TestVisitGreaterOrEqual) { CheckResult(result, {"file2", "file3", "file5"}); } -TEST_F(BTreeFileMetaSelectorTest, TestVisitEqual) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, - BTreeFileMetaSelector::Create(files_, key_type_, pool_)); +TEST_F(SortedFileMetaSelectorTest, TestVisitEqual) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + SortedFileMetaSelector::Create(files_, key_serializer_)); // 22 in [21,30] and [19,25] ASSERT_OK_AND_ASSIGN(auto result, selector->VisitEqual(Literal(22))); @@ -162,36 +168,36 @@ TEST_F(BTreeFileMetaSelectorTest, TestVisitEqual) { ASSERT_NOK(selector->VisitEqual(Literal(static_cast(1)))); } -TEST_F(BTreeFileMetaSelectorTest, TestVisitNotEqual) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, - BTreeFileMetaSelector::Create(files_, key_type_, pool_)); +TEST_F(SortedFileMetaSelectorTest, TestVisitNotEqual) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + SortedFileMetaSelector::Create(files_, key_serializer_)); // NotEqual cannot prune any file, returns all ASSERT_OK_AND_ASSIGN(auto result, selector->VisitNotEqual(Literal(22))); CheckResult(result, {"file1", "file2", "file3", "file4", "file5", "file6"}); } -TEST_F(BTreeFileMetaSelectorTest, TestVisitIsNull) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, - BTreeFileMetaSelector::Create(files_, key_type_, pool_)); +TEST_F(SortedFileMetaSelectorTest, TestVisitIsNull) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + SortedFileMetaSelector::Create(files_, key_serializer_)); // has_nulls: file1, file3, file5, file6 ASSERT_OK_AND_ASSIGN(auto result, selector->VisitIsNull()); CheckResult(result, {"file1", "file3", "file5", "file6"}); } -TEST_F(BTreeFileMetaSelectorTest, TestVisitIsNotNull) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, - BTreeFileMetaSelector::Create(files_, key_type_, pool_)); +TEST_F(SortedFileMetaSelectorTest, TestVisitIsNotNull) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + SortedFileMetaSelector::Create(files_, key_serializer_)); // !onlyNulls: file1..file5 (file6 is only-nulls) ASSERT_OK_AND_ASSIGN(auto result, selector->VisitIsNotNull()); CheckResult(result, {"file1", "file2", "file3", "file4", "file5"}); } -TEST_F(BTreeFileMetaSelectorTest, TestVisitIn) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, - BTreeFileMetaSelector::Create(files_, key_type_, pool_)); +TEST_F(SortedFileMetaSelectorTest, TestVisitIn) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + SortedFileMetaSelector::Create(files_, key_serializer_)); // IN(1, 2, 3, 26, 27, 28): // 1 in [1,10]=file1, [1,5]=file4 @@ -209,9 +215,9 @@ TEST_F(BTreeFileMetaSelectorTest, TestVisitIn) { ASSERT_TRUE(result.empty()); } -TEST_F(BTreeFileMetaSelectorTest, TestVisitNotIn) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, - BTreeFileMetaSelector::Create(files_, key_type_, pool_)); +TEST_F(SortedFileMetaSelectorTest, TestVisitNotIn) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + SortedFileMetaSelector::Create(files_, key_serializer_)); // NotIn cannot prune any file ASSERT_OK_AND_ASSIGN(auto result, @@ -219,9 +225,9 @@ TEST_F(BTreeFileMetaSelectorTest, TestVisitNotIn) { CheckResult(result, {"file1", "file2", "file3", "file4", "file5", "file6"}); } -TEST_F(BTreeFileMetaSelectorTest, TestOnlyNullsFileExcludedFromRangeQueries) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, - BTreeFileMetaSelector::Create(files_, key_type_, pool_)); +TEST_F(SortedFileMetaSelectorTest, TestOnlyNullsFileExcludedFromRangeQueries) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + SortedFileMetaSelector::Create(files_, key_serializer_)); // file6 is only-nulls, should be excluded from all range/equality queries ASSERT_OK_AND_ASSIGN(auto result, selector->VisitEqual(Literal(1))); @@ -242,29 +248,30 @@ TEST_F(BTreeFileMetaSelectorTest, TestOnlyNullsFileExcludedFromRangeQueries) { ASSERT_EQ(names.count("file6"), 1u); } -TEST_F(BTreeFileMetaSelectorTest, TestEmptyStringKeyDoesNotCrash) { +TEST_F(SortedFileMetaSelectorTest, TestEmptyStringKeyDoesNotCrash) { std::shared_ptr pool = GetDefaultPool(); std::shared_ptr key_type = arrow::utf8(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr key_serializer, + KeySerializer::Create(key_type, pool)); auto serialize = [&](const char* value, size_t size) { Literal literal(FieldType::STRING, value, size); - EXPECT_OK_AND_ASSIGN(std::shared_ptr result, - KeySerializer::SerializeKey(literal, key_type, pool.get())); + EXPECT_OK_AND_ASSIGN(std::shared_ptr result, key_serializer->Serialize(literal)); return result; }; - auto empty_meta = - std::make_shared(serialize("", 0), serialize("www.example.com", 15), false); - auto normal_meta = - std::make_shared(serialize("aaa.com", 7), serialize("zzz.com", 7), false); - auto null_meta = std::make_shared(nullptr, nullptr, true); + auto empty_meta = std::make_shared( + serialize("", 0), serialize("www.example.com", 15), false); + auto normal_meta = std::make_shared(serialize("aaa.com", 7), + serialize("zzz.com", 7), false); + auto null_meta = std::make_shared(nullptr, nullptr, true); std::vector files = { GlobalIndexIOMeta("file_empty", 1, empty_meta->Serialize(pool.get())), GlobalIndexIOMeta("file_normal", 1, normal_meta->Serialize(pool.get())), GlobalIndexIOMeta("file_nulls", 1, null_meta->Serialize(pool.get())), }; - ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, - BTreeFileMetaSelector::Create(files, key_type, pool)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + SortedFileMetaSelector::Create(files, key_serializer)); ASSERT_OK_AND_ASSIGN(std::vector result, selector->VisitEqual(Literal(FieldType::STRING, "www.example.com", 15))); @@ -282,38 +289,41 @@ TEST_F(BTreeFileMetaSelectorTest, TestEmptyStringKeyDoesNotCrash) { CheckResult(result, {"file_empty", "file_normal"}); } -TEST_F(BTreeFileMetaSelectorTest, RejectsMalformedFileMetadata) { +TEST_F(SortedFileMetaSelectorTest, RejectsMalformedFileMetadata) { std::shared_ptr pool = GetDefaultPool(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr key_serializer, + KeySerializer::Create(arrow::int32(), pool)); std::vector files = {GlobalIndexIOMeta("missing", 1, /*metadata=*/nullptr)}; - ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer)); auto truncated = std::make_shared(std::string(4, '\0'), pool.get()); files = {GlobalIndexIOMeta("truncated", 1, truncated)}; - ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer)); auto short_key = std::make_shared(std::string(1, '\0'), pool.get()); - auto invalid_key_meta = std::make_shared(short_key, short_key, false); + auto invalid_key_meta = std::make_shared(short_key, short_key, false); files = {GlobalIndexIOMeta("invalid-key", 1, invalid_key_meta->Serialize(pool.get()))}; - ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer)); - auto reversed_meta = std::make_shared(SerializeInt(10), SerializeInt(1), false); + auto reversed_meta = + std::make_shared(SerializeInt(10), SerializeInt(1), false); files = {GlobalIndexIOMeta("reversed-range", 1, reversed_meta->Serialize(pool.get()))}; - ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer)); auto only_first_meta = - std::make_shared(SerializeInt(1), /*last_key=*/nullptr, false); + std::make_shared(SerializeInt(1), /*last_key=*/nullptr, false); files = {GlobalIndexIOMeta("only-first-key", 1, only_first_meta->Serialize(pool.get()))}; - ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer)); auto only_last_meta = - std::make_shared(/*first_key=*/nullptr, SerializeInt(1), false); + std::make_shared(/*first_key=*/nullptr, SerializeInt(1), false); files = {GlobalIndexIOMeta("only-last-key", 1, only_last_meta->Serialize(pool.get()))}; - ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer)); auto empty_nonnull_meta = - std::make_shared(/*first_key=*/nullptr, /*last_key=*/nullptr, false); + std::make_shared(/*first_key=*/nullptr, /*last_key=*/nullptr, false); files = {GlobalIndexIOMeta("empty-nonnull", 1, empty_nonnull_meta->Serialize(pool.get()))}; - ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer)); } } // namespace paimon::test diff --git a/src/paimon/common/global_index/btree/btree_index_meta.cpp b/src/paimon/common/global_index/sorted_index_file_meta.cpp similarity index 72% rename from src/paimon/common/global_index/btree/btree_index_meta.cpp rename to src/paimon/common/global_index/sorted_index_file_meta.cpp index 4b6f6ce0e..a05e7b180 100644 --- a/src/paimon/common/global_index/btree/btree_index_meta.cpp +++ b/src/paimon/common/global_index/sorted_index_file_meta.cpp @@ -17,7 +17,7 @@ * under the License. */ -#include "paimon/common/global_index/btree/btree_index_meta.h" +#include "paimon/common/global_index/sorted_index_file_meta.h" #include #include @@ -32,14 +32,14 @@ Result> ReadKey(MemorySliceInput* input, int32_t key_leng int32_t required_remaining, const char* key_name, MemoryPool* pool) { if (key_length < 0) { - return Status::Invalid( - fmt::format("BTree index metadata has a negative {} length {}.", key_name, key_length)); + return Status::Invalid(fmt::format( + "Sorted index file metadata has a negative {} length {}.", key_name, key_length)); } if (input->Available() < required_remaining || key_length > input->Available() - required_remaining) { - return Status::Invalid( - fmt::format("BTree index metadata {} length {} exceeds the available payload bytes.", - key_name, key_length)); + return Status::Invalid(fmt::format( + "Sorted index file metadata {} length {} exceeds the available payload bytes.", + key_name, key_length)); } if (key_length == 0) { return std::make_shared(0, pool); @@ -49,24 +49,25 @@ Result> ReadKey(MemorySliceInput* input, int32_t key_leng } // namespace -Result> BTreeIndexMeta::Deserialize( +Result> SortedIndexFileMeta::Deserialize( const std::shared_ptr& meta, paimon::MemoryPool* pool) { if (meta == nullptr) { - return Status::Invalid("Cannot deserialize BTree index metadata from a null buffer."); + return Status::Invalid("Cannot deserialize Sorted index file metadata from a null buffer."); } if (pool == nullptr) { - return Status::Invalid("Cannot deserialize BTree index metadata with a null memory pool."); + return Status::Invalid( + "Cannot deserialize Sorted index file metadata with a null memory pool."); } // Legacy metadata contains two int32 lengths and one has-nulls byte. constexpr size_t kMinimumMetadataSize = 2 * sizeof(int32_t) + sizeof(int8_t); if (meta->size() < kMinimumMetadataSize) { return Status::Invalid(fmt::format( - "BTree index metadata is truncated: expected at least {} bytes, but found {}.", + "Sorted index file metadata is truncated: expected at least {} bytes, but found {}.", kMinimumMetadataSize, meta->size())); } if (meta->size() > static_cast(std::numeric_limits::max())) { return Status::Invalid( - fmt::format("BTree index metadata size {} exceeds the supported maximum {}.", + fmt::format("Sorted index file metadata size {} exceeds the supported maximum {}.", meta->size(), std::numeric_limits::max())); } @@ -83,8 +84,8 @@ Result> BTreeIndexMeta::Deserialize( ReadKey(&input, last_key_len, kRequiredAfterLastKey, "last key", pool)); int8_t has_nulls_byte = input.ReadByte(); if (has_nulls_byte != 0 && has_nulls_byte != 1) { - return Status::Invalid( - fmt::format("BTree index metadata has invalid has-nulls value {}.", has_nulls_byte)); + return Status::Invalid(fmt::format( + "Sorted index file metadata has invalid has-nulls value {}.", has_nulls_byte)); } bool has_nulls = has_nulls_byte == 1; @@ -92,23 +93,25 @@ Result> BTreeIndexMeta::Deserialize( int8_t format_version = input.ReadByte(); if (format_version != kFormatVersionWithNullFlags) { return Status::Invalid( - fmt::format("Unsupported BTree index metadata version {}.", format_version)); + fmt::format("Unsupported Sorted index file metadata version {}.", format_version)); } int8_t null_key_flags = input.ReadByte(); constexpr int8_t kKnownNullFlags = kFirstKeyIsNull | kLastKeyIsNull; if ((null_key_flags & ~kKnownNullFlags) != 0) { - return Status::Invalid( - fmt::format("BTree index metadata has invalid null-key flags {}.", null_key_flags)); + return Status::Invalid(fmt::format( + "Sorted index file metadata has invalid null-key flags {}.", null_key_flags)); } if ((null_key_flags & kFirstKeyIsNull) != 0) { if (first_key_len != 0) { - return Status::Invalid("BTree index metadata marks a non-empty first key as null."); + return Status::Invalid( + "Sorted index file metadata marks a non-empty first key as null."); } first_key.reset(); } if ((null_key_flags & kLastKeyIsNull) != 0) { if (last_key_len != 0) { - return Status::Invalid("BTree index metadata marks a non-empty last key as null."); + return Status::Invalid( + "Sorted index file metadata marks a non-empty last key as null."); } last_key.reset(); } @@ -118,16 +121,16 @@ Result> BTreeIndexMeta::Deserialize( first_key.reset(); last_key.reset(); } else if (input.Available() != 0) { - return Status::Invalid(fmt::format("BTree index metadata has {} unexpected trailing bytes.", - input.Available())); + return Status::Invalid(fmt::format( + "Sorted index file metadata has {} unexpected trailing bytes.", input.Available())); } - return std::make_shared(first_key, last_key, has_nulls); + return std::make_shared(first_key, last_key, has_nulls); } -std::shared_ptr BTreeIndexMeta::Serialize(paimon::MemoryPool* pool) const { +std::shared_ptr SortedIndexFileMeta::Serialize(paimon::MemoryPool* pool) const { int32_t first_key_size = first_key_ ? first_key_->size() : 0; int32_t last_key_size = last_key_ ? last_key_->size() : 0; - int32_t total_size = Size(); + int32_t total_size = MemorySize(); MemorySliceOutput output(total_size, pool); int8_t null_key_flags = 0; diff --git a/src/paimon/common/global_index/btree/btree_index_meta.h b/src/paimon/common/global_index/sorted_index_file_meta.h similarity index 82% rename from src/paimon/common/global_index/btree/btree_index_meta.h rename to src/paimon/common/global_index/sorted_index_file_meta.h index eb8158f12..88a7cdb7f 100644 --- a/src/paimon/common/global_index/btree/btree_index_meta.h +++ b/src/paimon/common/global_index/sorted_index_file_meta.h @@ -24,21 +24,22 @@ #include "paimon/common/memory/memory_slice_input.h" #include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" #include "paimon/result.h" namespace paimon { -/// Index metadata for each BTree index file. +/// Manifest-level min/max metadata for one global index file. /// /// Empty serialized keys are valid, so null boundary keys are encoded separately with flags. -class BTreeIndexMeta { +class SortedIndexFileMeta { public: - static Result> Deserialize(const std::shared_ptr& meta, - paimon::MemoryPool* pool); + static Result> Deserialize( + const std::shared_ptr& meta, paimon::MemoryPool* pool); std::shared_ptr Serialize(paimon::MemoryPool* pool) const; public: - BTreeIndexMeta(const std::shared_ptr& first_key, const std::shared_ptr& last_key, - bool has_nulls) + SortedIndexFileMeta(const std::shared_ptr& first_key, + const std::shared_ptr& last_key, bool has_nulls) : first_key_(first_key), last_key_(last_key), has_nulls_(has_nulls) {} const std::shared_ptr& FirstKey() const { @@ -58,7 +59,7 @@ class BTreeIndexMeta { } private: - int32_t Size() const { + int32_t MemorySize() const { // 11 bytes => key lengths (8) + has_nulls (1) + format version (1) + null flags (1). return (first_key_ ? first_key_->size() : 0) + (last_key_ ? last_key_->size() : 0) + 11; } diff --git a/src/paimon/common/global_index/btree/btree_index_meta_test.cpp b/src/paimon/common/global_index/sorted_index_file_meta_test.cpp similarity index 67% rename from src/paimon/common/global_index/btree/btree_index_meta_test.cpp rename to src/paimon/common/global_index/sorted_index_file_meta_test.cpp index b0d78fa3b..6c62fe5ba 100644 --- a/src/paimon/common/global_index/btree/btree_index_meta_test.cpp +++ b/src/paimon/common/global_index/sorted_index_file_meta_test.cpp @@ -17,7 +17,7 @@ * under the License. */ -#include "paimon/common/global_index/btree/btree_index_meta.h" +#include "paimon/common/global_index/sorted_index_file_meta.h" #include #include @@ -29,7 +29,7 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { -class BTreeIndexMetaTest : public ::testing::Test { +class SortedIndexFileMetaTest : public ::testing::Test { protected: void SetUp() override { pool_ = GetDefaultPool(); @@ -78,10 +78,10 @@ class BTreeIndexMetaTest : public ::testing::Test { std::shared_ptr pool_; }; -TEST_F(BTreeIndexMetaTest, SerializeDeserializeNormalKeys) { +TEST_F(SortedIndexFileMetaTest, SerializeDeserializeNormalKeys) { auto first_key = std::make_shared("first_key_data", pool_.get()); auto last_key = std::make_shared("last_key_data", pool_.get()); - auto meta = std::make_shared(first_key, last_key, true); + auto meta = std::make_shared(first_key, last_key, true); // Serialize auto serialized = meta->Serialize(pool_.get()); @@ -89,8 +89,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeNormalKeys) { ASSERT_GT(serialized->size(), 0u); // Deserialize - ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, - BTreeIndexMeta::Deserialize(serialized, pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + SortedIndexFileMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); // Verify first_key @@ -108,16 +108,16 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeNormalKeys) { ASSERT_TRUE(deserialized->HasNulls()); } -TEST_F(BTreeIndexMetaTest, SerializeDeserializeEmptyFirstKey) { +TEST_F(SortedIndexFileMetaTest, SerializeDeserializeEmptyFirstKey) { auto empty_key = std::make_shared(0, pool_.get()); auto last_key = std::make_shared("last_key_data", pool_.get()); - auto meta = std::make_shared(empty_key, last_key, false); + auto meta = std::make_shared(empty_key, last_key, false); auto serialized = meta->Serialize(pool_.get()); ASSERT_EQ(serialized->size(), 11 + last_key->size()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, - BTreeIndexMeta::Deserialize(serialized, pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + SortedIndexFileMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized->FirstKey()); ASSERT_EQ(deserialized->FirstKey()->size(), 0); ASSERT_TRUE(deserialized->LastKey()); @@ -127,15 +127,15 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeEmptyFirstKey) { ASSERT_FALSE(deserialized->OnlyNulls()); } -TEST_F(BTreeIndexMetaTest, SerializeDeserializeEmptyFirstAndLastKeysWithNulls) { +TEST_F(SortedIndexFileMetaTest, SerializeDeserializeEmptyFirstAndLastKeysWithNulls) { auto empty_key = std::make_shared(0, pool_.get()); - auto meta = std::make_shared(empty_key, empty_key, true); + auto meta = std::make_shared(empty_key, empty_key, true); auto serialized = meta->Serialize(pool_.get()); ASSERT_EQ(serialized->size(), 11); - ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, - BTreeIndexMeta::Deserialize(serialized, pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + SortedIndexFileMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized->FirstKey()); ASSERT_EQ(deserialized->FirstKey()->size(), 0); ASSERT_TRUE(deserialized->LastKey()); @@ -144,14 +144,14 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeEmptyFirstAndLastKeysWithNulls) { ASSERT_FALSE(deserialized->OnlyNulls()); } -TEST_F(BTreeIndexMetaTest, SerializeDeserializeOnlyNulls) { - auto meta = std::make_shared(nullptr, nullptr, true); +TEST_F(SortedIndexFileMetaTest, SerializeDeserializeOnlyNulls) { + auto meta = std::make_shared(nullptr, nullptr, true); auto serialized = meta->Serialize(pool_.get()); ASSERT_EQ(serialized->size(), 11); - ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, - BTreeIndexMeta::Deserialize(serialized, pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + SortedIndexFileMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); ASSERT_FALSE(deserialized->FirstKey()); ASSERT_FALSE(deserialized->LastKey()); @@ -159,25 +159,25 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeOnlyNulls) { ASSERT_TRUE(deserialized->OnlyNulls()); } -TEST_F(BTreeIndexMetaTest, DeserializeLegacyOnlyNulls) { +TEST_F(SortedIndexFileMetaTest, DeserializeLegacyOnlyNulls) { auto empty_key = std::make_shared(0, pool_.get()); auto serialized = LegacyMetaBytes(empty_key, empty_key, true); - ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, - BTreeIndexMeta::Deserialize(serialized, pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + SortedIndexFileMeta::Deserialize(serialized, pool_.get())); ASSERT_FALSE(deserialized->FirstKey()); ASSERT_FALSE(deserialized->LastKey()); ASSERT_TRUE(deserialized->HasNulls()); ASSERT_TRUE(deserialized->OnlyNulls()); } -TEST_F(BTreeIndexMetaTest, DeserializeLegacyEmptyFirstKey) { +TEST_F(SortedIndexFileMetaTest, DeserializeLegacyEmptyFirstKey) { auto empty_key = std::make_shared(0, pool_.get()); auto last_key = std::make_shared("last_key_data", pool_.get()); auto serialized = LegacyMetaBytes(empty_key, last_key, false); - ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, - BTreeIndexMeta::Deserialize(serialized, pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + SortedIndexFileMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized->FirstKey()); ASSERT_EQ(deserialized->FirstKey()->size(), 0); ASSERT_TRUE(deserialized->LastKey()); @@ -187,12 +187,12 @@ TEST_F(BTreeIndexMetaTest, DeserializeLegacyEmptyFirstKey) { ASSERT_FALSE(deserialized->OnlyNulls()); } -TEST_F(BTreeIndexMetaTest, DeserializeLegacyEmptyFirstAndLastKeysWithoutNulls) { +TEST_F(SortedIndexFileMetaTest, DeserializeLegacyEmptyFirstAndLastKeysWithoutNulls) { auto empty_key = std::make_shared(0, pool_.get()); auto serialized = LegacyMetaBytes(empty_key, empty_key, false); - ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, - BTreeIndexMeta::Deserialize(serialized, pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + SortedIndexFileMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized->FirstKey()); ASSERT_EQ(deserialized->FirstKey()->size(), 0); ASSERT_TRUE(deserialized->LastKey()); @@ -201,109 +201,109 @@ TEST_F(BTreeIndexMetaTest, DeserializeLegacyEmptyFirstAndLastKeysWithoutNulls) { ASSERT_FALSE(deserialized->OnlyNulls()); } -TEST_F(BTreeIndexMetaTest, DeserializeRejectsNullAndTruncatedMetadata) { - ASSERT_NOK(BTreeIndexMeta::Deserialize(nullptr, pool_.get())); +TEST_F(SortedIndexFileMetaTest, DeserializeRejectsNullAndTruncatedMetadata) { + ASSERT_NOK(SortedIndexFileMeta::Deserialize(nullptr, pool_.get())); for (size_t size = 0; size < 9; size++) { auto truncated = std::make_shared(std::string(size, '\0'), pool_.get()); - ASSERT_NOK(BTreeIndexMeta::Deserialize(truncated, pool_.get())); + ASSERT_NOK(SortedIndexFileMeta::Deserialize(truncated, pool_.get())); } } -TEST_F(BTreeIndexMetaTest, DeserializeRejectsInvalidKeyLengths) { - ASSERT_NOK(BTreeIndexMeta::Deserialize( +TEST_F(SortedIndexFileMetaTest, DeserializeRejectsInvalidKeyLengths) { + ASSERT_NOK(SortedIndexFileMeta::Deserialize( MetadataBytes(/*first_key_length=*/-1, "", /*last_key_length=*/0, "", /*has_nulls=*/0), pool_.get())); - ASSERT_NOK(BTreeIndexMeta::Deserialize( + ASSERT_NOK(SortedIndexFileMeta::Deserialize( MetadataBytes(/*first_key_length=*/100, "", /*last_key_length=*/0, "", /*has_nulls=*/0), pool_.get())); - ASSERT_NOK(BTreeIndexMeta::Deserialize( + ASSERT_NOK(SortedIndexFileMeta::Deserialize( MetadataBytes(/*first_key_length=*/0, "", /*last_key_length=*/-1, "", /*has_nulls=*/0), pool_.get())); - ASSERT_NOK(BTreeIndexMeta::Deserialize( + ASSERT_NOK(SortedIndexFileMeta::Deserialize( MetadataBytes(/*first_key_length=*/0, "", /*last_key_length=*/100, "", /*has_nulls=*/0), pool_.get())); } -TEST_F(BTreeIndexMetaTest, DeserializeRejectsMalformedFlagsAndTrailingBytes) { - ASSERT_NOK( - BTreeIndexMeta::Deserialize(MetadataBytes(0, "", 0, "", /*has_nulls=*/2), pool_.get())); - ASSERT_NOK(BTreeIndexMeta::Deserialize( +TEST_F(SortedIndexFileMetaTest, DeserializeRejectsMalformedFlagsAndTrailingBytes) { + ASSERT_NOK(SortedIndexFileMeta::Deserialize(MetadataBytes(0, "", 0, "", /*has_nulls=*/2), + pool_.get())); + ASSERT_NOK(SortedIndexFileMeta::Deserialize( MetadataBytes(0, "", 0, "", /*has_nulls=*/0, {/*truncated_version=*/1}), pool_.get())); - ASSERT_NOK(BTreeIndexMeta::Deserialize( + ASSERT_NOK(SortedIndexFileMeta::Deserialize( MetadataBytes(0, "", 0, "", /*has_nulls=*/0, {/*unsupported_version=*/2, /*flags=*/0}), pool_.get())); - ASSERT_NOK(BTreeIndexMeta::Deserialize( + ASSERT_NOK(SortedIndexFileMeta::Deserialize( MetadataBytes(0, "", 0, "", /*has_nulls=*/0, {/*version=*/1, /*unknown_flags=*/4}), pool_.get())); - ASSERT_NOK(BTreeIndexMeta::Deserialize( + ASSERT_NOK(SortedIndexFileMeta::Deserialize( MetadataBytes(0, "", 0, "", /*has_nulls=*/0, {/*version=*/1, /*flags=*/0, /*trailing=*/0}), pool_.get())); - ASSERT_NOK(BTreeIndexMeta::Deserialize( + ASSERT_NOK(SortedIndexFileMeta::Deserialize( MetadataBytes(1, "x", 0, "", /*has_nulls=*/0, {/*version=*/1, /*first_key_is_null=*/1}), pool_.get())); } -TEST_F(BTreeIndexMetaTest, HasNullsAndOnlyNulls) { +TEST_F(SortedIndexFileMetaTest, HasNullsAndOnlyNulls) { // Case 1: Has nulls with keys auto meta1 = - std::make_shared(std::make_shared("key", pool_.get()), - std::make_shared("key", pool_.get()), true); + std::make_shared(std::make_shared("key", pool_.get()), + std::make_shared("key", pool_.get()), true); ASSERT_TRUE(meta1->HasNulls()); ASSERT_FALSE(meta1->OnlyNulls()); // Case 2: No nulls with keys auto meta2 = - std::make_shared(std::make_shared("key", pool_.get()), - std::make_shared("key", pool_.get()), false); + std::make_shared(std::make_shared("key", pool_.get()), + std::make_shared("key", pool_.get()), false); ASSERT_FALSE(meta2->HasNulls()); ASSERT_FALSE(meta2->OnlyNulls()); // Case 3: Only nulls (no keys) - auto meta3 = std::make_shared(nullptr, nullptr, true); + auto meta3 = std::make_shared(nullptr, nullptr, true); ASSERT_TRUE(meta3->HasNulls()); ASSERT_TRUE(meta3->OnlyNulls()); // Case 4: No nulls and no keys (edge case) - auto meta4 = std::make_shared(nullptr, nullptr, false); + auto meta4 = std::make_shared(nullptr, nullptr, false); ASSERT_FALSE(meta4->HasNulls()); ASSERT_TRUE(meta4->OnlyNulls()); } -TEST_F(BTreeIndexMetaTest, SerializeDeserializeNoNulls) { - // Create a BTreeIndexMeta without nulls +TEST_F(SortedIndexFileMetaTest, SerializeDeserializeNoNulls) { + // Create a SortedIndexFileMeta without nulls auto first_key = std::make_shared("abc", pool_.get()); auto last_key = std::make_shared("xyz", pool_.get()); - auto meta = std::make_shared(first_key, last_key, false); + auto meta = std::make_shared(first_key, last_key, false); // Serialize auto serialized = meta->Serialize(pool_.get()); ASSERT_TRUE(serialized); // Deserialize - ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, - BTreeIndexMeta::Deserialize(serialized, pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + SortedIndexFileMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); // Verify has_nulls is false ASSERT_FALSE(deserialized->HasNulls()); } -TEST_F(BTreeIndexMetaTest, SerializeDeserializeWithOnlyFirstKey) { - // Create a BTreeIndexMeta with only first_key (edge case) +TEST_F(SortedIndexFileMetaTest, SerializeDeserializeWithOnlyFirstKey) { + // Create a SortedIndexFileMeta with only first_key (edge case) auto first_key = std::make_shared("first", pool_.get()); - auto meta = std::make_shared(first_key, nullptr, false); + auto meta = std::make_shared(first_key, nullptr, false); // Serialize auto serialized = meta->Serialize(pool_.get()); ASSERT_TRUE(serialized); // Deserialize - ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, - BTreeIndexMeta::Deserialize(serialized, pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + SortedIndexFileMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); // Verify first_key @@ -315,18 +315,18 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeWithOnlyFirstKey) { ASSERT_FALSE(deserialized->LastKey()); } -TEST_F(BTreeIndexMetaTest, SerializeDeserializeWithOnlyLastKey) { - // Create a BTreeIndexMeta with only last_key (edge case) +TEST_F(SortedIndexFileMetaTest, SerializeDeserializeWithOnlyLastKey) { + // Create a SortedIndexFileMeta with only last_key (edge case) auto last_key = std::make_shared("last", pool_.get()); - auto meta = std::make_shared(nullptr, last_key, false); + auto meta = std::make_shared(nullptr, last_key, false); // Serialize auto serialized = meta->Serialize(pool_.get()); ASSERT_TRUE(serialized); // Deserialize - ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, - BTreeIndexMeta::Deserialize(serialized, pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + SortedIndexFileMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); // Verify first_key is null @@ -338,21 +338,21 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeWithOnlyLastKey) { ASSERT_EQ(std::string(deserialized_last->data(), deserialized_last->size()), "last"); } -TEST_F(BTreeIndexMetaTest, SerializeDeserializeBinaryKeys) { - // Create a BTreeIndexMeta with binary keys containing null bytes +TEST_F(SortedIndexFileMetaTest, SerializeDeserializeBinaryKeys) { + // Create a SortedIndexFileMeta with binary keys containing null bytes std::string binary_first = std::string("key\0with\0nulls", 14); std::string binary_last = std::string("last\0key", 8); auto first_key = std::make_shared(binary_first, pool_.get()); auto last_key = std::make_shared(binary_last, pool_.get()); - auto meta = std::make_shared(first_key, last_key, true); + auto meta = std::make_shared(first_key, last_key, true); // Serialize auto serialized = meta->Serialize(pool_.get()); ASSERT_TRUE(serialized); // Deserialize - ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, - BTreeIndexMeta::Deserialize(serialized, pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + SortedIndexFileMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); // Verify first_key diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp index 7b14a5c4f..e4231cf5e 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp @@ -29,8 +29,8 @@ #include "arrow/api.h" #include "fmt/format.h" #include "gtest/gtest.h" -#include "paimon/common/global_index/btree/btree_index_meta.h" -#include "paimon/common/global_index/btree/key_serializer.h" +#include "paimon/common/global_index/key_serializer.h" +#include "paimon/common/global_index/sorted_index_file_meta.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/index/pk/primary_key_index_definitions.h" @@ -735,19 +735,21 @@ TEST_F(PrimaryKeySortedIndexScanTest, InvalidRowRangePayloadFallsBack) { TEST_F(PrimaryKeySortedIndexScanTest, MalformedBTreeMetadataFallsBack) { ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); const GlobalIndexMeta& meta = payload->GetGlobalIndexMeta().value(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr key_serializer, + KeySerializer::Create(arrow::int64(), pool_)); auto short_key = std::make_shared(std::string(1, '\0'), pool_.get()); - auto invalid_key_meta = std::make_shared(short_key, short_key, false); + auto invalid_key_meta = std::make_shared(short_key, short_key, false); ASSERT_OK_AND_ASSIGN(std::shared_ptr first_key, - KeySerializer::SerializeKey(Literal(static_cast(10)), - arrow::int64(), pool_.get())); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr last_key, - KeySerializer::SerializeKey(Literal(static_cast(1)), arrow::int64(), pool_.get())); - auto reversed_meta = std::make_shared(first_key, last_key, false); - auto only_first_meta = std::make_shared(first_key, /*last_key=*/nullptr, false); - auto only_last_meta = std::make_shared(/*first_key=*/nullptr, last_key, false); + key_serializer->Serialize(Literal(static_cast(10)))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr last_key, + key_serializer->Serialize(Literal(static_cast(1)))); + auto reversed_meta = std::make_shared(first_key, last_key, false); + auto only_first_meta = + std::make_shared(first_key, /*last_key=*/nullptr, false); + auto only_last_meta = + std::make_shared(/*first_key=*/nullptr, last_key, false); auto empty_nonnull_meta = - std::make_shared(/*first_key=*/nullptr, /*last_key=*/nullptr, false); + std::make_shared(/*first_key=*/nullptr, /*last_key=*/nullptr, false); std::vector> malformed_metadata = { nullptr, std::make_shared(std::string(4, '\0'), pool_.get()),