From 56c2d553e6864c95c4f3a12180ad5ef5990a9e28 Mon Sep 17 00:00:00 2001 From: csun5285 Date: Fri, 18 Sep 2026 17:44:19 +0800 Subject: [PATCH] [refactor](storage) Remove the tablet schema from the query read path `StorageReadOptions::tablet_schema` carried a whole `TabletSchema` down to every segment iterator, but it served two unrelated roles: - tablet-level facts the segment already knows (keys type, index storage format, whether a column is a key), and - the compaction *output* schema, which genuinely differs from the segment's own schema and is what variant compaction needs to lay out its subcolumns. Conflating them made the schema look like a general read input, so callers that only ever read on the query path had to produce one, and code that needed the compaction output schema looked like it worked on any read. This splits the two and deletes the field: 1. `StorageReadOptions::tablet_schema` is gone. Tablet-level facts are read from `Segment::_tablet_schema`, and the two facts a read cannot recover from the segment -- whether the tablet defines a sequence mapping, and whether it materializes variant subcolumns as extracted columns -- move onto `ReadSchema` as `tablet_has_sequence_map()` and `tablet_has_extracted_variant_columns()`. 2. `TabletSchema::_path_set_info_map` moves out into a new `VariantCompactionPaths` (`be/src/storage/variant_compaction_paths.h`). It was only ever built by compaction and only ever read by a compaction reader and writer, yet it lived on the schema, where it was never persisted and never survived a `RowsetMeta::set_tablet_schema` round trip. Living on the schema made `TabletSchema::inverted_indexs()` look like it needed it everywhere; it did not. The layout now travels explicitly through `RowsetWriterContext` / `TabletReaderParams` / `StorageReadOptions` as a `shared_ptr`, shared read-only by the one compaction that built it. 3. `ReadSchema`'s three public init methods collapse into a single `init_from_tablet_schema(schema, merge_by_sequence_mapping, map_row_binlog_columns)`, and `RowsetReaderContext::tablet_schema` is deleted. The two flags are decided by `BlockReader` / `VerticalBlockReader`, which is where the reader type, direct mode and MoW-ness are known. 4. `Segment::seek_and_read_by_rowid` takes the `TabletColumn` it reads instead of a whole `TabletSchema`, which is all it used. Its two callers (`PointQueryExecutor`, `RowIdFetcher`) resolve the column once and pass it in. 5. `OrderedDataCompactionTest` and `SegmentsKeyBoundsTruncationTest` now restore `ordered_data_compaction_min_segment_size`. Both leaked it, which sent every later suite's compaction down the ordered link-file path instead of a real merge -- this is what made `IndexStorageVariant*` fail depending on suite order. No behavior change intended on the query path. Tests: `be/test/storage/variant_compaction_paths_test.cpp` is new and ports the coverage that was deleted with `_path_set_info_map`; `read_schema_test.cpp` gains the sequence-map and extracted-variant cases; `segcompaction_test.cpp` gains `VariantRowsetIsNeverSegmentCompacted`, pinning the existing rule that a rowset with a variant column is never segment-compacted. --- be/src/cloud/cloud_schema_change_job.cpp | 4 +- be/src/exec/common/variant_util.cpp | 19 +- be/src/exec/common/variant_util.h | 24 ++- be/src/exec/rowid_fetcher.cpp | 29 +-- be/src/service/point_query_executor.cpp | 13 +- be/src/storage/compaction/compaction.cpp | 10 +- be/src/storage/compaction/compaction.h | 2 + be/src/storage/iterator/block_reader.cpp | 32 ++- .../iterator/vertical_block_reader.cpp | 4 + be/src/storage/iterators.h | 7 +- be/src/storage/merger.cpp | 2 + be/src/storage/rowset/beta_rowset_reader.cpp | 6 +- be/src/storage/rowset/rowset_reader_context.h | 4 +- be/src/storage/rowset/rowset_writer_context.h | 3 + be/src/storage/rowset/segcompaction.cpp | 1 - be/src/storage/schema.cpp | 22 +- be/src/storage/schema.h | 25 ++- .../storage/schema_change/schema_change.cpp | 4 +- be/src/storage/segment/segment.cpp | 30 +-- be/src/storage/segment/segment.h | 3 +- be/src/storage/segment/segment_iterator.cpp | 28 ++- be/src/storage/segment/segment_iterator.h | 3 + .../variant/sparse_column_merge_iterator.h | 3 +- .../segment/variant/variant_column_reader.cpp | 15 +- .../variant/variant_compaction_paths.cpp | 62 ++++++ .../variant/variant_compaction_paths.h | 54 +++++ .../segment/vertical_segment_writer.cpp | 8 + be/src/storage/tablet/tablet_reader.cpp | 2 +- be/src/storage/tablet/tablet_reader.h | 3 + be/src/storage/tablet/tablet_schema.cpp | 39 +--- be/src/storage/tablet/tablet_schema.h | 30 --- be/src/storage/task/index_builder.cpp | 2 - .../exec/common/schema_util_rowset_test.cpp | 7 +- be/test/exec/common/schema_util_test.cpp | 40 ++-- be/test/exec/scan/vgeneric_iterators_test.cpp | 2 - .../delta_writer_cluster_key_test.cpp | 1 - .../load/delta_writer/delta_writer_test.cpp | 3 - .../olap/rowset/group_rowset_writer_test.cpp | 12 +- .../ordered_data_compaction_test.cpp | 29 ++- .../compaction/segcompaction_mow_test.cpp | 6 +- .../storage/compaction/segcompaction_test.cpp | 201 +++++++++++++++++- .../compaction/vertical_compaction_test.cpp | 54 ++++- .../similarity/collection_statistics_test.cpp | 28 +-- .../block_reader_change_next_block_test.cpp | 6 +- be/test/storage/mow/mow_transform_test_base.h | 4 +- be/test/storage/read_schema_test.cpp | 159 +++++++++++++- .../row_binlog_vmerge_compaction_test.cpp | 6 +- be/test/storage/rowid_conversion_test.cpp | 18 +- .../rowset/segment_flusher_format_test.cpp | 1 - be/test/storage/segment/mock/mock_segment.h | 2 + .../storage/segment/segment_cache_test.cpp | 1 - ...ment_iterator_count_emit_shortcut_test.cpp | 9 +- .../segment_iterator_expr_zonemap_test.cpp | 5 - .../segment_iterator_lazy_pruned_test.cpp | 1 - .../segment_iterator_limit_opt_test.cpp | 1 - ...egment_iterator_no_need_read_data_test.cpp | 7 +- .../segment_writer_write_paths_test.cpp | 1 - .../segments_key_bounds_truncation_test.cpp | 8 + be/test/storage/tablet/tablet_schema_test.cpp | 142 ------------- ...storage_variant_compaction_schema_test.cpp | 28 +-- .../index_storage_variant_io_context_test.cpp | 1 - ...ndex_storage_variant_sparse_stats_test.cpp | 5 +- .../variant_column_writer_reader_test.cpp | 101 +++++---- .../variant/variant_compaction_paths_test.cpp | 152 +++++++++++++ .../variant_doc_mode_compaction_test.cpp | 12 +- be/test/testutil/index_storage_test_util.cpp | 6 +- 66 files changed, 1075 insertions(+), 477 deletions(-) create mode 100644 be/src/storage/segment/variant/variant_compaction_paths.cpp create mode 100644 be/src/storage/segment/variant/variant_compaction_paths.h create mode 100644 be/test/storage/variant/variant_compaction_paths_test.cpp diff --git a/be/src/cloud/cloud_schema_change_job.cpp b/be/src/cloud/cloud_schema_change_job.cpp index c269f7dcb1bca2..ae39f8856f07e6 100644 --- a/be/src/cloud/cloud_schema_change_job.cpp +++ b/be/src/cloud/cloud_schema_change_job.cpp @@ -244,6 +244,9 @@ Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& reque _new_tablet_schema = _new_tablet->tablet_schema(); ReadSchemaSPtr read_schema = std::make_shared(_base_tablet_schema->columns()); + RETURN_IF_ERROR(read_schema->init_from_tablet_schema(*_base_tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false)); // delete handlers to filter out deleted rows DeleteHandler delete_handler; @@ -262,7 +265,6 @@ Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& reque // reader_context is stack variables, it's lifetime MUST keep the same with rs_readers RowsetReaderContext reader_context; reader_context.reader_type = ReaderType::READER_ALTER_TABLE; - reader_context.tablet_schema = _base_tablet_schema; reader_context.need_ordered_result = true; reader_context.delete_handler = &delete_handler; reader_context.read_schema = read_schema; diff --git a/be/src/exec/common/variant_util.cpp b/be/src/exec/common/variant_util.cpp index e905602b695cab..e2b46cbe47d1a4 100644 --- a/be/src/exec/common/variant_util.cpp +++ b/be/src/exec/common/variant_util.cpp @@ -117,7 +117,7 @@ PathInData make_full_subcolumn_path(const TabletColumnPtr& parent_column, std::s return builder.append(parent_column->name_lower_case(), false).append("", false).build(); } -void append_empty_key_subcolumn_from_stats(TabletSchema::PathsSetInfo& paths_set_info, +void append_empty_key_subcolumn_from_stats(VariantCompactionPaths& paths_set_info, const TabletColumnPtr& parent_column, TabletSchemaSPtr& output_schema) { if (!paths_set_info.sub_path_set.contains("") || paths_set_info.sparse_path_set.contains("") || @@ -1002,7 +1002,7 @@ Status VariantCompactionUtil::aggregate_variant_extended_info( // get the subpaths and sparse paths for the variant column void VariantCompactionUtil::get_subpaths(int32_t max_subcolumns_count, const PathToNoneNullValues& stats, - TabletSchema::PathsSetInfo& paths_set_info) { + VariantCompactionPaths& paths_set_info) { // max_subcolumns_count is 0 means no limit if (max_subcolumns_count > 0 && stats.size() > max_subcolumns_count) { std::vector> paths_with_sizes; @@ -1148,7 +1148,7 @@ Status VariantCompactionUtil::check_path_stats(const std::vector& typed_paths, const TabletColumnPtr parent_column, TabletSchemaSPtr& output_schema, - TabletSchema::PathsSetInfo& paths_set_info) { + VariantCompactionPaths& paths_set_info) { if (parent_column->variant_enable_typed_paths_to_sparse()) { return Status::OK(); } @@ -1169,7 +1169,7 @@ Status VariantCompactionUtil::get_compaction_typed_columns( Status VariantCompactionUtil::get_compaction_nested_columns( const std::unordered_set& nested_paths, const PathToDataTypes& path_to_data_types, const TabletColumnPtr parent_column, - TabletSchemaSPtr& output_schema, TabletSchema::PathsSetInfo& paths_set_info) { + TabletSchemaSPtr& output_schema, VariantCompactionPaths& paths_set_info) { const auto& parent_indexes = output_schema->inverted_indexs(parent_column->unique_id()); for (const auto& path : nested_paths) { const auto& find_data_types = path_to_data_types.find(path); @@ -1200,7 +1200,7 @@ Status VariantCompactionUtil::get_compaction_nested_columns( } void VariantCompactionUtil::get_compaction_subcolumns_from_subpaths( - TabletSchema::PathsSetInfo& paths_set_info, const TabletColumnPtr parent_column, + VariantCompactionPaths& paths_set_info, const TabletColumnPtr parent_column, const TabletSchemaSPtr& target, const PathToDataTypes& path_to_data_types, const std::unordered_set& sparse_paths, TabletSchemaSPtr& output_schema) { auto& path_set = paths_set_info.sub_path_set; @@ -1265,7 +1265,7 @@ void VariantCompactionUtil::get_compaction_subcolumns_from_subpaths( } void VariantCompactionUtil::get_compaction_subcolumns_from_data_types( - TabletSchema::PathsSetInfo& paths_set_info, const TabletColumnPtr parent_column, + VariantCompactionPaths& paths_set_info, const TabletColumnPtr parent_column, const TabletSchemaSPtr& target, const PathToDataTypes& path_to_data_types, TabletSchemaSPtr& output_schema) { const auto& parent_indexes = target->inverted_indexs(parent_column->unique_id()); @@ -1305,7 +1305,8 @@ void VariantCompactionUtil::get_compaction_subcolumns_from_data_types( // ordinary extracted subcolumns. NG typed paths still use get_compaction_typed_columns(), keeping // typed-column rules out of the NG-specific regular-path filtering. Status VariantCompactionUtil::get_extended_compaction_schema( - const std::vector& rowsets, TabletSchemaSPtr& target) { + const std::vector& rowsets, TabletSchemaSPtr& target, + VariantCompactionPathsMap& paths) { std::unordered_map uid_to_variant_extended_info; const bool needs_variant_extended_info = std::ranges::any_of(target->columns(), [](const TabletColumnPtr& column) { @@ -1322,7 +1323,7 @@ Status VariantCompactionUtil::get_extended_compaction_schema( // build the output schema TabletSchemaSPtr output_schema = std::make_shared(); output_schema->shawdow_copy_without_columns(*target); - std::unordered_map uid_to_paths_set_info; + VariantCompactionPathsMap uid_to_paths_set_info; const auto ng_root_uids = collect_nested_group_compaction_root_uids(target, uid_to_variant_extended_info); for (const TabletColumnPtr& column : target->columns()) { @@ -1417,7 +1418,7 @@ Status VariantCompactionUtil::get_extended_compaction_schema( target = output_schema; // used to merge & filter path to sparse column during reading in compaction - target->set_path_set_info(std::move(uid_to_paths_set_info)); + paths = std::move(uid_to_paths_set_info); VLOG_DEBUG << "dump schema " << target->dump_full_schema(); return Status::OK(); } diff --git a/be/src/exec/common/variant_util.h b/be/src/exec/common/variant_util.h index 02a3592004e153..8cf0272de4595e 100644 --- a/be/src/exec/common/variant_util.h +++ b/be/src/exec/common/variant_util.h @@ -35,6 +35,7 @@ #include "core/string_ref.h" #include "core/types.h" #include "exprs/aggregate/aggregate_function.h" +#include "storage/segment/variant/variant_compaction_paths.h" #include "storage/tablet/tablet_fwd.h" #include "storage/tablet/tablet_schema.h" @@ -186,7 +187,7 @@ class VariantCompactionUtil { public: // get the subpaths and sparse paths for the variant column static void get_subpaths(int32_t max_subcolumns_count, const PathToNoneNullValues& path_stats, - TabletSchema::PathsSetInfo& paths_set_info); + VariantCompactionPaths& paths_set_info); // collect extended info from the variant column static Status aggregate_variant_extended_info( @@ -198,9 +199,11 @@ class VariantCompactionUtil { const RowsetSharedPtr& rs, std::unordered_map* uid_to_path_stats); - // Build the temporary schema for compaction, this will reduce the memory usage of compacting variant columns + // Build the temporary schema for compaction, this will reduce the memory usage of compacting + // variant columns. `paths` receives that schema's variant path layout. static Status get_extended_compaction_schema(const std::vector& rowsets, - TabletSchemaSPtr& target); + TabletSchemaSPtr& target, + VariantCompactionPathsMap& paths); // Used to collect all the subcolumns types of variant column from rowsets static TabletSchemaSPtr calculate_variant_extended_schema( @@ -218,25 +221,26 @@ class VariantCompactionUtil { size_t num_rows); static void get_compaction_subcolumns_from_subpaths( - TabletSchema::PathsSetInfo& paths_set_info, const TabletColumnPtr parent_column, + VariantCompactionPaths& paths_set_info, const TabletColumnPtr parent_column, const TabletSchemaSPtr& target, const PathToDataTypes& path_to_data_types, const std::unordered_set& sparse_paths, TabletSchemaSPtr& output_schema); - static void get_compaction_subcolumns_from_data_types( - TabletSchema::PathsSetInfo& paths_set_info, const TabletColumnPtr parent_column, - const TabletSchemaSPtr& target, const PathToDataTypes& path_to_data_types, - TabletSchemaSPtr& output_schema); + static void get_compaction_subcolumns_from_data_types(VariantCompactionPaths& paths_set_info, + const TabletColumnPtr parent_column, + const TabletSchemaSPtr& target, + const PathToDataTypes& path_to_data_types, + TabletSchemaSPtr& output_schema); static Status get_compaction_typed_columns(const TabletSchemaSPtr& target, const std::unordered_set& typed_paths, const TabletColumnPtr parent_column, TabletSchemaSPtr& output_schema, - TabletSchema::PathsSetInfo& paths_set_info); + VariantCompactionPaths& paths_set_info); static Status get_compaction_nested_columns( const std::unordered_set& nested_paths, const PathToDataTypes& path_to_data_types, const TabletColumnPtr parent_column, - TabletSchemaSPtr& output_schema, TabletSchema::PathsSetInfo& paths_set_info); + TabletSchemaSPtr& output_schema, VariantCompactionPaths& paths_set_info); }; } // namespace doris::variant_util diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp index fca26050618af2..4ac3eb249fbada 100644 --- a/be/src/exec/rowid_fetcher.cpp +++ b/be/src/exec/rowid_fetcher.cpp @@ -138,15 +138,11 @@ struct IteratorItem { StorageReadOptions storage_read_options; }; -static void set_slot_access_paths(const SlotDescriptor& slot, const TabletSchema& schema, +// read_column is what slot resolves to, or its variant parent column for a subpath slot. +static void set_slot_access_paths(const SlotDescriptor& slot, const TabletColumn& read_column, StorageReadOptions& storage_read_options) { - int32_t unique_id = slot.col_unique_id(); - const int field_index = - unique_id >= 0 ? schema.field_index(unique_id) : schema.field_index(slot.col_name()); - if (field_index >= 0) { - const auto& column = schema.column(field_index); - unique_id = column.unique_id() >= 0 ? column.unique_id() : column.parent_unique_id(); - } + const int32_t unique_id = + read_column.unique_id() >= 0 ? read_column.unique_id() : read_column.parent_unique_id(); if (unique_id < 0) { return; } @@ -1043,10 +1039,19 @@ Status RowIdStorageReader::read_doris_format_row( iterator_item.storage_read_options.io_ctx.file_cache_miss_policy = file_cache_miss_policy; } - set_slot_access_paths(slots[x], full_read_schema, iterator_item.storage_read_options); - RETURN_IF_ERROR(segment->seek_and_read_by_rowid( - full_read_schema, &slots[x], row_ids, column, - iterator_item.storage_read_options, iterator_item.iterator)); + int32_t index = slots[x].col_unique_id() >= 0 + ? full_read_schema.field_index(slots[x].col_unique_id()) + : full_read_schema.field_index(slots[x].col_name()); + if (index < 0) { + return Status::InternalError( + "field name is invalid. field={}, field_name_to_index={}", + slots[x].col_name(), full_read_schema.get_all_field_names()); + } + const auto& read_column = full_read_schema.column(index); + set_slot_access_paths(slots[x], read_column, iterator_item.storage_read_options); + RETURN_IF_ERROR(segment->seek_and_read_by_rowid(read_column, &slots[x], row_ids, column, + iterator_item.storage_read_options, + iterator_item.iterator)); } } return Status::OK(); diff --git a/be/src/service/point_query_executor.cpp b/be/src/service/point_query_executor.cpp index cee41c611e0ef5..7ba77f96ba2496 100644 --- a/be/src/service/point_query_executor.cpp +++ b/be/src/service/point_query_executor.cpp @@ -595,6 +595,7 @@ Status PointQueryExecutor::_lookup_row_data() { return seg->id() == row_loc.segment_id; }); const auto& segment = *it; + const auto tablet_schema = _tablet->tablet_schema(); for (int cid : _reusable->missing_col_uids()) { int pos = _reusable->get_col_uid_to_idx().at(cid); std::vector row_ids { @@ -602,11 +603,19 @@ Status PointQueryExecutor::_lookup_row_data() { auto& column = result_columns[pos]; std::unique_ptr iter; SlotDescriptor* slot = _reusable->tuple_desc()->slots()[pos]; + int32_t index = slot->col_unique_id() >= 0 + ? tablet_schema->field_index(slot->col_unique_id()) + : tablet_schema->field_index(slot->col_name()); + if (index < 0) { + return Status::InternalError( + "field name is invalid. field={}, field_name_to_index={}", + slot->col_name(), tablet_schema->get_all_field_names()); + } StorageReadOptions storage_read_options; storage_read_options.stats = &_read_stats; storage_read_options.io_ctx = io_ctx; - RETURN_IF_ERROR(segment->seek_and_read_by_rowid(*_tablet->tablet_schema(), slot, - row_ids, column, + RETURN_IF_ERROR(segment->seek_and_read_by_rowid(tablet_schema->column(index), + slot, row_ids, column, storage_read_options, iter)); } } diff --git a/be/src/storage/compaction/compaction.cpp b/be/src/storage/compaction/compaction.cpp index 0186fa9b0bfafa..e81e1c7f8e8da9 100644 --- a/be/src/storage/compaction/compaction.cpp +++ b/be/src/storage/compaction/compaction.cpp @@ -574,8 +574,10 @@ Status CompactionMixin::build_basic_info(bool is_ordered_compaction) { // so get_extended_compaction_schema will extended the schema for variant columns // for ordered compaction, we don't need to extend the schema for variant columns if (_enable_vertical_compact_variant_subcolumns && !is_ordered_compaction) { + auto paths = std::make_shared(); RETURN_IF_ERROR(variant_util::VariantCompactionUtil::get_extended_compaction_schema( - _input_rowsets, _cur_tablet_schema)); + _input_rowsets, _cur_tablet_schema, *paths)); + _cur_variant_compaction_paths = std::move(paths); } return Status::OK(); } @@ -1787,6 +1789,7 @@ Status CompactionMixin::construct_output_rowset_writer(RowsetWriterContext& ctx) ctx.rowset_state = VISIBLE; ctx.segments_overlap = _trigger_quick_merge_by_binlog ? OVERLAPPING : NONOVERLAPPING; ctx.tablet_schema = _cur_tablet_schema; + ctx.variant_compaction_paths = _cur_variant_compaction_paths; ctx.newest_write_timestamp = _newest_write_timestamp; ctx.write_type = DataWriteType::TYPE_COMPACTION; ctx.compaction_type = compaction_type(); @@ -2092,8 +2095,10 @@ Status CloudCompactionMixin::build_basic_info() { // if enable_vertical_compact_variant_subcolumns is true, we need to compact the variant subcolumns in seperate column groups // so get_extended_compaction_schema will extended the schema for variant columns if (_enable_vertical_compact_variant_subcolumns) { + auto paths = std::make_shared(); RETURN_IF_ERROR(variant_util::VariantCompactionUtil::get_extended_compaction_schema( - _input_rowsets, _cur_tablet_schema)); + _input_rowsets, _cur_tablet_schema, *paths)); + _cur_variant_compaction_paths = std::move(paths); } return Status::OK(); } @@ -2370,6 +2375,7 @@ Status CloudCompactionMixin::construct_output_rowset_writer(RowsetWriterContext& ctx.rowset_state = VISIBLE; ctx.segments_overlap = NONOVERLAPPING; ctx.tablet_schema = _cur_tablet_schema; + ctx.variant_compaction_paths = _cur_variant_compaction_paths; ctx.newest_write_timestamp = _newest_write_timestamp; ctx.write_type = DataWriteType::TYPE_COMPACTION; ctx.compaction_type = compaction_type(); diff --git a/be/src/storage/compaction/compaction.h b/be/src/storage/compaction/compaction.h index ebf361f0b61d2c..827f3bcf4c8294 100644 --- a/be/src/storage/compaction/compaction.h +++ b/be/src/storage/compaction/compaction.h @@ -208,6 +208,8 @@ class Compaction { int64_t _newest_write_timestamp {-1}; std::unique_ptr _rowid_conversion = nullptr; TabletSchemaSPtr _cur_tablet_schema; + // The variant path layout of _cur_tablet_schema; empty unless it is an extended schema. + VariantCompactionPathsSPtr _cur_variant_compaction_paths; std::unique_ptr _profile; diff --git a/be/src/storage/iterator/block_reader.cpp b/be/src/storage/iterator/block_reader.cpp index e91b16cd021212..0012fea97c713c 100644 --- a/be/src/storage/iterator/block_reader.cpp +++ b/be/src/storage/iterator/block_reader.cpp @@ -552,23 +552,19 @@ Status BlockReader::init(const ReaderParams& read_params) { SCOPED_RAW_TIMER(&_stats.tablet_reader_init_timer_ns); RETURN_IF_ERROR(TabletReader::init(read_params)); - const bool use_sequence_map = _tablet_schema->has_seq_map() && - _tablet_schema->keys_type() == UNIQUE_KEYS && !_direct_mode && - read_params.binlog_scan_type != TBinlogScanType::MIN_DELTA && - read_params.binlog_scan_type != TBinlogScanType::DETAIL && - !(read_params.reader_type == ReaderType::READER_QUERY && - _tablet->enable_unique_key_merge_on_write()); - if (use_sequence_map) { - auto read_schema = std::make_shared(*_read_schema); - RETURN_IF_ERROR(read_schema->init_sequence_map(*_tablet_schema)); - _read_schema = std::move(read_schema); - } - - if (read_params.binlog_scan_type == TBinlogScanType::MIN_DELTA || - read_params.binlog_scan_type == TBinlogScanType::DETAIL) { - auto read_schema = std::make_shared(*_read_schema); - read_schema->init_row_binlog_column_mappings(*_tablet_schema); - _read_schema = std::move(read_schema); + // A Row Binlog scan maps the before-image columns; every other read of this reader, which is + // the one that merges rows across rowsets, builds the sequence mapping instead. + const bool map_row_binlog_columns = + read_params.binlog_scan_type == TBinlogScanType::MIN_DELTA || + read_params.binlog_scan_type == TBinlogScanType::DETAIL; + const bool merge_by_sequence_mapping = + !map_row_binlog_columns && _tablet_schema->has_seq_map() && + _tablet_schema->keys_type() == UNIQUE_KEYS && !_direct_mode && + !(read_params.reader_type == ReaderType::READER_QUERY && + _tablet->enable_unique_key_merge_on_write()); + RETURN_IF_ERROR(_read_schema->init_from_tablet_schema( + *_tablet_schema, merge_by_sequence_mapping, map_row_binlog_columns)); + if (map_row_binlog_columns) { _min_delta_value_compare_unsupported = false; } @@ -615,7 +611,7 @@ Status BlockReader::init(const ReaderParams& read_params) { if (read_params.reader_type == ReaderType::READER_QUERY && _reader_context.enable_unique_key_merge_on_write) { _next_block_func = &BlockReader::_direct_next_block; - } else if (use_sequence_map) { + } else if (merge_by_sequence_mapping) { _next_block_func = &BlockReader::_replace_key_next_block; } else { _next_block_func = &BlockReader::_unique_key_next_block; diff --git a/be/src/storage/iterator/vertical_block_reader.cpp b/be/src/storage/iterator/vertical_block_reader.cpp index e3d1e38de40450..25b6607a8e6401 100644 --- a/be/src/storage/iterator/vertical_block_reader.cpp +++ b/be/src/storage/iterator/vertical_block_reader.cpp @@ -278,6 +278,10 @@ Status VerticalBlockReader::init(const ReaderParams& read_params, } RETURN_IF_ERROR(TabletReader::init(read_params)); + RETURN_IF_ERROR(_read_schema->init_from_tablet_schema(*_tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false)); + auto status = _init_collect_iter(read_params, sample_info); if (!status.ok()) [[unlikely]] { if (!config::is_cloud_mode()) { diff --git a/be/src/storage/iterators.h b/be/src/storage/iterators.h index 09f8be09cd4499..c7029f989c6350 100644 --- a/be/src/storage/iterators.h +++ b/be/src/storage/iterators.h @@ -33,6 +33,7 @@ #include "storage/predicate/column_predicate.h" #include "storage/row_cursor.h" #include "storage/segment/row_ranges.h" +#include "storage/segment/variant/variant_compaction_paths.h" #include "storage/tablet/tablet_schema.h" namespace doris { @@ -116,7 +117,11 @@ class StorageReadOptions { // Effective adaptive batch size byte budget. size_t preferred_block_size_bytes = 8388608UL; - TabletSchemaSPtr tablet_schema = nullptr; + // Whether the tablet schema this read targets materializes variant subcolumns as extracted + // columns. A compaction read then takes them as flat leaves. + bool tablet_has_extracted_variant_columns = false; + // The compaction output schema's variant path layout; null outside a variant compaction. + VariantCompactionPathsSPtr variant_compaction_paths; bool enable_unique_key_merge_on_write = false; bool record_rowids = false; std::vector topn_filter_source_node_ids; diff --git a/be/src/storage/merger.cpp b/be/src/storage/merger.cpp index 6999cb486930c1..566adea31ac949 100644 --- a/be/src/storage/merger.cpp +++ b/be/src/storage/merger.cpp @@ -140,6 +140,7 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type, reader_params.tablet_schema = std::make_shared(); reader_params.tablet_schema->copy_from(cur_tablet_schema); + reader_params.variant_compaction_paths = dst_rowset_writer->context().variant_compaction_paths; if (!tablet->tablet_schema()->cluster_key_uids().empty()) { reader_params.delete_bitmap = tablet->tablet_meta()->delete_bitmap_ptr(); } @@ -333,6 +334,7 @@ Status Merger::vertical_compact_one_group( reader_params.tablet_schema = std::make_shared(); reader_params.tablet_schema->copy_from(tablet_schema); + reader_params.variant_compaction_paths = dst_rowset_writer->context().variant_compaction_paths; bool has_cluster_key = false; if (!tablet->tablet_schema()->cluster_key_uids().empty()) { reader_params.delete_bitmap = tablet->tablet_meta()->delete_bitmap_ptr(); diff --git a/be/src/storage/rowset/beta_rowset_reader.cpp b/be/src/storage/rowset/beta_rowset_reader.cpp index fea4e22d6dcacd..79f62a9625158d 100644 --- a/be/src/storage/rowset/beta_rowset_reader.cpp +++ b/be/src/storage/rowset/beta_rowset_reader.cpp @@ -180,7 +180,7 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context if (_should_push_down_value_predicates()) { // sequence mapping currently only support merge on read, so can not push down value predicates if (_read_context->value_predicates != nullptr && - !read_context->tablet_schema->has_seq_map()) { + !_read_context->read_schema->tablet_has_sequence_map()) { _read_options.column_predicates.insert(_read_options.column_predicates.end(), _read_context->value_predicates->begin(), _read_context->value_predicates->end()); @@ -195,7 +195,9 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context } } _read_options.use_page_cache = _read_context->use_page_cache; - _read_options.tablet_schema = _read_context->tablet_schema; + _read_options.tablet_has_extracted_variant_columns = + _read_context->read_schema->tablet_has_extracted_variant_columns(); + _read_options.variant_compaction_paths = _read_context->variant_compaction_paths; _read_options.enable_unique_key_merge_on_write = _read_context->enable_unique_key_merge_on_write; _read_options.record_rowids = _read_context->record_rowids; diff --git a/be/src/storage/rowset/rowset_reader_context.h b/be/src/storage/rowset/rowset_reader_context.h index b72c5d58c4ceb0..8bf7dfc2a0ff15 100644 --- a/be/src/storage/rowset/rowset_reader_context.h +++ b/be/src/storage/rowset/rowset_reader_context.h @@ -33,6 +33,7 @@ #include "storage/row_cursor.h" #include "storage/rowid_conversion.h" #include "storage/schema.h" +#include "storage/segment/variant/variant_compaction_paths.h" namespace doris { @@ -44,7 +45,8 @@ struct RowsetReaderContext { ReaderType reader_type = ReaderType::READER_QUERY; bool read_row_binlog = false; Version version {-1, -1}; - TabletSchemaSPtr tablet_schema = nullptr; + // Set only by compaction, alongside its extended tablet_schema. + VariantCompactionPathsSPtr variant_compaction_paths; std::vector topn_filter_source_node_ids; // whether rowset should return ordered rows. bool need_ordered_result = true; diff --git a/be/src/storage/rowset/rowset_writer_context.h b/be/src/storage/rowset/rowset_writer_context.h index 08e7b31cace5a0..7c9d4add5ce7bc 100644 --- a/be/src/storage/rowset/rowset_writer_context.h +++ b/be/src/storage/rowset/rowset_writer_context.h @@ -40,6 +40,7 @@ #include "storage/olap_define.h" #include "storage/partial_update_info.h" #include "storage/segment/historical_row_retriever.h" +#include "storage/segment/variant/variant_compaction_paths.h" #include "storage/storage_policy.h" #include "storage/tablet/tablet.h" #include "storage/tablet/tablet_schema.h" @@ -73,6 +74,8 @@ struct RowsetWriterContext { RowsetTypePB rowset_type {BETA_ROWSET}; TabletSchemaSPtr tablet_schema; + // Set only by compaction, alongside its extended tablet_schema. + VariantCompactionPathsSPtr variant_compaction_paths; // Immutable inverted-index file format inherited from the owner tablet. std::optional inverted_index_storage_format; // Whether the owner tablet persists the format in its top-level metadata. diff --git a/be/src/storage/rowset/segcompaction.cpp b/be/src/storage/rowset/segcompaction.cpp index c3ec566c29f4cc..c18f1197d978fc 100644 --- a/be/src/storage/rowset/segcompaction.cpp +++ b/be/src/storage/rowset/segcompaction.cpp @@ -92,7 +92,6 @@ Status SegcompactionWorker::_get_segcompaction_reader( StorageReadOptions read_options; read_options.stats = stat; read_options.use_page_cache = false; - read_options.tablet_schema = ctx.tablet_schema; read_options.record_rowids = record_rowids; if (!tablet->tablet_schema()->cluster_key_uids().empty()) { DeleteBitmapPtr delete_bitmap = std::make_shared(tablet->tablet_id()); diff --git a/be/src/storage/schema.cpp b/be/src/storage/schema.cpp index c3fd1af402bed8..6cfee7a2523a34 100644 --- a/be/src/storage/schema.cpp +++ b/be/src/storage/schema.cpp @@ -17,6 +17,7 @@ #include "storage/schema.h" +#include #include #include "common/config.h" @@ -118,7 +119,7 @@ void ReadSchema::_init_before_column_ordinals() { } } -void ReadSchema::init_row_binlog_column_mappings(const TabletSchema& tablet_schema) { +void ReadSchema::_init_row_binlog_column_mappings(const TabletSchema& tablet_schema) { DORIS_CHECK_GE(_op_ordinal, 0); DORIS_CHECK_EQ(_before_column_ordinals.size(), _num_block_columns); @@ -221,7 +222,24 @@ std::string ReadSchema::read_columns_to_string() const { return result; } -Status ReadSchema::init_sequence_map(const TabletSchema& tablet_schema) { +Status ReadSchema::init_from_tablet_schema(const TabletSchema& tablet_schema, + bool merge_by_sequence_mapping, + bool map_row_binlog_columns) { + DORIS_CHECK(!(merge_by_sequence_mapping && map_row_binlog_columns)); + _tablet_has_sequence_map = tablet_schema.has_seq_map(); + _tablet_has_extracted_variant_columns = + std::ranges::any_of(tablet_schema.columns(), + [](const auto& column) { return column->is_extracted_column(); }); + if (merge_by_sequence_mapping) { + RETURN_IF_ERROR(_init_sequence_map(tablet_schema)); + } + if (map_row_binlog_columns) { + _init_row_binlog_column_mappings(tablet_schema); + } + return Status::OK(); +} + +Status ReadSchema::_init_sequence_map(const TabletSchema& tablet_schema) { if (tablet_schema.has_sequence_col()) { auto msg = "sequence columns conflict, both seq_col and seq_map are true!"; LOG(WARNING) << msg; diff --git a/be/src/storage/schema.h b/be/src/storage/schema.h index 473e603a05aa77..4e27e9eeef9746 100644 --- a/be/src/storage/schema.h +++ b/be/src/storage/schema.h @@ -85,16 +85,23 @@ class ReadSchema { std::string read_columns_to_string() const; - Status init_sequence_map(const TabletSchema& tablet_schema); + // Always sets the two facts below; the flags add the sequence mapping and the row-binlog + // before-image mapping. + Status init_from_tablet_schema(const TabletSchema& tablet_schema, + bool merge_by_sequence_mapping, bool map_row_binlog_columns); + + // Whether the tablet schema defines a sequence mapping -- not whether this read merges by + // one. A MoW query does not merge, yet a value predicate still must not be pushed down. + bool tablet_has_sequence_map() const { return _tablet_has_sequence_map; } + + // Whether the tablet schema materializes variant subcolumns as extracted columns. A + // compaction read then takes them as flat leaves. + bool tablet_has_extracted_variant_columns() const { + return _tablet_has_extracted_variant_columns; + } const SequenceMap& sequence_map() const { return _sequence_map; } - // Initialize all row-binlog column relationships from the physical tablet schema and map - // them to this ReadSchema's dense ordinals. Physical pairing avoids ambiguous column-name - // lookup, while schemas without a complete physical layout retain the name-based BEFORE - // mapping initialized by the constructor. - void init_row_binlog_column_mappings(const TabletSchema& tablet_schema); - // Return the matching before-image ordinal for a Row Binlog value column. For example, in // [v1, v2, __BEFORE__v1__, __BEFORE__v2__], 0 maps to 2 and 1 maps to 3. Columns without a // before image, including TSO/LSN/OP, map to themselves. @@ -153,6 +160,8 @@ class ReadSchema { private: void _init_read_types(); void _init_before_column_ordinals(); + Status _init_sequence_map(const TabletSchema& tablet_schema); + void _init_row_binlog_column_mappings(const TabletSchema& tablet_schema); void _init_descriptors() { DORIS_CHECK_LE(_num_block_columns, _read_columns.size()); @@ -240,6 +249,8 @@ class ReadSchema { std::vector _before_column_ordinals; RowBinlogValueColumnPairs _row_binlog_value_column_pairs; bool _row_binlog_value_pairs_complete = false; + bool _tablet_has_sequence_map = false; + bool _tablet_has_extracted_variant_columns = false; }; } // namespace doris diff --git a/be/src/storage/schema_change/schema_change.cpp b/be/src/storage/schema_change/schema_change.cpp index 59fbcb69c5852f..6295091eb942ef 100644 --- a/be/src/storage/schema_change/schema_change.cpp +++ b/be/src/storage/schema_change/schema_change.cpp @@ -937,6 +937,9 @@ Status SchemaChangeJob::_do_process_alter_tablet(const TAlterTabletReqV2& reques std::vector read_columns(_base_tablet_schema->columns().begin(), _base_tablet_schema->columns().begin() + num_cols); ReadSchemaSPtr read_schema = std::make_shared(std::move(read_columns)); + RETURN_IF_ERROR(read_schema->init_from_tablet_schema(*_base_tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false)); std::vector cluster_key_idxes; DBUG_EXECUTE_IF("SchemaChangeJob::_do_process_alter_tablet.block", DBUG_BLOCK); @@ -1046,7 +1049,6 @@ Status SchemaChangeJob::_do_process_alter_tablet(const TAlterTabletReqV2& reques read_schema->append_dropped_columns(std::move(dropped_columns)); reader_context.reader_type = ReaderType::READER_ALTER_TABLE; - reader_context.tablet_schema = _base_tablet_schema; reader_context.need_ordered_result = true; reader_context.delete_handler = &delete_handler; reader_context.read_schema = read_schema; diff --git a/be/src/storage/segment/segment.cpp b/be/src/storage/segment/segment.cpp index 9d7ae1563cbe20..8978d7e01eac03 100644 --- a/be/src/storage/segment/segment.cpp +++ b/be/src/storage/segment/segment.cpp @@ -27,7 +27,6 @@ #include #include #include -#include #include #include "cloud/config.h" @@ -972,13 +971,11 @@ Status Segment::new_column_iterator(const TabletColumn& tablet_column, // on every read path (projection / predicate / MIN-MAX zone-map) instead of the placeholder 0. // commit_tso == -1 means it is not assigned yet (before publish); keep the on-disk value then. // The value is constant per segment (a segment belongs to a single rowset), so caching the - // ConstantColumnReader does not cross-pollute other queries. Some internal read paths (e.g. MOW - // partial-update row fetch) build a bare StorageReadOptions without tablet_schema, so guard it. + // ConstantColumnReader does not cross-pollute other queries. std::optional const_value; - if (opt->tablet_schema != nullptr && opt->version.first == opt->version.second && - opt->commit_tso.end_tso() != -1) { - int32_t tso_idx = opt->tablet_schema->commit_tso_col_idx(); - if (tso_idx != -1 && opt->tablet_schema->column(tso_idx).unique_id() == unique_id) { + if (opt->version.first == opt->version.second && opt->commit_tso.end_tso() != -1) { + int32_t tso_idx = _tablet_schema->commit_tso_col_idx(); + if (tso_idx != -1 && _tablet_schema->column(tso_idx).unique_id() == unique_id) { const_value = Field::create_field(opt->commit_tso.end_tso()); } } @@ -1281,7 +1278,7 @@ Status Segment::read_key_by_rowid(uint32_t row_id, std::string* key) { return Status::OK(); } -Status Segment::seek_and_read_by_rowid(const TabletSchema& schema, SlotDescriptor* slot, +Status Segment::seek_and_read_by_rowid(const TabletColumn& read_column, SlotDescriptor* slot, const std::vector& row_ids, MutableColumnPtr& result, StorageReadOptions& storage_read_options, @@ -1313,8 +1310,7 @@ Status Segment::seek_and_read_by_rowid(const TabletSchema& schema, SlotDescripto RETURN_IF_ERROR( _create_column_meta_once(storage_read_options.stats, &storage_read_options.io_ctx)); - const PathInData path(schema.column_by_uid(slot->col_unique_id()).name_lower_case(), - slot->column_paths()); + const PathInData path(read_column.name_lower_case(), slot->column_paths()); TabletColumn column = variant_util::get_column_by_type( make_nullable(slot->type()), path.get_path(), variant_util::ExtraInfo {.parent_unique_id = slot->col_unique_id(), @@ -1343,20 +1339,12 @@ Status Segment::seek_and_read_by_rowid(const TabletSchema& schema, SlotDescripto } RETURN_IF_CATCH_EXCEPTION(result->insert_range_from(*source_ptr, 0, row_ids.size())); } else { - int index = (slot->col_unique_id() >= 0) ? schema.field_index(slot->col_unique_id()) - : schema.field_index(slot->col_name()); - if (index < 0) { - std::stringstream ss; - ss << "field name is invalid. field=" << slot->col_name() - << ", field_name_to_index=" << schema.get_all_field_names(); - return Status::InternalError(ss.str()); - } - TabletColumn column = schema.column(index); - if (column.type() == FieldType::OLAP_FIELD_TYPE_VARIANT) { + if (read_column.type() == FieldType::OLAP_FIELD_TYPE_VARIANT) { DORIS_CHECK(variant_v2_type != nullptr); } if (iterator_hint == nullptr) { - RETURN_IF_ERROR(new_column_iterator(column, &iterator_hint, &storage_read_options)); + RETURN_IF_ERROR( + new_column_iterator(read_column, &iterator_hint, &storage_read_options)); RETURN_IF_ERROR(iterator_hint->init(opt)); } RETURN_IF_ERROR(iterator_hint->read_by_rowids(row_ids.data(), row_ids.size(), result)); diff --git a/be/src/storage/segment/segment.h b/be/src/storage/segment/segment.h index ef1792ccdcf9d6..a39114187c2021 100644 --- a/be/src/storage/segment/segment.h +++ b/be/src/storage/segment/segment.h @@ -150,7 +150,8 @@ class Segment : public std::enable_shared_from_this, public MetadataAdd Status read_key_by_rowid(uint32_t row_id, std::string* key); // row_ids must be strictly increasing. - Status seek_and_read_by_rowid(const TabletSchema& schema, SlotDescriptor* slot, + // `read_column` is what `slot` resolves to, or its variant parent column for a subpath slot. + Status seek_and_read_by_rowid(const TabletColumn& read_column, SlotDescriptor* slot, const std::vector& row_ids, MutableColumnPtr& result, StorageReadOptions& storage_read_options, std::unique_ptr& iterator_hint); diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index 99d53fd7f868ea..ec6c180815c95b 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -437,7 +437,9 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) { } _storage_name_and_type.resize(_schema->num_read_columns()); - auto storage_format = _opts.tablet_schema->get_inverted_index_storage_format(); + // The field names built below must match the ones this segment's index files were written + // with, so take the format from the schema IndexFileReader was opened with. + auto storage_format = _segment->_tablet_schema->get_inverted_index_storage_format(); for (size_t i = 0; i < _schema->num_read_columns(); ++i) { const TabletColumn* col = _schema->column(i); auto storage_type = _segment->get_data_type_of(*col, _schema->data_type(i), _opts); @@ -714,10 +716,7 @@ Status SegmentIterator::_get_row_ranges_by_keys() { // groups must still apply the key range to read the same physical rows as the key group. if (_opts.io_ctx.reader_type != ReaderType::READER_BASE_COMPACTION && std::none_of(_schema->columns().begin(), _schema->columns().end(), - [&](const TabletColumnPtr& col) { - return col && - _opts.tablet_schema->column_by_uid(col->unique_id()).is_key(); - })) { + [](const TabletColumnPtr& col) { return col && col->is_key(); })) { return Status::OK(); } @@ -1401,9 +1400,7 @@ bool SegmentIterator::_count_on_index_fastpath_safe() const { facts.no_need_read_data_opt_enabled = _opts.runtime_state == nullptr || _opts.runtime_state->query_options().enable_no_need_read_data_opt; - facts.keys_type_supported = _opts.tablet_schema->keys_type() == KeysType::DUP_KEYS || - (_opts.tablet_schema->keys_type() == KeysType::UNIQUE_KEYS && - _opts.enable_unique_key_merge_on_write); + facts.keys_type_supported = _keys_type_allows_skipping_data(); return count_on_index_fastpath_safe(facts); } @@ -1595,6 +1592,12 @@ Status SegmentIterator::_apply_inverted_index_on_column_predicate( return Status::OK(); } +bool SegmentIterator::_keys_type_allows_skipping_data() const { + const KeysType keys_type = _segment->_tablet_schema->keys_type(); + return keys_type == KeysType::DUP_KEYS || + (keys_type == KeysType::UNIQUE_KEYS && _opts.enable_unique_key_merge_on_write); +} + bool SegmentIterator::_need_read_data(ColumnId cid) { if (_opts.runtime_state && !_opts.runtime_state->query_options().enable_no_need_read_data_opt) { return true; @@ -1602,10 +1605,7 @@ bool SegmentIterator::_need_read_data(ColumnId cid) { if (_can_skip_reading_extra_column(cid)) { return false; } - // only support DUP_KEYS and UNIQUE_KEYS with MOW - if (!((_opts.tablet_schema->keys_type() == KeysType::DUP_KEYS || - (_opts.tablet_schema->keys_type() == KeysType::UNIQUE_KEYS && - _opts.enable_unique_key_merge_on_write)))) { + if (!_keys_type_allows_skipping_data()) { return true; } // this is a virtual column, we always need to read data @@ -3538,9 +3538,7 @@ bool SegmentIterator::_no_need_read_key_data_eligible(ColumnId cid) { return false; } - if (!((_opts.tablet_schema->keys_type() == KeysType::DUP_KEYS || - (_opts.tablet_schema->keys_type() == KeysType::UNIQUE_KEYS && - _opts.enable_unique_key_merge_on_write)))) { + if (!_keys_type_allows_skipping_data()) { return false; } diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index 5602c6280ab774..91e4207573e041 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -265,6 +265,9 @@ class SegmentIterator : public RowwiseIterator { void _output_index_result_column(const VExprContextSPtrs& expr_ctxs, uint16_t* sel_rowid_idx, uint16_t select_size); + // False for MoR and AGG keys: the merge above this iterator needs the real values. + bool _keys_type_allows_skipping_data() const; + bool _need_read_data(ColumnId cid); bool _prune_column(ColumnId cid, MutableColumnPtr& column, size_t num_of_defaults); diff --git a/be/src/storage/segment/variant/sparse_column_merge_iterator.h b/be/src/storage/segment/variant/sparse_column_merge_iterator.h index 8f4b76bfa34444..f95363ea940628 100644 --- a/be/src/storage/segment/variant/sparse_column_merge_iterator.h +++ b/be/src/storage/segment/variant/sparse_column_merge_iterator.h @@ -44,6 +44,7 @@ #include "storage/segment/column_reader.h" #include "storage/segment/stream_reader.h" #include "storage/segment/variant/binary_column_extract_iterator.h" +#include "storage/segment/variant/variant_compaction_paths.h" #include "storage/tablet/tablet_schema.h" #include "util/json/path_in_data.h" @@ -52,7 +53,7 @@ namespace doris::segment_v2 { // Implementation for merge processor class SparseColumnMergeIterator : public BaseBinaryColumnProcessor { public: - SparseColumnMergeIterator(const TabletSchema::PathsSetInfo& path_set_info, + SparseColumnMergeIterator(const VariantCompactionPaths& path_set_info, BinaryColumnCacheSPtr sparse_column_cache, SubstreamReaderTree&& src_subcolumns_for_sparse, const StorageReadOptions* opts) diff --git a/be/src/storage/segment/variant/variant_column_reader.cpp b/be/src/storage/segment/variant/variant_column_reader.cpp index 831640a54d2318..1dd8d9c49e3df3 100644 --- a/be/src/storage/segment/variant/variant_column_reader.cpp +++ b/be/src/storage/segment/variant/variant_column_reader.cpp @@ -22,7 +22,6 @@ #include #include -#include #include #include #include @@ -306,8 +305,14 @@ Status VariantColumnReader::_create_sparse_merge_reader(ColumnIteratorUPtr* iter ColumnReaderCache* column_reader_cache, std::optional bucket_index) { std::shared_lock lock(_subcolumns_meta_mutex); - // Get subcolumns path set from tablet schema - const auto& path_set_info = opts->tablet_schema->path_set_info(target_col.parent_unique_id()); + // Only the flat-leaf plan reaches a sparse column, and only a compaction output schema has + // one, so the path sets are always present here. + DORIS_CHECK(opts->variant_compaction_paths != nullptr); + auto layout = opts->variant_compaction_paths->find(target_col.parent_unique_id()); + DORIS_CHECK(layout != opts->variant_compaction_paths->end()) + << "no compaction path layout for variant column, parent_unique_id=" + << target_col.parent_unique_id(); + const auto& path_set_info = layout->second; // Build substream reader tree for merging subcolumns into sparse column SubstreamReaderTree src_subcolumns_for_sparse; @@ -587,9 +592,7 @@ bool VariantColumnReader::_has_prefix_path_unlocked(const PathInData& relative_p } bool VariantColumnReader::_need_read_flat_leaves(const StorageReadOptions* opts) { - return opts != nullptr && opts->tablet_schema != nullptr && - std::ranges::any_of(opts->tablet_schema->columns(), - [](const auto& column) { return column->is_extracted_column(); }) && + return opts != nullptr && opts->tablet_has_extracted_variant_columns && is_compaction_or_checksum_reader(opts); } diff --git a/be/src/storage/segment/variant/variant_compaction_paths.cpp b/be/src/storage/segment/variant/variant_compaction_paths.cpp new file mode 100644 index 00000000000000..fcd1d84df8721b --- /dev/null +++ b/be/src/storage/segment/variant/variant_compaction_paths.cpp @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/segment/variant/variant_compaction_paths.h" + +#include "storage/index/index_writer.h" + +namespace doris { + +namespace { + +const TabletIndexes* indexes_of_path(const VariantCompactionPaths& paths, const TabletColumn& col, + const std::string& relative_path) { + if (col.path_info_ptr()->get_is_typed()) { + auto it = paths.typed_path_set.find(relative_path); + return it == paths.typed_path_set.end() ? nullptr : &it->second.indexes; + } + auto it = paths.subcolumn_indexes.find(relative_path); + return it == paths.subcolumn_indexes.end() ? nullptr : &it->second; +} + +} // namespace + +std::vector variant_subcolumn_indexes(const VariantCompactionPathsMap* paths, + const TabletColumn& col) { + // Some extracted types (JSONB, a nested variant, an array of them) cannot carry one. + if (paths == nullptr || !col.is_extracted_column() || + !segment_v2::IndexColumnWriter::check_support_inverted_index(col)) { + return {}; + } + auto column_paths = paths->find(col.parent_unique_id()); + if (column_paths == paths->end()) { + return {}; + } + const TabletIndexes* indexes = indexes_of_path( + column_paths->second, col, col.path_info_ptr()->copy_pop_front().get_path()); + if (indexes == nullptr) { + return {}; + } + std::vector result; + result.reserve(indexes->size()); + for (const auto& index : *indexes) { + result.push_back(index.get()); + } + return result; +} + +} // namespace doris diff --git a/be/src/storage/segment/variant/variant_compaction_paths.h b/be/src/storage/segment/variant/variant_compaction_paths.h new file mode 100644 index 00000000000000..c3e65fbfb51cbc --- /dev/null +++ b/be/src/storage/segment/variant/variant_compaction_paths.h @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "storage/tablet/tablet_schema.h" + +namespace doris { + +// How one variant column's paths are laid out in a compaction output schema: which become typed +// or extracted columns, which fall back to the sparse column, and the indexes on the materialized +// ones. Paths are relative to the parent variant column. Never persisted with the schema, so only +// a running compaction has one. +struct VariantCompactionPaths { + std::unordered_map typed_path_set; + std::unordered_map subcolumn_indexes; + // extracted columns + PathSet sub_path_set; + // paths left to the sparse column + PathSet sparse_path_set; +}; + +// Parent variant column unique id -> that column's layout. Shared, and read-only once built: a +// compaction's rowset writer and every reader it spawns hold the same layout. +using VariantCompactionPathsMap = std::unordered_map; +using VariantCompactionPathsSPtr = std::shared_ptr; + +// The inverted indexes `paths` attached to `col`, an extracted variant column. Empty when `paths` +// is null (any write that is not a compaction), when it does not cover this column's path, or +// when the column's type cannot carry an inverted index. +std::vector variant_subcolumn_indexes(const VariantCompactionPathsMap* paths, + const TabletColumn& col); + +} // namespace doris diff --git a/be/src/storage/segment/vertical_segment_writer.cpp b/be/src/storage/segment/vertical_segment_writer.cpp index 542cfe725be783..aacf62421cbada 100644 --- a/be/src/storage/segment/vertical_segment_writer.cpp +++ b/be/src/storage/segment/vertical_segment_writer.cpp @@ -69,6 +69,7 @@ #include "storage/segment/page_io.h" #include "storage/segment/page_pointer.h" #include "storage/segment/segment_loader.h" +#include "storage/segment/variant/variant_compaction_paths.h" #include "storage/segment/variant/variant_ext_meta_writer.h" #include "storage/segment/variant_stats_calculator.h" #include "storage/storage_engine.h" @@ -190,6 +191,13 @@ Status VerticalSegmentWriter::_create_column_writer(size_t pos, uint32_t cid, opts.is_direct_load = _opts.write_type == DataWriteType::TYPE_DIRECT; if (!skip_inverted_index) { auto inverted_indexs = tablet_schema->inverted_indexs(column); + if (inverted_indexs.empty() && column.is_extracted_column() && + _opts.rowset_ctx != nullptr) { + // A variant subcolumn that compaction materialized carries its indexes only in the + // compaction path layout, never on the output schema. + inverted_indexs = variant_subcolumn_indexes( + _opts.rowset_ctx->variant_compaction_paths.get(), column); + } // SNII splits index compaction per (column, index): indexes in the set // are produced by the postings merge, every sibling on the column still // raw-builds here. V2/V3 skip whole columns above instead. diff --git a/be/src/storage/tablet/tablet_reader.cpp b/be/src/storage/tablet/tablet_reader.cpp index 1ac7af12a571e4..db2744a3f9fcc1 100644 --- a/be/src/storage/tablet/tablet_reader.cpp +++ b/be/src/storage/tablet/tablet_reader.cpp @@ -159,7 +159,7 @@ Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) { _reader_context.reader_type = read_params.reader_type; _reader_context.read_row_binlog = read_params.read_row_binlog; _reader_context.version = read_params.version; - _reader_context.tablet_schema = _tablet_schema; + _reader_context.variant_compaction_paths = read_params.variant_compaction_paths; _reader_context.need_ordered_result = need_ordered_result || read_params.force_key_ordered_read; _reader_context.topn_filter_source_node_ids = read_params.topn_filter_source_node_ids; _reader_context.read_orderby_key_reverse = read_params.read_orderby_key_reverse; diff --git a/be/src/storage/tablet/tablet_reader.h b/be/src/storage/tablet/tablet_reader.h index db8e557ef5439b..82f3a707fdb2ad 100644 --- a/be/src/storage/tablet/tablet_reader.h +++ b/be/src/storage/tablet/tablet_reader.h @@ -45,6 +45,7 @@ #include "storage/rowset/rowset_meta.h" #include "storage/rowset/rowset_reader.h" #include "storage/rowset/rowset_reader_context.h" +#include "storage/segment/variant/variant_compaction_paths.h" #include "storage/tablet/base_tablet.h" #include "storage/tablet/tablet_fwd.h" @@ -127,6 +128,8 @@ class TabletReader { BaseTabletSPtr tablet; TabletSchemaSPtr tablet_schema; + // Set only by compaction, alongside its extended tablet_schema. + VariantCompactionPathsSPtr variant_compaction_paths = nullptr; ReaderType reader_type = ReaderType::READER_QUERY; bool read_row_binlog = false; bool direct_mode = false; diff --git a/be/src/storage/tablet/tablet_schema.cpp b/be/src/storage/tablet/tablet_schema.cpp index fd496809e78931..e607af932048ac 100644 --- a/be/src/storage/tablet/tablet_schema.cpp +++ b/be/src/storage/tablet/tablet_schema.cpp @@ -1187,7 +1187,6 @@ void TabletSchema::copy_from(const TabletSchema& tablet_schema) { tablet_schema.to_schema_pb(&tablet_schema_pb); init_from_pb(tablet_schema_pb); _table_id = tablet_schema.table_id(); - _path_set_info_map = tablet_schema._path_set_info_map; } void TabletSchema::shawdow_copy_without_columns(const TabletSchema& tablet_schema) { @@ -1630,43 +1629,7 @@ std::vector TabletSchema::inverted_indexs(const TabletColumn // TODO use more efficient impl // Use parent id if unique not assigned, this could happend when accessing subcolumns of variants int32_t col_unique_id = col.is_extracted_column() ? col.parent_unique_id() : col.unique_id(); - std::vector result; - if (result = inverted_indexs(col_unique_id, escape_for_path_name(col.suffix_path())); - !result.empty()) { - return result; - } - // variant's typed column has it's own index - else if (col.is_extracted_column() && col.path_info_ptr()->get_is_typed()) { - std::string relative_path = col.path_info_ptr()->copy_pop_front().get_path(); - if (_path_set_info_map.find(col_unique_id) == _path_set_info_map.end()) { - return result; - } - const auto& path_set_info = _path_set_info_map.at(col_unique_id); - if (path_set_info.typed_path_set.find(relative_path) == - path_set_info.typed_path_set.end()) { - return result; - } - for (const auto& index : path_set_info.typed_path_set.at(relative_path).indexes) { - result.push_back(index.get()); - } - return result; - } - // variant's subcolumns has it's own index - else if (col.is_extracted_column()) { - std::string relative_path = col.path_info_ptr()->copy_pop_front().get_path(); - if (_path_set_info_map.find(col_unique_id) == _path_set_info_map.end()) { - return result; - } - const auto& path_set_info = _path_set_info_map.at(col_unique_id); - if (path_set_info.subcolumn_indexes.find(relative_path) == - path_set_info.subcolumn_indexes.end()) { - return result; - } - for (const auto& index : path_set_info.subcolumn_indexes.at(relative_path)) { - result.push_back(index.get()); - } - } - return result; + return inverted_indexs(col_unique_id, escape_for_path_name(col.suffix_path())); } const TabletIndex* TabletSchema::ann_index(int32_t col_unique_id, diff --git a/be/src/storage/tablet/tablet_schema.h b/be/src/storage/tablet/tablet_schema.h index 01121657c725bf..db1db16ea4e0a7 100644 --- a/be/src/storage/tablet/tablet_schema.h +++ b/be/src/storage/tablet/tablet_schema.h @@ -723,33 +723,6 @@ class TabletSchema : public MetadataAdder { TabletIndexes indexes; }; - // all path in path_set_info are relative to the parent column - struct PathsSetInfo { - std::unordered_map typed_path_set; // typed columns - std::unordered_map subcolumn_indexes; // subcolumns indexes - PathSet sub_path_set; // extracted columns - PathSet sparse_path_set; // sparse columns - - // "Materialized regular path" means compaction chose to store this path as a dedicated - // column in the schema, either typed or extracted, instead of re-emitting it dynamically. - bool contains_materialized_regular_path(const std::string& path) const { - return typed_path_set.contains(path) || sub_path_set.contains(path); - } - }; - - void set_path_set_info(std::unordered_map&& path_set_info_map) { - _path_set_info_map = std::move(path_set_info_map); - } - - const PathsSetInfo& path_set_info(int32_t unique_id) const { - return _path_set_info_map.at(unique_id); - } - - const PathsSetInfo* try_path_set_info(int32_t unique_id) const { - auto it = _path_set_info_map.find(unique_id); - return it == _path_set_info_map.end() ? nullptr : &it->second; - } - bool need_record_variant_extended_schema() const { return variant_max_subcolumns_count() == 0; } int32_t variant_max_subcolumns_count() const { @@ -844,9 +817,6 @@ class TabletSchema : public MetadataAdder { std::map _vir_col_idx_to_unique_id; - // value: extracted path set and sparse path set - std::unordered_map _path_set_info_map; - // key: field_pattern // value: indexes using PatternToIndex = std::unordered_map>; diff --git a/be/src/storage/task/index_builder.cpp b/be/src/storage/task/index_builder.cpp index e436c1cbd11ff3..427c30cdcaf49c 100644 --- a/be/src/storage/task/index_builder.cpp +++ b/be/src/storage/task/index_builder.cpp @@ -718,7 +718,6 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta StorageReadOptions read_options; OlapReaderStatistics stats; read_options.stats = &stats; - read_options.tablet_schema = output_rowset_schema; auto schema = std::make_shared( project_columns_by_ordinal(output_rowset_schema->columns(), return_columns)); std::unique_ptr iter; @@ -954,7 +953,6 @@ Status IndexBuilder::_build_snii_indexes_for_segment(const TabletSchemaSPtr& out StorageReadOptions read_options; OlapReaderStatistics stats; read_options.stats = &stats; - read_options.tablet_schema = output_rowset_schema; auto schema = std::make_shared( project_columns_by_ordinal(output_rowset_schema->columns(), return_columns)); std::unique_ptr iter; diff --git a/be/test/exec/common/schema_util_rowset_test.cpp b/be/test/exec/common/schema_util_rowset_test.cpp index 66d7f79e591115..4d186b642fce07 100644 --- a/be/test/exec/common/schema_util_rowset_test.cpp +++ b/be/test/exec/common/schema_util_rowset_test.cpp @@ -532,7 +532,6 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_sche // key3 is in the sparse column, return variant type StorageReadOptions type_opts; - type_opts.tablet_schema = out_rowset->tablet_schema(); type_opts.io_ctx.reader_type = ReaderType::READER_QUERY; TabletColumn non_variant_column; non_variant_column.set_name("non_variant_column"); @@ -667,8 +666,9 @@ TabletSchemaSPtr create_compaction_schema_common(StorageEngine* _engine_ref, // 4. get compaction schema TabletSchemaSPtr compaction_schema = tablet_schema; + VariantCompactionPathsMap compaction_paths; auto st = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - rowsets, compaction_schema); + rowsets, compaction_schema, compaction_paths); EXPECT_TRUE(st.ok()) << st.msg(); // 5. check compaction schema @@ -785,8 +785,9 @@ TEST_F(SchemaUtilRowsetTest, typed_path_to_sparse_column) { // 4. get compaction schema TabletSchemaSPtr compaction_schema = tablet_schema; + VariantCompactionPathsMap compaction_paths; auto st = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - rowsets, compaction_schema); + rowsets, compaction_schema, compaction_paths); EXPECT_TRUE(st.ok()) << st.msg(); for (const auto& column : compaction_schema->columns()) { if (column->is_extracted_column()) { diff --git a/be/test/exec/common/schema_util_test.cpp b/be/test/exec/common/schema_util_test.cpp index 7e41be7d1f2c91..87620120557f64 100644 --- a/be/test/exec/common/schema_util_test.cpp +++ b/be/test/exec/common/schema_util_test.cpp @@ -379,7 +379,7 @@ TEST_F(SchemaUtilTest, get_subpaths) { {"path1", 1000}, {"path2", 800}, {"path3", 500}, {"path4", 300}, {"path5", 200}}; // get subpaths - std::unordered_map uid_to_paths_set_info; + VariantCompactionPathsMap uid_to_paths_set_info; variant_util::VariantCompactionUtil::get_subpaths(3, path_stats[1], uid_to_paths_set_info[1]); EXPECT_EQ(uid_to_paths_set_info[1].sub_path_set.size(), 3); @@ -408,7 +408,7 @@ TEST_F(SchemaUtilTest, get_subpaths_equal_to_max) { std::unordered_map path_stats; path_stats[1] = {{"path1", 1000}, {"path2", 800}, {"path3", 500}}; - std::unordered_map uid_to_paths_set_info; + VariantCompactionPathsMap uid_to_paths_set_info; variant_util::VariantCompactionUtil::get_subpaths(3, path_stats[1], uid_to_paths_set_info[1]); EXPECT_EQ(uid_to_paths_set_info[1].sub_path_set.size(), 3); @@ -426,7 +426,7 @@ TEST_F(SchemaUtilTest, get_subpaths_selects_empty_key_as_subpath) { variant_util::PathToNoneNullValues path_stats = { {"", 1000}, {"path1", 900}, {"path2", 800}, {"path3", 700}}; - TabletSchema::PathsSetInfo limited_paths; + VariantCompactionPaths limited_paths; variant_util::VariantCompactionUtil::get_subpaths(2, path_stats, limited_paths); EXPECT_TRUE(limited_paths.sub_path_set.contains("")); EXPECT_FALSE(limited_paths.sparse_path_set.contains("")); @@ -434,7 +434,7 @@ TEST_F(SchemaUtilTest, get_subpaths_selects_empty_key_as_subpath) { EXPECT_TRUE(limited_paths.sparse_path_set.contains("path2")); EXPECT_TRUE(limited_paths.sparse_path_set.contains("path3")); - TabletSchema::PathsSetInfo exact_limit_paths; + VariantCompactionPaths exact_limit_paths; variant_util::VariantCompactionUtil::get_subpaths(4, path_stats, exact_limit_paths); EXPECT_TRUE(exact_limit_paths.sub_path_set.contains("")); EXPECT_FALSE(exact_limit_paths.sparse_path_set.contains("")); @@ -442,7 +442,7 @@ TEST_F(SchemaUtilTest, get_subpaths_selects_empty_key_as_subpath) { EXPECT_TRUE(exact_limit_paths.sub_path_set.contains("path2")); EXPECT_TRUE(exact_limit_paths.sub_path_set.contains("path3")); - TabletSchema::PathsSetInfo unlimited_paths; + VariantCompactionPaths unlimited_paths; variant_util::VariantCompactionUtil::get_subpaths(0, path_stats, unlimited_paths); EXPECT_TRUE(unlimited_paths.sub_path_set.contains("")); EXPECT_TRUE(unlimited_paths.sparse_path_set.empty()); @@ -453,7 +453,7 @@ TEST_F(SchemaUtilTest, get_subpaths_selects_empty_key_as_subpath) { variant_util::PathToNoneNullValues low_rank_empty_key_stats = { {"path1", 1000}, {"path2", 900}, {"", 100}}; - TabletSchema::PathsSetInfo low_rank_empty_key_paths; + VariantCompactionPaths low_rank_empty_key_paths; variant_util::VariantCompactionUtil::get_subpaths(2, low_rank_empty_key_stats, low_rank_empty_key_paths); EXPECT_FALSE(low_rank_empty_key_paths.sub_path_set.contains("")); @@ -488,7 +488,7 @@ TEST_F(SchemaUtilTest, get_subpaths_multiple_variants) { path_stats[4] = { {"path1", 1000}, {"path2", 800}, {"path3", 500}, {"path4", 300}, {"path5", 200}}; - std::unordered_map uid_to_paths_set_info; + VariantCompactionPathsMap uid_to_paths_set_info; variant_util::VariantCompactionUtil::get_subpaths(3, path_stats[1], uid_to_paths_set_info[1]); variant_util::VariantCompactionUtil::get_subpaths(2, path_stats[2], uid_to_paths_set_info[2]); variant_util::VariantCompactionUtil::get_subpaths(4, path_stats[3], uid_to_paths_set_info[3]); @@ -539,7 +539,7 @@ TEST_F(SchemaUtilTest, get_subpaths_no_path_stats) { std::unordered_map path_stats; path_stats[2] = {{"path1", 1000}, {"path2", 800}}; - std::unordered_map uid_to_paths_set_info; + VariantCompactionPathsMap uid_to_paths_set_info; variant_util::VariantCompactionUtil::get_subpaths(3, path_stats[2], uid_to_paths_set_info[2]); EXPECT_EQ(uid_to_paths_set_info[1].sub_path_set.size(), 0); @@ -1227,8 +1227,9 @@ TEST_F(SchemaUtilTest, TestGetCompactionSchema) { auto target_schema = std::make_shared(); target_schema->init_from_pb(schema_pb); + VariantCompactionPathsMap compaction_paths; auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - rowsets, target_schema); + rowsets, target_schema, compaction_paths); EXPECT_TRUE(status.ok()); // Check that paths were properly distributed between subcolumns and sparse columns @@ -1265,8 +1266,9 @@ TEST_F(SchemaUtilTest, EXPECT_EQ(source_indexes[0]->index_name(), "v1_owner_idx"); std::vector rowsets; + VariantCompactionPathsMap compaction_paths; auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - rowsets, target_schema); + rowsets, target_schema, compaction_paths); ASSERT_TRUE(status.ok()) << status.to_string(); // get_extended_compaction_schema rebuilds from the base columns. Real compaction targets do @@ -1276,7 +1278,7 @@ TEST_F(SchemaUtilTest, const PathInData typed_path("v1.owner", true); EXPECT_EQ(target_schema->field_index(typed_path), -1); - const auto* path_set_info = target_schema->try_path_set_info(1); + const auto* path_set_info = compaction_paths.contains(1) ? &compaction_paths.at(1) : nullptr; ASSERT_NE(path_set_info, nullptr); EXPECT_FALSE(path_set_info->typed_path_set.contains("owner")); } @@ -1322,7 +1324,7 @@ TEST_F(SchemaUtilTest, get_compaction_typed_columns) { typed_paths.insert("profile.id.name"); TabletSchemaSPtr output_schema = std::make_shared(); TabletColumnPtr parent_column = std::make_shared(variant); - TabletSchema::PathsSetInfo paths_set_info; + VariantCompactionPaths paths_set_info; EXPECT_TRUE(variant_util::VariantCompactionUtil::get_compaction_typed_columns( schema, typed_paths, parent_column, output_schema, paths_set_info) .ok()); @@ -1351,7 +1353,7 @@ TEST_F(SchemaUtilTest, get_compaction_nested_columns) { nested_paths.insert(path2); TabletSchemaSPtr output_schema = std::make_shared(); - TabletSchema::PathsSetInfo paths_set_info; + VariantCompactionPaths paths_set_info; doris::variant_util::PathToDataTypes path_to_data_types; path_to_data_types[path1] = {std::make_shared(), @@ -1377,7 +1379,7 @@ TEST_F(SchemaUtilTest, get_compaction_nested_columns) { std::unordered_set bad_nested_paths; bad_nested_paths.insert(PathInData("not_exist")); TabletSchemaSPtr bad_output_schema = std::make_shared(); - TabletSchema::PathsSetInfo bad_paths_set_info; + VariantCompactionPaths bad_paths_set_info; Status st2 = variant_util::VariantCompactionUtil::get_compaction_nested_columns( bad_nested_paths, path_to_data_types, parent_column, bad_output_schema, bad_paths_set_info); @@ -1396,7 +1398,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_from_subpaths) { TabletColumnPtr parent_column = std::make_shared(variant); - TabletSchema::PathsSetInfo paths_set_info; + VariantCompactionPaths paths_set_info; paths_set_info.sub_path_set.insert(""); paths_set_info.sub_path_set.insert("a"); paths_set_info.sub_path_set.insert("b"); @@ -1492,7 +1494,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_advanced) { TabletColumnPtr parent_column = std::make_shared(variant); - TabletSchema::PathsSetInfo paths_set_info; + VariantCompactionPaths paths_set_info; paths_set_info.sub_path_set.insert("a"); paths_set_info.sub_path_set.insert("b"); paths_set_info.sub_path_set.insert("c"); @@ -1588,7 +1590,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_from_data_types) { path_to_data_types[PathInData()] = {std::make_shared()}; TabletSchemaSPtr output_schema = std::make_shared(); - TabletSchema::PathsSetInfo paths_set_info; + VariantCompactionPaths paths_set_info; variant_util::VariantCompactionUtil::get_compaction_subcolumns_from_data_types( paths_set_info, parent_column, target, path_to_data_types, output_schema); @@ -1657,7 +1659,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_from_data_types) { doris::variant_util::PathToDataTypes root_path_to_data_types; root_path_to_data_types[PathInData()] = {std::make_shared()}; TabletSchemaSPtr root_output_schema = std::make_shared(); - TabletSchema::PathsSetInfo root_paths_set_info; + VariantCompactionPaths root_paths_set_info; variant_util::VariantCompactionUtil::get_compaction_subcolumns_from_data_types( root_paths_set_info, parent_column, target, root_path_to_data_types, @@ -1668,7 +1670,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_from_data_types) { EXPECT_FALSE(root_paths_set_info.sub_path_set.contains("")); TabletSchemaSPtr empty_key_output_schema = std::make_shared(); - TabletSchema::PathsSetInfo empty_key_paths_set_info; + VariantCompactionPaths empty_key_paths_set_info; empty_key_paths_set_info.sub_path_set.insert(""); variant_util::VariantCompactionUtil::get_compaction_subcolumns_from_data_types( diff --git a/be/test/exec/scan/vgeneric_iterators_test.cpp b/be/test/exec/scan/vgeneric_iterators_test.cpp index d8f1a277efaa4f..1d8f8932d4a876 100644 --- a/be/test/exec/scan/vgeneric_iterators_test.cpp +++ b/be/test/exec/scan/vgeneric_iterators_test.cpp @@ -172,7 +172,6 @@ TEST(VGenericIteratorsTest, StatisticsIteratorPreservesNullForNullableChar) { OlapReaderStatistics stats; read_options.push_down_agg_type_opt = TPushAggOp::MINMAX; read_options.stats = &stats; - read_options.tablet_schema = tablet_schema; ASSERT_TRUE(iterator.init(read_options).ok()); Block block; @@ -277,7 +276,6 @@ class StatisticsIteratorStringBoundsTest : public testing::Test { StorageReadOptions read_options; read_options.push_down_agg_type_opt = agg; read_options.stats = &_stats; - read_options.tablet_schema = tablet_schema; if (with_delete) { auto del_pred = NullPredicate::create_shared(0, "c1", true, PrimitiveType::TYPE_INT); diff --git a/be/test/load/delta_writer/delta_writer_cluster_key_test.cpp b/be/test/load/delta_writer/delta_writer_cluster_key_test.cpp index ccb40950b0795c..8866a42f60e95a 100644 --- a/be/test/load/delta_writer/delta_writer_cluster_key_test.cpp +++ b/be/test/load/delta_writer/delta_writer_cluster_key_test.cpp @@ -349,7 +349,6 @@ TEST_F(TestDeltaWriterClusterKey, vec_sequence_col) { OlapReaderStatistics stats; StorageReadOptions opts; opts.stats = &stats; - opts.tablet_schema = rowset->tablet_schema(); std::unique_ptr iter; std::shared_ptr schema = create_full_schema(rowset->tablet_schema()); diff --git a/be/test/load/delta_writer/delta_writer_test.cpp b/be/test/load/delta_writer/delta_writer_test.cpp index 121b254cdba37e..1c6d7bd4522011 100644 --- a/be/test/load/delta_writer/delta_writer_test.cpp +++ b/be/test/load/delta_writer/delta_writer_test.cpp @@ -833,7 +833,6 @@ TEST_F(TestDeltaWriter, vec_sequence_col) { OlapReaderStatistics stats; StorageReadOptions opts; opts.stats = &stats; - opts.tablet_schema = rowset->tablet_schema(); std::unique_ptr iter; std::shared_ptr schema = create_full_schema(rowset->tablet_schema()); @@ -1040,7 +1039,6 @@ TEST_F(TestDeltaWriter, vec_sequence_col_concurrent_write) { OlapReaderStatistics stats; StorageReadOptions opts; opts.stats = &stats; - opts.tablet_schema = rowset1->tablet_schema(); opts.delete_bitmap.emplace(0, tablet->tablet_meta()->delete_bitmap().get_agg( {rowset1->rowset_id(), 0, cur_version})); std::unique_ptr iter; @@ -1068,7 +1066,6 @@ TEST_F(TestDeltaWriter, vec_sequence_col_concurrent_write) { OlapReaderStatistics stats; StorageReadOptions opts; opts.stats = &stats; - opts.tablet_schema = rowset2->tablet_schema(); opts.delete_bitmap.emplace(0, tablet->tablet_meta()->delete_bitmap().get_agg( {rowset2->rowset_id(), 0, cur_version})); std::unique_ptr iter; diff --git a/be/test/olap/rowset/group_rowset_writer_test.cpp b/be/test/olap/rowset/group_rowset_writer_test.cpp index 604c8366c41d4d..f832150f516c19 100644 --- a/be/test/olap/rowset/group_rowset_writer_test.cpp +++ b/be/test/olap/rowset/group_rowset_writer_test.cpp @@ -402,11 +402,15 @@ TEST_F(GroupRowsetWriterTest, partialUpdateSkipsHiddenNonKeyColumns) { const auto& row_binlog_schema = _row_binlog_tablet->tablet_schema(); ASSERT_EQ(7, row_binlog_schema->num_columns()); RowsetReaderContext reader_context; - reader_context.tablet_schema = row_binlog_schema; reader_context.need_ordered_result = false; // Read schema covers all row-binlog columns in order. auto read_schema = std::make_shared(row_binlog_schema->columns()); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*row_binlog_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr rowset_reader; ASSERT_TRUE(row_binlog_rowset->create_reader(&rowset_reader).ok()); @@ -471,9 +475,13 @@ TEST_F(GroupRowsetWriterTest, keyOnlyFixedPartialUpdatePreservesNarrowBlock) { auto read_schema = std::make_shared(project_columns_by_ordinal( row_binlog_schema->columns(), std::vector {0, 1, 2, 3, 4, 5, 6})); RowsetReaderContext reader_context; - reader_context.tablet_schema = row_binlog_schema; reader_context.need_ordered_result = false; reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*row_binlog_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr rowset_reader; ASSERT_TRUE(rowsets[1]->create_reader(&rowset_reader).ok()); diff --git a/be/test/storage/compaction/ordered_data_compaction_test.cpp b/be/test/storage/compaction/ordered_data_compaction_test.cpp index 685119d19ad630..8b39a01b873c66 100644 --- a/be/test/storage/compaction/ordered_data_compaction_test.cpp +++ b/be/test/storage/compaction/ordered_data_compaction_test.cpp @@ -77,6 +77,15 @@ static StorageEngine* engine_ref = nullptr; class OrderedDataCompactionTest : public ::testing::Test { protected: void SetUp() override { + _saved_enable_ordered_data_compaction = config::enable_ordered_data_compaction; + _saved_ordered_data_compaction_min_segment_size = + config::ordered_data_compaction_min_segment_size; + _saved_segments_key_bounds_truncation_threshold = + config::segments_key_bounds_truncation_threshold; + config::enable_ordered_data_compaction = true; + config::ordered_data_compaction_min_segment_size = 10; + config::segments_key_bounds_truncation_threshold = -1; + char buffer[MAX_PATH_LEN]; EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr); absolute_dir = std::string(buffer) + kTestDir; @@ -103,16 +112,24 @@ class OrderedDataCompactionTest : public ::testing::Test { _data_dir = std::make_unique(*engine_ref, absolute_dir); static_cast(_data_dir->update_capacity()); ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); - config::enable_ordered_data_compaction = true; - config::ordered_data_compaction_min_segment_size = 10; - config::segments_key_bounds_truncation_threshold = -1; } void TearDown() override { EXPECT_TRUE(io::global_local_filesystem()->delete_directory(absolute_dir).ok()); engine_ref = nullptr; ExecEnv::GetInstance()->set_storage_engine(nullptr); + // Otherwise the relaxed minimum segment size leaks into every later suite and sends its + // compactions down the ordered link-file path instead of a real merge. + config::enable_ordered_data_compaction = _saved_enable_ordered_data_compaction; + config::ordered_data_compaction_min_segment_size = + _saved_ordered_data_compaction_min_segment_size; + config::segments_key_bounds_truncation_threshold = + _saved_segments_key_bounds_truncation_threshold; } + bool _saved_enable_ordered_data_compaction = true; + int32_t _saved_ordered_data_compaction_min_segment_size = 0; + int32_t _saved_segments_key_bounds_truncation_threshold = 0; + TabletSchemaSPtr create_schema(KeysType keys_type = DUP_KEYS) { TabletSchemaSPtr tablet_schema = std::make_shared(); TabletSchemaPB tablet_schema_pb; @@ -492,11 +509,15 @@ TEST_F(OrderedDataCompactionTest, test_01) { // create output rowset reader RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); diff --git a/be/test/storage/compaction/segcompaction_mow_test.cpp b/be/test/storage/compaction/segcompaction_mow_test.cpp index bb32ca55d2c6ba..76100dffc11b19 100644 --- a/be/test/storage/compaction/segcompaction_mow_test.cpp +++ b/be/test/storage/compaction/segcompaction_mow_test.cpp @@ -241,7 +241,6 @@ class SegCompactionMoWTest : public ::testing::TestWithParam { int expect_total_rows, int rows_mark_deleted, bool skip_value_check = false) { RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; // use this type to avoid cache from other ut reader_context.reader_type = ReaderType::READER_QUERY; reader_context.need_ordered_result = true; @@ -249,6 +248,11 @@ class SegCompactionMoWTest : public ::testing::TestWithParam { auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), return_columns)); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); reader_context.stats = &_stats; reader_context.delete_bitmap = delete_bitmap; diff --git a/be/test/storage/compaction/segcompaction_test.cpp b/be/test/storage/compaction/segcompaction_test.cpp index fb7a27fee5d931..7e2438f604bb1d 100644 --- a/be/test/storage/compaction/segcompaction_test.cpp +++ b/be/test/storage/compaction/segcompaction_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#include #include #include #include @@ -26,6 +27,11 @@ #include #include "common/config.h" +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/column/variant_v2/column_variant_v2.h" #include "cpp/sync_point.h" #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" @@ -44,6 +50,7 @@ #include "storage/tablet/tablet_meta.h" #include "storage/tablet/tablet_schema.h" #include "storage/utils.h" +#include "testutil/variant_util.h" #include "util/debug_points.h" #include "util/defer_op.h" #include "util/slice.h" @@ -292,6 +299,139 @@ class SegCompactionTest : public testing::Test { EXPECT_EQ(Status::OK(), s); } + // ---- variant / segment-compaction interaction ---- + + static constexpr int kVariantSegments = 12; + static constexpr int kVariantRowsPerSegment = 200; + + // Each segment carries a key of its own ("s") on top of the shared ones, so a merge + // would have to re-split subcolumns and sparse paths rather than copy the input layout. + static std::string variant_json(int segment, int rid) { + return fmt::format(R"({{"a":{},"b":"mark_{}_{}","s{}":{}}})", rid, segment, rid, segment, + rid * 7); + } + + // (c1 INT key, v VARIANT or INT). The variant flavour is the one BetaRowsetWriter opts out + // of segment compaction. + TabletSchemaSPtr create_variant_tablet_schema(bool with_variant) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_num_short_key_columns(1); + schema_pb.set_num_rows_per_row_block(1024); + schema_pb.set_compress_kind(COMPRESS_NONE); + schema_pb.set_next_column_unique_id(3); + + ColumnPB* key = schema_pb.add_column(); + key->set_unique_id(1); + key->set_name("c1"); + key->set_type("INT"); + key->set_is_key(true); + key->set_length(4); + key->set_index_length(4); + key->set_is_nullable(false); + + ColumnPB* value = schema_pb.add_column(); + value->set_unique_id(2); + value->set_name("v"); + value->set_is_key(false); + value->set_is_nullable(true); + if (with_variant) { + value->set_type("VARIANT"); + // Small enough that only some paths stay extracted; the rest go to the sparse column. + value->set_variant_max_subcolumns_count(3); + value->set_variant_max_sparse_column_statistics_size(10000); + value->set_variant_sparse_hash_shard_count(1); + } else { + value->set_type("INT"); + value->set_length(4); + } + + auto tablet_schema = std::make_shared(); + tablet_schema->init_from_pb(schema_pb); + return tablet_schema; + } + + // `wait_for_segcompaction` sleeps between flushes so the async worker gets to start: + // BetaRowsetWriter::_close_file_writers cancels a task that has not started yet, which would + // leave a rowset uncompacted for timing reasons rather than for the reason under test. + void write_variant_rowset(int64_t id, const TabletSchemaSPtr& tablet_schema, + bool wait_for_segcompaction, RowsetSharedPtr* rowset) { + RowsetWriterContext writer_context; + create_rowset_writer_context(id, tablet_schema, &writer_context); + auto res = RowsetFactory::create_rowset_writer(*l_engine, writer_context, false); + ASSERT_TRUE(res.has_value()) << res.error(); + auto rowset_writer = std::move(res).value(); + + const bool with_variant = + tablet_schema->column(1).type() == FieldType::OLAP_FIELD_TYPE_VARIANT; + for (int seg = 0; seg < kVariantSegments; ++seg) { + Block block = tablet_schema->create_storage_block(); + auto columns = std::move(block).mutate_columns(); + auto raw_json = ColumnString::create(); + auto* nullable = assert_cast(columns[1].get()); + for (int rid = 0; rid < kVariantRowsPerSegment; ++rid) { + int32_t c1 = seg * kVariantRowsPerSegment + rid; + columns[0]->insert_data(reinterpret_cast(&c1), sizeof(c1)); + if (with_variant) { + std::string json = variant_json(seg, rid); + raw_json->insert_data(json.data(), json.size()); + } else { + nullable->get_nested_column().insert_data(reinterpret_cast(&c1), + sizeof(c1)); + } + nullable->get_null_map_data().push_back(0); + } + if (with_variant) { + VariantUtil::insert_json_rows( + assert_cast(nullable->get_nested_column()), *raw_json); + } + ASSERT_TRUE(add_block_with_columns(rowset_writer.get(), &block, &columns).ok()); + ASSERT_TRUE(rowset_writer->flush().ok()); + if (wait_for_segcompaction) { + sleep(1); + } + } + ASSERT_EQ(Status::OK(), rowset_writer->build(*rowset)); + } + + // (c1, stringified value) for every row, ordered by c1. + void read_all_rows(const RowsetSharedPtr& rowset, const TabletSchemaSPtr& tablet_schema, + std::vector>* rows) { + RowsetReaderContext reader_context; + reader_context.reader_type = ReaderType::READER_QUERY; + reader_context.need_ordered_result = true; + std::vector return_columns = {0, 1}; + auto read_schema = std::make_shared( + project_columns_by_ordinal(tablet_schema->columns(), return_columns)); + reader_context.read_schema = read_schema; + ASSERT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); + reader_context.stats = &_stats; + + RowsetReaderSharedPtr rowset_reader; + create_and_init_rowset_reader(rowset.get(), reader_context, &rowset_reader); + + while (true) { + auto block = read_schema->create_read_block(); + auto st = rowset_reader->next_batch(&block); + if (!st.ok()) { + ASSERT_TRUE(st.is()) << st; + break; + } + const auto& value_col = block.get_by_position(1); + const auto& keys = assert_cast(*block.get_by_position(0).column); + for (size_t i = 0; i < block.rows(); ++i) { + rows->emplace_back(keys.get_data()[i], + value_col.type->to_string(*value_col.column, i)); + } + } + std::sort(rows->begin(), rows->end(), + [](const auto& lhs, const auto& rhs) { return lhs.first < rhs.first; }); + } + private: std::unique_ptr _data_dir; std::unique_ptr _inverted_index_searcher_cache; @@ -410,7 +550,6 @@ TEST_F(SegCompactionTest, SegCompactionThenRead) { { // read RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; // use this type to avoid cache from other ut reader_context.reader_type = ReaderType::READER_CUMULATIVE_COMPACTION; reader_context.need_ordered_result = true; @@ -418,6 +557,11 @@ TEST_F(SegCompactionTest, SegCompactionThenRead) { auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), return_columns)); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); reader_context.stats = &_stats; // without predicates @@ -918,7 +1062,6 @@ TEST_F(SegCompactionTest, SegCompactionThenReadUniqueTableSmall) { { // read RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; // use this type to avoid cache from other ut reader_context.reader_type = ReaderType::READER_CUMULATIVE_COMPACTION; reader_context.need_ordered_result = true; @@ -926,6 +1069,11 @@ TEST_F(SegCompactionTest, SegCompactionThenReadUniqueTableSmall) { auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), return_columns)); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); reader_context.stats = &_stats; reader_context.is_unique = true; @@ -1186,7 +1334,6 @@ TEST_F(SegCompactionTest, SegCompactionThenReadAggTableSmall) { { // read RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; // use this type to avoid cache from other ut reader_context.reader_type = ReaderType::READER_CUMULATIVE_COMPACTION; reader_context.need_ordered_result = true; @@ -1194,6 +1341,11 @@ TEST_F(SegCompactionTest, SegCompactionThenReadAggTableSmall) { auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), return_columns)); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); reader_context.stats = &_stats; // reader_context.is_unique = true; @@ -1248,6 +1400,49 @@ TEST_F(SegCompactionTest, SegCompactionThenReadAggTableSmall) { } } +// BetaRowsetWriter::_segcompaction_if_necessary opts a rowset out of segment compaction as soon +// as its schema has a variant column, so a variant rowset keeps every segment it flushed. The +// variant-free rowset is the control: same rows, same thresholds, and that one does get merged. +TEST_F(SegCompactionTest, VariantRowsetIsNeverSegmentCompacted) { + const auto saved_candidate_max_rows = config::segcompaction_candidate_max_rows; + const auto saved_batch_size = config::segcompaction_batch_size; + Defer restore_config([&] { + config::segcompaction_candidate_max_rows = saved_candidate_max_rows; + config::segcompaction_batch_size = saved_batch_size; + }); + config::enable_segcompaction = true; + config::segcompaction_candidate_max_rows = kVariantRowsPerSegment * 2; + config::segcompaction_batch_size = 5; + + RowsetSharedPtr plain_rowset; + ASSERT_NO_FATAL_FAILURE(write_variant_rowset(10060, create_variant_tablet_schema(false), + /*wait_for_segcompaction=*/true, &plain_rowset)); + ASSERT_NE(plain_rowset, nullptr); + EXPECT_LT(plain_rowset->rowset_meta()->num_segments(), kVariantSegments); + EXPECT_EQ(kVariantSegments * kVariantRowsPerSegment, plain_rowset->rowset_meta()->num_rows()); + + // No task is ever submitted for the variant rowset, so there is nothing to wait for. + auto variant_schema = create_variant_tablet_schema(true); + RowsetSharedPtr variant_rowset; + ASSERT_NO_FATAL_FAILURE(write_variant_rowset(10061, variant_schema, + /*wait_for_segcompaction=*/false, + &variant_rowset)); + ASSERT_NE(variant_rowset, nullptr); + EXPECT_EQ(kVariantSegments, variant_rowset->rowset_meta()->num_segments()); + EXPECT_EQ(kVariantSegments * kVariantRowsPerSegment, variant_rowset->rowset_meta()->num_rows()); + + std::vector> rows; + ASSERT_NO_FATAL_FAILURE(read_all_rows(variant_rowset, variant_schema, &rows)); + ASSERT_EQ(kVariantSegments * kVariantRowsPerSegment, rows.size()); + for (size_t i = 0; i < rows.size(); ++i) { + ASSERT_EQ(static_cast(i), rows[i].first) << "row " << i; + EXPECT_NE(rows[i].second.find(fmt::format("mark_{}_{}", i / kVariantRowsPerSegment, + i % kVariantRowsPerSegment)), + std::string::npos) + << "row " << i << ": " << rows[i].second; + } +} + } // namespace doris // @brief Test Stub diff --git a/be/test/storage/compaction/vertical_compaction_test.cpp b/be/test/storage/compaction/vertical_compaction_test.cpp index 108ded0a8097bf..70706ebfa5fb7b 100644 --- a/be/test/storage/compaction/vertical_compaction_test.cpp +++ b/be/test/storage/compaction/vertical_compaction_test.cpp @@ -640,11 +640,15 @@ TEST_F(VerticalCompactionTest, TestDupKeyVerticalMerge) { // create output rowset reader RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -778,11 +782,15 @@ TEST_F(VerticalCompactionTest, MergeHonorsKeyRanges) { ASSERT_EQ(expected_end - expected_begin, output_rowset->num_rows()); RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_reader; create_and_init_rowset_reader(output_rowset.get(), reader_context, &output_reader); @@ -895,11 +903,15 @@ TEST_F(VerticalCompactionTest, TestDupWithoutKeyVerticalMerge) { // create output rowset reader RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -1002,11 +1014,15 @@ TEST_F(VerticalCompactionTest, TestUniqueKeyVerticalMerge) { // create output rowset reader RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -1263,11 +1279,15 @@ TEST_F(VerticalCompactionTest, TestUniqueKeySegmentContextMemoryAmplification) { ASSERT_EQ(total_rows, output_rowset->num_rows()); RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared(project_columns_by_ordinal( tablet_schema->columns(), std::vector {0, 1, 2})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; create_and_init_rowset_reader(output_rowset.get(), reader_context, &output_rs_reader); @@ -1382,11 +1402,15 @@ TEST_F(VerticalCompactionTest, TestDupKeyVerticalMergeWithDelete) { // create output rowset reader RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -1484,11 +1508,15 @@ TEST_F(VerticalCompactionTest, TestDupWithoutKeyVerticalMergeWithDelete) { // create output rowset reader RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -1577,11 +1605,15 @@ TEST_F(VerticalCompactionTest, TestAggKeyVerticalMerge) { // create output rowset reader RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -1772,11 +1804,15 @@ TEST_F(VerticalCompactionTest, TestUniqueKeyVerticalMergeWithNullableSparseColum ASSERT_EQ(Status::OK(), output_rs_writer->build(out_rowset)); RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1, 2})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); diff --git a/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp b/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp index f9c7693b67b09a..b2ea2c148235f2 100644 --- a/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp +++ b/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp @@ -1399,10 +1399,9 @@ TEST_F(CollectionStatisticsTest, CollectWithDoubleCastWrappedSlotRef) { EXPECT_TRUE(status.ok()) << status.msg(); } -// Regression for AIR-36: match score collection must resolve indexes for -// variant sub-columns whose indexes live in _path_set_info_map (typed paths or -// inherited sub-column indexes). The previous simple lookup using -// inverted_indexs(col_unique_id, suffix_path) missed those indexes. +// Regression for AIR-36: match score collection must resolve the index of a variant sub-column, +// which is registered on the parent column's unique id under the sub-column's suffix path rather +// than on a unique id of its own. TEST_F(CollectionStatisticsTest, ExtractCollectInfoForVariantSubcolumnIndex) { auto tablet_schema = std::make_shared(); @@ -1433,15 +1432,8 @@ TEST_F(CollectionStatisticsTest, ExtractCollectInfoForVariantSubcolumnIndex) { (*props)["parser"] = "standard"; (*props)["support_phrase"] = "true"; sub_index->init_from_pb(index_pb); - - TabletSchema::PathsSetInfo path_set_info; - TabletIndexes sub_indexes = {sub_index}; - path_set_info.subcolumn_indexes["host"] = sub_indexes; - std::unordered_map path_set_info_map; - path_set_info_map[kVariantUid] = std::move(path_set_info); - tablet_schema->set_path_set_info(std::move(path_set_info_map)); - - EXPECT_TRUE(tablet_schema->inverted_indexs(kVariantUid, "host").empty()); + sub_index->set_escaped_escaped_index_suffix_path(sub_col.suffix_path()); + tablet_schema->append_index(std::move(*sub_index)); auto found = tablet_schema->inverted_indexs(tablet_schema->column(/*ordinal=*/1)); ASSERT_EQ(found.size(), 1u); @@ -2317,9 +2309,6 @@ TEST_F(CollectionStatisticsTest, SearchTypedVariantBindingSelectsItsAnalyzerInde subcolumn.set_path_info(PathInData("v.host", true)); tablet_schema->append_column(subcolumn); - TabletSchema::PathsSetInfo path_set_info; - TabletSchema::SubColumnInfo typed_path_info; - typed_path_info.column = subcolumn; for (const auto& [index_id, parser] : {std::pair {3010, "standard"}, std::pair {3020, "english"}}) { auto index = std::make_shared(); @@ -2331,12 +2320,9 @@ TEST_F(CollectionStatisticsTest, SearchTypedVariantBindingSelectsItsAnalyzerInde (*index_pb.mutable_properties())["parser"] = parser; (*index_pb.mutable_properties())["support_phrase"] = "true"; index->init_from_pb(index_pb); - typed_path_info.indexes.push_back(std::move(index)); + index->set_escaped_escaped_index_suffix_path(subcolumn.suffix_path()); + tablet_schema->append_index(std::move(*index)); } - path_set_info.typed_path_set.emplace("host", std::move(typed_path_info)); - std::unordered_map path_set_info_map; - path_set_info_map.emplace(kVariantUid, std::move(path_set_info)); - tablet_schema->set_path_set_info(std::move(path_set_info_map)); TSearchClause clause; clause.clause_type = "TERM"; diff --git a/be/test/storage/iterator/block_reader_change_next_block_test.cpp b/be/test/storage/iterator/block_reader_change_next_block_test.cpp index a75bd4a6cfe4ee..1a9fbcbb1ee19f 100644 --- a/be/test/storage/iterator/block_reader_change_next_block_test.cpp +++ b/be/test/storage/iterator/block_reader_change_next_block_test.cpp @@ -371,7 +371,11 @@ void configure_reader(BlockReader& reader, std::shared_ptr source, size_t } auto read_schema = std::make_shared(reader._tablet_schema->columns(), std::move(read_types)); - read_schema->init_row_binlog_column_mappings(*reader._tablet_schema); + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*reader._tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/true) + .ok()); reader._read_schema = std::move(read_schema); reader._next_row.block = source; diff --git a/be/test/storage/mow/mow_transform_test_base.h b/be/test/storage/mow/mow_transform_test_base.h index 803f13f838cc90..ffaaf242999aac 100644 --- a/be/test/storage/mow/mow_transform_test_base.h +++ b/be/test/storage/mow/mow_transform_test_base.h @@ -241,8 +241,10 @@ class MowTransformTestBase : public testing::Test { Block* output) { auto read_schema = std::make_shared(schema->columns()); RowsetReaderContext context; - context.tablet_schema = schema; context.read_schema = read_schema; + static_cast(read_schema->init_from_tablet_schema(*schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false)); context.need_ordered_result = true; OlapReaderStatistics statistics; context.stats = &statistics; diff --git a/be/test/storage/read_schema_test.cpp b/be/test/storage/read_schema_test.cpp index 5ee2abd5f0ac49..cd27e5c753346a 100644 --- a/be/test/storage/read_schema_test.cpp +++ b/be/test/storage/read_schema_test.cpp @@ -15,11 +15,13 @@ // specific language governing permissions and limitations // under the License. +#include #include #include #include #include +#include #include #include @@ -28,6 +30,7 @@ #include "core/data_type/data_type_struct.h" #include "storage/binlog.h" #include "storage/schema.h" +#include "util/json/path_in_data.h" namespace doris { namespace { @@ -147,7 +150,11 @@ TEST(ReadSchemaTest, RowBinlogMappingsUsePhysicalSchemaOrdinals) { ReadSchema read_schema(project_columns_by_ordinal( tablet_schema->columns(), std::vector {0, 2, 1, 4, 3, 5, 6, 7})); - read_schema.init_row_binlog_column_mappings(*tablet_schema); + EXPECT_TRUE(read_schema + .init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/true) + .ok()); EXPECT_TRUE(read_schema.row_binlog_value_pairs_complete()); EXPECT_EQ(read_schema.row_binlog_value_column_pairs(), @@ -170,7 +177,10 @@ TEST(ReadSchemaTest, MalformedRowBinlogLayoutKeepsConservativeNameMapping) { tablet_schema.append_column(*create_int_column(16, BINLOG_OP_COL)); ReadSchema read_schema(tablet_schema.columns()); - read_schema.init_row_binlog_column_mappings(tablet_schema); + EXPECT_TRUE(read_schema + .init_from_tablet_schema(tablet_schema, /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/true) + .ok()); EXPECT_FALSE(read_schema.row_binlog_value_pairs_complete()); EXPECT_TRUE(read_schema.row_binlog_value_column_pairs().empty()); @@ -178,4 +188,149 @@ TEST(ReadSchemaTest, MalformedRowBinlogLayoutKeepsConservativeNameMapping) { } } // namespace + +namespace { + +// key k(10), value v(11) governed by sequence column s(12). +TabletSchemaSPtr create_sequence_mapped_schema() { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::UNIQUE_KEYS); + for (const auto& [uid, name, is_key] : {std::tuple {10, "k", true}, + {11, "v", false}, + {12, "s", false}}) { + ColumnPB* column_pb = schema_pb.add_column(); + column_pb->set_unique_id(uid); + column_pb->set_name(name); + column_pb->set_type("INT"); + column_pb->set_length(4); + column_pb->set_index_length(4); + column_pb->set_is_key(is_key); + column_pb->set_is_nullable(!is_key); + } + ColumnGroupPB* group = schema_pb.mutable_seq_map()->add_cg(); + group->set_sequence_column(12); + group->add_columns_in_group(11); + + auto tablet_schema = std::make_shared(); + tablet_schema->init_from_pb(schema_pb); + return tablet_schema; +} + +} // namespace + +// The two facts hold for the whole read: they are taken from the tablet schema, not from the +// projection, so a column group that excludes the columns they are about still reports them. +TEST(ReadSchemaTest, TabletHasSequenceMapIsIndependentOfTheProjection) { + auto tablet_schema = create_sequence_mapped_schema(); + + ReadSchema whole(tablet_schema->columns()); + ASSERT_TRUE(whole.init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); + EXPECT_TRUE(whole.tablet_has_sequence_map()); + + // The key column alone: neither the sequence column nor the value it governs is projected. + ReadSchema key_only( + project_columns_by_ordinal(tablet_schema->columns(), std::vector {0})); + ASSERT_TRUE(key_only.init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); + EXPECT_TRUE(key_only.tablet_has_sequence_map()); +} + +TEST(ReadSchemaTest, TabletWithoutSequenceMap) { + auto tablet_schema = std::make_shared(); + tablet_schema->append_column(*create_int_column(10, "k", true)); + tablet_schema->append_column(*create_int_column(11, "v")); + + ReadSchema read_schema(tablet_schema->columns()); + ASSERT_TRUE(read_schema + .init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); + EXPECT_FALSE(read_schema.tablet_has_sequence_map()); + EXPECT_FALSE(read_schema.tablet_has_extracted_variant_columns()); +} + +TEST(ReadSchemaTest, MergeBySequenceMappingBuildsTheMap) { + auto tablet_schema = create_sequence_mapped_schema(); + ReadSchema read_schema(tablet_schema->columns()); + ASSERT_TRUE(read_schema + .init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/true, + /*map_row_binlog_columns=*/false) + .ok()); + + // Ordinals are this ReadSchema's, not the tablet schema's column ids. + const auto& sequence_map = read_schema.sequence_map(); + ASSERT_EQ(1, sequence_map.size()); + auto group = sequence_map.find(2); + ASSERT_NE(group, sequence_map.end()); + EXPECT_EQ((std::vector {1}), group->second); + + // Without the flag the layout is left alone, even though the tablet has one. + ReadSchema untouched(tablet_schema->columns()); + ASSERT_TRUE(untouched + .init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); + EXPECT_TRUE(untouched.sequence_map().empty()); +} + +// A tablet cannot have both a sequence column and a sequence mapping; asking to merge by the +// mapping on such a schema must fail rather than build half a layout. +TEST(ReadSchemaTest, SequenceColumnAndSequenceMapConflict) { + auto tablet_schema = create_sequence_mapped_schema(); + tablet_schema->append_column(*create_int_column(13, SEQUENCE_COL)); + + ReadSchema read_schema(tablet_schema->columns()); + auto st = read_schema.init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/true, + /*map_row_binlog_columns=*/false); + EXPECT_FALSE(st.ok()) << st; + EXPECT_TRUE(read_schema.sequence_map().empty()); +} + +TEST(ReadSchemaTest, HasExtractedVariantColumns) { + auto tablet_schema = std::make_shared(); + tablet_schema->append_column(*create_int_column(10, "k", true)); + + TabletColumn variant; + variant.set_unique_id(11); + variant.set_name("v"); + variant.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); + variant.set_is_nullable(true); + tablet_schema->append_column(variant); + + ReadSchema without_extracted(tablet_schema->columns()); + ASSERT_TRUE(without_extracted + .init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); + EXPECT_FALSE(without_extracted.tablet_has_extracted_variant_columns()); + + TabletColumn extracted; + extracted.set_unique_id(-1); + extracted.set_name("v.a"); + extracted.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + extracted.set_parent_unique_id(11); + extracted.set_path_info(PathInData("v.a")); + tablet_schema->append_column(extracted); + + // Projected without the extracted column, which must not change the answer. + ReadSchema read_schema( + project_columns_by_ordinal(tablet_schema->columns(), std::vector {0})); + ASSERT_TRUE(read_schema + .init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); + EXPECT_TRUE(read_schema.tablet_has_extracted_variant_columns()); +} + } // namespace doris diff --git a/be/test/storage/row_binlog_vmerge_compaction_test.cpp b/be/test/storage/row_binlog_vmerge_compaction_test.cpp index 62bdd43963d235..3b3f5fbb5beca3 100644 --- a/be/test/storage/row_binlog_vmerge_compaction_test.cpp +++ b/be/test/storage/row_binlog_vmerge_compaction_test.cpp @@ -271,9 +271,13 @@ class RowBinlogVmergeCompactionTest : public testing::Test { project_columns_by_ordinal(tablet_schema->columns(), ordinals)); RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr reader; EXPECT_TRUE(rowset->create_reader(&reader).ok()); diff --git a/be/test/storage/rowid_conversion_test.cpp b/be/test/storage/rowid_conversion_test.cpp index 8c5fa1ab8f6554..2d22964eb6ef65 100644 --- a/be/test/storage/rowid_conversion_test.cpp +++ b/be/test/storage/rowid_conversion_test.cpp @@ -391,11 +391,15 @@ class TestRowIdConversion : public testing::TestWithParam( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -826,12 +830,16 @@ TEST_F(TestRowIdConversion, SingleRowsetGroupedCompactionRowIdConversionIsComple EXPECT_EQ(output_segment_count, output_rowset->num_segments()); RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; std::vector return_columns = {0, 1}; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), return_columns)); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_reader; create_and_init_rowset_reader(output_rowset.get(), reader_context, &output_reader); @@ -1028,9 +1036,13 @@ TEST_F(TestRowIdConversion, SingleRowsetGroupedCompactionRowIdConversionIsComple } RowsetReaderContext second_reader_context; - second_reader_context.tablet_schema = tablet_schema; second_reader_context.need_ordered_result = false; second_reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr second_output_reader; create_and_init_rowset_reader(second_output_rowset.get(), second_reader_context, &second_output_reader); diff --git a/be/test/storage/rowset/segment_flusher_format_test.cpp b/be/test/storage/rowset/segment_flusher_format_test.cpp index ccfa718d4f7013..22802570fd9de2 100644 --- a/be/test/storage/rowset/segment_flusher_format_test.cpp +++ b/be/test/storage/rowset/segment_flusher_format_test.cpp @@ -2228,7 +2228,6 @@ Result read_logical_segment(const std::string& path, uin OlapReaderStatistics stats; StorageReadOptions read_options; read_options.stats = &stats; - read_options.tablet_schema = schema; std::unique_ptr iterator; RETURN_IF_ERROR_RESULT(segment->new_iterator(read_schema, read_options, &iterator)); diff --git a/be/test/storage/segment/mock/mock_segment.h b/be/test/storage/segment/mock/mock_segment.h index a29404ee310098..052d37c8fc5e30 100644 --- a/be/test/storage/segment/mock/mock_segment.h +++ b/be/test/storage/segment/mock/mock_segment.h @@ -37,6 +37,8 @@ class ColumnReaderCache; class MockSegment : public Segment { public: MockSegment() : Segment(1, RowsetId(), std::make_shared(), {}) {} + explicit MockSegment(TabletSchemaSPtr tablet_schema) + : Segment(1, RowsetId(), std::move(tablet_schema), {}) {} ~MockSegment() override = default; // Mock methods for file reader diff --git a/be/test/storage/segment/segment_cache_test.cpp b/be/test/storage/segment/segment_cache_test.cpp index 6e69fa4afbc5eb..5a2ead29ffa80c 100644 --- a/be/test/storage/segment/segment_cache_test.cpp +++ b/be/test/storage/segment/segment_cache_test.cpp @@ -360,7 +360,6 @@ TEST_F(SegmentCacheTest, vec_sequence_col) { OlapReaderStatistics stats; StorageReadOptions opts; opts.stats = &stats; - opts.tablet_schema = rowset->tablet_schema(); std::unique_ptr iter; std::shared_ptr schema = create_full_schema(rowset->tablet_schema()); diff --git a/be/test/storage/segment/segment_iterator_count_emit_shortcut_test.cpp b/be/test/storage/segment/segment_iterator_count_emit_shortcut_test.cpp index 2ac1841ba04b02..0bb6646d54061a 100644 --- a/be/test/storage/segment/segment_iterator_count_emit_shortcut_test.cpp +++ b/be/test/storage/segment/segment_iterator_count_emit_shortcut_test.cpp @@ -23,8 +23,8 @@ // must admit only the provably emission-only configuration and refuse on any // deviation (falling through to today's loop). Uses the established // `#define private public` convention of segment_iterator_limit_opt_test.cpp -// over a bare SegmentIterator (no real segment needed: the shortcut never -// touches segment data). +// over a SegmentIterator whose segment carries only the tablet schema (the +// engage proof reads the keys type; nothing reads segment data). #include #include @@ -42,6 +42,7 @@ #include "storage/index/index_query_context.h" #include "storage/olap_common.h" #include "storage/segment/count_on_index_fastpath.h" +#include "storage/segment/mock/mock_segment.h" #include "storage/tablet/tablet_schema.h" #if defined(__clang__) @@ -111,8 +112,8 @@ struct Fixture { Fixture() { tablet_schema = make_tablet_schema(); read_schema = make_read_schema(tablet_schema); - iter = std::make_unique(nullptr, read_schema); - iter->_opts.tablet_schema = tablet_schema; + iter = std::make_unique(std::make_shared(tablet_schema), + read_schema); iter->_opts.push_down_agg_type_opt = TPushAggOp::COUNT_ON_INDEX; iter->_opts.stats = &stats; // State _lazy_init/_vec_init_lazy_materialization would have produced diff --git a/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp b/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp index 357e2ba876b133..94fa6398c1e161 100644 --- a/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp +++ b/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp @@ -247,7 +247,6 @@ TEST_F(SegmentIteratorExprZonemapTest, NewIteratorPrunesWholeSegmentByExprZonema StorageReadOptions read_options; read_options.stats = &_stats; read_options.runtime_state = &_runtime_state; - read_options.tablet_schema = _tablet_schema; read_options.common_expr_ctxs_push_down = {expr_ctx}; std::unique_ptr iter; @@ -270,7 +269,6 @@ TEST_F(SegmentIteratorExprZonemapTest, NewIteratorKeepsSegmentWhenExprZonemapMay StorageReadOptions read_options; read_options.stats = &_stats; read_options.runtime_state = &_runtime_state; - read_options.tablet_schema = _tablet_schema; read_options.common_expr_ctxs_push_down = {expr_ctx}; std::unique_ptr iter; @@ -290,7 +288,6 @@ TEST_F(SegmentIteratorExprZonemapTest, ApplyExprZonemapPrunesPageRowRanges) { SegmentIterator iter(segment, read_schema); iter._file_reader = segment->_file_reader; iter._opts.stats = &_stats; - iter._opts.tablet_schema = _tablet_schema; auto expr_ctx = std::make_shared(std::make_shared(1, 500)); VExprContextSPtrs conjuncts {expr_ctx}; @@ -313,7 +310,6 @@ TEST_F(SegmentIteratorExprZonemapTest, NewColumnIteratorReadsCommitTsoFromReadOp StorageReadOptions read_options; read_options.stats = &_stats; - read_options.tablet_schema = _tablet_schema; read_options.version = Version(7, 7); read_options.commit_tso = TsoRange(kCommitTso, kCommitTso); read_options.io_ctx.reader_type = ReaderType::READER_QUERY; @@ -354,7 +350,6 @@ TEST_F(SegmentIteratorExprZonemapTest, NewIteratorPrunesCommitTsoByReadOptionVal StorageReadOptions read_options; read_options.stats = &_stats; - read_options.tablet_schema = _tablet_schema; read_options.version = Version(7, 7); read_options.commit_tso = TsoRange(kCommitTso, kCommitTso); read_options.io_ctx.reader_type = ReaderType::READER_QUERY; diff --git a/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp b/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp index 2ae5574688c4f0..fe7df53369a810 100644 --- a/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp +++ b/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp @@ -117,7 +117,6 @@ class SegmentIteratorLazyPrunedTest : public ::testing::Test { std::unique_ptr make_iter(TrackingLazyColumnIterator** tracking_iter) { auto iter = std::make_unique(nullptr, _read_schema); - iter->_opts.tablet_schema = _tablet_schema; iter->_opts.stats = &_stats; iter->_lazy_pruned_ordinals.push_back(0); iter->_column_iterators.resize(1); diff --git a/be/test/storage/segment/segment_iterator_limit_opt_test.cpp b/be/test/storage/segment/segment_iterator_limit_opt_test.cpp index db69371b52b0ed..dac069a29a1799 100644 --- a/be/test/storage/segment/segment_iterator_limit_opt_test.cpp +++ b/be/test/storage/segment/segment_iterator_limit_opt_test.cpp @@ -77,7 +77,6 @@ class SegmentIteratorLimitOptTest : public ::testing::Test { // The segment pointer is null — only _opts and internal maps are accessed. std::unique_ptr make_iter() { auto iter = std::make_unique(nullptr, _read_schema); - iter->_opts.tablet_schema = _tablet_schema; iter->_opts.stats = &_stats; // delete_condition_predicates is default-constructed (empty) return iter; diff --git a/be/test/storage/segment/segment_iterator_no_need_read_data_test.cpp b/be/test/storage/segment/segment_iterator_no_need_read_data_test.cpp index 64627706e39050..0ab1df774fa372 100644 --- a/be/test/storage/segment/segment_iterator_no_need_read_data_test.cpp +++ b/be/test/storage/segment/segment_iterator_no_need_read_data_test.cpp @@ -19,6 +19,7 @@ #include "core/data_type/data_type_variant.h" #include "exec/common/variant_util.h" #include "gtest/gtest.h" +#include "storage/segment/mock/mock_segment.h" #include "storage/segment/segment_iterator.h" #include "storage/tablet/tablet_schema.h" #include "util/json/path_in_data.h" @@ -52,8 +53,7 @@ TEST(SegmentIteratorNoNeedReadDataTest, extracted_variant_count_on_index) { // Read schema covers all tablet columns in order, so ordinal == tablet cid. auto read_schema = std::make_shared(tablet_schema->columns()); - SegmentIterator iter(nullptr, read_schema); - iter._opts.tablet_schema = tablet_schema; + SegmentIterator iter(std::make_shared(tablet_schema), read_schema); iter._opts.push_down_agg_type_opt = TPushAggOp::COUNT_ON_INDEX; iter._column_states[subcol_cid].need_read_data = false; iter._output_column_uids.emplace(1); @@ -87,8 +87,7 @@ TEST(SegmentIteratorNoNeedReadDataTest, zonemap_always_true_predicate_column) { // Read schema covers all tablet columns in order, so ordinal == tablet cid. auto read_schema = std::make_shared(tablet_schema->columns()); - SegmentIterator iter(nullptr, read_schema); - iter._opts.tablet_schema = tablet_schema; + SegmentIterator iter(std::make_shared(tablet_schema), read_schema); iter._opts.zonemap_always_true_pred_cols.emplace(1); EXPECT_FALSE(iter._need_read_data(1)); diff --git a/be/test/storage/segment/segment_writer_write_paths_test.cpp b/be/test/storage/segment/segment_writer_write_paths_test.cpp index 4c70c00b252f43..dabfa2fbd22947 100644 --- a/be/test/storage/segment/segment_writer_write_paths_test.cpp +++ b/be/test/storage/segment/segment_writer_write_paths_test.cpp @@ -231,7 +231,6 @@ class VerticalSegmentWriterWritePathsTest : public testing::Test { OlapReaderStatistics stats; StorageReadOptions read_options; read_options.stats = &stats; - read_options.tablet_schema = schema; std::unique_ptr iterator; ASSERT_TRUE(segment->new_iterator(read_schema, read_options, &iterator).ok()); MutableBlock contents(schema->create_storage_block()); diff --git a/be/test/storage/segment/segments_key_bounds_truncation_test.cpp b/be/test/storage/segment/segments_key_bounds_truncation_test.cpp index 9b56c21abb7975..c077f17153eeae 100644 --- a/be/test/storage/segment/segments_key_bounds_truncation_test.cpp +++ b/be/test/storage/segment/segments_key_bounds_truncation_test.cpp @@ -38,6 +38,7 @@ #include "storage/tablet/tablet_meta.h" #include "storage/tablet/tablet_reader.h" #include "storage/tablet/tablet_schema.h" +#include "util/defer_op.h" namespace doris { static std::string kSegmentDir = "./ut_dir/segments_key_bounds_truncation_test"; @@ -659,6 +660,13 @@ TEST_F(SegmentsKeyBoundsTruncationTest, BlockReaderJudgeFuncTest) { TEST_F(SegmentsKeyBoundsTruncationTest, OrderedCompactionTest) { auto tablet_schema = create_schema(100); + Defer restore_config([saved_enable = config::enable_ordered_data_compaction, + saved_min_size = config::ordered_data_compaction_min_segment_size] { + // Otherwise a one-byte minimum segment size leaks into every later suite and sends its + // compactions down the ordered link-file path instead of a real merge. + config::enable_ordered_data_compaction = saved_enable; + config::ordered_data_compaction_min_segment_size = saved_min_size; + }); config::enable_ordered_data_compaction = true; config::ordered_data_compaction_min_segment_size = 1; diff --git a/be/test/storage/tablet/tablet_schema_test.cpp b/be/test/storage/tablet/tablet_schema_test.cpp index 8732c297db518d..34ad97e63058ef 100644 --- a/be/test/storage/tablet/tablet_schema_test.cpp +++ b/be/test/storage/tablet/tablet_schema_test.cpp @@ -689,148 +689,6 @@ TEST_F(TabletSchemaTest, test_tablet_schema_remove_and_clear_index) { EXPECT_EQ(0, indexes_after_clear.size()); } -TEST_F(TabletSchemaTest, test_tablet_schema_path_set_info_inverted_indexs) { - TabletSchema schema; - - TabletColumn variant_col; - variant_col.set_unique_id(9001); - variant_col.set_name("variant_col"); - variant_col.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); - schema.append_column(variant_col); - - auto create_index = [](int64_t id, const std::string& name, int32_t col_uid) { - auto index = std::make_shared(); - TabletIndexPB index_pb; - index_pb.set_index_id(id); - index_pb.set_index_name(name); - index_pb.set_index_type(IndexType::INVERTED); - index_pb.add_col_unique_id(col_uid); - index->init_from_pb(index_pb); - return index; - }; - - auto typed_index1 = create_index(1001, "typed_path_idx1", 9001); - auto typed_index2 = create_index(1002, "typed_path_idx2", 9001); - - auto subcolumn_index1 = create_index(2001, "subcolumn_idx1", 9001); - auto subcolumn_index2 = create_index(2002, "subcolumn_idx2", 9001); - - TabletSchema::PathsSetInfo path_set_info; - TabletSchema::SubColumnInfo typed_sub_col1; - typed_sub_col1.column = variant_col; - typed_sub_col1.indexes.push_back(typed_index1); - path_set_info.typed_path_set["user.name"] = typed_sub_col1; - - TabletSchema::SubColumnInfo typed_sub_col2; - typed_sub_col2.column = variant_col; - typed_sub_col2.indexes.push_back(typed_index2); - path_set_info.typed_path_set["user.age"] = typed_sub_col2; - - TabletIndexes subcolumn_indexes1 = {subcolumn_index1}; - TabletIndexes subcolumn_indexes2 = {subcolumn_index2}; - path_set_info.subcolumn_indexes["product.id"] = subcolumn_indexes1; - path_set_info.subcolumn_indexes["product.price"] = subcolumn_indexes2; - - std::unordered_map path_set_info_map; - path_set_info_map[9001] = std::move(path_set_info); - schema.set_path_set_info(std::move(path_set_info_map)); - - TabletColumn typed_extracted_col1; - typed_extracted_col1.set_unique_id(-1); - typed_extracted_col1.set_name("variant_col.user.name"); - typed_extracted_col1.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - typed_extracted_col1.set_parent_unique_id(9001); - - PathInData typed_path1("variant_col.user.name", true); - typed_extracted_col1.set_path_info(typed_path1); - - auto typed_indexes = schema.inverted_indexs(typed_extracted_col1); - EXPECT_EQ(1, typed_indexes.size()); - EXPECT_EQ("typed_path_idx1", typed_indexes[0]->index_name()); - EXPECT_EQ(1001, typed_indexes[0]->index_id()); - - TabletColumn typed_extracted_col2; - typed_extracted_col2.set_unique_id(-1); - typed_extracted_col2.set_name("variant_col.user.age"); - typed_extracted_col2.set_type(FieldType::OLAP_FIELD_TYPE_INT); - typed_extracted_col2.set_parent_unique_id(9001); - - PathInData typed_path2("variant_col.user.age", true); - typed_extracted_col2.set_path_info(typed_path2); - - auto typed_indexes2 = schema.inverted_indexs(typed_extracted_col2); - EXPECT_EQ(1, typed_indexes2.size()); - EXPECT_EQ("typed_path_idx2", typed_indexes2[0]->index_name()); - EXPECT_EQ(1002, typed_indexes2[0]->index_id()); - - // Test subcolumn path (non-typed) - TabletColumn subcolumn_extracted_col1; - subcolumn_extracted_col1.set_unique_id(-1); - subcolumn_extracted_col1.set_name("variant_col.product.id"); - subcolumn_extracted_col1.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - subcolumn_extracted_col1.set_parent_unique_id(9001); - - PathInData subcolumn_path1("variant_col.product.id"); - subcolumn_extracted_col1.set_path_info(subcolumn_path1); - - auto subcolumn_indexes = schema.inverted_indexs(subcolumn_extracted_col1); - EXPECT_EQ(1, subcolumn_indexes.size()); - EXPECT_EQ("subcolumn_idx1", subcolumn_indexes[0]->index_name()); - EXPECT_EQ(2001, subcolumn_indexes[0]->index_id()); - - TabletColumn non_existing_col; - non_existing_col.set_unique_id(-1); - non_existing_col.set_name("non_existing"); - non_existing_col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - non_existing_col.set_parent_unique_id(9001); - - PathInData non_existing_path("variant_col.non.existing"); - non_existing_col.set_path_info(non_existing_path); - - auto no_indexes = schema.inverted_indexs(non_existing_col); - EXPECT_EQ(0, no_indexes.size()); - - TabletColumn wrong_parent_col; - wrong_parent_col.set_unique_id(-1); - wrong_parent_col.set_name("wrong_parent"); - wrong_parent_col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - wrong_parent_col.set_parent_unique_id(9999); // Non-existing parent - - PathInData wrong_parent_path("wrong_variant.some.path"); - wrong_parent_col.set_path_info(wrong_parent_path); - - auto no_indexes_wrong_parent = schema.inverted_indexs(wrong_parent_col); - EXPECT_EQ(0, no_indexes_wrong_parent.size()); -} - -TEST_F(TabletSchemaTest, test_tablet_schema_path_set_info_accessors) { - TabletSchema schema; - - TabletColumn variant_col; - variant_col.set_unique_id(10001); - variant_col.set_name("json_data"); - variant_col.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); - schema.append_column(variant_col); - - TabletSchema::PathsSetInfo path_info; - path_info.sub_path_set.insert("extracted_path1"); - path_info.sub_path_set.insert("extracted_path2"); - path_info.sparse_path_set.insert("sparse_path1"); - path_info.sparse_path_set.insert("sparse_path2"); - - std::unordered_map path_map; - path_map[10001] = std::move(path_info); - schema.set_path_set_info(std::move(path_map)); - - const auto& retrieved_info = schema.path_set_info(10001); - EXPECT_EQ(2, retrieved_info.sub_path_set.size()); - EXPECT_EQ(2, retrieved_info.sparse_path_set.size()); - EXPECT_TRUE(retrieved_info.sub_path_set.count("extracted_path1") > 0); - EXPECT_TRUE(retrieved_info.sub_path_set.count("extracted_path2") > 0); - EXPECT_TRUE(retrieved_info.sparse_path_set.count("sparse_path1") > 0); - EXPECT_TRUE(retrieved_info.sparse_path_set.count("sparse_path2") > 0); -} - TEST_F(TabletSchemaTest, test_tablet_schema_inverted_index_by_field_pattern) { TabletSchema schema; diff --git a/be/test/storage/variant/index_storage_variant_compaction_schema_test.cpp b/be/test/storage/variant/index_storage_variant_compaction_schema_test.cpp index f9b50a077e41f2..eb0e1ce10ba041 100644 --- a/be/test/storage/variant/index_storage_variant_compaction_schema_test.cpp +++ b/be/test/storage/variant/index_storage_variant_compaction_schema_test.cpp @@ -33,14 +33,14 @@ TEST(IndexStorageVariantCompactionUtilTest, GetSubpathsHonorsZeroLimitAndTieOrde {"gamma", 1}, }; - TabletSchema::PathsSetInfo unlimited; + VariantCompactionPaths unlimited; variant_util::VariantCompactionUtil::get_subpaths(0, stats, unlimited); EXPECT_TRUE(unlimited.sub_path_set.contains(StringRef("alpha"))); EXPECT_TRUE(unlimited.sub_path_set.contains(StringRef("beta"))); EXPECT_TRUE(unlimited.sub_path_set.contains(StringRef("gamma"))); EXPECT_TRUE(unlimited.sparse_path_set.empty()); - TabletSchema::PathsSetInfo top_one; + VariantCompactionPaths top_one; variant_util::VariantCompactionUtil::get_subpaths(1, stats, top_one); EXPECT_TRUE(top_one.sub_path_set.contains(StringRef("beta"))); EXPECT_FALSE(top_one.sub_path_set.contains(StringRef("alpha"))); @@ -54,13 +54,13 @@ TEST(IndexStorageVariantCompactionUtilTest, GetSubpathsKeepsAllPathsAtLimitAndHa {"beta", 1}, }; - TabletSchema::PathsSetInfo at_limit; + VariantCompactionPaths at_limit; variant_util::VariantCompactionUtil::get_subpaths(2, exact_limit, at_limit); EXPECT_TRUE(at_limit.sub_path_set.contains(StringRef("alpha"))); EXPECT_TRUE(at_limit.sub_path_set.contains(StringRef("beta"))); EXPECT_TRUE(at_limit.sparse_path_set.empty()); - TabletSchema::PathsSetInfo empty; + VariantCompactionPaths empty; variant_util::VariantCompactionUtil::get_subpaths(1, {}, empty); EXPECT_TRUE(empty.sub_path_set.empty()); EXPECT_TRUE(empty.sparse_path_set.empty()); @@ -82,12 +82,13 @@ TEST(IndexStorageVariantCompactionUtilTest, EmptyInputsKeepVariantSchemaWithoutP nullptr); auto compaction_schema = std::make_shared(*base_schema); + VariantCompactionPathsMap compaction_paths; auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - {}, compaction_schema); + {}, compaction_schema, compaction_paths); ASSERT_TRUE(status.ok()) << status.to_string(); ASSERT_TRUE(compaction_schema->has_column_unique_id(2)); - const auto* path_set_info = compaction_schema->try_path_set_info(2); + const auto* path_set_info = compaction_paths.contains(2) ? &compaction_paths.at(2) : nullptr; ASSERT_NE(path_set_info, nullptr); EXPECT_TRUE(path_set_info->typed_path_set.empty()); EXPECT_TRUE(path_set_info->sub_path_set.empty()); @@ -114,11 +115,12 @@ TEST_F(IndexStorageVariantCompactionSchemaTest, ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); auto compaction_schema = std::make_shared(*tablet_schema()); + VariantCompactionPathsMap compaction_paths; auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - {rowset_result.value()}, compaction_schema); + {rowset_result.value()}, compaction_schema, compaction_paths); ASSERT_TRUE(status.ok()) << status.to_string(); - const auto* path_set_info = compaction_schema->try_path_set_info(2); + const auto* path_set_info = compaction_paths.contains(2) ? &compaction_paths.at(2) : nullptr; ASSERT_NE(path_set_info, nullptr); EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("alpha"))); EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("beta"))); @@ -149,11 +151,12 @@ TEST_F(IndexStorageVariantCompactionSchemaTest, ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); auto compaction_schema = std::make_shared(*tablet_schema()); + VariantCompactionPathsMap compaction_paths; auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - {rowset_result.value()}, compaction_schema); + {rowset_result.value()}, compaction_schema, compaction_paths); ASSERT_TRUE(status.ok()) << status.to_string(); - const auto* path_set_info = compaction_schema->try_path_set_info(2); + const auto* path_set_info = compaction_paths.contains(2) ? &compaction_paths.at(2) : nullptr; ASSERT_NE(path_set_info, nullptr); EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("alpha"))); EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("beta"))); @@ -190,11 +193,12 @@ TEST_F(IndexStorageVariantCompactionSchemaTest, ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); auto compaction_schema = std::make_shared(*tablet_schema()); + VariantCompactionPathsMap compaction_paths; auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - {rowset_result.value()}, compaction_schema); + {rowset_result.value()}, compaction_schema, compaction_paths); ASSERT_TRUE(status.ok()) << status.to_string(); - const auto* path_set_info = compaction_schema->try_path_set_info(2); + const auto* path_set_info = compaction_paths.contains(2) ? &compaction_paths.at(2) : nullptr; ASSERT_NE(path_set_info, nullptr); EXPECT_TRUE(path_set_info->typed_path_set.contains("typed_i")); EXPECT_FALSE(path_set_info->sub_path_set.contains(StringRef("typed_i"))); diff --git a/be/test/storage/variant/index_storage_variant_io_context_test.cpp b/be/test/storage/variant/index_storage_variant_io_context_test.cpp index cd3fbdaa5b38c7..603d8a6119bc91 100644 --- a/be/test/storage/variant/index_storage_variant_io_context_test.cpp +++ b/be/test/storage/variant/index_storage_variant_io_context_test.cpp @@ -74,7 +74,6 @@ TEST_F(IndexStorageVariantIoContextTest, OlapReaderStatistics stats; StorageReadOptions read_options; read_options.stats = &stats; - read_options.tablet_schema = reader_schema; read_options.io_ctx.reader_type = ReaderType::READER_QUERY; read_options.io_ctx.query_id = &query_id; read_options.io_ctx.file_cache_stats = &stats.file_cache_stats; diff --git a/be/test/storage/variant/index_storage_variant_sparse_stats_test.cpp b/be/test/storage/variant/index_storage_variant_sparse_stats_test.cpp index c1b8909ed9c1a4..67de811e830017 100644 --- a/be/test/storage/variant/index_storage_variant_sparse_stats_test.cpp +++ b/be/test/storage/variant/index_storage_variant_sparse_stats_test.cpp @@ -90,11 +90,12 @@ TEST_F(IndexStorageVariantSparseStatsTest, VariantCompactionSchemaTopNRecordsSpa ASSERT_TRUE(rowsets.has_value()) << rowsets.error(); auto compaction_schema = std::make_shared(*tablet_schema()); + VariantCompactionPathsMap compaction_paths; auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - rowsets.value(), compaction_schema); + rowsets.value(), compaction_schema, compaction_paths); ASSERT_TRUE(status.ok()) << status.to_string(); - const auto* path_set_info = compaction_schema->try_path_set_info(2); + const auto* path_set_info = compaction_paths.contains(2) ? &compaction_paths.at(2) : nullptr; ASSERT_NE(path_set_info, nullptr); EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("hot"))); EXPECT_FALSE(path_set_info->sub_path_set.contains(StringRef("warm"))); diff --git a/be/test/storage/variant/variant_column_writer_reader_test.cpp b/be/test/storage/variant/variant_column_writer_reader_test.cpp index cf0c86f211bc7e..1c4dda3f89cdaf 100644 --- a/be/test/storage/variant/variant_column_writer_reader_test.cpp +++ b/be/test/storage/variant/variant_column_writer_reader_test.cpp @@ -87,8 +87,23 @@ constexpr static std::string_view tmp_dir = "./ut_dir/tmp"; enum class VariantWriterInput : uint8_t { V2 }; +// Mirror of what BetaRowsetReader derives from the schema a read targets: it drives whether a +// compaction or checksum read flattens variant subcolumns instead of reading them hierarchically. +static void set_variant_read_facts(StorageReadOptions& opts, const TabletSchemaSPtr& schema) { + opts.tablet_has_extracted_variant_columns = std::ranges::any_of( + schema->columns(), [](const auto& column) { return column->is_extracted_column(); }); +} + enum class VariantIndexWritePolicy : uint8_t { NONE, BLOOM_AND_INVERTED }; +// Resolve a query slot against a schema the way the point-query and rowid-fetch callers do. +static const TabletColumn& read_column_of(const TabletSchemaSPtr& schema, SlotDescriptor* slot) { + int32_t index = slot->col_unique_id() >= 0 ? schema->field_index(slot->col_unique_id()) + : schema->field_index(slot->col_name()); + CHECK_GE(index, 0) << "slot not in schema: " << slot->col_name(); + return schema->column(index); +} + static std::string variant_writer_input_name(VariantWriterInput input) { return "V2"; } @@ -1371,11 +1386,15 @@ class VariantColumnWriterReaderTest : public testing::Test { RETURN_IF_ERROR(rowset->create_reader(&reader)); RowsetReaderContext reader_context; - reader_context.tablet_schema = _tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(_tablet_schema->columns(), std::vector {0})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*_tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RETURN_IF_ERROR(reader->init(&reader_context)); rows->clear(); @@ -1773,7 +1792,6 @@ class VariantColumnWriterReaderTest : public testing::Test { StorageReadOptions read_opts; read_opts.io_ctx.reader_type = ReaderType::READER_QUERY; - read_opts.tablet_schema = _tablet_schema; OlapReaderStatistics stats; read_opts.stats = &stats; @@ -2661,7 +2679,7 @@ TEST_F(VariantColumnWriterReaderTest, OlapReaderStatistics* stats) { StorageReadOptions read_opts; read_opts.io_ctx.reader_type = ReaderType::READER_BASE_COMPACTION; - read_opts.tablet_schema = compaction_schema; + set_variant_read_facts(read_opts, compaction_schema); read_opts.stats = stats; const size_t reader_calls = column_reader_cache.path_column_reader_calls(); ASSERT_TRUE( @@ -3646,16 +3664,17 @@ TEST_F(VariantColumnWriterReaderTest, test_segment_rowid_read_by_reader_version) StorageReadOptions read_options; read_options.stats = &stats; read_options.io_ctx.reader_type = ReaderType::READER_QUERY; - read_options.tablet_schema = _tablet_schema; MutableColumnPtr whole_result = slots[0]->type()->create_column(); ColumnIteratorUPtr whole_iterator; - auto st = segments[0]->seek_and_read_by_rowid(*_tablet_schema, slots[0], row_ids, whole_result, - read_options, whole_iterator); + auto st = segments[0]->seek_and_read_by_rowid(read_column_of(_tablet_schema, slots[0]), + slots[0], row_ids, whole_result, read_options, + whole_iterator); ASSERT_TRUE(st.ok()) << st.to_string(); auto* const whole_iterator_address = whole_iterator.get(); - st = segments[0]->seek_and_read_by_rowid(*_tablet_schema, slots[0], second_row_ids, - whole_result, read_options, whole_iterator); + st = segments[0]->seek_and_read_by_rowid(read_column_of(_tablet_schema, slots[0]), slots[0], + second_row_ids, whole_result, read_options, + whole_iterator); ASSERT_TRUE(st.ok()) << st.to_string(); EXPECT_EQ(whole_iterator.get(), whole_iterator_address); ASSERT_EQ(whole_result->size(), jsons.size()); @@ -3665,12 +3684,13 @@ TEST_F(VariantColumnWriterReaderTest, test_segment_rowid_read_by_reader_version) MutableColumnPtr hot_result = slots[1]->type()->create_column(); ColumnIteratorUPtr hot_iterator; - st = segments[0]->seek_and_read_by_rowid(*_tablet_schema, slots[1], row_ids, hot_result, - read_options, hot_iterator); + st = segments[0]->seek_and_read_by_rowid(read_column_of(_tablet_schema, slots[1]), slots[1], + row_ids, hot_result, read_options, hot_iterator); ASSERT_TRUE(st.ok()) << st.to_string(); auto* const hot_iterator_address = hot_iterator.get(); - st = segments[0]->seek_and_read_by_rowid(*_tablet_schema, slots[1], second_row_ids, hot_result, - read_options, hot_iterator); + st = segments[0]->seek_and_read_by_rowid(read_column_of(_tablet_schema, slots[1]), slots[1], + second_row_ids, hot_result, read_options, + hot_iterator); ASSERT_TRUE(st.ok()) << st.to_string(); EXPECT_EQ(hot_iterator.get(), hot_iterator_address); const auto& nullable_hot = assert_cast(*hot_result); @@ -3684,12 +3704,14 @@ TEST_F(VariantColumnWriterReaderTest, test_segment_rowid_read_by_reader_version) MutableColumnPtr subpath_result = slots[2]->type()->create_column(); ColumnIteratorUPtr subpath_iterator; - st = segments[0]->seek_and_read_by_rowid(*_tablet_schema, slots[2], row_ids, subpath_result, - read_options, subpath_iterator); + st = segments[0]->seek_and_read_by_rowid(read_column_of(_tablet_schema, slots[2]), slots[2], + row_ids, subpath_result, read_options, + subpath_iterator); ASSERT_TRUE(st.ok()) << st.to_string(); auto* const subpath_iterator_address = subpath_iterator.get(); - st = segments[0]->seek_and_read_by_rowid(*_tablet_schema, slots[2], second_row_ids, - subpath_result, read_options, subpath_iterator); + st = segments[0]->seek_and_read_by_rowid(read_column_of(_tablet_schema, slots[2]), slots[2], + second_row_ids, subpath_result, read_options, + subpath_iterator); ASSERT_TRUE(st.ok()) << st.to_string(); EXPECT_EQ(subpath_iterator.get(), subpath_iterator_address); const auto& nullable_subpath = assert_cast(*subpath_result); @@ -3751,13 +3773,12 @@ TEST_F(VariantColumnWriterReaderTest, test_segment_rowid_read_by_reader_version) StorageReadOptions empty_read_options; empty_read_options.stats = &empty_stats; empty_read_options.io_ctx.reader_type = ReaderType::READER_QUERY; - empty_read_options.tablet_schema = _tablet_schema; MutableColumnPtr empty_whole_result = empty_slots[0]->type()->create_column(); ColumnIteratorUPtr empty_whole_iterator; - st = empty_nested_segments[0]->seek_and_read_by_rowid(*_tablet_schema, empty_slots[0], - empty_row_ids, empty_whole_result, - empty_read_options, empty_whole_iterator); + st = empty_nested_segments[0]->seek_and_read_by_rowid( + read_column_of(_tablet_schema, empty_slots[0]), empty_slots[0], empty_row_ids, + empty_whole_result, empty_read_options, empty_whole_iterator); ASSERT_TRUE(st.ok()) << st.to_string(); ASSERT_EQ(empty_whole_result->size(), expected_whole.size()); for (size_t row = 0; row < expected_whole.size(); ++row) { @@ -3767,8 +3788,8 @@ TEST_F(VariantColumnWriterReaderTest, test_segment_rowid_read_by_reader_version) MutableColumnPtr empty_subpath_result = empty_slots[1]->type()->create_column(); ColumnIteratorUPtr empty_subpath_iterator; st = empty_nested_segments[0]->seek_and_read_by_rowid( - *_tablet_schema, empty_slots[1], empty_row_ids, empty_subpath_result, - empty_read_options, empty_subpath_iterator); + read_column_of(_tablet_schema, empty_slots[1]), empty_slots[1], empty_row_ids, + empty_subpath_result, empty_read_options, empty_subpath_iterator); ASSERT_TRUE(st.ok()) << st.to_string(); const auto& nullable = assert_cast(*empty_subpath_result); const auto& values = nullable.get_nested_column(); @@ -4196,9 +4217,8 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) { // construct tablet schema for compaction storage_read_opts.io_ctx.reader_type = ReaderType::READER_BASE_COMPACTION; - storage_read_opts.tablet_schema = _tablet_schema; - std::unordered_map uid_to_paths_set_info; - TabletSchema::PathsSetInfo paths_set_info; + VariantCompactionPathsMap uid_to_paths_set_info; + VariantCompactionPaths paths_set_info; paths_set_info.sub_path_set.insert("key0"); paths_set_info.sub_path_set.insert("key3"); paths_set_info.sub_path_set.insert("key4"); @@ -4210,7 +4230,8 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) { paths_set_info.sparse_path_set.insert("key8"); paths_set_info.sparse_path_set.insert("key9"); uid_to_paths_set_info[parent_column.unique_id()] = paths_set_info; - _tablet_schema->set_path_set_info(std::move(uid_to_paths_set_info)); + storage_read_opts.variant_compaction_paths = + std::make_shared(std::move(uid_to_paths_set_info)); // mock a subcolumn in compaction TabletColumn subcolumn_in_compaction; @@ -4220,6 +4241,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) { subcolumn_in_compaction.set_path_info(PathInData(parent_column.name_lower_case() + ".key10")); subcolumn_in_compaction.set_is_nullable(true); _tablet_schema->append_column(subcolumn_in_compaction); + set_variant_read_facts(storage_read_opts, _tablet_schema); // 14. check compaction subcolumn reader check_leaf_reader(); @@ -4714,7 +4736,6 @@ TEST_F(VariantColumnWriterReaderTest, TabletColumn parent_column_v2 = parent_column; StorageReadOptions v2_query_read_opts; v2_query_read_opts.io_ctx.reader_type = ReaderType::READER_QUERY; - v2_query_read_opts.tablet_schema = _tablet_schema; OlapReaderStatistics v2_query_stats; v2_query_read_opts.stats = &v2_query_stats; ColumnIteratorUPtr v2_root_it; @@ -4745,7 +4766,7 @@ TEST_F(VariantColumnWriterReaderTest, StorageReadOptions compact_read_opts; compact_read_opts.io_ctx.reader_type = ReaderType::READER_BASE_COMPACTION; - compact_read_opts.tablet_schema = _tablet_schema; + set_variant_read_facts(compact_read_opts, _tablet_schema); OlapReaderStatistics compact_stats; compact_read_opts.stats = &compact_stats; TabletColumn doc_bucket_col = variant_util::create_doc_value_column(parent_column, 0); @@ -4962,7 +4983,7 @@ TEST_F(VariantColumnWriterReaderTest, test_read_doc_compact_from_doc_value_bucke StorageReadOptions storage_read_opts; storage_read_opts.io_ctx.reader_type = ReaderType::READER_BASE_COMPACTION; - storage_read_opts.tablet_schema = compaction_schema; + set_variant_read_facts(storage_read_opts, compaction_schema); OlapReaderStatistics stats; storage_read_opts.stats = &stats; @@ -5188,7 +5209,7 @@ TEST_P(VariantSpecializedWriterCompatibilityTest, doc_compact_writer_round_trip) MockColumnReaderCache column_reader_cache(footer, file_reader, _tablet_schema); StorageReadOptions storage_read_opts; storage_read_opts.io_ctx.reader_type = ReaderType::READER_BASE_COMPACTION; - storage_read_opts.tablet_schema = _tablet_schema; + set_variant_read_facts(storage_read_opts, _tablet_schema); OlapReaderStatistics stats; storage_read_opts.stats = &stats; @@ -5954,11 +5975,12 @@ TEST_F(VariantColumnWriterReaderTest, std::vector input_rowsets {rowset}; auto compaction_schema = std::make_shared(*_tablet_schema); + VariantCompactionPathsMap compaction_paths; auto st = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - input_rowsets, compaction_schema); + input_rowsets, compaction_schema, compaction_paths); ASSERT_TRUE(st.ok()) << st.to_string(); - const auto* path_set_info = compaction_schema->try_path_set_info(1); + const auto* path_set_info = compaction_paths.contains(1) ? &compaction_paths.at(1) : nullptr; ASSERT_NE(path_set_info, nullptr); ASSERT_TRUE(path_set_info->typed_path_set.contains("a")); EXPECT_FALSE(path_set_info->sub_path_set.contains(StringRef("a"))); @@ -6935,7 +6957,6 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable) { StorageReadOptions v2_read_opts; v2_read_opts.stats = &v2_stats; v2_read_opts.io_ctx.reader_type = ReaderType::READER_QUERY; - v2_read_opts.tablet_schema = _tablet_schema; ColumnIteratorUPtr v2_it; st = variant_column_reader->new_iterator(&v2_it, &parent_column_v2, &v2_read_opts, &column_reader_cache); @@ -6967,7 +6988,6 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable) { StorageReadOptions v2_rowid_read_opts; v2_rowid_read_opts.stats = &v2_rowid_stats; v2_rowid_read_opts.io_ctx.reader_type = ReaderType::READER_QUERY; - v2_rowid_read_opts.tablet_schema = _tablet_schema; ColumnIteratorUPtr v2_rowid_it; st = variant_column_reader->new_iterator(&v2_rowid_it, &parent_column_v2, &v2_rowid_read_opts, &column_reader_cache); @@ -8125,8 +8145,6 @@ TEST_F(VariantColumnWriterReaderTest, test_read_with_checksum) { TabletColumn parent_column = _tablet_schema->column(0); StorageReadOptions storage_read_opts; - storage_read_opts.tablet_schema = _tablet_schema; - TabletColumn subcolumn; subcolumn.set_name(parent_column.name_lower_case() + ".b"); subcolumn.set_type((FieldType)(int)footer.columns(1).type()); @@ -8135,6 +8153,7 @@ TEST_F(VariantColumnWriterReaderTest, test_read_with_checksum) { subcolumn.set_variant_max_subcolumns_count(parent_column.variant_max_subcolumns_count()); subcolumn.set_is_nullable(true); _tablet_schema->append_column(subcolumn); + set_variant_read_facts(storage_read_opts, _tablet_schema); storage_read_opts.io_ctx.reader_type = ReaderType::READER_QUERY; OlapReaderStatistics stats; storage_read_opts.stats = &stats; @@ -8440,8 +8459,9 @@ TEST_F(VariantColumnWriterReaderTest, test_compaction_nokey_variant_uid0) { auto input_readers = create_rowset_readers(input_rowsets); auto compaction_schema = std::make_shared(*_tablet_schema); + auto compaction_paths = std::make_shared(); auto st = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - input_rowsets, compaction_schema); + input_rowsets, compaction_schema, *compaction_paths); ASSERT_TRUE(st.ok()) << st.to_string(); RowsetWriterContext ctx; @@ -8452,6 +8472,7 @@ TEST_F(VariantColumnWriterReaderTest, test_compaction_nokey_variant_uid0) { ctx.data_dir = _data_dir.get(); ctx.rowset_state = VISIBLE; ctx.tablet_schema = compaction_schema; + ctx.variant_compaction_paths = compaction_paths; ctx.tablet_path = _tablet->tablet_path(); ctx.tablet_id = _tablet->tablet_id(); ctx.tablet = _tablet; @@ -8535,8 +8556,9 @@ TEST_F(VariantColumnWriterReaderTest, legacy_v1_segment_compaction_preserves_ful auto input_readers = create_rowset_readers(input_rowsets); auto compaction_schema = std::make_shared(*_tablet_schema); - status = variant_util::VariantCompactionUtil::get_extended_compaction_schema(input_rowsets, - compaction_schema); + auto compaction_paths = std::make_shared(); + status = variant_util::VariantCompactionUtil::get_extended_compaction_schema( + input_rowsets, compaction_schema, *compaction_paths); ASSERT_TRUE(status.ok()) << status; RowsetWriterContext context; @@ -8547,6 +8569,7 @@ TEST_F(VariantColumnWriterReaderTest, legacy_v1_segment_compaction_preserves_ful context.data_dir = _data_dir.get(); context.rowset_state = VISIBLE; context.tablet_schema = compaction_schema; + context.variant_compaction_paths = compaction_paths; context.tablet_path = _tablet->tablet_path(); context.tablet_id = _tablet->tablet_id(); context.tablet = _tablet; diff --git a/be/test/storage/variant/variant_compaction_paths_test.cpp b/be/test/storage/variant/variant_compaction_paths_test.cpp new file mode 100644 index 00000000000000..eec0233c424a2a --- /dev/null +++ b/be/test/storage/variant/variant_compaction_paths_test.cpp @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/segment/variant/variant_compaction_paths.h" + +#include +#include + +#include +#include +#include + +#include "storage/tablet/tablet_schema.h" +#include "util/json/path_in_data.h" + +namespace doris { + +namespace { + +constexpr int32_t kVariantUid = 9001; + +TabletIndexPtr make_inverted_index(int64_t index_id, const std::string& name) { + auto index = std::make_shared(); + TabletIndexPB index_pb; + index_pb.set_index_id(index_id); + index_pb.set_index_name(name); + index_pb.set_index_type(IndexType::INVERTED); + index_pb.add_col_unique_id(kVariantUid); + index->init_from_pb(index_pb); + return index; +} + +// An extracted column of the variant above. `typed` selects which half of the layout the lookup +// is expected to consult. +TabletColumn make_extracted_column(const std::string& path, bool typed, + FieldType type = FieldType::OLAP_FIELD_TYPE_STRING, + int32_t parent_uid = kVariantUid) { + TabletColumn column; + column.set_unique_id(-1); + column.set_name("v." + path); + column.set_type(type); + column.set_parent_unique_id(parent_uid); + column.set_path_info(PathInData("v." + path, typed)); + return column; +} + +// typed_path_set["user.name"] -> index 1001, subcolumn_indexes["product.id"] -> index 2001. +VariantCompactionPathsMap make_layout() { + VariantCompactionPaths paths; + + TabletSchema::SubColumnInfo typed; + typed.indexes.push_back(make_inverted_index(1001, "typed_path_idx")); + paths.typed_path_set["user.name"] = std::move(typed); + + paths.subcolumn_indexes["product.id"] = {make_inverted_index(2001, "subcolumn_idx")}; + + VariantCompactionPathsMap layout; + layout[kVariantUid] = std::move(paths); + return layout; +} + +} // namespace + +TEST(VariantCompactionPathsTest, TypedPathTakesItsOwnIndexes) { + const auto layout = make_layout(); + auto indexes = variant_subcolumn_indexes(&layout, make_extracted_column("user.name", true)); + ASSERT_EQ(1, indexes.size()); + EXPECT_EQ(1001, indexes[0]->index_id()); + EXPECT_EQ("typed_path_idx", indexes[0]->index_name()); +} + +TEST(VariantCompactionPathsTest, SubcolumnPathTakesItsOwnIndexes) { + const auto layout = make_layout(); + auto indexes = variant_subcolumn_indexes(&layout, make_extracted_column("product.id", false)); + ASSERT_EQ(1, indexes.size()); + EXPECT_EQ(2001, indexes[0]->index_id()); + EXPECT_EQ("subcolumn_idx", indexes[0]->index_name()); +} + +// A typed path is looked up only in typed_path_set, and a plain one only in subcolumn_indexes, +// so asking for either under the wrong flavour finds nothing. +TEST(VariantCompactionPathsTest, PathIsNotFoundInTheOtherHalfOfTheLayout) { + const auto layout = make_layout(); + EXPECT_TRUE( + variant_subcolumn_indexes(&layout, make_extracted_column("user.name", false)).empty()); + EXPECT_TRUE( + variant_subcolumn_indexes(&layout, make_extracted_column("product.id", true)).empty()); +} + +TEST(VariantCompactionPathsTest, PathAbsentFromTheLayout) { + const auto layout = make_layout(); + EXPECT_TRUE(variant_subcolumn_indexes(&layout, make_extracted_column("non.existing", false)) + .empty()); +} + +TEST(VariantCompactionPathsTest, ParentColumnAbsentFromTheLayout) { + const auto layout = make_layout(); + auto column = make_extracted_column("user.name", true, FieldType::OLAP_FIELD_TYPE_STRING, + /*parent_uid=*/9999); + EXPECT_TRUE(variant_subcolumn_indexes(&layout, column).empty()); +} + +// Every write that is not a compaction passes no layout at all. +TEST(VariantCompactionPathsTest, NoLayout) { + EXPECT_TRUE( + variant_subcolumn_indexes(nullptr, make_extracted_column("user.name", true)).empty()); +} + +// A column that is not extracted never has a layout entry, whatever the layout holds. +// A plain column carries its indexes on the schema, never in the layout. STRING so that the +// extracted-column guard is the only thing that can refuse it. +TEST(VariantCompactionPathsTest, NonExtractedColumn) { + const auto layout = make_layout(); + TabletColumn column; + column.set_unique_id(kVariantUid); + column.set_name("user.name"); + column.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + ASSERT_FALSE(column.is_extracted_column()); + EXPECT_TRUE(variant_subcolumn_indexes(&layout, column).empty()); +} + +// Types that cannot carry an inverted index are refused before the layout is consulted, even +// though their path is in it: JSONB and variant are rejected outright, and an array only passes +// when its element type does (IndexColumnWriter::check_support_inverted_index). +TEST(VariantCompactionPathsTest, TypeThatCannotCarryAnInvertedIndex) { + const auto layout = make_layout(); + for (auto type : {FieldType::OLAP_FIELD_TYPE_JSONB, FieldType::OLAP_FIELD_TYPE_VARIANT}) { + auto column = make_extracted_column("user.name", true, type); + EXPECT_TRUE(variant_subcolumn_indexes(&layout, column).empty()) + << "type=" << static_cast(type); + } + + // A scalar element type does pass, so the path is found. + auto scalar = make_extracted_column("user.name", true, FieldType::OLAP_FIELD_TYPE_DOUBLE); + EXPECT_EQ(1, variant_subcolumn_indexes(&layout, scalar).size()); +} + +} // namespace doris diff --git a/be/test/storage/variant/variant_doc_mode_compaction_test.cpp b/be/test/storage/variant/variant_doc_mode_compaction_test.cpp index 6a4b3190111d8a..50898187de347e 100644 --- a/be/test/storage/variant/variant_doc_mode_compaction_test.cpp +++ b/be/test/storage/variant/variant_doc_mode_compaction_test.cpp @@ -423,12 +423,16 @@ TEST_F(VariantDocModeCompactionTest, variant_doc_mode_compaction_merge_10_segmen << " elapsed_ms=" << import_elapsed_ms << std::endl; if (i == 0) { RowsetReaderContext input_reader_context; - input_reader_context.tablet_schema = tablet_schema; input_reader_context.need_ordered_result = false; std::vector input_return_columns = {1}; auto input_read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), input_return_columns)); input_reader_context.read_schema = input_read_schema; + EXPECT_TRUE(input_read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr input_rs_reader; create_and_init_rowset_reader(rowset.get(), input_reader_context, &input_rs_reader); @@ -472,12 +476,16 @@ TEST_F(VariantDocModeCompactionTest, variant_doc_mode_compaction_merge_10_segmen ASSERT_EQ(static_cast(kRowsPerSegment) * 10, out_rowset->rowset_meta()->num_rows()); RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; std::vector return_columns = {0}; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), return_columns)); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); diff --git a/be/test/testutil/index_storage_test_util.cpp b/be/test/testutil/index_storage_test_util.cpp index a8e98149c572bb..e72a35a58ccc29 100644 --- a/be/test/testutil/index_storage_test_util.cpp +++ b/be/test/testutil/index_storage_test_util.cpp @@ -1303,9 +1303,13 @@ Result IndexStorageTestFixture::read_rowsets( RowsetReaderContext context; context.reader_type = options.reader_type; - context.tablet_schema = _tablet_schema; context.need_ordered_result = options.need_ordered_result; context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*_tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); context.predicates = &predicates; context.stats = &result.stats; context.target_cast_type_for_variants = options.target_cast_type_for_variants;