diff --git a/be/src/cloud/cloud_schema_change_job.cpp b/be/src/cloud/cloud_schema_change_job.cpp index c269f7dcb1bca2..ae39f8856f07e6 100644 --- a/be/src/cloud/cloud_schema_change_job.cpp +++ b/be/src/cloud/cloud_schema_change_job.cpp @@ -244,6 +244,9 @@ Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& reque _new_tablet_schema = _new_tablet->tablet_schema(); ReadSchemaSPtr read_schema = std::make_shared(_base_tablet_schema->columns()); + RETURN_IF_ERROR(read_schema->init_from_tablet_schema(*_base_tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false)); // delete handlers to filter out deleted rows DeleteHandler delete_handler; @@ -262,7 +265,6 @@ Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& reque // reader_context is stack variables, it's lifetime MUST keep the same with rs_readers RowsetReaderContext reader_context; reader_context.reader_type = ReaderType::READER_ALTER_TABLE; - reader_context.tablet_schema = _base_tablet_schema; reader_context.need_ordered_result = true; reader_context.delete_handler = &delete_handler; reader_context.read_schema = read_schema; diff --git a/be/src/exec/common/variant_util.cpp b/be/src/exec/common/variant_util.cpp index e905602b695cab..e2b46cbe47d1a4 100644 --- a/be/src/exec/common/variant_util.cpp +++ b/be/src/exec/common/variant_util.cpp @@ -117,7 +117,7 @@ PathInData make_full_subcolumn_path(const TabletColumnPtr& parent_column, std::s return builder.append(parent_column->name_lower_case(), false).append("", false).build(); } -void append_empty_key_subcolumn_from_stats(TabletSchema::PathsSetInfo& paths_set_info, +void append_empty_key_subcolumn_from_stats(VariantCompactionPaths& paths_set_info, const TabletColumnPtr& parent_column, TabletSchemaSPtr& output_schema) { if (!paths_set_info.sub_path_set.contains("") || paths_set_info.sparse_path_set.contains("") || @@ -1002,7 +1002,7 @@ Status VariantCompactionUtil::aggregate_variant_extended_info( // get the subpaths and sparse paths for the variant column void VariantCompactionUtil::get_subpaths(int32_t max_subcolumns_count, const PathToNoneNullValues& stats, - TabletSchema::PathsSetInfo& paths_set_info) { + VariantCompactionPaths& paths_set_info) { // max_subcolumns_count is 0 means no limit if (max_subcolumns_count > 0 && stats.size() > max_subcolumns_count) { std::vector> paths_with_sizes; @@ -1148,7 +1148,7 @@ Status VariantCompactionUtil::check_path_stats(const std::vector& typed_paths, const TabletColumnPtr parent_column, TabletSchemaSPtr& output_schema, - TabletSchema::PathsSetInfo& paths_set_info) { + VariantCompactionPaths& paths_set_info) { if (parent_column->variant_enable_typed_paths_to_sparse()) { return Status::OK(); } @@ -1169,7 +1169,7 @@ Status VariantCompactionUtil::get_compaction_typed_columns( Status VariantCompactionUtil::get_compaction_nested_columns( const std::unordered_set& nested_paths, const PathToDataTypes& path_to_data_types, const TabletColumnPtr parent_column, - TabletSchemaSPtr& output_schema, TabletSchema::PathsSetInfo& paths_set_info) { + TabletSchemaSPtr& output_schema, VariantCompactionPaths& paths_set_info) { const auto& parent_indexes = output_schema->inverted_indexs(parent_column->unique_id()); for (const auto& path : nested_paths) { const auto& find_data_types = path_to_data_types.find(path); @@ -1200,7 +1200,7 @@ Status VariantCompactionUtil::get_compaction_nested_columns( } void VariantCompactionUtil::get_compaction_subcolumns_from_subpaths( - TabletSchema::PathsSetInfo& paths_set_info, const TabletColumnPtr parent_column, + VariantCompactionPaths& paths_set_info, const TabletColumnPtr parent_column, const TabletSchemaSPtr& target, const PathToDataTypes& path_to_data_types, const std::unordered_set& sparse_paths, TabletSchemaSPtr& output_schema) { auto& path_set = paths_set_info.sub_path_set; @@ -1265,7 +1265,7 @@ void VariantCompactionUtil::get_compaction_subcolumns_from_subpaths( } void VariantCompactionUtil::get_compaction_subcolumns_from_data_types( - TabletSchema::PathsSetInfo& paths_set_info, const TabletColumnPtr parent_column, + VariantCompactionPaths& paths_set_info, const TabletColumnPtr parent_column, const TabletSchemaSPtr& target, const PathToDataTypes& path_to_data_types, TabletSchemaSPtr& output_schema) { const auto& parent_indexes = target->inverted_indexs(parent_column->unique_id()); @@ -1305,7 +1305,8 @@ void VariantCompactionUtil::get_compaction_subcolumns_from_data_types( // ordinary extracted subcolumns. NG typed paths still use get_compaction_typed_columns(), keeping // typed-column rules out of the NG-specific regular-path filtering. Status VariantCompactionUtil::get_extended_compaction_schema( - const std::vector& rowsets, TabletSchemaSPtr& target) { + const std::vector& rowsets, TabletSchemaSPtr& target, + VariantCompactionPathsMap& paths) { std::unordered_map uid_to_variant_extended_info; const bool needs_variant_extended_info = std::ranges::any_of(target->columns(), [](const TabletColumnPtr& column) { @@ -1322,7 +1323,7 @@ Status VariantCompactionUtil::get_extended_compaction_schema( // build the output schema TabletSchemaSPtr output_schema = std::make_shared(); output_schema->shawdow_copy_without_columns(*target); - std::unordered_map uid_to_paths_set_info; + VariantCompactionPathsMap uid_to_paths_set_info; const auto ng_root_uids = collect_nested_group_compaction_root_uids(target, uid_to_variant_extended_info); for (const TabletColumnPtr& column : target->columns()) { @@ -1417,7 +1418,7 @@ Status VariantCompactionUtil::get_extended_compaction_schema( target = output_schema; // used to merge & filter path to sparse column during reading in compaction - target->set_path_set_info(std::move(uid_to_paths_set_info)); + paths = std::move(uid_to_paths_set_info); VLOG_DEBUG << "dump schema " << target->dump_full_schema(); return Status::OK(); } diff --git a/be/src/exec/common/variant_util.h b/be/src/exec/common/variant_util.h index 02a3592004e153..8cf0272de4595e 100644 --- a/be/src/exec/common/variant_util.h +++ b/be/src/exec/common/variant_util.h @@ -35,6 +35,7 @@ #include "core/string_ref.h" #include "core/types.h" #include "exprs/aggregate/aggregate_function.h" +#include "storage/segment/variant/variant_compaction_paths.h" #include "storage/tablet/tablet_fwd.h" #include "storage/tablet/tablet_schema.h" @@ -186,7 +187,7 @@ class VariantCompactionUtil { public: // get the subpaths and sparse paths for the variant column static void get_subpaths(int32_t max_subcolumns_count, const PathToNoneNullValues& path_stats, - TabletSchema::PathsSetInfo& paths_set_info); + VariantCompactionPaths& paths_set_info); // collect extended info from the variant column static Status aggregate_variant_extended_info( @@ -198,9 +199,11 @@ class VariantCompactionUtil { const RowsetSharedPtr& rs, std::unordered_map* uid_to_path_stats); - // Build the temporary schema for compaction, this will reduce the memory usage of compacting variant columns + // Build the temporary schema for compaction, this will reduce the memory usage of compacting + // variant columns. `paths` receives that schema's variant path layout. static Status get_extended_compaction_schema(const std::vector& rowsets, - TabletSchemaSPtr& target); + TabletSchemaSPtr& target, + VariantCompactionPathsMap& paths); // Used to collect all the subcolumns types of variant column from rowsets static TabletSchemaSPtr calculate_variant_extended_schema( @@ -218,25 +221,26 @@ class VariantCompactionUtil { size_t num_rows); static void get_compaction_subcolumns_from_subpaths( - TabletSchema::PathsSetInfo& paths_set_info, const TabletColumnPtr parent_column, + VariantCompactionPaths& paths_set_info, const TabletColumnPtr parent_column, const TabletSchemaSPtr& target, const PathToDataTypes& path_to_data_types, const std::unordered_set& sparse_paths, TabletSchemaSPtr& output_schema); - static void get_compaction_subcolumns_from_data_types( - TabletSchema::PathsSetInfo& paths_set_info, const TabletColumnPtr parent_column, - const TabletSchemaSPtr& target, const PathToDataTypes& path_to_data_types, - TabletSchemaSPtr& output_schema); + static void get_compaction_subcolumns_from_data_types(VariantCompactionPaths& paths_set_info, + const TabletColumnPtr parent_column, + const TabletSchemaSPtr& target, + const PathToDataTypes& path_to_data_types, + TabletSchemaSPtr& output_schema); static Status get_compaction_typed_columns(const TabletSchemaSPtr& target, const std::unordered_set& typed_paths, const TabletColumnPtr parent_column, TabletSchemaSPtr& output_schema, - TabletSchema::PathsSetInfo& paths_set_info); + VariantCompactionPaths& paths_set_info); static Status get_compaction_nested_columns( const std::unordered_set& nested_paths, const PathToDataTypes& path_to_data_types, const TabletColumnPtr parent_column, - TabletSchemaSPtr& output_schema, TabletSchema::PathsSetInfo& paths_set_info); + TabletSchemaSPtr& output_schema, VariantCompactionPaths& paths_set_info); }; } // namespace doris::variant_util diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp index fca26050618af2..4ac3eb249fbada 100644 --- a/be/src/exec/rowid_fetcher.cpp +++ b/be/src/exec/rowid_fetcher.cpp @@ -138,15 +138,11 @@ struct IteratorItem { StorageReadOptions storage_read_options; }; -static void set_slot_access_paths(const SlotDescriptor& slot, const TabletSchema& schema, +// read_column is what slot resolves to, or its variant parent column for a subpath slot. +static void set_slot_access_paths(const SlotDescriptor& slot, const TabletColumn& read_column, StorageReadOptions& storage_read_options) { - int32_t unique_id = slot.col_unique_id(); - const int field_index = - unique_id >= 0 ? schema.field_index(unique_id) : schema.field_index(slot.col_name()); - if (field_index >= 0) { - const auto& column = schema.column(field_index); - unique_id = column.unique_id() >= 0 ? column.unique_id() : column.parent_unique_id(); - } + const int32_t unique_id = + read_column.unique_id() >= 0 ? read_column.unique_id() : read_column.parent_unique_id(); if (unique_id < 0) { return; } @@ -1043,10 +1039,19 @@ Status RowIdStorageReader::read_doris_format_row( iterator_item.storage_read_options.io_ctx.file_cache_miss_policy = file_cache_miss_policy; } - set_slot_access_paths(slots[x], full_read_schema, iterator_item.storage_read_options); - RETURN_IF_ERROR(segment->seek_and_read_by_rowid( - full_read_schema, &slots[x], row_ids, column, - iterator_item.storage_read_options, iterator_item.iterator)); + int32_t index = slots[x].col_unique_id() >= 0 + ? full_read_schema.field_index(slots[x].col_unique_id()) + : full_read_schema.field_index(slots[x].col_name()); + if (index < 0) { + return Status::InternalError( + "field name is invalid. field={}, field_name_to_index={}", + slots[x].col_name(), full_read_schema.get_all_field_names()); + } + const auto& read_column = full_read_schema.column(index); + set_slot_access_paths(slots[x], read_column, iterator_item.storage_read_options); + RETURN_IF_ERROR(segment->seek_and_read_by_rowid(read_column, &slots[x], row_ids, column, + iterator_item.storage_read_options, + iterator_item.iterator)); } } return Status::OK(); diff --git a/be/src/service/point_query_executor.cpp b/be/src/service/point_query_executor.cpp index cee41c611e0ef5..7ba77f96ba2496 100644 --- a/be/src/service/point_query_executor.cpp +++ b/be/src/service/point_query_executor.cpp @@ -595,6 +595,7 @@ Status PointQueryExecutor::_lookup_row_data() { return seg->id() == row_loc.segment_id; }); const auto& segment = *it; + const auto tablet_schema = _tablet->tablet_schema(); for (int cid : _reusable->missing_col_uids()) { int pos = _reusable->get_col_uid_to_idx().at(cid); std::vector row_ids { @@ -602,11 +603,19 @@ Status PointQueryExecutor::_lookup_row_data() { auto& column = result_columns[pos]; std::unique_ptr iter; SlotDescriptor* slot = _reusable->tuple_desc()->slots()[pos]; + int32_t index = slot->col_unique_id() >= 0 + ? tablet_schema->field_index(slot->col_unique_id()) + : tablet_schema->field_index(slot->col_name()); + if (index < 0) { + return Status::InternalError( + "field name is invalid. field={}, field_name_to_index={}", + slot->col_name(), tablet_schema->get_all_field_names()); + } StorageReadOptions storage_read_options; storage_read_options.stats = &_read_stats; storage_read_options.io_ctx = io_ctx; - RETURN_IF_ERROR(segment->seek_and_read_by_rowid(*_tablet->tablet_schema(), slot, - row_ids, column, + RETURN_IF_ERROR(segment->seek_and_read_by_rowid(tablet_schema->column(index), + slot, row_ids, column, storage_read_options, iter)); } } diff --git a/be/src/storage/compaction/compaction.cpp b/be/src/storage/compaction/compaction.cpp index 0186fa9b0bfafa..e81e1c7f8e8da9 100644 --- a/be/src/storage/compaction/compaction.cpp +++ b/be/src/storage/compaction/compaction.cpp @@ -574,8 +574,10 @@ Status CompactionMixin::build_basic_info(bool is_ordered_compaction) { // so get_extended_compaction_schema will extended the schema for variant columns // for ordered compaction, we don't need to extend the schema for variant columns if (_enable_vertical_compact_variant_subcolumns && !is_ordered_compaction) { + auto paths = std::make_shared(); RETURN_IF_ERROR(variant_util::VariantCompactionUtil::get_extended_compaction_schema( - _input_rowsets, _cur_tablet_schema)); + _input_rowsets, _cur_tablet_schema, *paths)); + _cur_variant_compaction_paths = std::move(paths); } return Status::OK(); } @@ -1787,6 +1789,7 @@ Status CompactionMixin::construct_output_rowset_writer(RowsetWriterContext& ctx) ctx.rowset_state = VISIBLE; ctx.segments_overlap = _trigger_quick_merge_by_binlog ? OVERLAPPING : NONOVERLAPPING; ctx.tablet_schema = _cur_tablet_schema; + ctx.variant_compaction_paths = _cur_variant_compaction_paths; ctx.newest_write_timestamp = _newest_write_timestamp; ctx.write_type = DataWriteType::TYPE_COMPACTION; ctx.compaction_type = compaction_type(); @@ -2092,8 +2095,10 @@ Status CloudCompactionMixin::build_basic_info() { // if enable_vertical_compact_variant_subcolumns is true, we need to compact the variant subcolumns in seperate column groups // so get_extended_compaction_schema will extended the schema for variant columns if (_enable_vertical_compact_variant_subcolumns) { + auto paths = std::make_shared(); RETURN_IF_ERROR(variant_util::VariantCompactionUtil::get_extended_compaction_schema( - _input_rowsets, _cur_tablet_schema)); + _input_rowsets, _cur_tablet_schema, *paths)); + _cur_variant_compaction_paths = std::move(paths); } return Status::OK(); } @@ -2370,6 +2375,7 @@ Status CloudCompactionMixin::construct_output_rowset_writer(RowsetWriterContext& ctx.rowset_state = VISIBLE; ctx.segments_overlap = NONOVERLAPPING; ctx.tablet_schema = _cur_tablet_schema; + ctx.variant_compaction_paths = _cur_variant_compaction_paths; ctx.newest_write_timestamp = _newest_write_timestamp; ctx.write_type = DataWriteType::TYPE_COMPACTION; ctx.compaction_type = compaction_type(); diff --git a/be/src/storage/compaction/compaction.h b/be/src/storage/compaction/compaction.h index ebf361f0b61d2c..827f3bcf4c8294 100644 --- a/be/src/storage/compaction/compaction.h +++ b/be/src/storage/compaction/compaction.h @@ -208,6 +208,8 @@ class Compaction { int64_t _newest_write_timestamp {-1}; std::unique_ptr _rowid_conversion = nullptr; TabletSchemaSPtr _cur_tablet_schema; + // The variant path layout of _cur_tablet_schema; empty unless it is an extended schema. + VariantCompactionPathsSPtr _cur_variant_compaction_paths; std::unique_ptr _profile; diff --git a/be/src/storage/iterator/block_reader.cpp b/be/src/storage/iterator/block_reader.cpp index e91b16cd021212..0012fea97c713c 100644 --- a/be/src/storage/iterator/block_reader.cpp +++ b/be/src/storage/iterator/block_reader.cpp @@ -552,23 +552,19 @@ Status BlockReader::init(const ReaderParams& read_params) { SCOPED_RAW_TIMER(&_stats.tablet_reader_init_timer_ns); RETURN_IF_ERROR(TabletReader::init(read_params)); - const bool use_sequence_map = _tablet_schema->has_seq_map() && - _tablet_schema->keys_type() == UNIQUE_KEYS && !_direct_mode && - read_params.binlog_scan_type != TBinlogScanType::MIN_DELTA && - read_params.binlog_scan_type != TBinlogScanType::DETAIL && - !(read_params.reader_type == ReaderType::READER_QUERY && - _tablet->enable_unique_key_merge_on_write()); - if (use_sequence_map) { - auto read_schema = std::make_shared(*_read_schema); - RETURN_IF_ERROR(read_schema->init_sequence_map(*_tablet_schema)); - _read_schema = std::move(read_schema); - } - - if (read_params.binlog_scan_type == TBinlogScanType::MIN_DELTA || - read_params.binlog_scan_type == TBinlogScanType::DETAIL) { - auto read_schema = std::make_shared(*_read_schema); - read_schema->init_row_binlog_column_mappings(*_tablet_schema); - _read_schema = std::move(read_schema); + // A Row Binlog scan maps the before-image columns; every other read of this reader, which is + // the one that merges rows across rowsets, builds the sequence mapping instead. + const bool map_row_binlog_columns = + read_params.binlog_scan_type == TBinlogScanType::MIN_DELTA || + read_params.binlog_scan_type == TBinlogScanType::DETAIL; + const bool merge_by_sequence_mapping = + !map_row_binlog_columns && _tablet_schema->has_seq_map() && + _tablet_schema->keys_type() == UNIQUE_KEYS && !_direct_mode && + !(read_params.reader_type == ReaderType::READER_QUERY && + _tablet->enable_unique_key_merge_on_write()); + RETURN_IF_ERROR(_read_schema->init_from_tablet_schema( + *_tablet_schema, merge_by_sequence_mapping, map_row_binlog_columns)); + if (map_row_binlog_columns) { _min_delta_value_compare_unsupported = false; } @@ -615,7 +611,7 @@ Status BlockReader::init(const ReaderParams& read_params) { if (read_params.reader_type == ReaderType::READER_QUERY && _reader_context.enable_unique_key_merge_on_write) { _next_block_func = &BlockReader::_direct_next_block; - } else if (use_sequence_map) { + } else if (merge_by_sequence_mapping) { _next_block_func = &BlockReader::_replace_key_next_block; } else { _next_block_func = &BlockReader::_unique_key_next_block; diff --git a/be/src/storage/iterator/vertical_block_reader.cpp b/be/src/storage/iterator/vertical_block_reader.cpp index e3d1e38de40450..25b6607a8e6401 100644 --- a/be/src/storage/iterator/vertical_block_reader.cpp +++ b/be/src/storage/iterator/vertical_block_reader.cpp @@ -278,6 +278,10 @@ Status VerticalBlockReader::init(const ReaderParams& read_params, } RETURN_IF_ERROR(TabletReader::init(read_params)); + RETURN_IF_ERROR(_read_schema->init_from_tablet_schema(*_tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false)); + auto status = _init_collect_iter(read_params, sample_info); if (!status.ok()) [[unlikely]] { if (!config::is_cloud_mode()) { diff --git a/be/src/storage/iterators.h b/be/src/storage/iterators.h index 09f8be09cd4499..c7029f989c6350 100644 --- a/be/src/storage/iterators.h +++ b/be/src/storage/iterators.h @@ -33,6 +33,7 @@ #include "storage/predicate/column_predicate.h" #include "storage/row_cursor.h" #include "storage/segment/row_ranges.h" +#include "storage/segment/variant/variant_compaction_paths.h" #include "storage/tablet/tablet_schema.h" namespace doris { @@ -116,7 +117,11 @@ class StorageReadOptions { // Effective adaptive batch size byte budget. size_t preferred_block_size_bytes = 8388608UL; - TabletSchemaSPtr tablet_schema = nullptr; + // Whether the tablet schema this read targets materializes variant subcolumns as extracted + // columns. A compaction read then takes them as flat leaves. + bool tablet_has_extracted_variant_columns = false; + // The compaction output schema's variant path layout; null outside a variant compaction. + VariantCompactionPathsSPtr variant_compaction_paths; bool enable_unique_key_merge_on_write = false; bool record_rowids = false; std::vector topn_filter_source_node_ids; diff --git a/be/src/storage/merger.cpp b/be/src/storage/merger.cpp index 6999cb486930c1..566adea31ac949 100644 --- a/be/src/storage/merger.cpp +++ b/be/src/storage/merger.cpp @@ -140,6 +140,7 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type, reader_params.tablet_schema = std::make_shared(); reader_params.tablet_schema->copy_from(cur_tablet_schema); + reader_params.variant_compaction_paths = dst_rowset_writer->context().variant_compaction_paths; if (!tablet->tablet_schema()->cluster_key_uids().empty()) { reader_params.delete_bitmap = tablet->tablet_meta()->delete_bitmap_ptr(); } @@ -333,6 +334,7 @@ Status Merger::vertical_compact_one_group( reader_params.tablet_schema = std::make_shared(); reader_params.tablet_schema->copy_from(tablet_schema); + reader_params.variant_compaction_paths = dst_rowset_writer->context().variant_compaction_paths; bool has_cluster_key = false; if (!tablet->tablet_schema()->cluster_key_uids().empty()) { reader_params.delete_bitmap = tablet->tablet_meta()->delete_bitmap_ptr(); diff --git a/be/src/storage/rowset/beta_rowset_reader.cpp b/be/src/storage/rowset/beta_rowset_reader.cpp index fea4e22d6dcacd..79f62a9625158d 100644 --- a/be/src/storage/rowset/beta_rowset_reader.cpp +++ b/be/src/storage/rowset/beta_rowset_reader.cpp @@ -180,7 +180,7 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context if (_should_push_down_value_predicates()) { // sequence mapping currently only support merge on read, so can not push down value predicates if (_read_context->value_predicates != nullptr && - !read_context->tablet_schema->has_seq_map()) { + !_read_context->read_schema->tablet_has_sequence_map()) { _read_options.column_predicates.insert(_read_options.column_predicates.end(), _read_context->value_predicates->begin(), _read_context->value_predicates->end()); @@ -195,7 +195,9 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context } } _read_options.use_page_cache = _read_context->use_page_cache; - _read_options.tablet_schema = _read_context->tablet_schema; + _read_options.tablet_has_extracted_variant_columns = + _read_context->read_schema->tablet_has_extracted_variant_columns(); + _read_options.variant_compaction_paths = _read_context->variant_compaction_paths; _read_options.enable_unique_key_merge_on_write = _read_context->enable_unique_key_merge_on_write; _read_options.record_rowids = _read_context->record_rowids; diff --git a/be/src/storage/rowset/rowset_reader_context.h b/be/src/storage/rowset/rowset_reader_context.h index b72c5d58c4ceb0..8bf7dfc2a0ff15 100644 --- a/be/src/storage/rowset/rowset_reader_context.h +++ b/be/src/storage/rowset/rowset_reader_context.h @@ -33,6 +33,7 @@ #include "storage/row_cursor.h" #include "storage/rowid_conversion.h" #include "storage/schema.h" +#include "storage/segment/variant/variant_compaction_paths.h" namespace doris { @@ -44,7 +45,8 @@ struct RowsetReaderContext { ReaderType reader_type = ReaderType::READER_QUERY; bool read_row_binlog = false; Version version {-1, -1}; - TabletSchemaSPtr tablet_schema = nullptr; + // Set only by compaction, alongside its extended tablet_schema. + VariantCompactionPathsSPtr variant_compaction_paths; std::vector topn_filter_source_node_ids; // whether rowset should return ordered rows. bool need_ordered_result = true; diff --git a/be/src/storage/rowset/rowset_writer_context.h b/be/src/storage/rowset/rowset_writer_context.h index 08e7b31cace5a0..7c9d4add5ce7bc 100644 --- a/be/src/storage/rowset/rowset_writer_context.h +++ b/be/src/storage/rowset/rowset_writer_context.h @@ -40,6 +40,7 @@ #include "storage/olap_define.h" #include "storage/partial_update_info.h" #include "storage/segment/historical_row_retriever.h" +#include "storage/segment/variant/variant_compaction_paths.h" #include "storage/storage_policy.h" #include "storage/tablet/tablet.h" #include "storage/tablet/tablet_schema.h" @@ -73,6 +74,8 @@ struct RowsetWriterContext { RowsetTypePB rowset_type {BETA_ROWSET}; TabletSchemaSPtr tablet_schema; + // Set only by compaction, alongside its extended tablet_schema. + VariantCompactionPathsSPtr variant_compaction_paths; // Immutable inverted-index file format inherited from the owner tablet. std::optional inverted_index_storage_format; // Whether the owner tablet persists the format in its top-level metadata. diff --git a/be/src/storage/rowset/segcompaction.cpp b/be/src/storage/rowset/segcompaction.cpp index c3ec566c29f4cc..c18f1197d978fc 100644 --- a/be/src/storage/rowset/segcompaction.cpp +++ b/be/src/storage/rowset/segcompaction.cpp @@ -92,7 +92,6 @@ Status SegcompactionWorker::_get_segcompaction_reader( StorageReadOptions read_options; read_options.stats = stat; read_options.use_page_cache = false; - read_options.tablet_schema = ctx.tablet_schema; read_options.record_rowids = record_rowids; if (!tablet->tablet_schema()->cluster_key_uids().empty()) { DeleteBitmapPtr delete_bitmap = std::make_shared(tablet->tablet_id()); diff --git a/be/src/storage/schema.cpp b/be/src/storage/schema.cpp index c3fd1af402bed8..6cfee7a2523a34 100644 --- a/be/src/storage/schema.cpp +++ b/be/src/storage/schema.cpp @@ -17,6 +17,7 @@ #include "storage/schema.h" +#include #include #include "common/config.h" @@ -118,7 +119,7 @@ void ReadSchema::_init_before_column_ordinals() { } } -void ReadSchema::init_row_binlog_column_mappings(const TabletSchema& tablet_schema) { +void ReadSchema::_init_row_binlog_column_mappings(const TabletSchema& tablet_schema) { DORIS_CHECK_GE(_op_ordinal, 0); DORIS_CHECK_EQ(_before_column_ordinals.size(), _num_block_columns); @@ -221,7 +222,24 @@ std::string ReadSchema::read_columns_to_string() const { return result; } -Status ReadSchema::init_sequence_map(const TabletSchema& tablet_schema) { +Status ReadSchema::init_from_tablet_schema(const TabletSchema& tablet_schema, + bool merge_by_sequence_mapping, + bool map_row_binlog_columns) { + DORIS_CHECK(!(merge_by_sequence_mapping && map_row_binlog_columns)); + _tablet_has_sequence_map = tablet_schema.has_seq_map(); + _tablet_has_extracted_variant_columns = + std::ranges::any_of(tablet_schema.columns(), + [](const auto& column) { return column->is_extracted_column(); }); + if (merge_by_sequence_mapping) { + RETURN_IF_ERROR(_init_sequence_map(tablet_schema)); + } + if (map_row_binlog_columns) { + _init_row_binlog_column_mappings(tablet_schema); + } + return Status::OK(); +} + +Status ReadSchema::_init_sequence_map(const TabletSchema& tablet_schema) { if (tablet_schema.has_sequence_col()) { auto msg = "sequence columns conflict, both seq_col and seq_map are true!"; LOG(WARNING) << msg; diff --git a/be/src/storage/schema.h b/be/src/storage/schema.h index 473e603a05aa77..4e27e9eeef9746 100644 --- a/be/src/storage/schema.h +++ b/be/src/storage/schema.h @@ -85,16 +85,23 @@ class ReadSchema { std::string read_columns_to_string() const; - Status init_sequence_map(const TabletSchema& tablet_schema); + // Always sets the two facts below; the flags add the sequence mapping and the row-binlog + // before-image mapping. + Status init_from_tablet_schema(const TabletSchema& tablet_schema, + bool merge_by_sequence_mapping, bool map_row_binlog_columns); + + // Whether the tablet schema defines a sequence mapping -- not whether this read merges by + // one. A MoW query does not merge, yet a value predicate still must not be pushed down. + bool tablet_has_sequence_map() const { return _tablet_has_sequence_map; } + + // Whether the tablet schema materializes variant subcolumns as extracted columns. A + // compaction read then takes them as flat leaves. + bool tablet_has_extracted_variant_columns() const { + return _tablet_has_extracted_variant_columns; + } const SequenceMap& sequence_map() const { return _sequence_map; } - // Initialize all row-binlog column relationships from the physical tablet schema and map - // them to this ReadSchema's dense ordinals. Physical pairing avoids ambiguous column-name - // lookup, while schemas without a complete physical layout retain the name-based BEFORE - // mapping initialized by the constructor. - void init_row_binlog_column_mappings(const TabletSchema& tablet_schema); - // Return the matching before-image ordinal for a Row Binlog value column. For example, in // [v1, v2, __BEFORE__v1__, __BEFORE__v2__], 0 maps to 2 and 1 maps to 3. Columns without a // before image, including TSO/LSN/OP, map to themselves. @@ -153,6 +160,8 @@ class ReadSchema { private: void _init_read_types(); void _init_before_column_ordinals(); + Status _init_sequence_map(const TabletSchema& tablet_schema); + void _init_row_binlog_column_mappings(const TabletSchema& tablet_schema); void _init_descriptors() { DORIS_CHECK_LE(_num_block_columns, _read_columns.size()); @@ -240,6 +249,8 @@ class ReadSchema { std::vector _before_column_ordinals; RowBinlogValueColumnPairs _row_binlog_value_column_pairs; bool _row_binlog_value_pairs_complete = false; + bool _tablet_has_sequence_map = false; + bool _tablet_has_extracted_variant_columns = false; }; } // namespace doris diff --git a/be/src/storage/schema_change/schema_change.cpp b/be/src/storage/schema_change/schema_change.cpp index 59fbcb69c5852f..6295091eb942ef 100644 --- a/be/src/storage/schema_change/schema_change.cpp +++ b/be/src/storage/schema_change/schema_change.cpp @@ -937,6 +937,9 @@ Status SchemaChangeJob::_do_process_alter_tablet(const TAlterTabletReqV2& reques std::vector read_columns(_base_tablet_schema->columns().begin(), _base_tablet_schema->columns().begin() + num_cols); ReadSchemaSPtr read_schema = std::make_shared(std::move(read_columns)); + RETURN_IF_ERROR(read_schema->init_from_tablet_schema(*_base_tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false)); std::vector cluster_key_idxes; DBUG_EXECUTE_IF("SchemaChangeJob::_do_process_alter_tablet.block", DBUG_BLOCK); @@ -1046,7 +1049,6 @@ Status SchemaChangeJob::_do_process_alter_tablet(const TAlterTabletReqV2& reques read_schema->append_dropped_columns(std::move(dropped_columns)); reader_context.reader_type = ReaderType::READER_ALTER_TABLE; - reader_context.tablet_schema = _base_tablet_schema; reader_context.need_ordered_result = true; reader_context.delete_handler = &delete_handler; reader_context.read_schema = read_schema; diff --git a/be/src/storage/segment/segment.cpp b/be/src/storage/segment/segment.cpp index 9d7ae1563cbe20..8978d7e01eac03 100644 --- a/be/src/storage/segment/segment.cpp +++ b/be/src/storage/segment/segment.cpp @@ -27,7 +27,6 @@ #include #include #include -#include #include #include "cloud/config.h" @@ -972,13 +971,11 @@ Status Segment::new_column_iterator(const TabletColumn& tablet_column, // on every read path (projection / predicate / MIN-MAX zone-map) instead of the placeholder 0. // commit_tso == -1 means it is not assigned yet (before publish); keep the on-disk value then. // The value is constant per segment (a segment belongs to a single rowset), so caching the - // ConstantColumnReader does not cross-pollute other queries. Some internal read paths (e.g. MOW - // partial-update row fetch) build a bare StorageReadOptions without tablet_schema, so guard it. + // ConstantColumnReader does not cross-pollute other queries. std::optional const_value; - if (opt->tablet_schema != nullptr && opt->version.first == opt->version.second && - opt->commit_tso.end_tso() != -1) { - int32_t tso_idx = opt->tablet_schema->commit_tso_col_idx(); - if (tso_idx != -1 && opt->tablet_schema->column(tso_idx).unique_id() == unique_id) { + if (opt->version.first == opt->version.second && opt->commit_tso.end_tso() != -1) { + int32_t tso_idx = _tablet_schema->commit_tso_col_idx(); + if (tso_idx != -1 && _tablet_schema->column(tso_idx).unique_id() == unique_id) { const_value = Field::create_field(opt->commit_tso.end_tso()); } } @@ -1281,7 +1278,7 @@ Status Segment::read_key_by_rowid(uint32_t row_id, std::string* key) { return Status::OK(); } -Status Segment::seek_and_read_by_rowid(const TabletSchema& schema, SlotDescriptor* slot, +Status Segment::seek_and_read_by_rowid(const TabletColumn& read_column, SlotDescriptor* slot, const std::vector& row_ids, MutableColumnPtr& result, StorageReadOptions& storage_read_options, @@ -1313,8 +1310,7 @@ Status Segment::seek_and_read_by_rowid(const TabletSchema& schema, SlotDescripto RETURN_IF_ERROR( _create_column_meta_once(storage_read_options.stats, &storage_read_options.io_ctx)); - const PathInData path(schema.column_by_uid(slot->col_unique_id()).name_lower_case(), - slot->column_paths()); + const PathInData path(read_column.name_lower_case(), slot->column_paths()); TabletColumn column = variant_util::get_column_by_type( make_nullable(slot->type()), path.get_path(), variant_util::ExtraInfo {.parent_unique_id = slot->col_unique_id(), @@ -1343,20 +1339,12 @@ Status Segment::seek_and_read_by_rowid(const TabletSchema& schema, SlotDescripto } RETURN_IF_CATCH_EXCEPTION(result->insert_range_from(*source_ptr, 0, row_ids.size())); } else { - int index = (slot->col_unique_id() >= 0) ? schema.field_index(slot->col_unique_id()) - : schema.field_index(slot->col_name()); - if (index < 0) { - std::stringstream ss; - ss << "field name is invalid. field=" << slot->col_name() - << ", field_name_to_index=" << schema.get_all_field_names(); - return Status::InternalError(ss.str()); - } - TabletColumn column = schema.column(index); - if (column.type() == FieldType::OLAP_FIELD_TYPE_VARIANT) { + if (read_column.type() == FieldType::OLAP_FIELD_TYPE_VARIANT) { DORIS_CHECK(variant_v2_type != nullptr); } if (iterator_hint == nullptr) { - RETURN_IF_ERROR(new_column_iterator(column, &iterator_hint, &storage_read_options)); + RETURN_IF_ERROR( + new_column_iterator(read_column, &iterator_hint, &storage_read_options)); RETURN_IF_ERROR(iterator_hint->init(opt)); } RETURN_IF_ERROR(iterator_hint->read_by_rowids(row_ids.data(), row_ids.size(), result)); diff --git a/be/src/storage/segment/segment.h b/be/src/storage/segment/segment.h index ef1792ccdcf9d6..a39114187c2021 100644 --- a/be/src/storage/segment/segment.h +++ b/be/src/storage/segment/segment.h @@ -150,7 +150,8 @@ class Segment : public std::enable_shared_from_this, public MetadataAdd Status read_key_by_rowid(uint32_t row_id, std::string* key); // row_ids must be strictly increasing. - Status seek_and_read_by_rowid(const TabletSchema& schema, SlotDescriptor* slot, + // `read_column` is what `slot` resolves to, or its variant parent column for a subpath slot. + Status seek_and_read_by_rowid(const TabletColumn& read_column, SlotDescriptor* slot, const std::vector& row_ids, MutableColumnPtr& result, StorageReadOptions& storage_read_options, std::unique_ptr& iterator_hint); diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index 99d53fd7f868ea..ec6c180815c95b 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -437,7 +437,9 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) { } _storage_name_and_type.resize(_schema->num_read_columns()); - auto storage_format = _opts.tablet_schema->get_inverted_index_storage_format(); + // The field names built below must match the ones this segment's index files were written + // with, so take the format from the schema IndexFileReader was opened with. + auto storage_format = _segment->_tablet_schema->get_inverted_index_storage_format(); for (size_t i = 0; i < _schema->num_read_columns(); ++i) { const TabletColumn* col = _schema->column(i); auto storage_type = _segment->get_data_type_of(*col, _schema->data_type(i), _opts); @@ -714,10 +716,7 @@ Status SegmentIterator::_get_row_ranges_by_keys() { // groups must still apply the key range to read the same physical rows as the key group. if (_opts.io_ctx.reader_type != ReaderType::READER_BASE_COMPACTION && std::none_of(_schema->columns().begin(), _schema->columns().end(), - [&](const TabletColumnPtr& col) { - return col && - _opts.tablet_schema->column_by_uid(col->unique_id()).is_key(); - })) { + [](const TabletColumnPtr& col) { return col && col->is_key(); })) { return Status::OK(); } @@ -1401,9 +1400,7 @@ bool SegmentIterator::_count_on_index_fastpath_safe() const { facts.no_need_read_data_opt_enabled = _opts.runtime_state == nullptr || _opts.runtime_state->query_options().enable_no_need_read_data_opt; - facts.keys_type_supported = _opts.tablet_schema->keys_type() == KeysType::DUP_KEYS || - (_opts.tablet_schema->keys_type() == KeysType::UNIQUE_KEYS && - _opts.enable_unique_key_merge_on_write); + facts.keys_type_supported = _keys_type_allows_skipping_data(); return count_on_index_fastpath_safe(facts); } @@ -1595,6 +1592,12 @@ Status SegmentIterator::_apply_inverted_index_on_column_predicate( return Status::OK(); } +bool SegmentIterator::_keys_type_allows_skipping_data() const { + const KeysType keys_type = _segment->_tablet_schema->keys_type(); + return keys_type == KeysType::DUP_KEYS || + (keys_type == KeysType::UNIQUE_KEYS && _opts.enable_unique_key_merge_on_write); +} + bool SegmentIterator::_need_read_data(ColumnId cid) { if (_opts.runtime_state && !_opts.runtime_state->query_options().enable_no_need_read_data_opt) { return true; @@ -1602,10 +1605,7 @@ bool SegmentIterator::_need_read_data(ColumnId cid) { if (_can_skip_reading_extra_column(cid)) { return false; } - // only support DUP_KEYS and UNIQUE_KEYS with MOW - if (!((_opts.tablet_schema->keys_type() == KeysType::DUP_KEYS || - (_opts.tablet_schema->keys_type() == KeysType::UNIQUE_KEYS && - _opts.enable_unique_key_merge_on_write)))) { + if (!_keys_type_allows_skipping_data()) { return true; } // this is a virtual column, we always need to read data @@ -3538,9 +3538,7 @@ bool SegmentIterator::_no_need_read_key_data_eligible(ColumnId cid) { return false; } - if (!((_opts.tablet_schema->keys_type() == KeysType::DUP_KEYS || - (_opts.tablet_schema->keys_type() == KeysType::UNIQUE_KEYS && - _opts.enable_unique_key_merge_on_write)))) { + if (!_keys_type_allows_skipping_data()) { return false; } diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index 5602c6280ab774..91e4207573e041 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -265,6 +265,9 @@ class SegmentIterator : public RowwiseIterator { void _output_index_result_column(const VExprContextSPtrs& expr_ctxs, uint16_t* sel_rowid_idx, uint16_t select_size); + // False for MoR and AGG keys: the merge above this iterator needs the real values. + bool _keys_type_allows_skipping_data() const; + bool _need_read_data(ColumnId cid); bool _prune_column(ColumnId cid, MutableColumnPtr& column, size_t num_of_defaults); diff --git a/be/src/storage/segment/variant/sparse_column_merge_iterator.h b/be/src/storage/segment/variant/sparse_column_merge_iterator.h index 8f4b76bfa34444..f95363ea940628 100644 --- a/be/src/storage/segment/variant/sparse_column_merge_iterator.h +++ b/be/src/storage/segment/variant/sparse_column_merge_iterator.h @@ -44,6 +44,7 @@ #include "storage/segment/column_reader.h" #include "storage/segment/stream_reader.h" #include "storage/segment/variant/binary_column_extract_iterator.h" +#include "storage/segment/variant/variant_compaction_paths.h" #include "storage/tablet/tablet_schema.h" #include "util/json/path_in_data.h" @@ -52,7 +53,7 @@ namespace doris::segment_v2 { // Implementation for merge processor class SparseColumnMergeIterator : public BaseBinaryColumnProcessor { public: - SparseColumnMergeIterator(const TabletSchema::PathsSetInfo& path_set_info, + SparseColumnMergeIterator(const VariantCompactionPaths& path_set_info, BinaryColumnCacheSPtr sparse_column_cache, SubstreamReaderTree&& src_subcolumns_for_sparse, const StorageReadOptions* opts) diff --git a/be/src/storage/segment/variant/variant_column_reader.cpp b/be/src/storage/segment/variant/variant_column_reader.cpp index 831640a54d2318..1dd8d9c49e3df3 100644 --- a/be/src/storage/segment/variant/variant_column_reader.cpp +++ b/be/src/storage/segment/variant/variant_column_reader.cpp @@ -22,7 +22,6 @@ #include #include -#include #include #include #include @@ -306,8 +305,14 @@ Status VariantColumnReader::_create_sparse_merge_reader(ColumnIteratorUPtr* iter ColumnReaderCache* column_reader_cache, std::optional bucket_index) { std::shared_lock lock(_subcolumns_meta_mutex); - // Get subcolumns path set from tablet schema - const auto& path_set_info = opts->tablet_schema->path_set_info(target_col.parent_unique_id()); + // Only the flat-leaf plan reaches a sparse column, and only a compaction output schema has + // one, so the path sets are always present here. + DORIS_CHECK(opts->variant_compaction_paths != nullptr); + auto layout = opts->variant_compaction_paths->find(target_col.parent_unique_id()); + DORIS_CHECK(layout != opts->variant_compaction_paths->end()) + << "no compaction path layout for variant column, parent_unique_id=" + << target_col.parent_unique_id(); + const auto& path_set_info = layout->second; // Build substream reader tree for merging subcolumns into sparse column SubstreamReaderTree src_subcolumns_for_sparse; @@ -587,9 +592,7 @@ bool VariantColumnReader::_has_prefix_path_unlocked(const PathInData& relative_p } bool VariantColumnReader::_need_read_flat_leaves(const StorageReadOptions* opts) { - return opts != nullptr && opts->tablet_schema != nullptr && - std::ranges::any_of(opts->tablet_schema->columns(), - [](const auto& column) { return column->is_extracted_column(); }) && + return opts != nullptr && opts->tablet_has_extracted_variant_columns && is_compaction_or_checksum_reader(opts); } diff --git a/be/src/storage/segment/variant/variant_compaction_paths.cpp b/be/src/storage/segment/variant/variant_compaction_paths.cpp new file mode 100644 index 00000000000000..fcd1d84df8721b --- /dev/null +++ b/be/src/storage/segment/variant/variant_compaction_paths.cpp @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/segment/variant/variant_compaction_paths.h" + +#include "storage/index/index_writer.h" + +namespace doris { + +namespace { + +const TabletIndexes* indexes_of_path(const VariantCompactionPaths& paths, const TabletColumn& col, + const std::string& relative_path) { + if (col.path_info_ptr()->get_is_typed()) { + auto it = paths.typed_path_set.find(relative_path); + return it == paths.typed_path_set.end() ? nullptr : &it->second.indexes; + } + auto it = paths.subcolumn_indexes.find(relative_path); + return it == paths.subcolumn_indexes.end() ? nullptr : &it->second; +} + +} // namespace + +std::vector variant_subcolumn_indexes(const VariantCompactionPathsMap* paths, + const TabletColumn& col) { + // Some extracted types (JSONB, a nested variant, an array of them) cannot carry one. + if (paths == nullptr || !col.is_extracted_column() || + !segment_v2::IndexColumnWriter::check_support_inverted_index(col)) { + return {}; + } + auto column_paths = paths->find(col.parent_unique_id()); + if (column_paths == paths->end()) { + return {}; + } + const TabletIndexes* indexes = indexes_of_path( + column_paths->second, col, col.path_info_ptr()->copy_pop_front().get_path()); + if (indexes == nullptr) { + return {}; + } + std::vector result; + result.reserve(indexes->size()); + for (const auto& index : *indexes) { + result.push_back(index.get()); + } + return result; +} + +} // namespace doris diff --git a/be/src/storage/segment/variant/variant_compaction_paths.h b/be/src/storage/segment/variant/variant_compaction_paths.h new file mode 100644 index 00000000000000..c3e65fbfb51cbc --- /dev/null +++ b/be/src/storage/segment/variant/variant_compaction_paths.h @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "storage/tablet/tablet_schema.h" + +namespace doris { + +// How one variant column's paths are laid out in a compaction output schema: which become typed +// or extracted columns, which fall back to the sparse column, and the indexes on the materialized +// ones. Paths are relative to the parent variant column. Never persisted with the schema, so only +// a running compaction has one. +struct VariantCompactionPaths { + std::unordered_map typed_path_set; + std::unordered_map subcolumn_indexes; + // extracted columns + PathSet sub_path_set; + // paths left to the sparse column + PathSet sparse_path_set; +}; + +// Parent variant column unique id -> that column's layout. Shared, and read-only once built: a +// compaction's rowset writer and every reader it spawns hold the same layout. +using VariantCompactionPathsMap = std::unordered_map; +using VariantCompactionPathsSPtr = std::shared_ptr; + +// The inverted indexes `paths` attached to `col`, an extracted variant column. Empty when `paths` +// is null (any write that is not a compaction), when it does not cover this column's path, or +// when the column's type cannot carry an inverted index. +std::vector variant_subcolumn_indexes(const VariantCompactionPathsMap* paths, + const TabletColumn& col); + +} // namespace doris diff --git a/be/src/storage/segment/vertical_segment_writer.cpp b/be/src/storage/segment/vertical_segment_writer.cpp index 542cfe725be783..aacf62421cbada 100644 --- a/be/src/storage/segment/vertical_segment_writer.cpp +++ b/be/src/storage/segment/vertical_segment_writer.cpp @@ -69,6 +69,7 @@ #include "storage/segment/page_io.h" #include "storage/segment/page_pointer.h" #include "storage/segment/segment_loader.h" +#include "storage/segment/variant/variant_compaction_paths.h" #include "storage/segment/variant/variant_ext_meta_writer.h" #include "storage/segment/variant_stats_calculator.h" #include "storage/storage_engine.h" @@ -190,6 +191,13 @@ Status VerticalSegmentWriter::_create_column_writer(size_t pos, uint32_t cid, opts.is_direct_load = _opts.write_type == DataWriteType::TYPE_DIRECT; if (!skip_inverted_index) { auto inverted_indexs = tablet_schema->inverted_indexs(column); + if (inverted_indexs.empty() && column.is_extracted_column() && + _opts.rowset_ctx != nullptr) { + // A variant subcolumn that compaction materialized carries its indexes only in the + // compaction path layout, never on the output schema. + inverted_indexs = variant_subcolumn_indexes( + _opts.rowset_ctx->variant_compaction_paths.get(), column); + } // SNII splits index compaction per (column, index): indexes in the set // are produced by the postings merge, every sibling on the column still // raw-builds here. V2/V3 skip whole columns above instead. diff --git a/be/src/storage/tablet/tablet_reader.cpp b/be/src/storage/tablet/tablet_reader.cpp index 1ac7af12a571e4..db2744a3f9fcc1 100644 --- a/be/src/storage/tablet/tablet_reader.cpp +++ b/be/src/storage/tablet/tablet_reader.cpp @@ -159,7 +159,7 @@ Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) { _reader_context.reader_type = read_params.reader_type; _reader_context.read_row_binlog = read_params.read_row_binlog; _reader_context.version = read_params.version; - _reader_context.tablet_schema = _tablet_schema; + _reader_context.variant_compaction_paths = read_params.variant_compaction_paths; _reader_context.need_ordered_result = need_ordered_result || read_params.force_key_ordered_read; _reader_context.topn_filter_source_node_ids = read_params.topn_filter_source_node_ids; _reader_context.read_orderby_key_reverse = read_params.read_orderby_key_reverse; diff --git a/be/src/storage/tablet/tablet_reader.h b/be/src/storage/tablet/tablet_reader.h index db8e557ef5439b..82f3a707fdb2ad 100644 --- a/be/src/storage/tablet/tablet_reader.h +++ b/be/src/storage/tablet/tablet_reader.h @@ -45,6 +45,7 @@ #include "storage/rowset/rowset_meta.h" #include "storage/rowset/rowset_reader.h" #include "storage/rowset/rowset_reader_context.h" +#include "storage/segment/variant/variant_compaction_paths.h" #include "storage/tablet/base_tablet.h" #include "storage/tablet/tablet_fwd.h" @@ -127,6 +128,8 @@ class TabletReader { BaseTabletSPtr tablet; TabletSchemaSPtr tablet_schema; + // Set only by compaction, alongside its extended tablet_schema. + VariantCompactionPathsSPtr variant_compaction_paths = nullptr; ReaderType reader_type = ReaderType::READER_QUERY; bool read_row_binlog = false; bool direct_mode = false; diff --git a/be/src/storage/tablet/tablet_schema.cpp b/be/src/storage/tablet/tablet_schema.cpp index fd496809e78931..e607af932048ac 100644 --- a/be/src/storage/tablet/tablet_schema.cpp +++ b/be/src/storage/tablet/tablet_schema.cpp @@ -1187,7 +1187,6 @@ void TabletSchema::copy_from(const TabletSchema& tablet_schema) { tablet_schema.to_schema_pb(&tablet_schema_pb); init_from_pb(tablet_schema_pb); _table_id = tablet_schema.table_id(); - _path_set_info_map = tablet_schema._path_set_info_map; } void TabletSchema::shawdow_copy_without_columns(const TabletSchema& tablet_schema) { @@ -1630,43 +1629,7 @@ std::vector TabletSchema::inverted_indexs(const TabletColumn // TODO use more efficient impl // Use parent id if unique not assigned, this could happend when accessing subcolumns of variants int32_t col_unique_id = col.is_extracted_column() ? col.parent_unique_id() : col.unique_id(); - std::vector result; - if (result = inverted_indexs(col_unique_id, escape_for_path_name(col.suffix_path())); - !result.empty()) { - return result; - } - // variant's typed column has it's own index - else if (col.is_extracted_column() && col.path_info_ptr()->get_is_typed()) { - std::string relative_path = col.path_info_ptr()->copy_pop_front().get_path(); - if (_path_set_info_map.find(col_unique_id) == _path_set_info_map.end()) { - return result; - } - const auto& path_set_info = _path_set_info_map.at(col_unique_id); - if (path_set_info.typed_path_set.find(relative_path) == - path_set_info.typed_path_set.end()) { - return result; - } - for (const auto& index : path_set_info.typed_path_set.at(relative_path).indexes) { - result.push_back(index.get()); - } - return result; - } - // variant's subcolumns has it's own index - else if (col.is_extracted_column()) { - std::string relative_path = col.path_info_ptr()->copy_pop_front().get_path(); - if (_path_set_info_map.find(col_unique_id) == _path_set_info_map.end()) { - return result; - } - const auto& path_set_info = _path_set_info_map.at(col_unique_id); - if (path_set_info.subcolumn_indexes.find(relative_path) == - path_set_info.subcolumn_indexes.end()) { - return result; - } - for (const auto& index : path_set_info.subcolumn_indexes.at(relative_path)) { - result.push_back(index.get()); - } - } - return result; + return inverted_indexs(col_unique_id, escape_for_path_name(col.suffix_path())); } const TabletIndex* TabletSchema::ann_index(int32_t col_unique_id, diff --git a/be/src/storage/tablet/tablet_schema.h b/be/src/storage/tablet/tablet_schema.h index 01121657c725bf..db1db16ea4e0a7 100644 --- a/be/src/storage/tablet/tablet_schema.h +++ b/be/src/storage/tablet/tablet_schema.h @@ -723,33 +723,6 @@ class TabletSchema : public MetadataAdder { TabletIndexes indexes; }; - // all path in path_set_info are relative to the parent column - struct PathsSetInfo { - std::unordered_map typed_path_set; // typed columns - std::unordered_map subcolumn_indexes; // subcolumns indexes - PathSet sub_path_set; // extracted columns - PathSet sparse_path_set; // sparse columns - - // "Materialized regular path" means compaction chose to store this path as a dedicated - // column in the schema, either typed or extracted, instead of re-emitting it dynamically. - bool contains_materialized_regular_path(const std::string& path) const { - return typed_path_set.contains(path) || sub_path_set.contains(path); - } - }; - - void set_path_set_info(std::unordered_map&& path_set_info_map) { - _path_set_info_map = std::move(path_set_info_map); - } - - const PathsSetInfo& path_set_info(int32_t unique_id) const { - return _path_set_info_map.at(unique_id); - } - - const PathsSetInfo* try_path_set_info(int32_t unique_id) const { - auto it = _path_set_info_map.find(unique_id); - return it == _path_set_info_map.end() ? nullptr : &it->second; - } - bool need_record_variant_extended_schema() const { return variant_max_subcolumns_count() == 0; } int32_t variant_max_subcolumns_count() const { @@ -844,9 +817,6 @@ class TabletSchema : public MetadataAdder { std::map _vir_col_idx_to_unique_id; - // value: extracted path set and sparse path set - std::unordered_map _path_set_info_map; - // key: field_pattern // value: indexes using PatternToIndex = std::unordered_map>; diff --git a/be/src/storage/task/index_builder.cpp b/be/src/storage/task/index_builder.cpp index e436c1cbd11ff3..427c30cdcaf49c 100644 --- a/be/src/storage/task/index_builder.cpp +++ b/be/src/storage/task/index_builder.cpp @@ -718,7 +718,6 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta StorageReadOptions read_options; OlapReaderStatistics stats; read_options.stats = &stats; - read_options.tablet_schema = output_rowset_schema; auto schema = std::make_shared( project_columns_by_ordinal(output_rowset_schema->columns(), return_columns)); std::unique_ptr iter; @@ -954,7 +953,6 @@ Status IndexBuilder::_build_snii_indexes_for_segment(const TabletSchemaSPtr& out StorageReadOptions read_options; OlapReaderStatistics stats; read_options.stats = &stats; - read_options.tablet_schema = output_rowset_schema; auto schema = std::make_shared( project_columns_by_ordinal(output_rowset_schema->columns(), return_columns)); std::unique_ptr iter; diff --git a/be/test/exec/common/schema_util_rowset_test.cpp b/be/test/exec/common/schema_util_rowset_test.cpp index 66d7f79e591115..4d186b642fce07 100644 --- a/be/test/exec/common/schema_util_rowset_test.cpp +++ b/be/test/exec/common/schema_util_rowset_test.cpp @@ -532,7 +532,6 @@ TEST_F(SchemaUtilRowsetTest, collect_path_stats_and_get_extended_compaction_sche // key3 is in the sparse column, return variant type StorageReadOptions type_opts; - type_opts.tablet_schema = out_rowset->tablet_schema(); type_opts.io_ctx.reader_type = ReaderType::READER_QUERY; TabletColumn non_variant_column; non_variant_column.set_name("non_variant_column"); @@ -667,8 +666,9 @@ TabletSchemaSPtr create_compaction_schema_common(StorageEngine* _engine_ref, // 4. get compaction schema TabletSchemaSPtr compaction_schema = tablet_schema; + VariantCompactionPathsMap compaction_paths; auto st = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - rowsets, compaction_schema); + rowsets, compaction_schema, compaction_paths); EXPECT_TRUE(st.ok()) << st.msg(); // 5. check compaction schema @@ -785,8 +785,9 @@ TEST_F(SchemaUtilRowsetTest, typed_path_to_sparse_column) { // 4. get compaction schema TabletSchemaSPtr compaction_schema = tablet_schema; + VariantCompactionPathsMap compaction_paths; auto st = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - rowsets, compaction_schema); + rowsets, compaction_schema, compaction_paths); EXPECT_TRUE(st.ok()) << st.msg(); for (const auto& column : compaction_schema->columns()) { if (column->is_extracted_column()) { diff --git a/be/test/exec/common/schema_util_test.cpp b/be/test/exec/common/schema_util_test.cpp index 7e41be7d1f2c91..87620120557f64 100644 --- a/be/test/exec/common/schema_util_test.cpp +++ b/be/test/exec/common/schema_util_test.cpp @@ -379,7 +379,7 @@ TEST_F(SchemaUtilTest, get_subpaths) { {"path1", 1000}, {"path2", 800}, {"path3", 500}, {"path4", 300}, {"path5", 200}}; // get subpaths - std::unordered_map uid_to_paths_set_info; + VariantCompactionPathsMap uid_to_paths_set_info; variant_util::VariantCompactionUtil::get_subpaths(3, path_stats[1], uid_to_paths_set_info[1]); EXPECT_EQ(uid_to_paths_set_info[1].sub_path_set.size(), 3); @@ -408,7 +408,7 @@ TEST_F(SchemaUtilTest, get_subpaths_equal_to_max) { std::unordered_map path_stats; path_stats[1] = {{"path1", 1000}, {"path2", 800}, {"path3", 500}}; - std::unordered_map uid_to_paths_set_info; + VariantCompactionPathsMap uid_to_paths_set_info; variant_util::VariantCompactionUtil::get_subpaths(3, path_stats[1], uid_to_paths_set_info[1]); EXPECT_EQ(uid_to_paths_set_info[1].sub_path_set.size(), 3); @@ -426,7 +426,7 @@ TEST_F(SchemaUtilTest, get_subpaths_selects_empty_key_as_subpath) { variant_util::PathToNoneNullValues path_stats = { {"", 1000}, {"path1", 900}, {"path2", 800}, {"path3", 700}}; - TabletSchema::PathsSetInfo limited_paths; + VariantCompactionPaths limited_paths; variant_util::VariantCompactionUtil::get_subpaths(2, path_stats, limited_paths); EXPECT_TRUE(limited_paths.sub_path_set.contains("")); EXPECT_FALSE(limited_paths.sparse_path_set.contains("")); @@ -434,7 +434,7 @@ TEST_F(SchemaUtilTest, get_subpaths_selects_empty_key_as_subpath) { EXPECT_TRUE(limited_paths.sparse_path_set.contains("path2")); EXPECT_TRUE(limited_paths.sparse_path_set.contains("path3")); - TabletSchema::PathsSetInfo exact_limit_paths; + VariantCompactionPaths exact_limit_paths; variant_util::VariantCompactionUtil::get_subpaths(4, path_stats, exact_limit_paths); EXPECT_TRUE(exact_limit_paths.sub_path_set.contains("")); EXPECT_FALSE(exact_limit_paths.sparse_path_set.contains("")); @@ -442,7 +442,7 @@ TEST_F(SchemaUtilTest, get_subpaths_selects_empty_key_as_subpath) { EXPECT_TRUE(exact_limit_paths.sub_path_set.contains("path2")); EXPECT_TRUE(exact_limit_paths.sub_path_set.contains("path3")); - TabletSchema::PathsSetInfo unlimited_paths; + VariantCompactionPaths unlimited_paths; variant_util::VariantCompactionUtil::get_subpaths(0, path_stats, unlimited_paths); EXPECT_TRUE(unlimited_paths.sub_path_set.contains("")); EXPECT_TRUE(unlimited_paths.sparse_path_set.empty()); @@ -453,7 +453,7 @@ TEST_F(SchemaUtilTest, get_subpaths_selects_empty_key_as_subpath) { variant_util::PathToNoneNullValues low_rank_empty_key_stats = { {"path1", 1000}, {"path2", 900}, {"", 100}}; - TabletSchema::PathsSetInfo low_rank_empty_key_paths; + VariantCompactionPaths low_rank_empty_key_paths; variant_util::VariantCompactionUtil::get_subpaths(2, low_rank_empty_key_stats, low_rank_empty_key_paths); EXPECT_FALSE(low_rank_empty_key_paths.sub_path_set.contains("")); @@ -488,7 +488,7 @@ TEST_F(SchemaUtilTest, get_subpaths_multiple_variants) { path_stats[4] = { {"path1", 1000}, {"path2", 800}, {"path3", 500}, {"path4", 300}, {"path5", 200}}; - std::unordered_map uid_to_paths_set_info; + VariantCompactionPathsMap uid_to_paths_set_info; variant_util::VariantCompactionUtil::get_subpaths(3, path_stats[1], uid_to_paths_set_info[1]); variant_util::VariantCompactionUtil::get_subpaths(2, path_stats[2], uid_to_paths_set_info[2]); variant_util::VariantCompactionUtil::get_subpaths(4, path_stats[3], uid_to_paths_set_info[3]); @@ -539,7 +539,7 @@ TEST_F(SchemaUtilTest, get_subpaths_no_path_stats) { std::unordered_map path_stats; path_stats[2] = {{"path1", 1000}, {"path2", 800}}; - std::unordered_map uid_to_paths_set_info; + VariantCompactionPathsMap uid_to_paths_set_info; variant_util::VariantCompactionUtil::get_subpaths(3, path_stats[2], uid_to_paths_set_info[2]); EXPECT_EQ(uid_to_paths_set_info[1].sub_path_set.size(), 0); @@ -1227,8 +1227,9 @@ TEST_F(SchemaUtilTest, TestGetCompactionSchema) { auto target_schema = std::make_shared(); target_schema->init_from_pb(schema_pb); + VariantCompactionPathsMap compaction_paths; auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - rowsets, target_schema); + rowsets, target_schema, compaction_paths); EXPECT_TRUE(status.ok()); // Check that paths were properly distributed between subcolumns and sparse columns @@ -1265,8 +1266,9 @@ TEST_F(SchemaUtilTest, EXPECT_EQ(source_indexes[0]->index_name(), "v1_owner_idx"); std::vector rowsets; + VariantCompactionPathsMap compaction_paths; auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - rowsets, target_schema); + rowsets, target_schema, compaction_paths); ASSERT_TRUE(status.ok()) << status.to_string(); // get_extended_compaction_schema rebuilds from the base columns. Real compaction targets do @@ -1276,7 +1278,7 @@ TEST_F(SchemaUtilTest, const PathInData typed_path("v1.owner", true); EXPECT_EQ(target_schema->field_index(typed_path), -1); - const auto* path_set_info = target_schema->try_path_set_info(1); + const auto* path_set_info = compaction_paths.contains(1) ? &compaction_paths.at(1) : nullptr; ASSERT_NE(path_set_info, nullptr); EXPECT_FALSE(path_set_info->typed_path_set.contains("owner")); } @@ -1322,7 +1324,7 @@ TEST_F(SchemaUtilTest, get_compaction_typed_columns) { typed_paths.insert("profile.id.name"); TabletSchemaSPtr output_schema = std::make_shared(); TabletColumnPtr parent_column = std::make_shared(variant); - TabletSchema::PathsSetInfo paths_set_info; + VariantCompactionPaths paths_set_info; EXPECT_TRUE(variant_util::VariantCompactionUtil::get_compaction_typed_columns( schema, typed_paths, parent_column, output_schema, paths_set_info) .ok()); @@ -1351,7 +1353,7 @@ TEST_F(SchemaUtilTest, get_compaction_nested_columns) { nested_paths.insert(path2); TabletSchemaSPtr output_schema = std::make_shared(); - TabletSchema::PathsSetInfo paths_set_info; + VariantCompactionPaths paths_set_info; doris::variant_util::PathToDataTypes path_to_data_types; path_to_data_types[path1] = {std::make_shared(), @@ -1377,7 +1379,7 @@ TEST_F(SchemaUtilTest, get_compaction_nested_columns) { std::unordered_set bad_nested_paths; bad_nested_paths.insert(PathInData("not_exist")); TabletSchemaSPtr bad_output_schema = std::make_shared(); - TabletSchema::PathsSetInfo bad_paths_set_info; + VariantCompactionPaths bad_paths_set_info; Status st2 = variant_util::VariantCompactionUtil::get_compaction_nested_columns( bad_nested_paths, path_to_data_types, parent_column, bad_output_schema, bad_paths_set_info); @@ -1396,7 +1398,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_from_subpaths) { TabletColumnPtr parent_column = std::make_shared(variant); - TabletSchema::PathsSetInfo paths_set_info; + VariantCompactionPaths paths_set_info; paths_set_info.sub_path_set.insert(""); paths_set_info.sub_path_set.insert("a"); paths_set_info.sub_path_set.insert("b"); @@ -1492,7 +1494,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_advanced) { TabletColumnPtr parent_column = std::make_shared(variant); - TabletSchema::PathsSetInfo paths_set_info; + VariantCompactionPaths paths_set_info; paths_set_info.sub_path_set.insert("a"); paths_set_info.sub_path_set.insert("b"); paths_set_info.sub_path_set.insert("c"); @@ -1588,7 +1590,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_from_data_types) { path_to_data_types[PathInData()] = {std::make_shared()}; TabletSchemaSPtr output_schema = std::make_shared(); - TabletSchema::PathsSetInfo paths_set_info; + VariantCompactionPaths paths_set_info; variant_util::VariantCompactionUtil::get_compaction_subcolumns_from_data_types( paths_set_info, parent_column, target, path_to_data_types, output_schema); @@ -1657,7 +1659,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_from_data_types) { doris::variant_util::PathToDataTypes root_path_to_data_types; root_path_to_data_types[PathInData()] = {std::make_shared()}; TabletSchemaSPtr root_output_schema = std::make_shared(); - TabletSchema::PathsSetInfo root_paths_set_info; + VariantCompactionPaths root_paths_set_info; variant_util::VariantCompactionUtil::get_compaction_subcolumns_from_data_types( root_paths_set_info, parent_column, target, root_path_to_data_types, @@ -1668,7 +1670,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_from_data_types) { EXPECT_FALSE(root_paths_set_info.sub_path_set.contains("")); TabletSchemaSPtr empty_key_output_schema = std::make_shared(); - TabletSchema::PathsSetInfo empty_key_paths_set_info; + VariantCompactionPaths empty_key_paths_set_info; empty_key_paths_set_info.sub_path_set.insert(""); variant_util::VariantCompactionUtil::get_compaction_subcolumns_from_data_types( diff --git a/be/test/exec/scan/vgeneric_iterators_test.cpp b/be/test/exec/scan/vgeneric_iterators_test.cpp index d8f1a277efaa4f..1d8f8932d4a876 100644 --- a/be/test/exec/scan/vgeneric_iterators_test.cpp +++ b/be/test/exec/scan/vgeneric_iterators_test.cpp @@ -172,7 +172,6 @@ TEST(VGenericIteratorsTest, StatisticsIteratorPreservesNullForNullableChar) { OlapReaderStatistics stats; read_options.push_down_agg_type_opt = TPushAggOp::MINMAX; read_options.stats = &stats; - read_options.tablet_schema = tablet_schema; ASSERT_TRUE(iterator.init(read_options).ok()); Block block; @@ -277,7 +276,6 @@ class StatisticsIteratorStringBoundsTest : public testing::Test { StorageReadOptions read_options; read_options.push_down_agg_type_opt = agg; read_options.stats = &_stats; - read_options.tablet_schema = tablet_schema; if (with_delete) { auto del_pred = NullPredicate::create_shared(0, "c1", true, PrimitiveType::TYPE_INT); diff --git a/be/test/load/delta_writer/delta_writer_cluster_key_test.cpp b/be/test/load/delta_writer/delta_writer_cluster_key_test.cpp index ccb40950b0795c..8866a42f60e95a 100644 --- a/be/test/load/delta_writer/delta_writer_cluster_key_test.cpp +++ b/be/test/load/delta_writer/delta_writer_cluster_key_test.cpp @@ -349,7 +349,6 @@ TEST_F(TestDeltaWriterClusterKey, vec_sequence_col) { OlapReaderStatistics stats; StorageReadOptions opts; opts.stats = &stats; - opts.tablet_schema = rowset->tablet_schema(); std::unique_ptr iter; std::shared_ptr schema = create_full_schema(rowset->tablet_schema()); diff --git a/be/test/load/delta_writer/delta_writer_test.cpp b/be/test/load/delta_writer/delta_writer_test.cpp index 121b254cdba37e..1c6d7bd4522011 100644 --- a/be/test/load/delta_writer/delta_writer_test.cpp +++ b/be/test/load/delta_writer/delta_writer_test.cpp @@ -833,7 +833,6 @@ TEST_F(TestDeltaWriter, vec_sequence_col) { OlapReaderStatistics stats; StorageReadOptions opts; opts.stats = &stats; - opts.tablet_schema = rowset->tablet_schema(); std::unique_ptr iter; std::shared_ptr schema = create_full_schema(rowset->tablet_schema()); @@ -1040,7 +1039,6 @@ TEST_F(TestDeltaWriter, vec_sequence_col_concurrent_write) { OlapReaderStatistics stats; StorageReadOptions opts; opts.stats = &stats; - opts.tablet_schema = rowset1->tablet_schema(); opts.delete_bitmap.emplace(0, tablet->tablet_meta()->delete_bitmap().get_agg( {rowset1->rowset_id(), 0, cur_version})); std::unique_ptr iter; @@ -1068,7 +1066,6 @@ TEST_F(TestDeltaWriter, vec_sequence_col_concurrent_write) { OlapReaderStatistics stats; StorageReadOptions opts; opts.stats = &stats; - opts.tablet_schema = rowset2->tablet_schema(); opts.delete_bitmap.emplace(0, tablet->tablet_meta()->delete_bitmap().get_agg( {rowset2->rowset_id(), 0, cur_version})); std::unique_ptr iter; diff --git a/be/test/olap/rowset/group_rowset_writer_test.cpp b/be/test/olap/rowset/group_rowset_writer_test.cpp index 604c8366c41d4d..f832150f516c19 100644 --- a/be/test/olap/rowset/group_rowset_writer_test.cpp +++ b/be/test/olap/rowset/group_rowset_writer_test.cpp @@ -402,11 +402,15 @@ TEST_F(GroupRowsetWriterTest, partialUpdateSkipsHiddenNonKeyColumns) { const auto& row_binlog_schema = _row_binlog_tablet->tablet_schema(); ASSERT_EQ(7, row_binlog_schema->num_columns()); RowsetReaderContext reader_context; - reader_context.tablet_schema = row_binlog_schema; reader_context.need_ordered_result = false; // Read schema covers all row-binlog columns in order. auto read_schema = std::make_shared(row_binlog_schema->columns()); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*row_binlog_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr rowset_reader; ASSERT_TRUE(row_binlog_rowset->create_reader(&rowset_reader).ok()); @@ -471,9 +475,13 @@ TEST_F(GroupRowsetWriterTest, keyOnlyFixedPartialUpdatePreservesNarrowBlock) { auto read_schema = std::make_shared(project_columns_by_ordinal( row_binlog_schema->columns(), std::vector {0, 1, 2, 3, 4, 5, 6})); RowsetReaderContext reader_context; - reader_context.tablet_schema = row_binlog_schema; reader_context.need_ordered_result = false; reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*row_binlog_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr rowset_reader; ASSERT_TRUE(rowsets[1]->create_reader(&rowset_reader).ok()); diff --git a/be/test/storage/compaction/ordered_data_compaction_test.cpp b/be/test/storage/compaction/ordered_data_compaction_test.cpp index 685119d19ad630..8b39a01b873c66 100644 --- a/be/test/storage/compaction/ordered_data_compaction_test.cpp +++ b/be/test/storage/compaction/ordered_data_compaction_test.cpp @@ -77,6 +77,15 @@ static StorageEngine* engine_ref = nullptr; class OrderedDataCompactionTest : public ::testing::Test { protected: void SetUp() override { + _saved_enable_ordered_data_compaction = config::enable_ordered_data_compaction; + _saved_ordered_data_compaction_min_segment_size = + config::ordered_data_compaction_min_segment_size; + _saved_segments_key_bounds_truncation_threshold = + config::segments_key_bounds_truncation_threshold; + config::enable_ordered_data_compaction = true; + config::ordered_data_compaction_min_segment_size = 10; + config::segments_key_bounds_truncation_threshold = -1; + char buffer[MAX_PATH_LEN]; EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr); absolute_dir = std::string(buffer) + kTestDir; @@ -103,16 +112,24 @@ class OrderedDataCompactionTest : public ::testing::Test { _data_dir = std::make_unique(*engine_ref, absolute_dir); static_cast(_data_dir->update_capacity()); ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); - config::enable_ordered_data_compaction = true; - config::ordered_data_compaction_min_segment_size = 10; - config::segments_key_bounds_truncation_threshold = -1; } void TearDown() override { EXPECT_TRUE(io::global_local_filesystem()->delete_directory(absolute_dir).ok()); engine_ref = nullptr; ExecEnv::GetInstance()->set_storage_engine(nullptr); + // Otherwise the relaxed minimum segment size leaks into every later suite and sends its + // compactions down the ordered link-file path instead of a real merge. + config::enable_ordered_data_compaction = _saved_enable_ordered_data_compaction; + config::ordered_data_compaction_min_segment_size = + _saved_ordered_data_compaction_min_segment_size; + config::segments_key_bounds_truncation_threshold = + _saved_segments_key_bounds_truncation_threshold; } + bool _saved_enable_ordered_data_compaction = true; + int32_t _saved_ordered_data_compaction_min_segment_size = 0; + int32_t _saved_segments_key_bounds_truncation_threshold = 0; + TabletSchemaSPtr create_schema(KeysType keys_type = DUP_KEYS) { TabletSchemaSPtr tablet_schema = std::make_shared(); TabletSchemaPB tablet_schema_pb; @@ -492,11 +509,15 @@ TEST_F(OrderedDataCompactionTest, test_01) { // create output rowset reader RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); diff --git a/be/test/storage/compaction/segcompaction_mow_test.cpp b/be/test/storage/compaction/segcompaction_mow_test.cpp index bb32ca55d2c6ba..76100dffc11b19 100644 --- a/be/test/storage/compaction/segcompaction_mow_test.cpp +++ b/be/test/storage/compaction/segcompaction_mow_test.cpp @@ -241,7 +241,6 @@ class SegCompactionMoWTest : public ::testing::TestWithParam { int expect_total_rows, int rows_mark_deleted, bool skip_value_check = false) { RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; // use this type to avoid cache from other ut reader_context.reader_type = ReaderType::READER_QUERY; reader_context.need_ordered_result = true; @@ -249,6 +248,11 @@ class SegCompactionMoWTest : public ::testing::TestWithParam { auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), return_columns)); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); reader_context.stats = &_stats; reader_context.delete_bitmap = delete_bitmap; diff --git a/be/test/storage/compaction/segcompaction_test.cpp b/be/test/storage/compaction/segcompaction_test.cpp index fb7a27fee5d931..7e2438f604bb1d 100644 --- a/be/test/storage/compaction/segcompaction_test.cpp +++ b/be/test/storage/compaction/segcompaction_test.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#include #include #include #include @@ -26,6 +27,11 @@ #include #include "common/config.h" +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/column/variant_v2/column_variant_v2.h" #include "cpp/sync_point.h" #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" @@ -44,6 +50,7 @@ #include "storage/tablet/tablet_meta.h" #include "storage/tablet/tablet_schema.h" #include "storage/utils.h" +#include "testutil/variant_util.h" #include "util/debug_points.h" #include "util/defer_op.h" #include "util/slice.h" @@ -292,6 +299,139 @@ class SegCompactionTest : public testing::Test { EXPECT_EQ(Status::OK(), s); } + // ---- variant / segment-compaction interaction ---- + + static constexpr int kVariantSegments = 12; + static constexpr int kVariantRowsPerSegment = 200; + + // Each segment carries a key of its own ("s") on top of the shared ones, so a merge + // would have to re-split subcolumns and sparse paths rather than copy the input layout. + static std::string variant_json(int segment, int rid) { + return fmt::format(R"({{"a":{},"b":"mark_{}_{}","s{}":{}}})", rid, segment, rid, segment, + rid * 7); + } + + // (c1 INT key, v VARIANT or INT). The variant flavour is the one BetaRowsetWriter opts out + // of segment compaction. + TabletSchemaSPtr create_variant_tablet_schema(bool with_variant) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_num_short_key_columns(1); + schema_pb.set_num_rows_per_row_block(1024); + schema_pb.set_compress_kind(COMPRESS_NONE); + schema_pb.set_next_column_unique_id(3); + + ColumnPB* key = schema_pb.add_column(); + key->set_unique_id(1); + key->set_name("c1"); + key->set_type("INT"); + key->set_is_key(true); + key->set_length(4); + key->set_index_length(4); + key->set_is_nullable(false); + + ColumnPB* value = schema_pb.add_column(); + value->set_unique_id(2); + value->set_name("v"); + value->set_is_key(false); + value->set_is_nullable(true); + if (with_variant) { + value->set_type("VARIANT"); + // Small enough that only some paths stay extracted; the rest go to the sparse column. + value->set_variant_max_subcolumns_count(3); + value->set_variant_max_sparse_column_statistics_size(10000); + value->set_variant_sparse_hash_shard_count(1); + } else { + value->set_type("INT"); + value->set_length(4); + } + + auto tablet_schema = std::make_shared(); + tablet_schema->init_from_pb(schema_pb); + return tablet_schema; + } + + // `wait_for_segcompaction` sleeps between flushes so the async worker gets to start: + // BetaRowsetWriter::_close_file_writers cancels a task that has not started yet, which would + // leave a rowset uncompacted for timing reasons rather than for the reason under test. + void write_variant_rowset(int64_t id, const TabletSchemaSPtr& tablet_schema, + bool wait_for_segcompaction, RowsetSharedPtr* rowset) { + RowsetWriterContext writer_context; + create_rowset_writer_context(id, tablet_schema, &writer_context); + auto res = RowsetFactory::create_rowset_writer(*l_engine, writer_context, false); + ASSERT_TRUE(res.has_value()) << res.error(); + auto rowset_writer = std::move(res).value(); + + const bool with_variant = + tablet_schema->column(1).type() == FieldType::OLAP_FIELD_TYPE_VARIANT; + for (int seg = 0; seg < kVariantSegments; ++seg) { + Block block = tablet_schema->create_storage_block(); + auto columns = std::move(block).mutate_columns(); + auto raw_json = ColumnString::create(); + auto* nullable = assert_cast(columns[1].get()); + for (int rid = 0; rid < kVariantRowsPerSegment; ++rid) { + int32_t c1 = seg * kVariantRowsPerSegment + rid; + columns[0]->insert_data(reinterpret_cast(&c1), sizeof(c1)); + if (with_variant) { + std::string json = variant_json(seg, rid); + raw_json->insert_data(json.data(), json.size()); + } else { + nullable->get_nested_column().insert_data(reinterpret_cast(&c1), + sizeof(c1)); + } + nullable->get_null_map_data().push_back(0); + } + if (with_variant) { + VariantUtil::insert_json_rows( + assert_cast(nullable->get_nested_column()), *raw_json); + } + ASSERT_TRUE(add_block_with_columns(rowset_writer.get(), &block, &columns).ok()); + ASSERT_TRUE(rowset_writer->flush().ok()); + if (wait_for_segcompaction) { + sleep(1); + } + } + ASSERT_EQ(Status::OK(), rowset_writer->build(*rowset)); + } + + // (c1, stringified value) for every row, ordered by c1. + void read_all_rows(const RowsetSharedPtr& rowset, const TabletSchemaSPtr& tablet_schema, + std::vector>* rows) { + RowsetReaderContext reader_context; + reader_context.reader_type = ReaderType::READER_QUERY; + reader_context.need_ordered_result = true; + std::vector return_columns = {0, 1}; + auto read_schema = std::make_shared( + project_columns_by_ordinal(tablet_schema->columns(), return_columns)); + reader_context.read_schema = read_schema; + ASSERT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); + reader_context.stats = &_stats; + + RowsetReaderSharedPtr rowset_reader; + create_and_init_rowset_reader(rowset.get(), reader_context, &rowset_reader); + + while (true) { + auto block = read_schema->create_read_block(); + auto st = rowset_reader->next_batch(&block); + if (!st.ok()) { + ASSERT_TRUE(st.is()) << st; + break; + } + const auto& value_col = block.get_by_position(1); + const auto& keys = assert_cast(*block.get_by_position(0).column); + for (size_t i = 0; i < block.rows(); ++i) { + rows->emplace_back(keys.get_data()[i], + value_col.type->to_string(*value_col.column, i)); + } + } + std::sort(rows->begin(), rows->end(), + [](const auto& lhs, const auto& rhs) { return lhs.first < rhs.first; }); + } + private: std::unique_ptr _data_dir; std::unique_ptr _inverted_index_searcher_cache; @@ -410,7 +550,6 @@ TEST_F(SegCompactionTest, SegCompactionThenRead) { { // read RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; // use this type to avoid cache from other ut reader_context.reader_type = ReaderType::READER_CUMULATIVE_COMPACTION; reader_context.need_ordered_result = true; @@ -418,6 +557,11 @@ TEST_F(SegCompactionTest, SegCompactionThenRead) { auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), return_columns)); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); reader_context.stats = &_stats; // without predicates @@ -918,7 +1062,6 @@ TEST_F(SegCompactionTest, SegCompactionThenReadUniqueTableSmall) { { // read RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; // use this type to avoid cache from other ut reader_context.reader_type = ReaderType::READER_CUMULATIVE_COMPACTION; reader_context.need_ordered_result = true; @@ -926,6 +1069,11 @@ TEST_F(SegCompactionTest, SegCompactionThenReadUniqueTableSmall) { auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), return_columns)); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); reader_context.stats = &_stats; reader_context.is_unique = true; @@ -1186,7 +1334,6 @@ TEST_F(SegCompactionTest, SegCompactionThenReadAggTableSmall) { { // read RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; // use this type to avoid cache from other ut reader_context.reader_type = ReaderType::READER_CUMULATIVE_COMPACTION; reader_context.need_ordered_result = true; @@ -1194,6 +1341,11 @@ TEST_F(SegCompactionTest, SegCompactionThenReadAggTableSmall) { auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), return_columns)); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); reader_context.stats = &_stats; // reader_context.is_unique = true; @@ -1248,6 +1400,49 @@ TEST_F(SegCompactionTest, SegCompactionThenReadAggTableSmall) { } } +// BetaRowsetWriter::_segcompaction_if_necessary opts a rowset out of segment compaction as soon +// as its schema has a variant column, so a variant rowset keeps every segment it flushed. The +// variant-free rowset is the control: same rows, same thresholds, and that one does get merged. +TEST_F(SegCompactionTest, VariantRowsetIsNeverSegmentCompacted) { + const auto saved_candidate_max_rows = config::segcompaction_candidate_max_rows; + const auto saved_batch_size = config::segcompaction_batch_size; + Defer restore_config([&] { + config::segcompaction_candidate_max_rows = saved_candidate_max_rows; + config::segcompaction_batch_size = saved_batch_size; + }); + config::enable_segcompaction = true; + config::segcompaction_candidate_max_rows = kVariantRowsPerSegment * 2; + config::segcompaction_batch_size = 5; + + RowsetSharedPtr plain_rowset; + ASSERT_NO_FATAL_FAILURE(write_variant_rowset(10060, create_variant_tablet_schema(false), + /*wait_for_segcompaction=*/true, &plain_rowset)); + ASSERT_NE(plain_rowset, nullptr); + EXPECT_LT(plain_rowset->rowset_meta()->num_segments(), kVariantSegments); + EXPECT_EQ(kVariantSegments * kVariantRowsPerSegment, plain_rowset->rowset_meta()->num_rows()); + + // No task is ever submitted for the variant rowset, so there is nothing to wait for. + auto variant_schema = create_variant_tablet_schema(true); + RowsetSharedPtr variant_rowset; + ASSERT_NO_FATAL_FAILURE(write_variant_rowset(10061, variant_schema, + /*wait_for_segcompaction=*/false, + &variant_rowset)); + ASSERT_NE(variant_rowset, nullptr); + EXPECT_EQ(kVariantSegments, variant_rowset->rowset_meta()->num_segments()); + EXPECT_EQ(kVariantSegments * kVariantRowsPerSegment, variant_rowset->rowset_meta()->num_rows()); + + std::vector> rows; + ASSERT_NO_FATAL_FAILURE(read_all_rows(variant_rowset, variant_schema, &rows)); + ASSERT_EQ(kVariantSegments * kVariantRowsPerSegment, rows.size()); + for (size_t i = 0; i < rows.size(); ++i) { + ASSERT_EQ(static_cast(i), rows[i].first) << "row " << i; + EXPECT_NE(rows[i].second.find(fmt::format("mark_{}_{}", i / kVariantRowsPerSegment, + i % kVariantRowsPerSegment)), + std::string::npos) + << "row " << i << ": " << rows[i].second; + } +} + } // namespace doris // @brief Test Stub diff --git a/be/test/storage/compaction/vertical_compaction_test.cpp b/be/test/storage/compaction/vertical_compaction_test.cpp index 108ded0a8097bf..70706ebfa5fb7b 100644 --- a/be/test/storage/compaction/vertical_compaction_test.cpp +++ b/be/test/storage/compaction/vertical_compaction_test.cpp @@ -640,11 +640,15 @@ TEST_F(VerticalCompactionTest, TestDupKeyVerticalMerge) { // create output rowset reader RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -778,11 +782,15 @@ TEST_F(VerticalCompactionTest, MergeHonorsKeyRanges) { ASSERT_EQ(expected_end - expected_begin, output_rowset->num_rows()); RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_reader; create_and_init_rowset_reader(output_rowset.get(), reader_context, &output_reader); @@ -895,11 +903,15 @@ TEST_F(VerticalCompactionTest, TestDupWithoutKeyVerticalMerge) { // create output rowset reader RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -1002,11 +1014,15 @@ TEST_F(VerticalCompactionTest, TestUniqueKeyVerticalMerge) { // create output rowset reader RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -1263,11 +1279,15 @@ TEST_F(VerticalCompactionTest, TestUniqueKeySegmentContextMemoryAmplification) { ASSERT_EQ(total_rows, output_rowset->num_rows()); RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared(project_columns_by_ordinal( tablet_schema->columns(), std::vector {0, 1, 2})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; create_and_init_rowset_reader(output_rowset.get(), reader_context, &output_rs_reader); @@ -1382,11 +1402,15 @@ TEST_F(VerticalCompactionTest, TestDupKeyVerticalMergeWithDelete) { // create output rowset reader RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -1484,11 +1508,15 @@ TEST_F(VerticalCompactionTest, TestDupWithoutKeyVerticalMergeWithDelete) { // create output rowset reader RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -1577,11 +1605,15 @@ TEST_F(VerticalCompactionTest, TestAggKeyVerticalMerge) { // create output rowset reader RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -1772,11 +1804,15 @@ TEST_F(VerticalCompactionTest, TestUniqueKeyVerticalMergeWithNullableSparseColum ASSERT_EQ(Status::OK(), output_rs_writer->build(out_rowset)); RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1, 2})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); diff --git a/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp b/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp index f9c7693b67b09a..b2ea2c148235f2 100644 --- a/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp +++ b/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp @@ -1399,10 +1399,9 @@ TEST_F(CollectionStatisticsTest, CollectWithDoubleCastWrappedSlotRef) { EXPECT_TRUE(status.ok()) << status.msg(); } -// Regression for AIR-36: match score collection must resolve indexes for -// variant sub-columns whose indexes live in _path_set_info_map (typed paths or -// inherited sub-column indexes). The previous simple lookup using -// inverted_indexs(col_unique_id, suffix_path) missed those indexes. +// Regression for AIR-36: match score collection must resolve the index of a variant sub-column, +// which is registered on the parent column's unique id under the sub-column's suffix path rather +// than on a unique id of its own. TEST_F(CollectionStatisticsTest, ExtractCollectInfoForVariantSubcolumnIndex) { auto tablet_schema = std::make_shared(); @@ -1433,15 +1432,8 @@ TEST_F(CollectionStatisticsTest, ExtractCollectInfoForVariantSubcolumnIndex) { (*props)["parser"] = "standard"; (*props)["support_phrase"] = "true"; sub_index->init_from_pb(index_pb); - - TabletSchema::PathsSetInfo path_set_info; - TabletIndexes sub_indexes = {sub_index}; - path_set_info.subcolumn_indexes["host"] = sub_indexes; - std::unordered_map path_set_info_map; - path_set_info_map[kVariantUid] = std::move(path_set_info); - tablet_schema->set_path_set_info(std::move(path_set_info_map)); - - EXPECT_TRUE(tablet_schema->inverted_indexs(kVariantUid, "host").empty()); + sub_index->set_escaped_escaped_index_suffix_path(sub_col.suffix_path()); + tablet_schema->append_index(std::move(*sub_index)); auto found = tablet_schema->inverted_indexs(tablet_schema->column(/*ordinal=*/1)); ASSERT_EQ(found.size(), 1u); @@ -2317,9 +2309,6 @@ TEST_F(CollectionStatisticsTest, SearchTypedVariantBindingSelectsItsAnalyzerInde subcolumn.set_path_info(PathInData("v.host", true)); tablet_schema->append_column(subcolumn); - TabletSchema::PathsSetInfo path_set_info; - TabletSchema::SubColumnInfo typed_path_info; - typed_path_info.column = subcolumn; for (const auto& [index_id, parser] : {std::pair {3010, "standard"}, std::pair {3020, "english"}}) { auto index = std::make_shared(); @@ -2331,12 +2320,9 @@ TEST_F(CollectionStatisticsTest, SearchTypedVariantBindingSelectsItsAnalyzerInde (*index_pb.mutable_properties())["parser"] = parser; (*index_pb.mutable_properties())["support_phrase"] = "true"; index->init_from_pb(index_pb); - typed_path_info.indexes.push_back(std::move(index)); + index->set_escaped_escaped_index_suffix_path(subcolumn.suffix_path()); + tablet_schema->append_index(std::move(*index)); } - path_set_info.typed_path_set.emplace("host", std::move(typed_path_info)); - std::unordered_map path_set_info_map; - path_set_info_map.emplace(kVariantUid, std::move(path_set_info)); - tablet_schema->set_path_set_info(std::move(path_set_info_map)); TSearchClause clause; clause.clause_type = "TERM"; diff --git a/be/test/storage/iterator/block_reader_change_next_block_test.cpp b/be/test/storage/iterator/block_reader_change_next_block_test.cpp index a75bd4a6cfe4ee..1a9fbcbb1ee19f 100644 --- a/be/test/storage/iterator/block_reader_change_next_block_test.cpp +++ b/be/test/storage/iterator/block_reader_change_next_block_test.cpp @@ -371,7 +371,11 @@ void configure_reader(BlockReader& reader, std::shared_ptr source, size_t } auto read_schema = std::make_shared(reader._tablet_schema->columns(), std::move(read_types)); - read_schema->init_row_binlog_column_mappings(*reader._tablet_schema); + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*reader._tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/true) + .ok()); reader._read_schema = std::move(read_schema); reader._next_row.block = source; diff --git a/be/test/storage/mow/mow_transform_test_base.h b/be/test/storage/mow/mow_transform_test_base.h index 803f13f838cc90..ffaaf242999aac 100644 --- a/be/test/storage/mow/mow_transform_test_base.h +++ b/be/test/storage/mow/mow_transform_test_base.h @@ -241,8 +241,10 @@ class MowTransformTestBase : public testing::Test { Block* output) { auto read_schema = std::make_shared(schema->columns()); RowsetReaderContext context; - context.tablet_schema = schema; context.read_schema = read_schema; + static_cast(read_schema->init_from_tablet_schema(*schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false)); context.need_ordered_result = true; OlapReaderStatistics statistics; context.stats = &statistics; diff --git a/be/test/storage/read_schema_test.cpp b/be/test/storage/read_schema_test.cpp index 5ee2abd5f0ac49..cd27e5c753346a 100644 --- a/be/test/storage/read_schema_test.cpp +++ b/be/test/storage/read_schema_test.cpp @@ -15,11 +15,13 @@ // specific language governing permissions and limitations // under the License. +#include #include #include #include #include +#include #include #include @@ -28,6 +30,7 @@ #include "core/data_type/data_type_struct.h" #include "storage/binlog.h" #include "storage/schema.h" +#include "util/json/path_in_data.h" namespace doris { namespace { @@ -147,7 +150,11 @@ TEST(ReadSchemaTest, RowBinlogMappingsUsePhysicalSchemaOrdinals) { ReadSchema read_schema(project_columns_by_ordinal( tablet_schema->columns(), std::vector {0, 2, 1, 4, 3, 5, 6, 7})); - read_schema.init_row_binlog_column_mappings(*tablet_schema); + EXPECT_TRUE(read_schema + .init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/true) + .ok()); EXPECT_TRUE(read_schema.row_binlog_value_pairs_complete()); EXPECT_EQ(read_schema.row_binlog_value_column_pairs(), @@ -170,7 +177,10 @@ TEST(ReadSchemaTest, MalformedRowBinlogLayoutKeepsConservativeNameMapping) { tablet_schema.append_column(*create_int_column(16, BINLOG_OP_COL)); ReadSchema read_schema(tablet_schema.columns()); - read_schema.init_row_binlog_column_mappings(tablet_schema); + EXPECT_TRUE(read_schema + .init_from_tablet_schema(tablet_schema, /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/true) + .ok()); EXPECT_FALSE(read_schema.row_binlog_value_pairs_complete()); EXPECT_TRUE(read_schema.row_binlog_value_column_pairs().empty()); @@ -178,4 +188,149 @@ TEST(ReadSchemaTest, MalformedRowBinlogLayoutKeepsConservativeNameMapping) { } } // namespace + +namespace { + +// key k(10), value v(11) governed by sequence column s(12). +TabletSchemaSPtr create_sequence_mapped_schema() { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::UNIQUE_KEYS); + for (const auto& [uid, name, is_key] : {std::tuple {10, "k", true}, + {11, "v", false}, + {12, "s", false}}) { + ColumnPB* column_pb = schema_pb.add_column(); + column_pb->set_unique_id(uid); + column_pb->set_name(name); + column_pb->set_type("INT"); + column_pb->set_length(4); + column_pb->set_index_length(4); + column_pb->set_is_key(is_key); + column_pb->set_is_nullable(!is_key); + } + ColumnGroupPB* group = schema_pb.mutable_seq_map()->add_cg(); + group->set_sequence_column(12); + group->add_columns_in_group(11); + + auto tablet_schema = std::make_shared(); + tablet_schema->init_from_pb(schema_pb); + return tablet_schema; +} + +} // namespace + +// The two facts hold for the whole read: they are taken from the tablet schema, not from the +// projection, so a column group that excludes the columns they are about still reports them. +TEST(ReadSchemaTest, TabletHasSequenceMapIsIndependentOfTheProjection) { + auto tablet_schema = create_sequence_mapped_schema(); + + ReadSchema whole(tablet_schema->columns()); + ASSERT_TRUE(whole.init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); + EXPECT_TRUE(whole.tablet_has_sequence_map()); + + // The key column alone: neither the sequence column nor the value it governs is projected. + ReadSchema key_only( + project_columns_by_ordinal(tablet_schema->columns(), std::vector {0})); + ASSERT_TRUE(key_only.init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); + EXPECT_TRUE(key_only.tablet_has_sequence_map()); +} + +TEST(ReadSchemaTest, TabletWithoutSequenceMap) { + auto tablet_schema = std::make_shared(); + tablet_schema->append_column(*create_int_column(10, "k", true)); + tablet_schema->append_column(*create_int_column(11, "v")); + + ReadSchema read_schema(tablet_schema->columns()); + ASSERT_TRUE(read_schema + .init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); + EXPECT_FALSE(read_schema.tablet_has_sequence_map()); + EXPECT_FALSE(read_schema.tablet_has_extracted_variant_columns()); +} + +TEST(ReadSchemaTest, MergeBySequenceMappingBuildsTheMap) { + auto tablet_schema = create_sequence_mapped_schema(); + ReadSchema read_schema(tablet_schema->columns()); + ASSERT_TRUE(read_schema + .init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/true, + /*map_row_binlog_columns=*/false) + .ok()); + + // Ordinals are this ReadSchema's, not the tablet schema's column ids. + const auto& sequence_map = read_schema.sequence_map(); + ASSERT_EQ(1, sequence_map.size()); + auto group = sequence_map.find(2); + ASSERT_NE(group, sequence_map.end()); + EXPECT_EQ((std::vector {1}), group->second); + + // Without the flag the layout is left alone, even though the tablet has one. + ReadSchema untouched(tablet_schema->columns()); + ASSERT_TRUE(untouched + .init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); + EXPECT_TRUE(untouched.sequence_map().empty()); +} + +// A tablet cannot have both a sequence column and a sequence mapping; asking to merge by the +// mapping on such a schema must fail rather than build half a layout. +TEST(ReadSchemaTest, SequenceColumnAndSequenceMapConflict) { + auto tablet_schema = create_sequence_mapped_schema(); + tablet_schema->append_column(*create_int_column(13, SEQUENCE_COL)); + + ReadSchema read_schema(tablet_schema->columns()); + auto st = read_schema.init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/true, + /*map_row_binlog_columns=*/false); + EXPECT_FALSE(st.ok()) << st; + EXPECT_TRUE(read_schema.sequence_map().empty()); +} + +TEST(ReadSchemaTest, HasExtractedVariantColumns) { + auto tablet_schema = std::make_shared(); + tablet_schema->append_column(*create_int_column(10, "k", true)); + + TabletColumn variant; + variant.set_unique_id(11); + variant.set_name("v"); + variant.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); + variant.set_is_nullable(true); + tablet_schema->append_column(variant); + + ReadSchema without_extracted(tablet_schema->columns()); + ASSERT_TRUE(without_extracted + .init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); + EXPECT_FALSE(without_extracted.tablet_has_extracted_variant_columns()); + + TabletColumn extracted; + extracted.set_unique_id(-1); + extracted.set_name("v.a"); + extracted.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + extracted.set_parent_unique_id(11); + extracted.set_path_info(PathInData("v.a")); + tablet_schema->append_column(extracted); + + // Projected without the extracted column, which must not change the answer. + ReadSchema read_schema( + project_columns_by_ordinal(tablet_schema->columns(), std::vector {0})); + ASSERT_TRUE(read_schema + .init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); + EXPECT_TRUE(read_schema.tablet_has_extracted_variant_columns()); +} + } // namespace doris diff --git a/be/test/storage/row_binlog_vmerge_compaction_test.cpp b/be/test/storage/row_binlog_vmerge_compaction_test.cpp index 62bdd43963d235..3b3f5fbb5beca3 100644 --- a/be/test/storage/row_binlog_vmerge_compaction_test.cpp +++ b/be/test/storage/row_binlog_vmerge_compaction_test.cpp @@ -271,9 +271,13 @@ class RowBinlogVmergeCompactionTest : public testing::Test { project_columns_by_ordinal(tablet_schema->columns(), ordinals)); RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr reader; EXPECT_TRUE(rowset->create_reader(&reader).ok()); diff --git a/be/test/storage/rowid_conversion_test.cpp b/be/test/storage/rowid_conversion_test.cpp index 8c5fa1ab8f6554..2d22964eb6ef65 100644 --- a/be/test/storage/rowid_conversion_test.cpp +++ b/be/test/storage/rowid_conversion_test.cpp @@ -391,11 +391,15 @@ class TestRowIdConversion : public testing::TestWithParam( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0, 1})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -826,12 +830,16 @@ TEST_F(TestRowIdConversion, SingleRowsetGroupedCompactionRowIdConversionIsComple EXPECT_EQ(output_segment_count, output_rowset->num_segments()); RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; std::vector return_columns = {0, 1}; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), return_columns)); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_reader; create_and_init_rowset_reader(output_rowset.get(), reader_context, &output_reader); @@ -1028,9 +1036,13 @@ TEST_F(TestRowIdConversion, SingleRowsetGroupedCompactionRowIdConversionIsComple } RowsetReaderContext second_reader_context; - second_reader_context.tablet_schema = tablet_schema; second_reader_context.need_ordered_result = false; second_reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr second_output_reader; create_and_init_rowset_reader(second_output_rowset.get(), second_reader_context, &second_output_reader); diff --git a/be/test/storage/rowset/segment_flusher_format_test.cpp b/be/test/storage/rowset/segment_flusher_format_test.cpp index ccfa718d4f7013..22802570fd9de2 100644 --- a/be/test/storage/rowset/segment_flusher_format_test.cpp +++ b/be/test/storage/rowset/segment_flusher_format_test.cpp @@ -2228,7 +2228,6 @@ Result read_logical_segment(const std::string& path, uin OlapReaderStatistics stats; StorageReadOptions read_options; read_options.stats = &stats; - read_options.tablet_schema = schema; std::unique_ptr iterator; RETURN_IF_ERROR_RESULT(segment->new_iterator(read_schema, read_options, &iterator)); diff --git a/be/test/storage/segment/mock/mock_segment.h b/be/test/storage/segment/mock/mock_segment.h index a29404ee310098..052d37c8fc5e30 100644 --- a/be/test/storage/segment/mock/mock_segment.h +++ b/be/test/storage/segment/mock/mock_segment.h @@ -37,6 +37,8 @@ class ColumnReaderCache; class MockSegment : public Segment { public: MockSegment() : Segment(1, RowsetId(), std::make_shared(), {}) {} + explicit MockSegment(TabletSchemaSPtr tablet_schema) + : Segment(1, RowsetId(), std::move(tablet_schema), {}) {} ~MockSegment() override = default; // Mock methods for file reader diff --git a/be/test/storage/segment/segment_cache_test.cpp b/be/test/storage/segment/segment_cache_test.cpp index 6e69fa4afbc5eb..5a2ead29ffa80c 100644 --- a/be/test/storage/segment/segment_cache_test.cpp +++ b/be/test/storage/segment/segment_cache_test.cpp @@ -360,7 +360,6 @@ TEST_F(SegmentCacheTest, vec_sequence_col) { OlapReaderStatistics stats; StorageReadOptions opts; opts.stats = &stats; - opts.tablet_schema = rowset->tablet_schema(); std::unique_ptr iter; std::shared_ptr schema = create_full_schema(rowset->tablet_schema()); diff --git a/be/test/storage/segment/segment_iterator_count_emit_shortcut_test.cpp b/be/test/storage/segment/segment_iterator_count_emit_shortcut_test.cpp index 2ac1841ba04b02..0bb6646d54061a 100644 --- a/be/test/storage/segment/segment_iterator_count_emit_shortcut_test.cpp +++ b/be/test/storage/segment/segment_iterator_count_emit_shortcut_test.cpp @@ -23,8 +23,8 @@ // must admit only the provably emission-only configuration and refuse on any // deviation (falling through to today's loop). Uses the established // `#define private public` convention of segment_iterator_limit_opt_test.cpp -// over a bare SegmentIterator (no real segment needed: the shortcut never -// touches segment data). +// over a SegmentIterator whose segment carries only the tablet schema (the +// engage proof reads the keys type; nothing reads segment data). #include #include @@ -42,6 +42,7 @@ #include "storage/index/index_query_context.h" #include "storage/olap_common.h" #include "storage/segment/count_on_index_fastpath.h" +#include "storage/segment/mock/mock_segment.h" #include "storage/tablet/tablet_schema.h" #if defined(__clang__) @@ -111,8 +112,8 @@ struct Fixture { Fixture() { tablet_schema = make_tablet_schema(); read_schema = make_read_schema(tablet_schema); - iter = std::make_unique(nullptr, read_schema); - iter->_opts.tablet_schema = tablet_schema; + iter = std::make_unique(std::make_shared(tablet_schema), + read_schema); iter->_opts.push_down_agg_type_opt = TPushAggOp::COUNT_ON_INDEX; iter->_opts.stats = &stats; // State _lazy_init/_vec_init_lazy_materialization would have produced diff --git a/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp b/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp index 357e2ba876b133..94fa6398c1e161 100644 --- a/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp +++ b/be/test/storage/segment/segment_iterator_expr_zonemap_test.cpp @@ -247,7 +247,6 @@ TEST_F(SegmentIteratorExprZonemapTest, NewIteratorPrunesWholeSegmentByExprZonema StorageReadOptions read_options; read_options.stats = &_stats; read_options.runtime_state = &_runtime_state; - read_options.tablet_schema = _tablet_schema; read_options.common_expr_ctxs_push_down = {expr_ctx}; std::unique_ptr iter; @@ -270,7 +269,6 @@ TEST_F(SegmentIteratorExprZonemapTest, NewIteratorKeepsSegmentWhenExprZonemapMay StorageReadOptions read_options; read_options.stats = &_stats; read_options.runtime_state = &_runtime_state; - read_options.tablet_schema = _tablet_schema; read_options.common_expr_ctxs_push_down = {expr_ctx}; std::unique_ptr iter; @@ -290,7 +288,6 @@ TEST_F(SegmentIteratorExprZonemapTest, ApplyExprZonemapPrunesPageRowRanges) { SegmentIterator iter(segment, read_schema); iter._file_reader = segment->_file_reader; iter._opts.stats = &_stats; - iter._opts.tablet_schema = _tablet_schema; auto expr_ctx = std::make_shared(std::make_shared(1, 500)); VExprContextSPtrs conjuncts {expr_ctx}; @@ -313,7 +310,6 @@ TEST_F(SegmentIteratorExprZonemapTest, NewColumnIteratorReadsCommitTsoFromReadOp StorageReadOptions read_options; read_options.stats = &_stats; - read_options.tablet_schema = _tablet_schema; read_options.version = Version(7, 7); read_options.commit_tso = TsoRange(kCommitTso, kCommitTso); read_options.io_ctx.reader_type = ReaderType::READER_QUERY; @@ -354,7 +350,6 @@ TEST_F(SegmentIteratorExprZonemapTest, NewIteratorPrunesCommitTsoByReadOptionVal StorageReadOptions read_options; read_options.stats = &_stats; - read_options.tablet_schema = _tablet_schema; read_options.version = Version(7, 7); read_options.commit_tso = TsoRange(kCommitTso, kCommitTso); read_options.io_ctx.reader_type = ReaderType::READER_QUERY; diff --git a/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp b/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp index 2ae5574688c4f0..fe7df53369a810 100644 --- a/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp +++ b/be/test/storage/segment/segment_iterator_lazy_pruned_test.cpp @@ -117,7 +117,6 @@ class SegmentIteratorLazyPrunedTest : public ::testing::Test { std::unique_ptr make_iter(TrackingLazyColumnIterator** tracking_iter) { auto iter = std::make_unique(nullptr, _read_schema); - iter->_opts.tablet_schema = _tablet_schema; iter->_opts.stats = &_stats; iter->_lazy_pruned_ordinals.push_back(0); iter->_column_iterators.resize(1); diff --git a/be/test/storage/segment/segment_iterator_limit_opt_test.cpp b/be/test/storage/segment/segment_iterator_limit_opt_test.cpp index db69371b52b0ed..dac069a29a1799 100644 --- a/be/test/storage/segment/segment_iterator_limit_opt_test.cpp +++ b/be/test/storage/segment/segment_iterator_limit_opt_test.cpp @@ -77,7 +77,6 @@ class SegmentIteratorLimitOptTest : public ::testing::Test { // The segment pointer is null — only _opts and internal maps are accessed. std::unique_ptr make_iter() { auto iter = std::make_unique(nullptr, _read_schema); - iter->_opts.tablet_schema = _tablet_schema; iter->_opts.stats = &_stats; // delete_condition_predicates is default-constructed (empty) return iter; diff --git a/be/test/storage/segment/segment_iterator_no_need_read_data_test.cpp b/be/test/storage/segment/segment_iterator_no_need_read_data_test.cpp index 64627706e39050..0ab1df774fa372 100644 --- a/be/test/storage/segment/segment_iterator_no_need_read_data_test.cpp +++ b/be/test/storage/segment/segment_iterator_no_need_read_data_test.cpp @@ -19,6 +19,7 @@ #include "core/data_type/data_type_variant.h" #include "exec/common/variant_util.h" #include "gtest/gtest.h" +#include "storage/segment/mock/mock_segment.h" #include "storage/segment/segment_iterator.h" #include "storage/tablet/tablet_schema.h" #include "util/json/path_in_data.h" @@ -52,8 +53,7 @@ TEST(SegmentIteratorNoNeedReadDataTest, extracted_variant_count_on_index) { // Read schema covers all tablet columns in order, so ordinal == tablet cid. auto read_schema = std::make_shared(tablet_schema->columns()); - SegmentIterator iter(nullptr, read_schema); - iter._opts.tablet_schema = tablet_schema; + SegmentIterator iter(std::make_shared(tablet_schema), read_schema); iter._opts.push_down_agg_type_opt = TPushAggOp::COUNT_ON_INDEX; iter._column_states[subcol_cid].need_read_data = false; iter._output_column_uids.emplace(1); @@ -87,8 +87,7 @@ TEST(SegmentIteratorNoNeedReadDataTest, zonemap_always_true_predicate_column) { // Read schema covers all tablet columns in order, so ordinal == tablet cid. auto read_schema = std::make_shared(tablet_schema->columns()); - SegmentIterator iter(nullptr, read_schema); - iter._opts.tablet_schema = tablet_schema; + SegmentIterator iter(std::make_shared(tablet_schema), read_schema); iter._opts.zonemap_always_true_pred_cols.emplace(1); EXPECT_FALSE(iter._need_read_data(1)); diff --git a/be/test/storage/segment/segment_writer_write_paths_test.cpp b/be/test/storage/segment/segment_writer_write_paths_test.cpp index 4c70c00b252f43..dabfa2fbd22947 100644 --- a/be/test/storage/segment/segment_writer_write_paths_test.cpp +++ b/be/test/storage/segment/segment_writer_write_paths_test.cpp @@ -231,7 +231,6 @@ class VerticalSegmentWriterWritePathsTest : public testing::Test { OlapReaderStatistics stats; StorageReadOptions read_options; read_options.stats = &stats; - read_options.tablet_schema = schema; std::unique_ptr iterator; ASSERT_TRUE(segment->new_iterator(read_schema, read_options, &iterator).ok()); MutableBlock contents(schema->create_storage_block()); diff --git a/be/test/storage/segment/segments_key_bounds_truncation_test.cpp b/be/test/storage/segment/segments_key_bounds_truncation_test.cpp index 9b56c21abb7975..c077f17153eeae 100644 --- a/be/test/storage/segment/segments_key_bounds_truncation_test.cpp +++ b/be/test/storage/segment/segments_key_bounds_truncation_test.cpp @@ -38,6 +38,7 @@ #include "storage/tablet/tablet_meta.h" #include "storage/tablet/tablet_reader.h" #include "storage/tablet/tablet_schema.h" +#include "util/defer_op.h" namespace doris { static std::string kSegmentDir = "./ut_dir/segments_key_bounds_truncation_test"; @@ -659,6 +660,13 @@ TEST_F(SegmentsKeyBoundsTruncationTest, BlockReaderJudgeFuncTest) { TEST_F(SegmentsKeyBoundsTruncationTest, OrderedCompactionTest) { auto tablet_schema = create_schema(100); + Defer restore_config([saved_enable = config::enable_ordered_data_compaction, + saved_min_size = config::ordered_data_compaction_min_segment_size] { + // Otherwise a one-byte minimum segment size leaks into every later suite and sends its + // compactions down the ordered link-file path instead of a real merge. + config::enable_ordered_data_compaction = saved_enable; + config::ordered_data_compaction_min_segment_size = saved_min_size; + }); config::enable_ordered_data_compaction = true; config::ordered_data_compaction_min_segment_size = 1; diff --git a/be/test/storage/tablet/tablet_schema_test.cpp b/be/test/storage/tablet/tablet_schema_test.cpp index 8732c297db518d..34ad97e63058ef 100644 --- a/be/test/storage/tablet/tablet_schema_test.cpp +++ b/be/test/storage/tablet/tablet_schema_test.cpp @@ -689,148 +689,6 @@ TEST_F(TabletSchemaTest, test_tablet_schema_remove_and_clear_index) { EXPECT_EQ(0, indexes_after_clear.size()); } -TEST_F(TabletSchemaTest, test_tablet_schema_path_set_info_inverted_indexs) { - TabletSchema schema; - - TabletColumn variant_col; - variant_col.set_unique_id(9001); - variant_col.set_name("variant_col"); - variant_col.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); - schema.append_column(variant_col); - - auto create_index = [](int64_t id, const std::string& name, int32_t col_uid) { - auto index = std::make_shared(); - TabletIndexPB index_pb; - index_pb.set_index_id(id); - index_pb.set_index_name(name); - index_pb.set_index_type(IndexType::INVERTED); - index_pb.add_col_unique_id(col_uid); - index->init_from_pb(index_pb); - return index; - }; - - auto typed_index1 = create_index(1001, "typed_path_idx1", 9001); - auto typed_index2 = create_index(1002, "typed_path_idx2", 9001); - - auto subcolumn_index1 = create_index(2001, "subcolumn_idx1", 9001); - auto subcolumn_index2 = create_index(2002, "subcolumn_idx2", 9001); - - TabletSchema::PathsSetInfo path_set_info; - TabletSchema::SubColumnInfo typed_sub_col1; - typed_sub_col1.column = variant_col; - typed_sub_col1.indexes.push_back(typed_index1); - path_set_info.typed_path_set["user.name"] = typed_sub_col1; - - TabletSchema::SubColumnInfo typed_sub_col2; - typed_sub_col2.column = variant_col; - typed_sub_col2.indexes.push_back(typed_index2); - path_set_info.typed_path_set["user.age"] = typed_sub_col2; - - TabletIndexes subcolumn_indexes1 = {subcolumn_index1}; - TabletIndexes subcolumn_indexes2 = {subcolumn_index2}; - path_set_info.subcolumn_indexes["product.id"] = subcolumn_indexes1; - path_set_info.subcolumn_indexes["product.price"] = subcolumn_indexes2; - - std::unordered_map path_set_info_map; - path_set_info_map[9001] = std::move(path_set_info); - schema.set_path_set_info(std::move(path_set_info_map)); - - TabletColumn typed_extracted_col1; - typed_extracted_col1.set_unique_id(-1); - typed_extracted_col1.set_name("variant_col.user.name"); - typed_extracted_col1.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - typed_extracted_col1.set_parent_unique_id(9001); - - PathInData typed_path1("variant_col.user.name", true); - typed_extracted_col1.set_path_info(typed_path1); - - auto typed_indexes = schema.inverted_indexs(typed_extracted_col1); - EXPECT_EQ(1, typed_indexes.size()); - EXPECT_EQ("typed_path_idx1", typed_indexes[0]->index_name()); - EXPECT_EQ(1001, typed_indexes[0]->index_id()); - - TabletColumn typed_extracted_col2; - typed_extracted_col2.set_unique_id(-1); - typed_extracted_col2.set_name("variant_col.user.age"); - typed_extracted_col2.set_type(FieldType::OLAP_FIELD_TYPE_INT); - typed_extracted_col2.set_parent_unique_id(9001); - - PathInData typed_path2("variant_col.user.age", true); - typed_extracted_col2.set_path_info(typed_path2); - - auto typed_indexes2 = schema.inverted_indexs(typed_extracted_col2); - EXPECT_EQ(1, typed_indexes2.size()); - EXPECT_EQ("typed_path_idx2", typed_indexes2[0]->index_name()); - EXPECT_EQ(1002, typed_indexes2[0]->index_id()); - - // Test subcolumn path (non-typed) - TabletColumn subcolumn_extracted_col1; - subcolumn_extracted_col1.set_unique_id(-1); - subcolumn_extracted_col1.set_name("variant_col.product.id"); - subcolumn_extracted_col1.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - subcolumn_extracted_col1.set_parent_unique_id(9001); - - PathInData subcolumn_path1("variant_col.product.id"); - subcolumn_extracted_col1.set_path_info(subcolumn_path1); - - auto subcolumn_indexes = schema.inverted_indexs(subcolumn_extracted_col1); - EXPECT_EQ(1, subcolumn_indexes.size()); - EXPECT_EQ("subcolumn_idx1", subcolumn_indexes[0]->index_name()); - EXPECT_EQ(2001, subcolumn_indexes[0]->index_id()); - - TabletColumn non_existing_col; - non_existing_col.set_unique_id(-1); - non_existing_col.set_name("non_existing"); - non_existing_col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - non_existing_col.set_parent_unique_id(9001); - - PathInData non_existing_path("variant_col.non.existing"); - non_existing_col.set_path_info(non_existing_path); - - auto no_indexes = schema.inverted_indexs(non_existing_col); - EXPECT_EQ(0, no_indexes.size()); - - TabletColumn wrong_parent_col; - wrong_parent_col.set_unique_id(-1); - wrong_parent_col.set_name("wrong_parent"); - wrong_parent_col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - wrong_parent_col.set_parent_unique_id(9999); // Non-existing parent - - PathInData wrong_parent_path("wrong_variant.some.path"); - wrong_parent_col.set_path_info(wrong_parent_path); - - auto no_indexes_wrong_parent = schema.inverted_indexs(wrong_parent_col); - EXPECT_EQ(0, no_indexes_wrong_parent.size()); -} - -TEST_F(TabletSchemaTest, test_tablet_schema_path_set_info_accessors) { - TabletSchema schema; - - TabletColumn variant_col; - variant_col.set_unique_id(10001); - variant_col.set_name("json_data"); - variant_col.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT); - schema.append_column(variant_col); - - TabletSchema::PathsSetInfo path_info; - path_info.sub_path_set.insert("extracted_path1"); - path_info.sub_path_set.insert("extracted_path2"); - path_info.sparse_path_set.insert("sparse_path1"); - path_info.sparse_path_set.insert("sparse_path2"); - - std::unordered_map path_map; - path_map[10001] = std::move(path_info); - schema.set_path_set_info(std::move(path_map)); - - const auto& retrieved_info = schema.path_set_info(10001); - EXPECT_EQ(2, retrieved_info.sub_path_set.size()); - EXPECT_EQ(2, retrieved_info.sparse_path_set.size()); - EXPECT_TRUE(retrieved_info.sub_path_set.count("extracted_path1") > 0); - EXPECT_TRUE(retrieved_info.sub_path_set.count("extracted_path2") > 0); - EXPECT_TRUE(retrieved_info.sparse_path_set.count("sparse_path1") > 0); - EXPECT_TRUE(retrieved_info.sparse_path_set.count("sparse_path2") > 0); -} - TEST_F(TabletSchemaTest, test_tablet_schema_inverted_index_by_field_pattern) { TabletSchema schema; diff --git a/be/test/storage/variant/index_storage_variant_compaction_schema_test.cpp b/be/test/storage/variant/index_storage_variant_compaction_schema_test.cpp index f9b50a077e41f2..eb0e1ce10ba041 100644 --- a/be/test/storage/variant/index_storage_variant_compaction_schema_test.cpp +++ b/be/test/storage/variant/index_storage_variant_compaction_schema_test.cpp @@ -33,14 +33,14 @@ TEST(IndexStorageVariantCompactionUtilTest, GetSubpathsHonorsZeroLimitAndTieOrde {"gamma", 1}, }; - TabletSchema::PathsSetInfo unlimited; + VariantCompactionPaths unlimited; variant_util::VariantCompactionUtil::get_subpaths(0, stats, unlimited); EXPECT_TRUE(unlimited.sub_path_set.contains(StringRef("alpha"))); EXPECT_TRUE(unlimited.sub_path_set.contains(StringRef("beta"))); EXPECT_TRUE(unlimited.sub_path_set.contains(StringRef("gamma"))); EXPECT_TRUE(unlimited.sparse_path_set.empty()); - TabletSchema::PathsSetInfo top_one; + VariantCompactionPaths top_one; variant_util::VariantCompactionUtil::get_subpaths(1, stats, top_one); EXPECT_TRUE(top_one.sub_path_set.contains(StringRef("beta"))); EXPECT_FALSE(top_one.sub_path_set.contains(StringRef("alpha"))); @@ -54,13 +54,13 @@ TEST(IndexStorageVariantCompactionUtilTest, GetSubpathsKeepsAllPathsAtLimitAndHa {"beta", 1}, }; - TabletSchema::PathsSetInfo at_limit; + VariantCompactionPaths at_limit; variant_util::VariantCompactionUtil::get_subpaths(2, exact_limit, at_limit); EXPECT_TRUE(at_limit.sub_path_set.contains(StringRef("alpha"))); EXPECT_TRUE(at_limit.sub_path_set.contains(StringRef("beta"))); EXPECT_TRUE(at_limit.sparse_path_set.empty()); - TabletSchema::PathsSetInfo empty; + VariantCompactionPaths empty; variant_util::VariantCompactionUtil::get_subpaths(1, {}, empty); EXPECT_TRUE(empty.sub_path_set.empty()); EXPECT_TRUE(empty.sparse_path_set.empty()); @@ -82,12 +82,13 @@ TEST(IndexStorageVariantCompactionUtilTest, EmptyInputsKeepVariantSchemaWithoutP nullptr); auto compaction_schema = std::make_shared(*base_schema); + VariantCompactionPathsMap compaction_paths; auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - {}, compaction_schema); + {}, compaction_schema, compaction_paths); ASSERT_TRUE(status.ok()) << status.to_string(); ASSERT_TRUE(compaction_schema->has_column_unique_id(2)); - const auto* path_set_info = compaction_schema->try_path_set_info(2); + const auto* path_set_info = compaction_paths.contains(2) ? &compaction_paths.at(2) : nullptr; ASSERT_NE(path_set_info, nullptr); EXPECT_TRUE(path_set_info->typed_path_set.empty()); EXPECT_TRUE(path_set_info->sub_path_set.empty()); @@ -114,11 +115,12 @@ TEST_F(IndexStorageVariantCompactionSchemaTest, ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); auto compaction_schema = std::make_shared(*tablet_schema()); + VariantCompactionPathsMap compaction_paths; auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - {rowset_result.value()}, compaction_schema); + {rowset_result.value()}, compaction_schema, compaction_paths); ASSERT_TRUE(status.ok()) << status.to_string(); - const auto* path_set_info = compaction_schema->try_path_set_info(2); + const auto* path_set_info = compaction_paths.contains(2) ? &compaction_paths.at(2) : nullptr; ASSERT_NE(path_set_info, nullptr); EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("alpha"))); EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("beta"))); @@ -149,11 +151,12 @@ TEST_F(IndexStorageVariantCompactionSchemaTest, ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); auto compaction_schema = std::make_shared(*tablet_schema()); + VariantCompactionPathsMap compaction_paths; auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - {rowset_result.value()}, compaction_schema); + {rowset_result.value()}, compaction_schema, compaction_paths); ASSERT_TRUE(status.ok()) << status.to_string(); - const auto* path_set_info = compaction_schema->try_path_set_info(2); + const auto* path_set_info = compaction_paths.contains(2) ? &compaction_paths.at(2) : nullptr; ASSERT_NE(path_set_info, nullptr); EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("alpha"))); EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("beta"))); @@ -190,11 +193,12 @@ TEST_F(IndexStorageVariantCompactionSchemaTest, ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error(); auto compaction_schema = std::make_shared(*tablet_schema()); + VariantCompactionPathsMap compaction_paths; auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - {rowset_result.value()}, compaction_schema); + {rowset_result.value()}, compaction_schema, compaction_paths); ASSERT_TRUE(status.ok()) << status.to_string(); - const auto* path_set_info = compaction_schema->try_path_set_info(2); + const auto* path_set_info = compaction_paths.contains(2) ? &compaction_paths.at(2) : nullptr; ASSERT_NE(path_set_info, nullptr); EXPECT_TRUE(path_set_info->typed_path_set.contains("typed_i")); EXPECT_FALSE(path_set_info->sub_path_set.contains(StringRef("typed_i"))); diff --git a/be/test/storage/variant/index_storage_variant_io_context_test.cpp b/be/test/storage/variant/index_storage_variant_io_context_test.cpp index cd3fbdaa5b38c7..603d8a6119bc91 100644 --- a/be/test/storage/variant/index_storage_variant_io_context_test.cpp +++ b/be/test/storage/variant/index_storage_variant_io_context_test.cpp @@ -74,7 +74,6 @@ TEST_F(IndexStorageVariantIoContextTest, OlapReaderStatistics stats; StorageReadOptions read_options; read_options.stats = &stats; - read_options.tablet_schema = reader_schema; read_options.io_ctx.reader_type = ReaderType::READER_QUERY; read_options.io_ctx.query_id = &query_id; read_options.io_ctx.file_cache_stats = &stats.file_cache_stats; diff --git a/be/test/storage/variant/index_storage_variant_sparse_stats_test.cpp b/be/test/storage/variant/index_storage_variant_sparse_stats_test.cpp index c1b8909ed9c1a4..67de811e830017 100644 --- a/be/test/storage/variant/index_storage_variant_sparse_stats_test.cpp +++ b/be/test/storage/variant/index_storage_variant_sparse_stats_test.cpp @@ -90,11 +90,12 @@ TEST_F(IndexStorageVariantSparseStatsTest, VariantCompactionSchemaTopNRecordsSpa ASSERT_TRUE(rowsets.has_value()) << rowsets.error(); auto compaction_schema = std::make_shared(*tablet_schema()); + VariantCompactionPathsMap compaction_paths; auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - rowsets.value(), compaction_schema); + rowsets.value(), compaction_schema, compaction_paths); ASSERT_TRUE(status.ok()) << status.to_string(); - const auto* path_set_info = compaction_schema->try_path_set_info(2); + const auto* path_set_info = compaction_paths.contains(2) ? &compaction_paths.at(2) : nullptr; ASSERT_NE(path_set_info, nullptr); EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("hot"))); EXPECT_FALSE(path_set_info->sub_path_set.contains(StringRef("warm"))); diff --git a/be/test/storage/variant/variant_column_writer_reader_test.cpp b/be/test/storage/variant/variant_column_writer_reader_test.cpp index cf0c86f211bc7e..1c4dda3f89cdaf 100644 --- a/be/test/storage/variant/variant_column_writer_reader_test.cpp +++ b/be/test/storage/variant/variant_column_writer_reader_test.cpp @@ -87,8 +87,23 @@ constexpr static std::string_view tmp_dir = "./ut_dir/tmp"; enum class VariantWriterInput : uint8_t { V2 }; +// Mirror of what BetaRowsetReader derives from the schema a read targets: it drives whether a +// compaction or checksum read flattens variant subcolumns instead of reading them hierarchically. +static void set_variant_read_facts(StorageReadOptions& opts, const TabletSchemaSPtr& schema) { + opts.tablet_has_extracted_variant_columns = std::ranges::any_of( + schema->columns(), [](const auto& column) { return column->is_extracted_column(); }); +} + enum class VariantIndexWritePolicy : uint8_t { NONE, BLOOM_AND_INVERTED }; +// Resolve a query slot against a schema the way the point-query and rowid-fetch callers do. +static const TabletColumn& read_column_of(const TabletSchemaSPtr& schema, SlotDescriptor* slot) { + int32_t index = slot->col_unique_id() >= 0 ? schema->field_index(slot->col_unique_id()) + : schema->field_index(slot->col_name()); + CHECK_GE(index, 0) << "slot not in schema: " << slot->col_name(); + return schema->column(index); +} + static std::string variant_writer_input_name(VariantWriterInput input) { return "V2"; } @@ -1371,11 +1386,15 @@ class VariantColumnWriterReaderTest : public testing::Test { RETURN_IF_ERROR(rowset->create_reader(&reader)); RowsetReaderContext reader_context; - reader_context.tablet_schema = _tablet_schema; reader_context.need_ordered_result = false; auto read_schema = std::make_shared( project_columns_by_ordinal(_tablet_schema->columns(), std::vector {0})); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*_tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RETURN_IF_ERROR(reader->init(&reader_context)); rows->clear(); @@ -1773,7 +1792,6 @@ class VariantColumnWriterReaderTest : public testing::Test { StorageReadOptions read_opts; read_opts.io_ctx.reader_type = ReaderType::READER_QUERY; - read_opts.tablet_schema = _tablet_schema; OlapReaderStatistics stats; read_opts.stats = &stats; @@ -2661,7 +2679,7 @@ TEST_F(VariantColumnWriterReaderTest, OlapReaderStatistics* stats) { StorageReadOptions read_opts; read_opts.io_ctx.reader_type = ReaderType::READER_BASE_COMPACTION; - read_opts.tablet_schema = compaction_schema; + set_variant_read_facts(read_opts, compaction_schema); read_opts.stats = stats; const size_t reader_calls = column_reader_cache.path_column_reader_calls(); ASSERT_TRUE( @@ -3646,16 +3664,17 @@ TEST_F(VariantColumnWriterReaderTest, test_segment_rowid_read_by_reader_version) StorageReadOptions read_options; read_options.stats = &stats; read_options.io_ctx.reader_type = ReaderType::READER_QUERY; - read_options.tablet_schema = _tablet_schema; MutableColumnPtr whole_result = slots[0]->type()->create_column(); ColumnIteratorUPtr whole_iterator; - auto st = segments[0]->seek_and_read_by_rowid(*_tablet_schema, slots[0], row_ids, whole_result, - read_options, whole_iterator); + auto st = segments[0]->seek_and_read_by_rowid(read_column_of(_tablet_schema, slots[0]), + slots[0], row_ids, whole_result, read_options, + whole_iterator); ASSERT_TRUE(st.ok()) << st.to_string(); auto* const whole_iterator_address = whole_iterator.get(); - st = segments[0]->seek_and_read_by_rowid(*_tablet_schema, slots[0], second_row_ids, - whole_result, read_options, whole_iterator); + st = segments[0]->seek_and_read_by_rowid(read_column_of(_tablet_schema, slots[0]), slots[0], + second_row_ids, whole_result, read_options, + whole_iterator); ASSERT_TRUE(st.ok()) << st.to_string(); EXPECT_EQ(whole_iterator.get(), whole_iterator_address); ASSERT_EQ(whole_result->size(), jsons.size()); @@ -3665,12 +3684,13 @@ TEST_F(VariantColumnWriterReaderTest, test_segment_rowid_read_by_reader_version) MutableColumnPtr hot_result = slots[1]->type()->create_column(); ColumnIteratorUPtr hot_iterator; - st = segments[0]->seek_and_read_by_rowid(*_tablet_schema, slots[1], row_ids, hot_result, - read_options, hot_iterator); + st = segments[0]->seek_and_read_by_rowid(read_column_of(_tablet_schema, slots[1]), slots[1], + row_ids, hot_result, read_options, hot_iterator); ASSERT_TRUE(st.ok()) << st.to_string(); auto* const hot_iterator_address = hot_iterator.get(); - st = segments[0]->seek_and_read_by_rowid(*_tablet_schema, slots[1], second_row_ids, hot_result, - read_options, hot_iterator); + st = segments[0]->seek_and_read_by_rowid(read_column_of(_tablet_schema, slots[1]), slots[1], + second_row_ids, hot_result, read_options, + hot_iterator); ASSERT_TRUE(st.ok()) << st.to_string(); EXPECT_EQ(hot_iterator.get(), hot_iterator_address); const auto& nullable_hot = assert_cast(*hot_result); @@ -3684,12 +3704,14 @@ TEST_F(VariantColumnWriterReaderTest, test_segment_rowid_read_by_reader_version) MutableColumnPtr subpath_result = slots[2]->type()->create_column(); ColumnIteratorUPtr subpath_iterator; - st = segments[0]->seek_and_read_by_rowid(*_tablet_schema, slots[2], row_ids, subpath_result, - read_options, subpath_iterator); + st = segments[0]->seek_and_read_by_rowid(read_column_of(_tablet_schema, slots[2]), slots[2], + row_ids, subpath_result, read_options, + subpath_iterator); ASSERT_TRUE(st.ok()) << st.to_string(); auto* const subpath_iterator_address = subpath_iterator.get(); - st = segments[0]->seek_and_read_by_rowid(*_tablet_schema, slots[2], second_row_ids, - subpath_result, read_options, subpath_iterator); + st = segments[0]->seek_and_read_by_rowid(read_column_of(_tablet_schema, slots[2]), slots[2], + second_row_ids, subpath_result, read_options, + subpath_iterator); ASSERT_TRUE(st.ok()) << st.to_string(); EXPECT_EQ(subpath_iterator.get(), subpath_iterator_address); const auto& nullable_subpath = assert_cast(*subpath_result); @@ -3751,13 +3773,12 @@ TEST_F(VariantColumnWriterReaderTest, test_segment_rowid_read_by_reader_version) StorageReadOptions empty_read_options; empty_read_options.stats = &empty_stats; empty_read_options.io_ctx.reader_type = ReaderType::READER_QUERY; - empty_read_options.tablet_schema = _tablet_schema; MutableColumnPtr empty_whole_result = empty_slots[0]->type()->create_column(); ColumnIteratorUPtr empty_whole_iterator; - st = empty_nested_segments[0]->seek_and_read_by_rowid(*_tablet_schema, empty_slots[0], - empty_row_ids, empty_whole_result, - empty_read_options, empty_whole_iterator); + st = empty_nested_segments[0]->seek_and_read_by_rowid( + read_column_of(_tablet_schema, empty_slots[0]), empty_slots[0], empty_row_ids, + empty_whole_result, empty_read_options, empty_whole_iterator); ASSERT_TRUE(st.ok()) << st.to_string(); ASSERT_EQ(empty_whole_result->size(), expected_whole.size()); for (size_t row = 0; row < expected_whole.size(); ++row) { @@ -3767,8 +3788,8 @@ TEST_F(VariantColumnWriterReaderTest, test_segment_rowid_read_by_reader_version) MutableColumnPtr empty_subpath_result = empty_slots[1]->type()->create_column(); ColumnIteratorUPtr empty_subpath_iterator; st = empty_nested_segments[0]->seek_and_read_by_rowid( - *_tablet_schema, empty_slots[1], empty_row_ids, empty_subpath_result, - empty_read_options, empty_subpath_iterator); + read_column_of(_tablet_schema, empty_slots[1]), empty_slots[1], empty_row_ids, + empty_subpath_result, empty_read_options, empty_subpath_iterator); ASSERT_TRUE(st.ok()) << st.to_string(); const auto& nullable = assert_cast(*empty_subpath_result); const auto& values = nullable.get_nested_column(); @@ -4196,9 +4217,8 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) { // construct tablet schema for compaction storage_read_opts.io_ctx.reader_type = ReaderType::READER_BASE_COMPACTION; - storage_read_opts.tablet_schema = _tablet_schema; - std::unordered_map uid_to_paths_set_info; - TabletSchema::PathsSetInfo paths_set_info; + VariantCompactionPathsMap uid_to_paths_set_info; + VariantCompactionPaths paths_set_info; paths_set_info.sub_path_set.insert("key0"); paths_set_info.sub_path_set.insert("key3"); paths_set_info.sub_path_set.insert("key4"); @@ -4210,7 +4230,8 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) { paths_set_info.sparse_path_set.insert("key8"); paths_set_info.sparse_path_set.insert("key9"); uid_to_paths_set_info[parent_column.unique_id()] = paths_set_info; - _tablet_schema->set_path_set_info(std::move(uid_to_paths_set_info)); + storage_read_opts.variant_compaction_paths = + std::make_shared(std::move(uid_to_paths_set_info)); // mock a subcolumn in compaction TabletColumn subcolumn_in_compaction; @@ -4220,6 +4241,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) { subcolumn_in_compaction.set_path_info(PathInData(parent_column.name_lower_case() + ".key10")); subcolumn_in_compaction.set_is_nullable(true); _tablet_schema->append_column(subcolumn_in_compaction); + set_variant_read_facts(storage_read_opts, _tablet_schema); // 14. check compaction subcolumn reader check_leaf_reader(); @@ -4714,7 +4736,6 @@ TEST_F(VariantColumnWriterReaderTest, TabletColumn parent_column_v2 = parent_column; StorageReadOptions v2_query_read_opts; v2_query_read_opts.io_ctx.reader_type = ReaderType::READER_QUERY; - v2_query_read_opts.tablet_schema = _tablet_schema; OlapReaderStatistics v2_query_stats; v2_query_read_opts.stats = &v2_query_stats; ColumnIteratorUPtr v2_root_it; @@ -4745,7 +4766,7 @@ TEST_F(VariantColumnWriterReaderTest, StorageReadOptions compact_read_opts; compact_read_opts.io_ctx.reader_type = ReaderType::READER_BASE_COMPACTION; - compact_read_opts.tablet_schema = _tablet_schema; + set_variant_read_facts(compact_read_opts, _tablet_schema); OlapReaderStatistics compact_stats; compact_read_opts.stats = &compact_stats; TabletColumn doc_bucket_col = variant_util::create_doc_value_column(parent_column, 0); @@ -4962,7 +4983,7 @@ TEST_F(VariantColumnWriterReaderTest, test_read_doc_compact_from_doc_value_bucke StorageReadOptions storage_read_opts; storage_read_opts.io_ctx.reader_type = ReaderType::READER_BASE_COMPACTION; - storage_read_opts.tablet_schema = compaction_schema; + set_variant_read_facts(storage_read_opts, compaction_schema); OlapReaderStatistics stats; storage_read_opts.stats = &stats; @@ -5188,7 +5209,7 @@ TEST_P(VariantSpecializedWriterCompatibilityTest, doc_compact_writer_round_trip) MockColumnReaderCache column_reader_cache(footer, file_reader, _tablet_schema); StorageReadOptions storage_read_opts; storage_read_opts.io_ctx.reader_type = ReaderType::READER_BASE_COMPACTION; - storage_read_opts.tablet_schema = _tablet_schema; + set_variant_read_facts(storage_read_opts, _tablet_schema); OlapReaderStatistics stats; storage_read_opts.stats = &stats; @@ -5954,11 +5975,12 @@ TEST_F(VariantColumnWriterReaderTest, std::vector input_rowsets {rowset}; auto compaction_schema = std::make_shared(*_tablet_schema); + VariantCompactionPathsMap compaction_paths; auto st = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - input_rowsets, compaction_schema); + input_rowsets, compaction_schema, compaction_paths); ASSERT_TRUE(st.ok()) << st.to_string(); - const auto* path_set_info = compaction_schema->try_path_set_info(1); + const auto* path_set_info = compaction_paths.contains(1) ? &compaction_paths.at(1) : nullptr; ASSERT_NE(path_set_info, nullptr); ASSERT_TRUE(path_set_info->typed_path_set.contains("a")); EXPECT_FALSE(path_set_info->sub_path_set.contains(StringRef("a"))); @@ -6935,7 +6957,6 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable) { StorageReadOptions v2_read_opts; v2_read_opts.stats = &v2_stats; v2_read_opts.io_ctx.reader_type = ReaderType::READER_QUERY; - v2_read_opts.tablet_schema = _tablet_schema; ColumnIteratorUPtr v2_it; st = variant_column_reader->new_iterator(&v2_it, &parent_column_v2, &v2_read_opts, &column_reader_cache); @@ -6967,7 +6988,6 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_nullable) { StorageReadOptions v2_rowid_read_opts; v2_rowid_read_opts.stats = &v2_rowid_stats; v2_rowid_read_opts.io_ctx.reader_type = ReaderType::READER_QUERY; - v2_rowid_read_opts.tablet_schema = _tablet_schema; ColumnIteratorUPtr v2_rowid_it; st = variant_column_reader->new_iterator(&v2_rowid_it, &parent_column_v2, &v2_rowid_read_opts, &column_reader_cache); @@ -8125,8 +8145,6 @@ TEST_F(VariantColumnWriterReaderTest, test_read_with_checksum) { TabletColumn parent_column = _tablet_schema->column(0); StorageReadOptions storage_read_opts; - storage_read_opts.tablet_schema = _tablet_schema; - TabletColumn subcolumn; subcolumn.set_name(parent_column.name_lower_case() + ".b"); subcolumn.set_type((FieldType)(int)footer.columns(1).type()); @@ -8135,6 +8153,7 @@ TEST_F(VariantColumnWriterReaderTest, test_read_with_checksum) { subcolumn.set_variant_max_subcolumns_count(parent_column.variant_max_subcolumns_count()); subcolumn.set_is_nullable(true); _tablet_schema->append_column(subcolumn); + set_variant_read_facts(storage_read_opts, _tablet_schema); storage_read_opts.io_ctx.reader_type = ReaderType::READER_QUERY; OlapReaderStatistics stats; storage_read_opts.stats = &stats; @@ -8440,8 +8459,9 @@ TEST_F(VariantColumnWriterReaderTest, test_compaction_nokey_variant_uid0) { auto input_readers = create_rowset_readers(input_rowsets); auto compaction_schema = std::make_shared(*_tablet_schema); + auto compaction_paths = std::make_shared(); auto st = variant_util::VariantCompactionUtil::get_extended_compaction_schema( - input_rowsets, compaction_schema); + input_rowsets, compaction_schema, *compaction_paths); ASSERT_TRUE(st.ok()) << st.to_string(); RowsetWriterContext ctx; @@ -8452,6 +8472,7 @@ TEST_F(VariantColumnWriterReaderTest, test_compaction_nokey_variant_uid0) { ctx.data_dir = _data_dir.get(); ctx.rowset_state = VISIBLE; ctx.tablet_schema = compaction_schema; + ctx.variant_compaction_paths = compaction_paths; ctx.tablet_path = _tablet->tablet_path(); ctx.tablet_id = _tablet->tablet_id(); ctx.tablet = _tablet; @@ -8535,8 +8556,9 @@ TEST_F(VariantColumnWriterReaderTest, legacy_v1_segment_compaction_preserves_ful auto input_readers = create_rowset_readers(input_rowsets); auto compaction_schema = std::make_shared(*_tablet_schema); - status = variant_util::VariantCompactionUtil::get_extended_compaction_schema(input_rowsets, - compaction_schema); + auto compaction_paths = std::make_shared(); + status = variant_util::VariantCompactionUtil::get_extended_compaction_schema( + input_rowsets, compaction_schema, *compaction_paths); ASSERT_TRUE(status.ok()) << status; RowsetWriterContext context; @@ -8547,6 +8569,7 @@ TEST_F(VariantColumnWriterReaderTest, legacy_v1_segment_compaction_preserves_ful context.data_dir = _data_dir.get(); context.rowset_state = VISIBLE; context.tablet_schema = compaction_schema; + context.variant_compaction_paths = compaction_paths; context.tablet_path = _tablet->tablet_path(); context.tablet_id = _tablet->tablet_id(); context.tablet = _tablet; diff --git a/be/test/storage/variant/variant_compaction_paths_test.cpp b/be/test/storage/variant/variant_compaction_paths_test.cpp new file mode 100644 index 00000000000000..eec0233c424a2a --- /dev/null +++ b/be/test/storage/variant/variant_compaction_paths_test.cpp @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/segment/variant/variant_compaction_paths.h" + +#include +#include + +#include +#include +#include + +#include "storage/tablet/tablet_schema.h" +#include "util/json/path_in_data.h" + +namespace doris { + +namespace { + +constexpr int32_t kVariantUid = 9001; + +TabletIndexPtr make_inverted_index(int64_t index_id, const std::string& name) { + auto index = std::make_shared(); + TabletIndexPB index_pb; + index_pb.set_index_id(index_id); + index_pb.set_index_name(name); + index_pb.set_index_type(IndexType::INVERTED); + index_pb.add_col_unique_id(kVariantUid); + index->init_from_pb(index_pb); + return index; +} + +// An extracted column of the variant above. `typed` selects which half of the layout the lookup +// is expected to consult. +TabletColumn make_extracted_column(const std::string& path, bool typed, + FieldType type = FieldType::OLAP_FIELD_TYPE_STRING, + int32_t parent_uid = kVariantUid) { + TabletColumn column; + column.set_unique_id(-1); + column.set_name("v." + path); + column.set_type(type); + column.set_parent_unique_id(parent_uid); + column.set_path_info(PathInData("v." + path, typed)); + return column; +} + +// typed_path_set["user.name"] -> index 1001, subcolumn_indexes["product.id"] -> index 2001. +VariantCompactionPathsMap make_layout() { + VariantCompactionPaths paths; + + TabletSchema::SubColumnInfo typed; + typed.indexes.push_back(make_inverted_index(1001, "typed_path_idx")); + paths.typed_path_set["user.name"] = std::move(typed); + + paths.subcolumn_indexes["product.id"] = {make_inverted_index(2001, "subcolumn_idx")}; + + VariantCompactionPathsMap layout; + layout[kVariantUid] = std::move(paths); + return layout; +} + +} // namespace + +TEST(VariantCompactionPathsTest, TypedPathTakesItsOwnIndexes) { + const auto layout = make_layout(); + auto indexes = variant_subcolumn_indexes(&layout, make_extracted_column("user.name", true)); + ASSERT_EQ(1, indexes.size()); + EXPECT_EQ(1001, indexes[0]->index_id()); + EXPECT_EQ("typed_path_idx", indexes[0]->index_name()); +} + +TEST(VariantCompactionPathsTest, SubcolumnPathTakesItsOwnIndexes) { + const auto layout = make_layout(); + auto indexes = variant_subcolumn_indexes(&layout, make_extracted_column("product.id", false)); + ASSERT_EQ(1, indexes.size()); + EXPECT_EQ(2001, indexes[0]->index_id()); + EXPECT_EQ("subcolumn_idx", indexes[0]->index_name()); +} + +// A typed path is looked up only in typed_path_set, and a plain one only in subcolumn_indexes, +// so asking for either under the wrong flavour finds nothing. +TEST(VariantCompactionPathsTest, PathIsNotFoundInTheOtherHalfOfTheLayout) { + const auto layout = make_layout(); + EXPECT_TRUE( + variant_subcolumn_indexes(&layout, make_extracted_column("user.name", false)).empty()); + EXPECT_TRUE( + variant_subcolumn_indexes(&layout, make_extracted_column("product.id", true)).empty()); +} + +TEST(VariantCompactionPathsTest, PathAbsentFromTheLayout) { + const auto layout = make_layout(); + EXPECT_TRUE(variant_subcolumn_indexes(&layout, make_extracted_column("non.existing", false)) + .empty()); +} + +TEST(VariantCompactionPathsTest, ParentColumnAbsentFromTheLayout) { + const auto layout = make_layout(); + auto column = make_extracted_column("user.name", true, FieldType::OLAP_FIELD_TYPE_STRING, + /*parent_uid=*/9999); + EXPECT_TRUE(variant_subcolumn_indexes(&layout, column).empty()); +} + +// Every write that is not a compaction passes no layout at all. +TEST(VariantCompactionPathsTest, NoLayout) { + EXPECT_TRUE( + variant_subcolumn_indexes(nullptr, make_extracted_column("user.name", true)).empty()); +} + +// A column that is not extracted never has a layout entry, whatever the layout holds. +// A plain column carries its indexes on the schema, never in the layout. STRING so that the +// extracted-column guard is the only thing that can refuse it. +TEST(VariantCompactionPathsTest, NonExtractedColumn) { + const auto layout = make_layout(); + TabletColumn column; + column.set_unique_id(kVariantUid); + column.set_name("user.name"); + column.set_type(FieldType::OLAP_FIELD_TYPE_STRING); + ASSERT_FALSE(column.is_extracted_column()); + EXPECT_TRUE(variant_subcolumn_indexes(&layout, column).empty()); +} + +// Types that cannot carry an inverted index are refused before the layout is consulted, even +// though their path is in it: JSONB and variant are rejected outright, and an array only passes +// when its element type does (IndexColumnWriter::check_support_inverted_index). +TEST(VariantCompactionPathsTest, TypeThatCannotCarryAnInvertedIndex) { + const auto layout = make_layout(); + for (auto type : {FieldType::OLAP_FIELD_TYPE_JSONB, FieldType::OLAP_FIELD_TYPE_VARIANT}) { + auto column = make_extracted_column("user.name", true, type); + EXPECT_TRUE(variant_subcolumn_indexes(&layout, column).empty()) + << "type=" << static_cast(type); + } + + // A scalar element type does pass, so the path is found. + auto scalar = make_extracted_column("user.name", true, FieldType::OLAP_FIELD_TYPE_DOUBLE); + EXPECT_EQ(1, variant_subcolumn_indexes(&layout, scalar).size()); +} + +} // namespace doris diff --git a/be/test/storage/variant/variant_doc_mode_compaction_test.cpp b/be/test/storage/variant/variant_doc_mode_compaction_test.cpp index 6a4b3190111d8a..50898187de347e 100644 --- a/be/test/storage/variant/variant_doc_mode_compaction_test.cpp +++ b/be/test/storage/variant/variant_doc_mode_compaction_test.cpp @@ -423,12 +423,16 @@ TEST_F(VariantDocModeCompactionTest, variant_doc_mode_compaction_merge_10_segmen << " elapsed_ms=" << import_elapsed_ms << std::endl; if (i == 0) { RowsetReaderContext input_reader_context; - input_reader_context.tablet_schema = tablet_schema; input_reader_context.need_ordered_result = false; std::vector input_return_columns = {1}; auto input_read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), input_return_columns)); input_reader_context.read_schema = input_read_schema; + EXPECT_TRUE(input_read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr input_rs_reader; create_and_init_rowset_reader(rowset.get(), input_reader_context, &input_rs_reader); @@ -472,12 +476,16 @@ TEST_F(VariantDocModeCompactionTest, variant_doc_mode_compaction_merge_10_segmen ASSERT_EQ(static_cast(kRowsPerSegment) * 10, out_rowset->rowset_meta()->num_rows()); RowsetReaderContext reader_context; - reader_context.tablet_schema = tablet_schema; reader_context.need_ordered_result = false; std::vector return_columns = {0}; auto read_schema = std::make_shared( project_columns_by_ordinal(tablet_schema->columns(), return_columns)); reader_context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); RowsetReaderSharedPtr output_rs_reader; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); diff --git a/be/test/testutil/index_storage_test_util.cpp b/be/test/testutil/index_storage_test_util.cpp index a8e98149c572bb..e72a35a58ccc29 100644 --- a/be/test/testutil/index_storage_test_util.cpp +++ b/be/test/testutil/index_storage_test_util.cpp @@ -1303,9 +1303,13 @@ Result IndexStorageTestFixture::read_rowsets( RowsetReaderContext context; context.reader_type = options.reader_type; - context.tablet_schema = _tablet_schema; context.need_ordered_result = options.need_ordered_result; context.read_schema = read_schema; + EXPECT_TRUE(read_schema + ->init_from_tablet_schema(*_tablet_schema, + /*merge_by_sequence_mapping=*/false, + /*map_row_binlog_columns=*/false) + .ok()); context.predicates = &predicates; context.stats = &result.stats; context.target_cast_type_for_variants = options.target_cast_type_for_variants;