diff --git a/be/src/core/column/column_nullable.h b/be/src/core/column/column_nullable.h index 2c611e2732419b..89648635ebc4a1 100644 --- a/be/src/core/column/column_nullable.h +++ b/be/src/core/column/column_nullable.h @@ -190,6 +190,14 @@ class ColumnNullable final : public COWHelper { get_nested_column().insert_many_continuous_binary_data(data, offsets, num); } + void insert_offsets_from_lengths(const uint32_t* lengths, size_t num) override { + if (UNLIKELY(num == 0)) { + return; + } + push_false_to_nullmap(num); + get_nested_column().insert_offsets_from_lengths(lengths, num); + } + // Default value in `ColumnNullable` is null void insert_default() override { get_nested_column().insert_default(); diff --git a/be/src/storage/segment/binary_dict_page.cpp b/be/src/storage/segment/binary_dict_page.cpp index ebfedcb4afe522..0ec1531f3da87c 100644 --- a/be/src/storage/segment/binary_dict_page.cpp +++ b/be/src/storage/segment/binary_dict_page.cpp @@ -29,6 +29,7 @@ #include "common/logging.h" #include "common/status.h" #include "core/column/column.h" +#include "core/column/column_string.h" #include "storage/segment/binary_plain_page_v2.h" #include "storage/segment/bitshuffle_page.h" #include "storage/segment/encoding_info.h" @@ -280,11 +281,28 @@ Status BinaryDictPageDecoder::next_batch(size_t* n, MutableColumnPtr& dst) { _bit_shuffle_ptr->_cur_index)); *n = max_fetch; - const auto* data_array = reinterpret_cast(_bit_shuffle_ptr->get_data(0)); - size_t start_index = _bit_shuffle_ptr->_cur_index; + if (_options.only_read_offsets) { + // OFFSET_ONLY mode: resolve dict codes to get real string lengths + // without copying actual char data. This allows length() to work. + const auto* data_array = reinterpret_cast(_bit_shuffle_ptr->get_data(0)); + size_t start_index = _bit_shuffle_ptr->_cur_index; + // Reuse _buffer (int32_t vector) to store uint32_t lengths. + // int32_t and uint32_t have the same size/alignment, and string + // lengths are always non-negative, so the bit patterns are identical. + _buffer.resize(max_fetch); + for (size_t i = 0; i < max_fetch; ++i) { + int32_t codeword = data_array[start_index + i]; + _buffer[i] = static_cast(_dict_word_info[codeword].size); + } + dst->insert_offsets_from_lengths(reinterpret_cast(_buffer.data()), + max_fetch); + } else { + const auto* data_array = reinterpret_cast(_bit_shuffle_ptr->get_data(0)); + size_t start_index = _bit_shuffle_ptr->_cur_index; - dst->insert_many_dict_data(data_array, start_index, _dict_word_info, max_fetch, - _num_dict_items); + dst->insert_many_dict_data(data_array, start_index, _dict_word_info, max_fetch, + _num_dict_items); + } _bit_shuffle_ptr->_cur_index += max_fetch; @@ -305,8 +323,32 @@ Status BinaryDictPageDecoder::read_by_rowids(const rowid_t* rowids, ordinal_t pa return Status::OK(); } - const auto* data_array = reinterpret_cast(_bit_shuffle_ptr->get_data(0)); auto total = *n; + + if (_options.only_read_offsets) { + // OFFSET_ONLY mode: resolve dict codes to get real string lengths + // without copying actual char data. This allows length() to work correctly. + const auto* data_array = reinterpret_cast(_bit_shuffle_ptr->get_data(0)); + size_t read_count = 0; + _buffer.resize(total); + for (size_t i = 0; i < total; ++i) { + ordinal_t ord = rowids[i] - page_first_ordinal; + if (ord >= _bit_shuffle_ptr->_num_elements) [[unlikely]] { + break; + } + int32_t codeword = data_array[ord]; + _buffer[read_count] = static_cast(_dict_word_info[codeword].size); + read_count++; + } + if (read_count > 0) { + dst->insert_offsets_from_lengths(reinterpret_cast(_buffer.data()), + read_count); + } + *n = read_count; + return Status::OK(); + } + + const auto* data_array = reinterpret_cast(_bit_shuffle_ptr->get_data(0)); size_t read_count = 0; _buffer.resize(total); for (size_t i = 0; i < total; ++i) { diff --git a/be/src/storage/segment/binary_plain_page.h b/be/src/storage/segment/binary_plain_page.h index 0d15a32c0a79ca..a1870e9dc99ecb 100644 --- a/be/src/storage/segment/binary_plain_page.h +++ b/be/src/storage/segment/binary_plain_page.h @@ -30,7 +30,6 @@ #include "common/logging.h" #include "core/column/column_complex.h" -#include "core/column/column_nullable.h" #include "storage/olap_common.h" #include "storage/segment/options.h" #include "storage/segment/page_builder.h" @@ -202,6 +201,21 @@ class BinaryPlainPageDecoder : public PageDecoder { } const size_t max_fetch = std::min(*n, static_cast(_num_elems - _cur_idx)); + if (_options.only_read_offsets) { + // OFFSET_ONLY mode: read string lengths from page offset trailer + // without copying actual char data. This allows length() to work. + _offsets.resize(max_fetch); + for (size_t i = 0; i < max_fetch; ++i) { + uint32_t str_start = offset(_cur_idx + i); + uint32_t str_end = offset(_cur_idx + i + 1); + _offsets[i] = str_end - str_start; + } + dst->insert_offsets_from_lengths(_offsets.data(), max_fetch); + _cur_idx += max_fetch; + *n = max_fetch; + return Status::OK(); + } + uint32_t last_offset = guarded_offset(_cur_idx); _offsets.resize(max_fetch + 1); _offsets[0] = last_offset; @@ -237,6 +251,29 @@ class BinaryPlainPageDecoder : public PageDecoder { } auto total = *n; + + if (_options.only_read_offsets) { + // OFFSET_ONLY mode: read string lengths from page offset trailer + // without copying actual char data. This allows length() to work. + size_t read_count = 0; + _offsets.resize(total); + for (size_t i = 0; i < total; ++i) { + ordinal_t ord = rowids[i] - page_first_ordinal; + if (UNLIKELY(ord >= _num_elems)) { + break; + } + uint32_t str_start = offset(ord); + uint32_t str_end = offset(ord + 1); + _offsets[read_count] = str_end - str_start; + read_count++; + } + if (read_count > 0) { + dst->insert_offsets_from_lengths(_offsets.data(), read_count); + } + *n = read_count; + return Status::OK(); + } + size_t read_count = 0; _binary_data.resize(total); for (size_t i = 0; i < total; ++i) { diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp index a3fa0329b68a72..7e2ee007f47474 100644 --- a/be/src/storage/segment/column_reader.cpp +++ b/be/src/storage/segment/column_reader.cpp @@ -762,7 +762,11 @@ Status ColumnReader::new_iterator(ColumnIteratorUPtr* iterator, const TabletColu return Status::OK(); } if (is_scalar_type(_meta_type)) { - *iterator = std::make_unique(shared_from_this()); + if (is_string_type(_meta_type)) { + *iterator = std::make_unique(shared_from_this()); + } else { + *iterator = std::make_unique(shared_from_this()); + } (*iterator)->set_column_name(tablet_column ? tablet_column->name() : ""); return Status::OK(); } else { @@ -944,10 +948,22 @@ Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) { return Status::OK(); } + if (read_null_map_only()) { + // In NULL_MAP_ONLY mode, only seek the null iterator; skip offset/key/val iterators + if (_map_reader->is_nullable() && _null_iterator) { + RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord)); + } + return Status::OK(); + } + if (_map_reader->is_nullable()) { RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord)); } RETURN_IF_ERROR(_offsets_iterator->seek_to_ordinal(ord)); + if (read_offset_only()) { + // In OFFSET_ONLY mode, key/value iterators are SKIP_READING, no need to seek them + return Status::OK(); + } // here to use offset info ordinal_t offset = 0; RETURN_IF_ERROR(_offsets_iterator->_peek_one_offset(&offset)); @@ -986,6 +1002,36 @@ Status MapFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* return Status::OK(); } + if (read_null_map_only()) { + // NULL_MAP_ONLY mode: read null map, fill nested ColumnMap with empty defaults + DORIS_CHECK(dst->is_nullable()); + auto& nullable_col = assert_cast(*dst); + auto null_map_ptr = nullable_col.get_null_map_column_ptr(); + size_t num_read = *n; + if (_null_iterator) { + bool null_signs_has_null = false; + // branch-4.2: ColumnNullable::get_null_map_column_ptr() returns ColumnUInt8::MutablePtr + // here (master returns MutableColumnPtr), so it must be converted explicitly before + // being handed to next_batch(). + MutableColumnPtr null_map_column = std::move(null_map_ptr); + RETURN_IF_ERROR( + _null_iterator->next_batch(&num_read, null_map_column, &null_signs_has_null)); + } else { + // schema-change: column became nullable but old segment has no null data + // branch-4.2: ColumnNullable::get_null_map_column_ptr() already yields a + // ColumnUInt8::MutablePtr (master returns MutableColumnPtr), so `*null_map_ptr` + // is a ColumnUInt8& and needs no assert_cast. + null_map_ptr->insert_many_vals(0, num_read); + } + DCHECK(num_read == *n); + // fill nested ColumnMap with empty (zero-element) maps + auto& column_map = assert_cast( + nullable_col.get_nested_column()); + column_map.insert_many_defaults(num_read); + *has_null = true; + return Status::OK(); + } + auto& column_map = assert_cast( dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); auto column_offsets_ptr = IColumn::mutate(std::move(column_map.get_offsets_ptr())); @@ -1013,12 +1059,18 @@ Status MapFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* auto val_ptr = IColumn::mutate(std::move(column_map.get_values_ptr())); Defer defer_keys {[&] { column_map.get_keys_ptr() = std::move(key_ptr); }}; Defer defer_values {[&] { column_map.get_values_ptr() = std::move(val_ptr); }}; - size_t num_read = num_items; - bool key_has_null = false; - bool val_has_null = false; - RETURN_IF_ERROR(_key_iterator->next_batch(&num_read, key_ptr, &key_has_null)); - RETURN_IF_ERROR(_val_iterator->next_batch(&num_read, val_ptr, &val_has_null)); - DCHECK(num_read == num_items); + if (read_offset_only()) { + // OFFSET_ONLY mode: skip reading actual key/value data, fill with defaults + key_ptr->insert_many_defaults(num_items); + val_ptr->insert_many_defaults(num_items); + } else { + size_t num_read = num_items; + bool key_has_null = false; + bool val_has_null = false; + RETURN_IF_ERROR(_key_iterator->next_batch(&num_read, key_ptr, &key_has_null)); + RETURN_IF_ERROR(_val_iterator->next_batch(&num_read, val_ptr, &val_has_null)); + DCHECK(num_read == num_items); + } } if (dst->is_nullable()) { @@ -1048,6 +1100,31 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t dst->resize(count); return Status::OK(); } + + if (read_null_map_only()) { + // NULL_MAP_ONLY mode: read null map by rowids, fill nested ColumnMap with empty defaults + DORIS_CHECK(dst->is_nullable()); + auto& nullable_col = assert_cast(*dst); + if (_null_iterator) { + auto null_map_ptr = nullable_col.get_null_map_column_ptr(); + // branch-4.2: get_null_map_column_ptr() returns ColumnUInt8::MutablePtr here + // (master returns MutableColumnPtr), so convert before read_by_rowids(). + MutableColumnPtr null_map_column = std::move(null_map_ptr); + RETURN_IF_ERROR(_null_iterator->read_by_rowids(rowids, count, null_map_column)); + } else { + // schema-change: column became nullable but old segment has no null data + // branch-4.2: get_null_map_column_ptr() already yields a ColumnUInt8::MutablePtr, + // so `*null_map_ptr` is a ColumnUInt8& and needs no assert_cast. + auto null_map_ptr = nullable_col.get_null_map_column_ptr(); + null_map_ptr->insert_many_vals(0, count); + } + // fill nested ColumnMap with empty (zero-element) maps + auto& column_map = assert_cast( + nullable_col.get_nested_column()); + column_map.insert_many_defaults(count); + return Status::OK(); + } + if (count == 0) { return Status::OK(); } @@ -1243,21 +1320,47 @@ Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_acc return Status::OK(); } + // Check for meta-only modes (OFFSET_ONLY or NULL_MAP_ONLY) + _check_and_set_meta_read_mode(sub_all_access_paths); + if (read_offset_only()) { + _key_iterator->set_reading_flag(ReadingFlag::SKIP_READING); + _val_iterator->set_reading_flag(ReadingFlag::SKIP_READING); + DLOG(INFO) << "Map column iterator set column " << _column_name + << " to OFFSET_ONLY reading mode, key/value columns set to SKIP_READING"; + return Status::OK(); + } + if (read_null_map_only()) { + _key_iterator->set_reading_flag(ReadingFlag::SKIP_READING); + _val_iterator->set_reading_flag(ReadingFlag::SKIP_READING); + DLOG(INFO) << "Map column iterator set column " << _column_name + << " to NULL_MAP_ONLY reading mode, key/value columns set to SKIP_READING"; + return Status::OK(); + } + TColumnAccessPaths key_all_access_paths; TColumnAccessPaths val_all_access_paths; TColumnAccessPaths key_predicate_access_paths; TColumnAccessPaths val_predicate_access_paths; for (auto paths : sub_all_access_paths) { - if (paths.data_access_path.path[0] == "*") { - paths.data_access_path.path[0] = _key_iterator->column_name(); - key_all_access_paths.emplace_back(paths); + if (paths.data_access_path.path[0] == ACCESS_ALL) { + // ACCESS_ALL means element_at(map, key) style access: the key column must be + // fully read so that the runtime can match the requested key, while any sub-path + // qualifiers (e.g. OFFSET) apply only to the value column. + // For key: create a path with just the column name (= full data access). + TColumnAccessPath key_path; + key_path.__set_type(paths.type); + TDataAccessPath key_data_path; + key_data_path.__set_path({_key_iterator->column_name()}); + key_path.__set_data_access_path(key_data_path); + key_all_access_paths.emplace_back(std::move(key_path)); + // For value: pass the full sub-path so qualifiers like OFFSET propagate. paths.data_access_path.path[0] = _val_iterator->column_name(); val_all_access_paths.emplace_back(paths); - } else if (paths.data_access_path.path[0] == "KEYS") { + } else if (paths.data_access_path.path[0] == ACCESS_MAP_KEYS) { paths.data_access_path.path[0] = _key_iterator->column_name(); key_all_access_paths.emplace_back(paths); - } else if (paths.data_access_path.path[0] == "VALUES") { + } else if (paths.data_access_path.path[0] == ACCESS_MAP_VALUES) { paths.data_access_path.path[0] = _val_iterator->column_name(); val_all_access_paths.emplace_back(paths); } @@ -1266,15 +1369,20 @@ Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_acc const auto need_read_values = !val_all_access_paths.empty(); for (auto paths : sub_predicate_access_paths) { - if (paths.data_access_path.path[0] == "*") { - paths.data_access_path.path[0] = _key_iterator->column_name(); - key_predicate_access_paths.emplace_back(paths); + if (paths.data_access_path.path[0] == ACCESS_ALL) { + // Same logic as above: key needs full data, value gets the sub-path. + TColumnAccessPath key_path; + key_path.__set_type(paths.type); + TDataAccessPath key_data_path; + key_data_path.__set_path({_key_iterator->column_name()}); + key_path.__set_data_access_path(key_data_path); + key_predicate_access_paths.emplace_back(std::move(key_path)); paths.data_access_path.path[0] = _val_iterator->column_name(); val_predicate_access_paths.emplace_back(paths); - } else if (paths.data_access_path.path[0] == "KEYS") { + } else if (paths.data_access_path.path[0] == ACCESS_MAP_KEYS) { paths.data_access_path.path[0] = _key_iterator->column_name(); key_predicate_access_paths.emplace_back(paths); - } else if (paths.data_access_path.path[0] == "VALUES") { + } else if (paths.data_access_path.path[0] == ACCESS_MAP_VALUES) { paths.data_access_path.path[0] = _val_iterator->column_name(); val_predicate_access_paths.emplace_back(paths); } @@ -1333,6 +1441,36 @@ Status StructFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bo return Status::OK(); } + if (read_null_map_only()) { + // NULL_MAP_ONLY mode: read null map, fill nested ColumnStruct with empty defaults + DORIS_CHECK(dst->is_nullable()); + auto& nullable_col = assert_cast(*dst); + auto null_map_ptr = nullable_col.get_null_map_column_ptr(); + size_t num_read = *n; + if (_null_iterator) { + bool null_signs_has_null = false; + // branch-4.2: ColumnNullable::get_null_map_column_ptr() returns ColumnUInt8::MutablePtr + // here (master returns MutableColumnPtr), so it must be converted explicitly before + // being handed to next_batch(). + MutableColumnPtr null_map_column = std::move(null_map_ptr); + RETURN_IF_ERROR( + _null_iterator->next_batch(&num_read, null_map_column, &null_signs_has_null)); + } else { + // schema-change: column became nullable but old segment has no null data + // branch-4.2: ColumnNullable::get_null_map_column_ptr() already yields a + // ColumnUInt8::MutablePtr (master returns MutableColumnPtr), so `*null_map_ptr` + // is a ColumnUInt8& and needs no assert_cast. + null_map_ptr->insert_many_vals(0, num_read); + } + DCHECK(num_read == *n); + // fill nested ColumnStruct with defaults to maintain consistent column sizes + auto& column_struct = assert_cast( + nullable_col.get_nested_column()); + column_struct.insert_many_defaults(num_read); + *has_null = true; + return Status::OK(); + } + auto& column_struct = assert_cast( dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); for (size_t i = 0; i < column_struct.tuple_size(); i++) { @@ -1373,6 +1511,14 @@ Status StructFileColumnIterator::seek_to_ordinal(ordinal_t ord) { return Status::OK(); } + if (read_null_map_only()) { + // In NULL_MAP_ONLY mode, only seek the null iterator; skip all sub-column iterators + if (_struct_reader->is_nullable() && _null_iterator) { + RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord)); + } + return Status::OK(); + } + for (auto& column_iterator : _sub_column_iterators) { RETURN_IF_ERROR(column_iterator->seek_to_ordinal(ord)); } @@ -1426,7 +1572,7 @@ Status StructFileColumnIterator::read_by_rowids(const rowid_t* rowids, const siz } RETURN_IF_ERROR(seek_to_ordinal(start_idx)); size_t num_read = this_run; - RETURN_IF_ERROR(next_batch(&num_read, dst, nullptr)); + RETURN_IF_ERROR(next_batch(&num_read, dst)); DCHECK_EQ(num_read, this_run); start_idx = rowids[i]; @@ -1436,7 +1582,7 @@ Status StructFileColumnIterator::read_by_rowids(const rowid_t* rowids, const siz RETURN_IF_ERROR(seek_to_ordinal(start_idx)); size_t num_read = this_run; - RETURN_IF_ERROR(next_batch(&num_read, dst, nullptr)); + RETURN_IF_ERROR(next_batch(&num_read, dst)); DCHECK_EQ(num_read, this_run); return Status::OK(); } @@ -1477,6 +1623,17 @@ Status StructFileColumnIterator::set_access_paths( auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths)); auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths)); + // Check for NULL_MAP_ONLY mode: only read null map, skip all sub-columns + _check_and_set_meta_read_mode(sub_all_access_paths); + if (read_null_map_only()) { + for (auto& sub_iterator : _sub_column_iterators) { + sub_iterator->set_reading_flag(ReadingFlag::SKIP_READING); + } + DLOG(INFO) << "Struct column iterator set column " << _column_name + << " to NULL_MAP_ONLY reading mode, all sub-columns set to SKIP_READING"; + return Status::OK(); + } + const auto no_sub_column_to_skip = sub_all_access_paths.empty(); const auto no_predicate_sub_column = sub_predicate_access_paths.empty(); @@ -1615,6 +1772,10 @@ Status ArrayFileColumnIterator::init(const ColumnIteratorOptions& opts) { } Status ArrayFileColumnIterator::_seek_by_offsets(ordinal_t ord) { + if (read_offset_only()) { + // In OFFSET_ONLY mode, item iterator is SKIP_READING, no need to seek it + return Status::OK(); + } // using offsets info ordinal_t offset = 0; RETURN_IF_ERROR(_offset_iterator->_peek_one_offset(&offset)); @@ -1628,6 +1789,14 @@ Status ArrayFileColumnIterator::seek_to_ordinal(ordinal_t ord) { return Status::OK(); } + if (read_null_map_only()) { + // In NULL_MAP_ONLY mode, only seek the null iterator; skip offset and item iterators + if (_array_reader->is_nullable() && _null_iterator) { + RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord)); + } + return Status::OK(); + } + RETURN_IF_ERROR(_offset_iterator->seek_to_ordinal(ord)); if (_array_reader->is_nullable()) { RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord)); @@ -1642,6 +1811,36 @@ Status ArrayFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, boo return Status::OK(); } + if (read_null_map_only()) { + // NULL_MAP_ONLY mode: read null map, fill nested ColumnArray with empty defaults + DORIS_CHECK(dst->is_nullable()); + auto& nullable_col = assert_cast(*dst); + auto null_map_ptr = nullable_col.get_null_map_column_ptr(); + size_t num_read = *n; + if (_null_iterator) { + bool null_signs_has_null = false; + // branch-4.2: ColumnNullable::get_null_map_column_ptr() returns ColumnUInt8::MutablePtr + // here (master returns MutableColumnPtr), so it must be converted explicitly before + // being handed to next_batch(). + MutableColumnPtr null_map_column = std::move(null_map_ptr); + RETURN_IF_ERROR( + _null_iterator->next_batch(&num_read, null_map_column, &null_signs_has_null)); + } else { + // schema-change: column became nullable but old segment has no null data + // branch-4.2: ColumnNullable::get_null_map_column_ptr() already yields a + // ColumnUInt8::MutablePtr (master returns MutableColumnPtr), so `*null_map_ptr` + // is a ColumnUInt8& and needs no assert_cast. + null_map_ptr->insert_many_vals(0, num_read); + } + DCHECK(num_read == *n); + // fill nested ColumnArray with empty (zero-length) arrays + auto& column_array = assert_cast( + nullable_col.get_nested_column()); + column_array.insert_many_defaults(num_read); + *has_null = true; + return Status::OK(); + } + auto& column_array = assert_cast( dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); @@ -1666,10 +1865,16 @@ Status ArrayFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, boo if (num_items > 0) { auto column_items_ptr = IColumn::mutate(std::move(column_array.get_data_ptr())); Defer defer_items {[&] { column_array.get_data_ptr() = std::move(column_items_ptr); }}; - size_t num_read = num_items; - bool items_has_null = false; - RETURN_IF_ERROR(_item_iterator->next_batch(&num_read, column_items_ptr, &items_has_null)); - DCHECK(num_read == num_items); + if (read_offset_only()) { + // OFFSET_ONLY mode: skip reading actual item data, fill with defaults + column_items_ptr->insert_many_defaults(num_items); + } else { + size_t num_read = num_items; + bool items_has_null = false; + RETURN_IF_ERROR( + _item_iterator->next_batch(&num_read, column_items_ptr, &items_has_null)); + DCHECK(num_read == num_items); + } } if (dst->is_nullable()) { @@ -1723,11 +1928,10 @@ Status ArrayFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size } for (size_t i = 0; i < count; ++i) { - // TODO(cambyzju): now read array one by one, need optimize later + // TODO(cambyszju): now read array one by one, need optimize later RETURN_IF_ERROR(seek_to_ordinal(rowids[i])); size_t num_read = 1; - RETURN_IF_ERROR(next_batch(&num_read, dst, nullptr)); - DCHECK(num_read == 1); + RETURN_IF_ERROR(next_batch(&num_read, dst)); } return Status::OK(); } @@ -1756,12 +1960,27 @@ Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_a auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths)); auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths)); + // Check for meta-only modes (OFFSET_ONLY or NULL_MAP_ONLY) + _check_and_set_meta_read_mode(sub_all_access_paths); + if (read_offset_only()) { + _item_iterator->set_reading_flag(ReadingFlag::SKIP_READING); + DLOG(INFO) << "Array column iterator set column " << _column_name + << " to OFFSET_ONLY reading mode, item column set to SKIP_READING"; + return Status::OK(); + } + if (read_null_map_only()) { + _item_iterator->set_reading_flag(ReadingFlag::SKIP_READING); + DLOG(INFO) << "Array column iterator set column " << _column_name + << " to NULL_MAP_ONLY reading mode, item column set to SKIP_READING"; + return Status::OK(); + } + const auto no_sub_column_to_skip = sub_all_access_paths.empty(); const auto no_predicate_sub_column = sub_predicate_access_paths.empty(); if (!no_sub_column_to_skip) { for (auto& path : sub_all_access_paths) { - if (path.data_access_path.path[0] == "*") { + if (path.data_access_path.path[0] == ACCESS_ALL) { path.data_access_path.path[0] = _item_iterator->column_name(); } } @@ -1769,7 +1988,7 @@ Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_a if (!no_predicate_sub_column) { for (auto& path : sub_predicate_access_paths) { - if (path.data_access_path.path[0] == "*") { + if (path.data_access_path.path[0] == ACCESS_ALL) { path.data_access_path.path[0] = _item_iterator->column_name(); } } @@ -1783,10 +2002,68 @@ Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_a return Status::OK(); } +//////////////////////////////////////////////////////////////////////////////// +// StringFileColumnIterator implementation +//////////////////////////////////////////////////////////////////////////////// + +StringFileColumnIterator::StringFileColumnIterator(std::shared_ptr reader) + : FileColumnIterator(std::move(reader)) {} + +Status StringFileColumnIterator::init(const ColumnIteratorOptions& opts) { + if (read_offset_only()) { + // Propagate only_read_offsets to the FileColumnIterator's options + auto modified_opts = opts; + modified_opts.only_read_offsets = true; + return FileColumnIterator::init(modified_opts); + } + return FileColumnIterator::init(opts); +} + +Status StringFileColumnIterator::set_access_paths( + const TColumnAccessPaths& all_access_paths, + const TColumnAccessPaths& predicate_access_paths) { + if (all_access_paths.empty()) { + return Status::OK(); + } + + if (!predicate_access_paths.empty()) { + set_reading_flag(ReadingFlag::READING_FOR_PREDICATE); + } + + // Strip the column name from path[0] before checking for meta-only modes. + // Raw paths look like ["col_name", "OFFSET"] or ["col_name", "NULL"]. + auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths)); + _check_and_set_meta_read_mode(sub_all_access_paths); + if (read_offset_only()) { + DLOG(INFO) << "String column iterator set column " << _column_name + << " to OFFSET_ONLY reading mode"; + } else if (read_null_map_only()) { + DLOG(INFO) << "String column iterator set column " << _column_name + << " to NULL_MAP_ONLY reading mode"; + } + + return Status::OK(); +} + //////////////////////////////////////////////////////////////////////////////// FileColumnIterator::FileColumnIterator(std::shared_ptr reader) : _reader(reader) {} +void ColumnIterator::_check_and_set_meta_read_mode(const TColumnAccessPaths& sub_all_access_paths) { + for (const auto& path : sub_all_access_paths) { + if (!path.data_access_path.path.empty()) { + if (StringCaseEqual()(path.data_access_path.path[0], ACCESS_OFFSET)) { + _read_mode = ReadMode::OFFSET_ONLY; + return; + } else if (StringCaseEqual()(path.data_access_path.path[0], ACCESS_NULL)) { + _read_mode = ReadMode::NULL_MAP_ONLY; + return; + } + } + } + _read_mode = ReadMode::DEFAULT; +} + Status FileColumnIterator::init(const ColumnIteratorOptions& opts) { if (_reading_flag == ReadingFlag::SKIP_READING) { DLOG(INFO) << "File column iterator column " << _column_name << " skip reading."; @@ -1894,6 +2171,54 @@ Status FileColumnIterator::next_batch_of_zone_map(size_t* n, MutableColumnPtr& d } Status FileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) { + if (read_null_map_only()) { + DLOG(INFO) << "File column iterator column " << _column_name + << " in NULL_MAP_ONLY mode, reading only null map."; + DORIS_CHECK(dst->is_nullable()); + auto& nullable_col = assert_cast(*dst); + auto& null_map_data = nullable_col.get_null_map_data(); + + size_t remaining = *n; + *has_null = false; + while (remaining > 0) { + if (!_page.has_remaining()) { + bool eos = false; + RETURN_IF_ERROR(_load_next_page(&eos)); + if (eos) { + break; + } + } + + size_t nrows_in_page = std::min(remaining, _page.remaining()); + size_t nrows_to_read = nrows_in_page; + if (_page.has_null) { + while (nrows_to_read > 0) { + bool is_null = false; + size_t this_run = _page.null_decoder.GetNextRun(&is_null, nrows_to_read); + const size_t cur_size = null_map_data.size(); + null_map_data.resize(cur_size + this_run); + memset(null_map_data.data() + cur_size, is_null ? 1 : 0, this_run); + if (is_null) { + *has_null = true; + } + nrows_to_read -= this_run; + _page.offset_in_page += this_run; + _current_ordinal += this_run; + } + } else { + const size_t cur_size = null_map_data.size(); + null_map_data.resize(cur_size + nrows_to_read); + memset(null_map_data.data() + cur_size, 0, nrows_to_read); + _page.offset_in_page += nrows_to_read; + _current_ordinal += nrows_to_read; + } + remaining -= nrows_in_page; + } + *n -= remaining; + nullable_col.get_nested_column().insert_many_defaults(*n); + return Status::OK(); + } + if (_reading_flag == ReadingFlag::SKIP_READING) { DLOG(INFO) << "File column iterator column " << _column_name << " skip reading."; dst->resize(dst->size() + *n); @@ -1959,6 +2284,71 @@ Status FileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* ha Status FileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count, MutableColumnPtr& dst) { + if (read_null_map_only()) { + DLOG(INFO) << "File column iterator column " << _column_name + << " in NULL_MAP_ONLY mode, reading only null map by rowids."; + + DORIS_CHECK(dst->is_nullable()); + auto& nullable_col = assert_cast(*dst); + auto& null_map_data = nullable_col.get_null_map_data(); + const size_t base_size = null_map_data.size(); + null_map_data.resize(base_size + count); + + nullable_col.get_nested_column().insert_many_defaults(count); + + size_t remaining = count; + size_t total_read_count = 0; + size_t nrows_to_read = 0; + while (remaining > 0) { + RETURN_IF_ERROR(seek_to_ordinal(rowids[total_read_count])); + + nrows_to_read = std::min(remaining, _page.remaining()); + + if (_page.has_null) { + size_t already_read = 0; + while ((nrows_to_read - already_read) > 0) { + bool is_null = false; + size_t this_run = std::min(nrows_to_read - already_read, _page.remaining()); + if (UNLIKELY(this_run == 0)) { + break; + } + this_run = _page.null_decoder.GetNextRun(&is_null, this_run); + + size_t offset = total_read_count + already_read; + size_t this_read_count = 0; + rowid_t current_ordinal_in_page = + cast_set(_page.offset_in_page + _page.first_ordinal); + for (size_t i = 0; i < this_run; ++i) { + if (rowids[offset + i] - current_ordinal_in_page >= this_run) { + break; + } + this_read_count++; + } + + if (this_read_count > 0) { + memset(null_map_data.data() + base_size + offset, is_null ? 1 : 0, + this_read_count); + } + + already_read += this_read_count; + _page.offset_in_page += this_run; + } + + nrows_to_read = already_read; + total_read_count += nrows_to_read; + remaining -= nrows_to_read; + } else { + memset(null_map_data.data() + base_size + total_read_count, 0, nrows_to_read); + total_read_count += nrows_to_read; + remaining -= nrows_to_read; + } + } + + null_map_data.resize(base_size + total_read_count); + nullable_col.get_nested_column().insert_many_defaults(total_read_count); + return Status::OK(); + } + if (_reading_flag == ReadingFlag::SKIP_READING) { DLOG(INFO) << "File column iterator column " << _column_name << " skip reading."; dst->resize(count); @@ -2057,11 +2447,14 @@ Status FileColumnIterator::_read_data_page(const OrdinalPageIndexIterator& iter) Slice page_body; PageFooterPB footer; _opts.type = DATA_PAGE; + PageDecoderOptions decoder_opts; + decoder_opts.only_read_offsets = _opts.only_read_offsets; RETURN_IF_ERROR( _reader->read_page(_opts, iter.page(), &handle, &page_body, &footer, _compress_codec)); // parse data page auto st = ParsedPage::create(std::move(handle), page_body, footer.data_page_footer(), - _reader->encoding_info(), iter.page(), iter.page_index(), &_page); + _reader->encoding_info(), iter.page(), iter.page_index(), &_page, + decoder_opts); if (!st.ok()) { LOG(WARNING) << "failed to create ParsedPage, file=" << _opts.file_reader->path().native() << ", page_offset=" << iter.page().offset << ", page_size=" << iter.page().size diff --git a/be/src/storage/segment/column_reader.h b/be/src/storage/segment/column_reader.h index 03390543366a8a..8c1831aa7cb9c9 100644 --- a/be/src/storage/segment/column_reader.h +++ b/be/src/storage/segment/column_reader.h @@ -109,6 +109,7 @@ struct ColumnIteratorOptions { // reader statistics OlapReaderStatistics* stats = nullptr; // Ref io::IOContext io_ctx; + bool only_read_offsets = false; void sanity_check() const { CHECK_NOTNULL(file_reader); @@ -418,17 +419,38 @@ class ColumnIterator { std::map>& prefetchers, PrefetcherInitMethod init_method) {} + static constexpr const char* ACCESS_OFFSET = "OFFSET"; + static constexpr const char* ACCESS_ALL = "*"; + static constexpr const char* ACCESS_MAP_KEYS = "KEYS"; + static constexpr const char* ACCESS_MAP_VALUES = "VALUES"; + static constexpr const char* ACCESS_NULL = "NULL"; + + // Meta-only read modes: + // - OFFSET_ONLY: only read offset information (e.g., for array_size/map_size/string_length) + // - NULL_MAP_ONLY: only read null map (e.g., for IS NULL / IS NOT NULL predicates) + // When these modes are enabled, actual content data is skipped. + enum class ReadMode : int { DEFAULT, OFFSET_ONLY, NULL_MAP_ONLY }; + + bool read_offset_only() const { return _read_mode == ReadMode::OFFSET_ONLY; } + bool read_null_map_only() const { return _read_mode == ReadMode::NULL_MAP_ONLY; } + protected: + // Checks sub access paths for OFFSET or NULL meta-only modes and + // updates _read_mode accordingly. Use the accessor helpers + // read_offset_only() / read_null_map_only() to query the current mode. + void _check_and_set_meta_read_mode(const TColumnAccessPaths& sub_all_access_paths); + Result _get_sub_access_paths(const TColumnAccessPaths& access_paths); ColumnIteratorOptions _opts; ReadingFlag _reading_flag {ReadingFlag::NORMAL_READING}; + ReadMode _read_mode = ReadMode::DEFAULT; std::string _column_name; }; // This iterator is used to read column data from file // for scalar type -class FileColumnIterator final : public ColumnIterator { +class FileColumnIterator : public ColumnIterator { public: explicit FileColumnIterator(std::shared_ptr reader); ~FileColumnIterator() override; @@ -482,7 +504,6 @@ class FileColumnIterator final : public ColumnIterator { std::shared_ptr _reader = nullptr; - // iterator owned compress codec, should NOT be shared by threads, initialized in init() BlockCompressionCodec* _compress_codec = nullptr; // 1. The _page represents current page. @@ -518,6 +539,21 @@ class EmptyFileColumnIterator final : public ColumnIterator { ordinal_t get_current_ordinal() const override { return 0; } }; +// StringFileColumnIterator extends FileColumnIterator with meta-only reading +// support for string/binary column types. When the OFFSET path is detected in +// set_access_paths, it sets only_read_offsets on the ColumnIteratorOptions so +// that the BinaryPlainPageDecoder skips chars memcpy and only fills offsets. +class StringFileColumnIterator final : public FileColumnIterator { +public: + explicit StringFileColumnIterator(std::shared_ptr reader); + ~StringFileColumnIterator() override = default; + + Status init(const ColumnIteratorOptions& opts) override; + + Status set_access_paths(const TColumnAccessPaths& all_access_paths, + const TColumnAccessPaths& predicate_access_paths) override; +}; + // This iterator make offset operation write once for class OffsetFileColumnIterator final : public ColumnIterator { public: @@ -530,6 +566,12 @@ class OffsetFileColumnIterator final : public ColumnIterator { Status init(const ColumnIteratorOptions& opts) override; Status next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) override; + + Status next_batch(size_t* n, MutableColumnPtr& dst) { + bool has_null; + return next_batch(n, dst, &has_null); + } + ordinal_t get_current_ordinal() const override { return _offset_iterator->get_current_ordinal(); } @@ -579,6 +621,9 @@ class MapFileColumnIterator final : public ColumnIterator { Status seek_to_ordinal(ordinal_t ord) override; ordinal_t get_current_ordinal() const override { + if (read_null_map_only() && _null_iterator) { + return _null_iterator->get_current_ordinal(); + } return _offsets_iterator->get_current_ordinal(); } Status init_prefetcher(const SegmentPrefetchParams& params) override; @@ -613,12 +658,20 @@ class StructFileColumnIterator final : public ColumnIterator { Status next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) override; + Status next_batch(size_t* n, MutableColumnPtr& dst) { + bool has_null; + return next_batch(n, dst, &has_null); + } + Status read_by_rowids(const rowid_t* rowids, const size_t count, MutableColumnPtr& dst) override; Status seek_to_ordinal(ordinal_t ord) override; ordinal_t get_current_ordinal() const override { + if (read_null_map_only() && _null_iterator) { + return _null_iterator->get_current_ordinal(); + } return _sub_column_iterators[0]->get_current_ordinal(); } @@ -653,12 +706,20 @@ class ArrayFileColumnIterator final : public ColumnIterator { Status next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) override; + Status next_batch(size_t* n, MutableColumnPtr& dst) { + bool has_null; + return next_batch(n, dst, &has_null); + } + Status read_by_rowids(const rowid_t* rowids, const size_t count, MutableColumnPtr& dst) override; Status seek_to_ordinal(ordinal_t ord) override; ordinal_t get_current_ordinal() const override { + if (read_null_map_only() && _null_iterator) { + return _null_iterator->get_current_ordinal(); + } return _offset_iterator->get_current_ordinal(); } diff --git a/be/src/storage/segment/options.h b/be/src/storage/segment/options.h index 3941be17716bd3..a7c1ef0fa64b8c 100644 --- a/be/src/storage/segment/options.h +++ b/be/src/storage/segment/options.h @@ -46,6 +46,7 @@ struct PageBuilderOptions { struct PageDecoderOptions { bool need_check_bitmap = true; + bool only_read_offsets = false; }; } // namespace segment_v2 diff --git a/be/test/storage/segment/column_reader_test.cpp b/be/test/storage/segment/column_reader_test.cpp index 49924f0907bca2..de8b5aadf1f3de 100644 --- a/be/test/storage/segment/column_reader_test.cpp +++ b/be/test/storage/segment/column_reader_test.cpp @@ -302,4 +302,42 @@ TEST_F(ColumnReaderTest, MapReadByRowidsSkipReadingResizesDestination) { ASSERT_TRUE(st.ok()) << "read_by_rowids failed: " << st.to_string(); ASSERT_EQ(count, dst->size()); } -} // namespace doris::segment_v2 \ No newline at end of file +TEST_F(ColumnReaderTest, MapAccessAllWithOffsetDoesNotPropagateOffsetToKey) { + // Regression test: when the access path is [map_col, *, OFFSET] + // (e.g. length(map_col['some_key'])), the key column must be fully read + // so that element_at() can match the key. Only the value column should + // enter OFFSET_ONLY mode. + auto map_reader = std::make_shared(); + auto null_iter = std::make_unique(std::make_shared()); + auto offsets_iter = std::make_unique( + std::make_unique(std::make_shared())); + auto key_iter = std::make_unique(std::make_shared()); + key_iter->set_column_name("key"); + auto val_iter = std::make_unique(std::make_shared()); + val_iter->set_column_name("value"); + + MapFileColumnIterator map_iter(map_reader, std::move(null_iter), std::move(offsets_iter), + std::move(key_iter), std::move(val_iter)); + map_iter.set_column_name("map_col"); + + // path: [map_col, *, OFFSET] — simulates length(map_col['c_phone']) + TColumnAccessPaths all_access_paths; + all_access_paths.emplace_back(); + all_access_paths[0].data_access_path.path = {"map_col", "*", "OFFSET"}; + TColumnAccessPaths predicate_access_paths; + + auto st = map_iter.set_access_paths(all_access_paths, predicate_access_paths); + ASSERT_TRUE(st.ok()) << "set_access_paths failed: " << st.to_string(); + + // Key must be fully readable (NEED_TO_READ), NOT in OFFSET_ONLY mode. + auto* key_ptr = static_cast(map_iter._key_iterator.get()); + ASSERT_EQ(key_ptr->_reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ); + ASSERT_FALSE(key_ptr->read_offset_only()); + + // Value should be in OFFSET_ONLY mode since we only need string lengths. + auto* val_ptr = static_cast(map_iter._val_iterator.get()); + ASSERT_EQ(val_ptr->_reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ); + ASSERT_TRUE(val_ptr->read_offset_only()); +} + +} // namespace doris::segment_v2 diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AccessPathInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AccessPathInfo.java index 77321c64c6f2db..a6d1cd3dd84a05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AccessPathInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AccessPathInfo.java @@ -31,6 +31,11 @@ public class AccessPathInfo { public static final String ACCESS_MAP_KEYS = "KEYS"; public static final String ACCESS_MAP_VALUES = "VALUES"; public static final String ACCESS_OFFSET = "OFFSET"; + // Suffix appended to a string-column path to indicate that only the offset array + // (not the char data) is needed — agreed with BE as the special path component name. + public static final String ACCESS_STRING_OFFSET = ACCESS_OFFSET; + // Suffix appended to a column path to indicate that only the null flag + // (not the actual data) is needed — used when the column is only accessed via IS NULL / IS NOT NULL. public static final String ACCESS_NULL = "NULL"; private DataType prunedType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java index af0231c4f094b3..ecb765be3c7e0f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java @@ -68,6 +68,10 @@ public SlotDescriptor(SlotId id, TupleDescriptor parent) { this.id = id; this.parent = parent; this.isNullable = true; + this.allAccessPaths = Collections.emptyList(); + this.predicateAccessPaths = Collections.emptyList(); + this.displayAllAccessPaths = Collections.emptyList(); + this.displayPredicateAccessPaths = Collections.emptyList(); } public SlotDescriptor(SlotId id, TupleDescriptor parent, SlotDescriptor src) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index d72a4b034e9309..c9ef6a38d5a819 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -109,6 +109,7 @@ public class CascadesContext implements ScheduleContext { private Optional outerScope = Optional.empty(); private boolean isRewriteRoot; + private boolean isMaterializedViewRewritePlanFragment; private volatile boolean isTimeout = false; // current process subtree, represent outer plan if empty @@ -409,6 +410,14 @@ public boolean isRewriteRoot() { return isRewriteRoot; } + public void setMaterializedViewRewritePlanFragment(boolean materializedViewRewritePlanFragment) { + isMaterializedViewRewritePlanFragment = materializedViewRewritePlanFragment; + } + + public boolean isMaterializedViewRewritePlanFragment() { + return isMaterializedViewRewritePlanFragment; + } + public Optional getOuterScope() { return outerScope; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 4ff0cd0ed3d7ae..dcb6d15060edf9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -149,6 +149,7 @@ public enum TableFrom { private boolean isDpHyp = false; + private boolean isDelete = false; private boolean hasNondeterministic = false; // hasUnknownColStats true if any column stats in the tables used by this sql is @@ -326,8 +327,6 @@ public enum TableFrom { private final Set> materializationRewrittenSuccessSet = new HashSet<>(); private boolean isInsert = false; - private boolean skipPrunePredicate = false; - private Optional>> mvRefreshPredicates = Optional.empty(); // For Iceberg rewrite operations: store file scan tasks to be used by @@ -1728,14 +1727,6 @@ public boolean isUseGatherForIcebergRewrite() { return this.useGatherForIcebergRewrite; } - public boolean isSkipPrunePredicate() { - return skipPrunePredicate; - } - - public void setSkipPrunePredicate(boolean skipPrunePredicate) { - this.skipPrunePredicate = skipPrunePredicate; - } - public boolean hasNestedColumns() { return hasNestedColumns; } @@ -1770,6 +1761,14 @@ public Optional getIcebergWriteSchemaContext() { return icebergWriteSchemaContext; } + public boolean isDelete() { + return isDelete; + } + + public void setIsDelete(boolean del) { + isDelete = del; + } + public void setIcebergWriteSchemaContext( Optional icebergWriteSchemaContext) { this.icebergWriteSchemaContext = Objects.requireNonNull( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 9167a0aed681dd..5d2b5e3b48921a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -125,6 +125,7 @@ import org.apache.doris.nereids.rules.rewrite.PullUpCteAnchor; import org.apache.doris.nereids.rules.rewrite.PullUpJoinFromUnionAll; import org.apache.doris.nereids.rules.rewrite.PullUpProjectBetweenTopNAndAgg; +import org.apache.doris.nereids.rules.rewrite.PullUpProjectExprUnderTopN; import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderApply; import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderLimit; import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderTopN; @@ -712,7 +713,9 @@ public class Rewriter extends AbstractBatchJobExecutor { topDown( new PullUpProjectUnderTopN(), new PullUpProjectUnderLimit() - ) + ), + custom(RuleType.PULL_UP_PROJECT_EXPR_UNDER_TOPN, + PullUpProjectExprUnderTopN::new) ), // TODO: these rules should be implementation rules, and generate alternative physical plans. topic("Table/Physical optimization", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java index 9cb6ce33e7bfa6..01ecf88e595ac4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java @@ -63,7 +63,7 @@ public Plan visitPhysicalFilter(PhysicalFilter filter, CascadesC return filter; } boolean skipPrunePredicate = context.getConnectContext().getSessionVariable().skipPrunePredicate - || context.getStatementContext().isSkipPrunePredicate(); + || context.getStatementContext().isDelete(); if (skipPrunePredicate) { return filter; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterContext.java index 4fa902203a77d2..d52c5b71ac4367 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterContext.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.algebra.TopN; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterializeOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; import org.apache.doris.nereids.trees.plans.physical.TopnFilter; import org.apache.doris.planner.ScanNode; @@ -71,13 +72,22 @@ public List getTopnFilters() { public void translateTarget(PhysicalRelation relation, ScanNode legacyScan, PlanTranslatorContext translatorContext) { for (TopnFilter filter : filters.values()) { - if (filter.hasTargetRelation(relation)) { - Expr expr = ExpressionTranslator.translate(filter.targets.get(relation), translatorContext); - filter.legacyTargets.put(legacyScan, expr); + translateTarget(filter, relation, legacyScan, translatorContext); + if (relation instanceof PhysicalLazyMaterializeOlapScan) { + translateTarget(filter, ((PhysicalLazyMaterializeOlapScan) relation).getScan(), + legacyScan, translatorContext); } } } + private void translateTarget(TopnFilter filter, PhysicalRelation relation, ScanNode legacyScan, + PlanTranslatorContext translatorContext) { + if (filter.hasTargetRelation(relation)) { + Expr expr = ExpressionTranslator.translate(filter.targets.get(relation), translatorContext); + filter.legacyTargets.put(legacyScan, expr); + } + } + /** * translate topn-filter */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java index 94e176f83d2fec..f3b8f3c5c71fc3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java @@ -19,11 +19,15 @@ import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Type; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.processor.post.PlanPostProcessor; +import org.apache.doris.nereids.processor.post.Validator; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; @@ -53,7 +57,8 @@ import java.util.Set; /** - * post rule to do lazy materialize + * Post rule to insert MaterializeNode for TopN lazy materialization. + * Expression pull-up is handled by PullUpProjectExprUnderTopN in the logical phase. */ public class LazyMaterializeTopN extends PlanPostProcessor { /* @@ -73,7 +78,21 @@ public class LazyMaterializeTopN extends PlanPostProcessor { private boolean hasMaterialized = false; @Override - public Plan visitPhysicalTopN(PhysicalTopN topN, CascadesContext ctx) { + public Plan visitPhysicalTopN(PhysicalTopN topN, CascadesContext ctx) { + try { + Plan result = computeTopN(topN, ctx); + if (SessionVariable.isFeDebug()) { + Validator validator = new Validator(); + validator.processRoot(result, ctx); + } + return result; + } catch (Exception e) { + LOG.warn("lazy materialize topn failed", e); + return topN; + } + } + + private Plan computeTopN(PhysicalTopN topN, CascadesContext ctx) { if (hasMaterialized) { return topN; } @@ -83,24 +102,53 @@ public Plan visitPhysicalTopN(PhysicalTopN topN, CascadesContext ctx) { if (!sessionVariable.enableLanceLazyMaterialization && !enableOtherTables) { return topN; } - /* - * topn(output=[x] orderkey=[b]) - * ->project(a as x) - * ->T(a, b) - * 'x' can be lazy materialized. - * materializeMap: x->(T, a) - */ + try { + List userVisibleOutput = ImmutableList.copyOf(topN.getOutput()); + List effectiveOutput = ImmutableList.copyOf(topN.getOutput()); + Plan result = doComputeTopN(topN, ctx, effectiveOutput, sessionVariable, enableOtherTables); + if (result == topN) { + return topN; + } + result = new PhysicalProject(ImmutableList.copyOf(userVisibleOutput), null, result); + return result; + } catch (RuntimeException e) { + LOG.warn("lazy materialize topn failed for plan: {}", topN.shapeInfo(), e); + return topN; + } + } + + private Plan doComputeTopN(PhysicalTopN topN, CascadesContext ctx, List effectiveOutput, + SessionVariable sessionVariable, boolean enableOtherTables) { Map materializeMap = new HashMap<>(); List materializedSlots = new ArrayList<>(); - // find the slots which can be lazy materialized - for (Slot slot : topN.getOutput()) { - // Decide per source so a Lance relation does not bypass the threshold for other tables. - Optional source = computeMaterializeSource(topN, (SlotReference) slot) + Set requiredMaterializedSlots = new HashSet<>(); + collectProjectExprInputSlots(topN.child(), requiredMaterializedSlots); + + /* + * requiredMaterializedSlots only records slots consumed by Project/final-projection expressions inside the + * TopN subtree. Other mandatory slots, such as TopN order keys or Filter predicates, are rejected by + * MaterializeProbeVisitor while tracing each output slot from TopN down to the source relation: + * + * Project(b) -> TopN(order by id) -> Filter(a > 0) -> Scan(id, a, b, c) + * + * For id, the probe stops at TopN because id is in TopN.getInputSlots(); for a, it stops at Filter because + * a is in Filter.getInputSlots(). Both return Optional.empty() and are appended to materializedSlots below. + * Therefore an empty requiredMaterializedSlots set does not mean every scan column can be delayed; it only + * means no extra Project/final-projection input must be forced materialized by this local safety check. + * + * `x` can be lazy materialized: topn(output=[x] orderkey=[b]) -> project(a as x) -> T(a, b); + * materializeMap: x->(T, a). The per-source Lance predicate is kept so a Lance relation does not + * bypass the threshold for other tables. + */ + for (Slot slot : effectiveOutput) { + Optional source = computeMaterializeSource(topN, (SlotReference) slot, + requiredMaterializedSlots) .filter(candidate -> MaterializeProbeVisitor.isLanceExternalSearch(candidate.relation) ? sessionVariable.enableLanceLazyMaterialization : enableOtherTables); if (source.isPresent()) { SlotReference baseSlot = source.get().baseSlot; - if (source.get().baseSlot.hasSubColPath()) { + if (source.get().baseSlot.hasSubColPath() + || source.get().baseSlot.getAllAccessPaths().isPresent()) { slot = baseSlot.withExprId(slot.getExprId()); } materializeMap.put(slot, source.get()); @@ -108,7 +156,19 @@ public Plan visitPhysicalTopN(PhysicalTopN topN, CascadesContext ctx) { materializedSlots.add(slot); } } - // find out the slots which are worth doing lazy materialization + List requiredOutputSlots = new ArrayList<>(); + for (Map.Entry entry : materializeMap.entrySet()) { + if (requiredMaterializedSlots.contains(entry.getKey()) + || requiredMaterializedSlots.contains(entry.getValue().baseSlot)) { + requiredOutputSlots.add(entry.getKey()); + } + } + for (Slot slot : requiredOutputSlots) { + if (materializeMap.remove(slot) != null) { + materializedSlots.add(slot); + } + } + List lazyMaterializeSlots = filterSlotsForLazyMaterialization(materializeMap); if (lazyMaterializeSlots.isEmpty()) { return topN; @@ -121,7 +181,6 @@ public Plan visitPhysicalTopN(PhysicalTopN topN, CascadesContext ctx) { } Plan result = topN; - List originOutput = topN.getOutput(); BiMap relationToRowId = HashBiMap.create(relationToLazySlotMap.size()); HashSet rowIdSet = new HashSet<>(); // we should use threadStatementContext, not ctx.getStatementContext(), because @@ -150,9 +209,9 @@ public Plan visitPhysicalTopN(PhysicalTopN topN, CascadesContext ctx) { catalogRelation.getTable().getName() + ".global_row_id", false, Integer.MAX_VALUE); SlotReference rowIdSlot = SlotReference.fromColumn(threadStatementContext.getNextExprId(), catalogRelation.getTable(), rowIdCol, catalogRelation.getQualifier()); - result = result.accept(new LazySlotPruning(), - new LazySlotPruning.Context((PhysicalCatalogRelation) relation, - rowIdSlot, relationToLazySlotMap.get(relation))); + result = result.accept(new LazySlotPruning(), new LazySlotPruning.Context( + (PhysicalCatalogRelation) relation, + rowIdSlot, relationToLazySlotMap.get(relation))); relationToRowId.put(catalogRelation, rowIdSlot); rowIdSet.add(rowIdSlot); } else if (relation instanceof PhysicalTVFRelation) { @@ -162,20 +221,16 @@ public Plan visitPhysicalTopN(PhysicalTopN topN, CascadesContext ctx) { tvfRelation.getFunction().getName() + ".global_row_id", false, Integer.MAX_VALUE); SlotReference rowIdSlot = SlotReference.fromColumn(threadStatementContext.getNextExprId(), tvfRelation.getFunction().getTable(), rowIdCol, ImmutableList.of()); - result = result.accept(new LazySlotPruning(), - new LazySlotPruning.Context((PhysicalTVFRelation) relation, - rowIdSlot, relationToLazySlotMap.get(relation))); + result = result.accept(new LazySlotPruning(), new LazySlotPruning.Context( + (PhysicalTVFRelation) relation, + rowIdSlot, relationToLazySlotMap.get(relation))); relationToRowId.put(tvfRelation, rowIdSlot); rowIdSet.add(rowIdSlot); } else { - // should not reach here. throw new RuntimeException("LazyMaterializeTopN not support this relation." + relation); } } - // materialize.child.output requires - // rowId only appears once. - // that is [a, rowId1, b rowId1] is not acceptable List materializeInput = moveRowIdsToTail(result.getOutput(), rowIdSet); if (materializeInput == null) { @@ -188,8 +243,17 @@ public Plan visitPhysicalTopN(PhysicalTopN topN, CascadesContext ctx) { * -->topn * -->any */ + // Row IDs are already at the tail in the correct order. + // Keep materialized slots in the same order as the child tuple layout. + List reOrderedMaterializedSlots = new ArrayList<>(); + for (Slot slot : result.getOutput()) { + if (rowIdSet.contains(slot)) { + break; + } + reOrderedMaterializedSlots.add(slot); + } result = new PhysicalLazyMaterialize(result, result.getOutput(), - materializedSlots, relationToLazySlotMap, relationToRowId, materializeMap, + reOrderedMaterializedSlots, relationToLazySlotMap, relationToRowId, materializeMap, null, ((AbstractPlan) result).getStats()); hasMaterialized = true; } else { @@ -216,10 +280,54 @@ public Plan visitPhysicalTopN(PhysicalTopN topN, CascadesContext ctx) { null, ((AbstractPlan) result).getStats()); hasMaterialized = true; } - result = new PhysicalProject(originOutput, null, result); return result; } + private void collectProjectExprInputSlots(Plan plan, Set requiredMaterializedSlots) { + if (plan instanceof PhysicalProject) { + PhysicalProject project = (PhysicalProject) plan; + for (NamedExpression projectExpr : project.getProjects()) { + if (projectExpr instanceof SlotReference) { + continue; + } + if (projectExpr instanceof Alias && ((Alias) projectExpr).child() instanceof SlotReference) { + SlotReference childSlot = (SlotReference) ((Alias) projectExpr).child(); + if (!childSlot.getOriginalColumn().isPresent()) { + requiredMaterializedSlots.addAll(project.getInputSlots()); + } + continue; + } + requiredMaterializedSlots.addAll(projectExpr.getInputSlots()); + } + } else if (plan instanceof PhysicalCatalogRelation) { + PhysicalCatalogRelation relation = (PhysicalCatalogRelation) plan; + if (relation.getTable() instanceof OlapTable) { + OlapTable table = (OlapTable) relation.getTable(); + if (KeysType.UNIQUE_KEYS.equals(table.getKeysType()) + && !table.getTableProperty().getEnableUniqueKeyMergeOnWrite() + || KeysType.AGG_KEYS.equals(table.getKeysType()) + || KeysType.PRIMARY_KEYS.equals(table.getKeysType())) { + for (Slot slot : relation.getOutput()) { + SlotReference slotReference = (SlotReference) slot; + if (slotReference.getOriginalColumn().isPresent() + && slotReference.getOriginalColumn().get().isKey()) { + requiredMaterializedSlots.add(slotReference); + } + } + } + } + for (Slot slot : plan.getOutput()) { + if (slot instanceof SlotReference && !((SlotReference) slot).getOriginalColumn().isPresent()) { + requiredMaterializedSlots.addAll(plan.getOutputSet()); + break; + } + } + } + for (Plan child : plan.children()) { + collectProjectExprInputSlots(child, requiredMaterializedSlots); + } + } + /* * [a, r1, r2, b, r2] => [a, b, r1, r2] * move all rowIds to tail, and remove duplicated rowIds @@ -253,10 +361,11 @@ private List filterSlotsForLazyMaterialization(Map(materializeMap.keySet()); } - private Optional computeMaterializeSource(PhysicalTopN topN, SlotReference slot) { + private Optional computeMaterializeSource(PhysicalTopN topN, SlotReference slot, + Set requiredMaterializedSlots) { MaterializeProbeVisitor probe = new MaterializeProbeVisitor(); - MaterializeProbeVisitor.ProbeContext context = new MaterializeProbeVisitor.ProbeContext(slot); + MaterializeProbeVisitor.ProbeContext context = new MaterializeProbeVisitor.ProbeContext(slot, + requiredMaterializedSlots); return probe.visit(topN, context); } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java index c7b89519f1c353..e4abccb40d356d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java @@ -79,12 +79,23 @@ public void updateRowIdSlot(SlotReference rowIdSlot) { } } - @Override + /** + * Whether the given child should be pruned. Default checks if the child's + * output contains all lazy slots. Override to bypass when logical properties + * are stale after plan restructuring. + */ + protected boolean shouldPruneChild(Plan child, Context context) { + return child.getOutput().containsAll(context.lazySlots); + } + + /** + * visit + */ public Plan visit(Plan plan, Context context) { ImmutableList.Builder newChildren = ImmutableList.builderWithExpectedSize(plan.arity()); boolean hasNewChildren = false; for (Plan child : plan.children()) { - if (child.getOutput().containsAll(context.lazySlots)) { + if (shouldPruneChild(child, context)) { Plan newChild = child.accept(this, context); if (newChild != child) { hasNewChildren = true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java index effc2bc69ccc11..a89f35f47941fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.processor.post.materialize.MaterializeProbeVisitor.ProbeContext; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Relation; @@ -45,6 +46,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.HashSet; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -67,13 +69,20 @@ public class MaterializeProbeVisitor extends DefaultPlanVisitor requiredMaterializedSlots; /** * constructor */ public ProbeContext(SlotReference slot) { + this(slot, new HashSet<>()); + } + + public ProbeContext(SlotReference slot, Set requiredMaterializedSlots) { this.slot = slot; + this.requiredMaterializedSlots = requiredMaterializedSlots; } + } @Override @@ -194,7 +203,7 @@ public Optional visitPhysicalOlapScan(PhysicalOlapScan scan, if (!supportOlapTopnLazyMaterialize(table)) { return Optional.empty(); } - if (scan.getOperativeSlots().contains(context.slot)) { + if (context.requiredMaterializedSlots.contains(context.slot)) { return Optional.empty(); } return Optional.of( @@ -206,12 +215,14 @@ public Optional visitPhysicalCatalogRelation( PhysicalCatalogRelation relation, ProbeContext context) { if (checkRelationTableSupportedType(relation) && relation.getOutput().contains(context.slot) - && !relation.getOperativeSlots().contains(context.slot)) { - // lazy materialize slot must be a passive slot + && !relation.getOperativeSlots().contains(context.slot) + && !context.requiredMaterializedSlots.contains(context.slot)) { + // lazy materialize slot must be backed by a base column. if (context.slot.getOriginalColumn().isPresent()) { return Optional.of(new MaterializeSource( relation, findRelationOutputSlot(relation, context.slot).orElse(context.slot))); } else { + context.requiredMaterializedSlots.addAll(relation.getOutputSet()); LOG.info("lazy materialize {} failed, because its column is empty", context.slot); } } @@ -227,8 +238,9 @@ public Optional visitPhysicalTVFRelation( return Optional.empty(); } if (checkTVFRelationTableSupportedType(tvfRelation) && tvfRelation.getOutput().contains(context.slot) - && !tvfRelation.getOperativeSlots().contains(context.slot)) { - // lazy materialize slot must be a passive slot + && !tvfRelation.getOperativeSlots().contains(context.slot) + && !context.requiredMaterializedSlots.contains(context.slot)) { + // lazy materialize slot must be backed by a base column. if (context.slot.getOriginalColumn().isPresent()) { return Optional.of(new MaterializeSource( tvfRelation, findRelationOutputSlot(tvfRelation, context.slot).orElse(context.slot))); @@ -270,9 +282,22 @@ public Optional visitPhysicalProject( // projectExpr is alias Alias alias = (Alias) projectExpr; if (alias.child() instanceof SlotReference && !SessionVariable.getTopNLazyMaterializationUsingIndex()) { - ProbeContext childContext = new ProbeContext((SlotReference) alias.child()); - return project.child().accept(this, childContext); + SlotReference childSlot = (SlotReference) alias.child(); + ProbeContext childContext = new ProbeContext(childSlot, context.requiredMaterializedSlots); + Optional source = project.child().accept(this, childContext); + if (!source.isPresent() && !childSlot.getOriginalColumn().isPresent()) { + context.requiredMaterializedSlots.addAll(project.getInputSlots()); + } + return source; } else { + for (Slot inputSlot : projectExpr.getInputSlots()) { + context.requiredMaterializedSlots.add(inputSlot); + if (inputSlot instanceof SlotReference) { + ProbeContext childContext = new ProbeContext((SlotReference) inputSlot, + context.requiredMaterializedSlots); + project.child().accept(this, childContext); + } + } return Optional.empty(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 25f46eba8e563c..eb0b014a50ceb1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -187,6 +187,7 @@ public enum RuleType { PULL_UP_PROJECT_UNDER_APPLY(RuleTypeClass.REWRITE), PULL_UP_PROJECT_UNDER_LIMIT(RuleTypeClass.REWRITE), PULL_UP_PROJECT_UNDER_TOPN(RuleTypeClass.REWRITE), + PULL_UP_PROJECT_EXPR_UNDER_TOPN(RuleTypeClass.REWRITE), AGG_SCALAR_SUBQUERY_TO_WINDOW_FUNCTION(RuleTypeClass.REWRITE), UN_CORRELATED_APPLY_FILTER(RuleTypeClass.REWRITE), UN_CORRELATED_APPLY_PROJECT_FILTER(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index f1bfe8d89e33a0..51e1ab2af4e458 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -315,7 +315,7 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca childContext -> { Rewriter.getWholeTreeRewriter(childContext).execute(); return childContext.getRewritePlan(); - }, rewrittenPlan, queryPlan, false); + }, rewrittenPlan, queryPlan, false, true); if (rewrittenPlan == null) { continue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 1fe2bcc4274ab4..71d36282c3403e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -343,6 +343,16 @@ public static Plan generateMvScanPlan(OlapTable table, long indexId, public static Plan rewriteByRules( CascadesContext cascadesContext, Function planRewriter, Plan rewrittenPlan, Plan originPlan, boolean mvRewrite) { + return rewriteByRules(cascadesContext, planRewriter, rewrittenPlan, originPlan, mvRewrite, false); + } + + /** + * Optimize by rules, this support optimize by custom rules by define different rewriter according to different + * rules, this method is only for materialized view rewrite + */ + public static Plan rewriteByRules( + CascadesContext cascadesContext, Function planRewriter, + Plan rewrittenPlan, Plan originPlan, boolean mvRewrite, boolean materializedViewRewritePlanFragment) { if (originPlan == null || rewrittenPlan == null) { return null; } @@ -358,6 +368,7 @@ public static Plan rewriteByRules( CascadesContext rewrittenPlanContext = CascadesContext.initContext( cascadesContext.getStatementContext(), rewrittenPlan, cascadesContext.getCurrentJobContext().getRequiredProperties()); + rewrittenPlanContext.setMaterializedViewRewritePlanFragment(materializedViewRewritePlanFragment); // Tmp old disable rule variable Set oldDisableRuleNames = rewrittenPlanContext.getStatementContext().getConnectContext() .getSessionVariable() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java index d098c00e40ba67..c08d65e777f8e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java @@ -69,6 +69,7 @@ public class PreMaterializedViewRewriter { NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.PROCESS_SCALAR_AGG_MUST_USE_MULTI_DISTINCT.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM.ordinal()); NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.SALT_JOIN.ordinal()); + NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.PULL_UP_PROJECT_EXPR_UNDER_TOPN.ordinal()); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index a45863aa32c3c7..d4e0e3f50031d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -981,7 +981,7 @@ public static Pair addFilterOnTableScan(Plan queryPlan, return Pair.of(MaterializedViewUtils.rewriteByRules(parentCascadesContext, context -> { Rewriter.getWholeTreeRewriter(context).execute(); return context.getRewritePlan(); - }, queryPlanWithUnionFilter, queryPlan, false), true); + }, queryPlanWithUnionFilter, queryPlan, false, true), true); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java index ca06e93f453a9a..ca7cdb6aac8705 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java @@ -36,6 +36,7 @@ import org.apache.doris.nereids.rules.expression.rules.SimplifyRange; import org.apache.doris.nereids.rules.expression.rules.SimplifySelfComparison; import org.apache.doris.nereids.rules.expression.rules.SimplifyTimeFieldFromUnixtime; +import org.apache.doris.nereids.rules.expression.rules.StringEmptyToLengthRule; import com.google.common.collect.ImmutableList; @@ -69,7 +70,8 @@ public class ExpressionOptimization extends ExpressionRewrite { PushIntoCaseWhenBranch.INSTANCE, NullSafeEqualToEqual.INSTANCE, LikeToEqualRewrite.INSTANCE, - BetweenToEqual.INSTANCE + BetweenToEqual.INSTANCE, + StringEmptyToLengthRule.INSTANCE ) ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java index 7fcb6285875727..03586beee68844 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java @@ -60,6 +60,7 @@ public enum ExpressionRuleType { SIMPLIFY_EQUAL_BOOLEAN_LITERAL, SIMPLIFY_IN_PREDICATE, SIMPLIFY_NOT_EXPR, + STRING_EMPTY_TO_LENGTH, SIMPLIFY_RANGE, SIMPLIFY_SELF_COMPARISON, SUPPORT_JAVA_DATE_FORMATTER, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/StringEmptyToLengthRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/StringEmptyToLengthRule.java new file mode 100644 index 00000000000000..5321eed9300229 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/StringEmptyToLengthRule.java @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.expression.rules; + +import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher; +import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory; +import org.apache.doris.nereids.rules.expression.ExpressionRuleType; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Length; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * Rewrites comparisons with empty strings to equivalent length()-based expressions so that + * the NestedColumnPruning OFFSET optimization can apply. + * + *
    + *
  • {@code str_col = ''} → {@code length(str_col) = 0}
    • + *
  • {@code str_col <> ''} → {@code length(str_col) != 0} + * (represented as {@code NOT(length(str_col) = 0)})
    • + *
  • {@code element_at(struct_col, 'f3') = ''} → {@code length(element_at(struct_col, 'f3')) = 0}
    • + *
+ * + * This is a semantics-preserving rewrite: for any non-NULL string, {@code s = ''} is equivalent + * to {@code length(s) = 0}; for NULL, both sides evaluate to NULL. + * + * Applies when the compared expression is a non-literal expression of string-like type (e.g. a + * {@code SlotReference}, a {@code StructElement} field access, etc.), so that the resulting + * {@code length(expr)} call can benefit from OFFSET-only column reading. + */ +public class StringEmptyToLengthRule implements ExpressionPatternRuleFactory { + public static final StringEmptyToLengthRule INSTANCE = new StringEmptyToLengthRule(); + + @Override + public List> buildRules() { + return ImmutableList.of( + // str_col = '' → length(str_col) = 0 + matchesType(EqualTo.class) + .thenApply(ctx -> rewriteEqualToEmpty(ctx.expr)) + .toRule(ExpressionRuleType.STRING_EMPTY_TO_LENGTH), + + // NOT(str_col = '') → NOT(length(str_col) = 0) (i.e. str_col <> '') + matchesType(Not.class) + .thenApply(ctx -> { + Not not = ctx.expr; + if (!(not.child() instanceof EqualTo)) { + return not; + } + Expression rewritten = rewriteEqualToEmpty((EqualTo) not.child()); + if (rewritten == not.child()) { + return not; + } + return new Not(rewritten); + }) + .toRule(ExpressionRuleType.STRING_EMPTY_TO_LENGTH) + ); + } + + /** + * If {@code equalTo} compares a string-typed expression against an empty-string literal, + * rewrites it to {@code length(expr) = 0}. Returns the original expression unchanged otherwise. + */ + private static Expression rewriteEqualToEmpty(EqualTo equalTo) { + if (ConnectContext.get().getStatementContext().isDelete()) { + return equalTo; + } + Expression left = equalTo.left(); + Expression right = equalTo.right(); + + Expression stringExpr = null; + if (isStringExpression(left) && isEmptyStringLiteral(right)) { + stringExpr = left; + } else if (isStringExpression(right) && isEmptyStringLiteral(left)) { + stringExpr = right; + } + + if (stringExpr == null) { + return equalTo; + } + return new EqualTo(new Length(stringExpr), new IntegerLiteral(0)); + } + + private static boolean isStringExpression(Expression expr) { + return !(expr instanceof Literal) && expr.getDataType().isStringLikeType(); + } + + private static boolean isEmptyStringLiteral(Expression expr) { + return expr instanceof Literal + && expr.getDataType().isStringLikeType() + && ((Literal) expr).getStringValue().isEmpty(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java index 0a64dbbc157b94..74a9934e249d7e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java @@ -26,6 +26,8 @@ import org.apache.doris.nereids.trees.expressions.ArrayItemReference.ArrayItemSlot; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayCount; import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayExists; @@ -41,12 +43,16 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySort; import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySortBy; import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySplit; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Cardinality; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.If; import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Length; import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsEntry; import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsKey; import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsValue; import org.apache.doris.nereids.trees.expressions.functions.scalar.MapKeys; +import org.apache.doris.nereids.trees.expressions.functions.scalar.MapSize; import org.apache.doris.nereids.trees.expressions.functions.scalar.MapValues; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor; @@ -115,7 +121,14 @@ public Void visitSlotReference(SlotReference slotReference, CollectorContext con if (slotReference.hasSubColPath()) { path.addAll(slotReference.getSubPath()); } - path.addAll(context.accessPathBuilder.getPathList()); + // Strip NULL suffix for variant sub-column access — null-flag-only optimization + // does not apply to variant sub-column data layout. + List builderPath = context.accessPathBuilder.getPathList(); + if (builderPath.size() > 1 + && AccessPathInfo.ACCESS_NULL.equals(builderPath.get(builderPath.size() - 1))) { + builderPath = new ArrayList<>(builderPath.subList(0, builderPath.size() - 1)); + } + path.addAll(builderPath); int slotId = slotReference.getExprId().asInt(); slotToAccessPaths.put(slotId, new CollectAccessPathResult( path, context.bottomFilter, TAccessPathType.DATA)); @@ -136,9 +149,99 @@ public Void visitSlotReference(SlotReference slotReference, CollectorContext con int slotId = slotReference.getExprId().asInt(); slotToAccessPaths.put(slotId, new CollectAccessPathResult(path, context.bottomFilter, context.type)); } + if (dataType.isStringLikeType()) { + int slotId = slotReference.getExprId().asInt(); + if (!context.accessPathBuilder.isEmpty()) { + // Accessed via an offset-only function (e.g. length()) or null-check (IS NULL). + // Builder already has "OFFSET"/"NULL" at the tail; add the column name as prefix. + context.accessPathBuilder.addPrefix(slotReference.getName()); + ImmutableList path = ImmutableList.copyOf(context.accessPathBuilder.accessPath); + slotToAccessPaths.put(slotId, + new CollectAccessPathResult(path, context.bottomFilter, TAccessPathType.DATA)); + } else { + // Direct access to the string column → record a DATA path so that any + // concurrent offset-only path for the same slot is suppressed. + List path = ImmutableList.of(slotReference.getName()); + slotToAccessPaths.put(slotId, + new CollectAccessPathResult(path, context.bottomFilter, TAccessPathType.DATA)); + } + return null; + } + // For any other nullable column type (e.g. INT, BIGINT) accessed via IS NULL / IS NOT NULL: + // record the [col_name, NULL] path so NestedColumnPruning can emit null-only access paths. + // Skip NestedColumnPrunable types (already handled above) and string types (handled above). + if (!(dataType instanceof NestedColumnPrunable) && !dataType.isStringLikeType() + && !context.accessPathBuilder.isEmpty() && slotReference.nullable()) { + context.accessPathBuilder.addPrefix(slotReference.getName()); + ImmutableList path = ImmutableList.copyOf(context.accessPathBuilder.accessPath); + int slotId = slotReference.getExprId().asInt(); + slotToAccessPaths.put(slotId, + new CollectAccessPathResult(path, context.bottomFilter, TAccessPathType.DATA)); + } + // For any other nullable column type accessed directly (not via IS NULL / length / etc.): + // record a [col_name] full-access path so that when the column is also used via IS NULL, + // stripNullSuffixPaths correctly suppresses the null-only optimization. + if (!(dataType instanceof NestedColumnPrunable) && !dataType.isStringLikeType() + && !(dataType instanceof VariantType) + && context.accessPathBuilder.isEmpty() && slotReference.nullable()) { + int slotId = slotReference.getExprId().asInt(); + slotToAccessPaths.put(slotId, + new CollectAccessPathResult( + ImmutableList.of(slotReference.getName()), + context.bottomFilter, TAccessPathType.DATA)); + } return null; } + @Override + public Void visitLength(Length length, CollectorContext context) { + Expression arg = length.child(0); + // length() only needs the offset array, not the chars data. + // Add ACCESS_STRING_OFFSET as a suffix so the path builder accumulates + // e.g. ["str_col", "OFFSET"] or ["c_struct", "f3", "OFFSET"]. + if (arg.getDataType().isStringLikeType() && context.accessPathBuilder.isEmpty()) { + CollectorContext offsetContext = + new CollectorContext(context.statementContext, context.bottomFilter); + offsetContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_STRING_OFFSET); + return arg.accept(this, offsetContext); + } + // fall through to default (recurse into children with fresh contexts) + return visit(length, context); + } + + @Override + public Void visitMapSize(MapSize mapSize, CollectorContext context) { + Expression arg = mapSize.child(); + DataType argType = arg.getDataType(); + if (argType.isMapType() && context.accessPathBuilder.isEmpty()) { + CollectorContext offsetContext = + new CollectorContext(context.statementContext, context.bottomFilter); + offsetContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_STRING_OFFSET); + return arg.accept(this, offsetContext); + } + return visit(mapSize, context); + } + + @Override + public Void visitCardinality(Cardinality cardinality, CollectorContext context) { + Expression arg = cardinality.child(0); + // cardinality(arr) / cardinality(map) only needs the offset array, not element data. + // Arrays and maps share the same offset-array + data storage layout as strings on the BE. + DataType argType = arg.getDataType(); + if ((argType.isArrayType() || argType.isMapType()) && context.accessPathBuilder.isEmpty()) { + CollectorContext offsetContext = + new CollectorContext(context.statementContext, context.bottomFilter); + offsetContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_STRING_OFFSET); + // cardinality(map_keys(m)) == cardinality(m) == cardinality(map_values(m)): + // all three count map entries, so emit the same [map_col, OFFSET] path. + Expression effectiveArg = (arg instanceof MapKeys || arg instanceof MapValues) + ? arg.child(0) : arg; + return effectiveArg.accept(this, offsetContext); + } + // fall through to default + return visit(cardinality, context); + } + @Override public Void visitArrayItemSlot(ArrayItemSlot arrayItemSlot, CollectorContext context) { if (nameToLambdaArguments.isEmpty()) { @@ -230,7 +333,19 @@ public Void visitElementAt(ElementAt elementAt, CollectorContext context) { @Override public Void visitMapKeys(MapKeys mapKeys, CollectorContext context) { - context = new CollectorContext(context.statementContext, context.bottomFilter); + LinkedList suffixPath = context.accessPathBuilder.accessPath; + if (isFunctionNullCheckPath(suffixPath)) { + // map_keys(nullable_map) returns a NULL array only when the parent map is NULL. + // The NULL suffix therefore belongs to the map itself, not to the KEYS child. + return continueCollectAccessPath(mapKeys.getArgument(0), context); + } + if (!suffixPath.isEmpty() && suffixPath.get(0).equals(AccessPathInfo.ACCESS_ALL)) { + CollectorContext removeStarContext + = new CollectorContext(context.statementContext, context.bottomFilter); + removeStarContext.accessPathBuilder.accessPath.addAll(suffixPath.subList(1, suffixPath.size())); + removeStarContext.accessPathBuilder.addPrefix(AccessPathInfo.ACCESS_MAP_KEYS); + return continueCollectAccessPath(mapKeys.getArgument(0), removeStarContext); + } context.accessPathBuilder.addPrefix(AccessPathInfo.ACCESS_MAP_KEYS); return continueCollectAccessPath(mapKeys.getArgument(0), context); } @@ -238,6 +353,11 @@ public Void visitMapKeys(MapKeys mapKeys, CollectorContext context) { @Override public Void visitMapValues(MapValues mapValues, CollectorContext context) { LinkedList suffixPath = context.accessPathBuilder.accessPath; + if (isFunctionNullCheckPath(suffixPath)) { + // map_values(nullable_map) returns a NULL array only when the parent map is NULL. + // A map entry whose value is NULL still produces a non-NULL values array. + return continueCollectAccessPath(mapValues.getArgument(0), context); + } if (!suffixPath.isEmpty() && suffixPath.get(0).equals(AccessPathInfo.ACCESS_ALL)) { CollectorContext removeStarContext = new CollectorContext(context.statementContext, context.bottomFilter); @@ -249,6 +369,10 @@ public Void visitMapValues(MapValues mapValues, CollectorContext context) { return continueCollectAccessPath(mapValues.getArgument(0), context); } + private static boolean isFunctionNullCheckPath(List suffixPath) { + return suffixPath.size() == 1 && AccessPathInfo.ACCESS_NULL.equals(suffixPath.get(0)); + } + @Override public Void visitMapContainsKey(MapContainsKey mapContainsKey, CollectorContext context) { context.accessPathBuilder.addPrefix(AccessPathInfo.ACCESS_MAP_KEYS); @@ -421,14 +545,53 @@ public Void visitArraySortBy(ArraySortBy arraySortBy, CollectorContext context) return visit(arraySortBy, context); } - // @Override - // public Void visitIsNull(IsNull isNull, CollectorContext context) { - // if (context.accessPathBuilder.isEmpty()) { - // context.setType(TAccessPathType.META); - // return continueCollectAccessPath(isNull.child(), context); - // } - // return visit(isNull, context); - // } + @Override + public Void visitIsNull(IsNull isNull, CollectorContext context) { + Expression arg = isNull.child(); + // Skip variant sub-column paths (v['k'] IS NULL): the sub-column path is already baked + // into the SlotReference, so null-only access doesn't apply the same way. + if (arg instanceof SlotReference && ((SlotReference) arg).hasSubColPath()) { + return visit(isNull, context); + } + // Optimize IS NULL on nullable expressions: create a context with NULL suffix to indicate + // only the null flag is needed. Works for top-level columns (col IS NULL → [col, NULL]) + // and nested access (struct_element(s, 'city') IS NULL → [s, city, NULL]). + // For unrecognized expressions, the default visitor resets context, safely discarding NULL. + if (arg.nullable() && context.accessPathBuilder.isEmpty()) { + CollectorContext nullContext = + new CollectorContext(context.statementContext, context.bottomFilter); + nullContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_NULL); + return continueCollectAccessPath(arg, nullContext); + } + return visit(isNull, context); + } + + @Override + public Void visitIf(If ifExpr, CollectorContext context) { + if (isFunctionNullCheckPath(context.accessPathBuilder.accessPath)) { + ifExpr.getCondition().accept(this, new CollectorContext(context.statementContext, context.bottomFilter)); + ifExpr.getTrueValue().accept(this, copyContext(context)); + ifExpr.getFalseValue().accept(this, copyContext(context)); + return null; + } + return visit(ifExpr, context); + } + + private static CollectorContext copyContext(CollectorContext context) { + CollectorContext copy = new CollectorContext(context.statementContext, context.bottomFilter); + copy.accessPathBuilder.addSuffix(context.accessPathBuilder.getPathList()); + copy.type = context.type; + return copy; + } + + @Override + public Void visitNot(Not not, CollectorContext context) { + // NOT(IS NULL) == IS NOT NULL: same null-only access pattern + if (not.child() instanceof IsNull) { + return not.child().accept(this, context); + } + return visit(not, context); + } private Void collectArrayPathInLambda(Lambda lambda, CollectorContext context) { List arguments = lambda.getArguments(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java index 9f6b170a156654..f3e92216afc57f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java @@ -35,6 +35,7 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplodeOuter; import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; @@ -48,6 +49,7 @@ import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.nereids.types.NestedColumnPrunable; import org.apache.doris.nereids.types.VariantType; +import org.apache.doris.thrift.TAccessPathType; import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.Multimap; @@ -72,7 +74,10 @@ public Map> collect(Plan root, StatementCont } private boolean shouldCollectAccessPath(Slot slot) { - return slot.getDataType() instanceof NestedColumnPrunable || slot.getDataType().isVariantType(); + return slot.getDataType() instanceof NestedColumnPrunable + || slot.getDataType().isVariantType() + || slot.getDataType().isStringLikeType() + || slot.nullable(); } @Override @@ -280,6 +285,15 @@ public Void visitLogicalFilter(LogicalFilter filter, StatementCo return filter.child().accept(this, context); } + @Override + public Void visitLogicalAggregate(LogicalAggregate aggregate, StatementContext context) { + // Collect access paths from aggregate expressions (e.g. sum(length(str_col))) before + // visiting children so that when the bottom project is processed next, str_col's offset + // path is already recorded and the direct-DATA suppression guard can fire correctly. + collectByExpressions(aggregate, context); + return aggregate.child().accept(this, context); + } + @Override public Void visitLogicalCTEAnchor( LogicalCTEAnchor cteAnchor, StatementContext context) { @@ -352,7 +366,7 @@ public Void visitLogicalFileScan(LogicalFileScan fileScan, StatementContext cont } Collection accessPaths = allSlotToAccessPaths.get(slot.getExprId().asInt()); if (!accessPaths.isEmpty()) { - scanSlotToAccessPaths.put(slot, new ArrayList<>(accessPaths)); + scanSlotToAccessPaths.put(slot, normalizeDataSkippingOnlyAccessPaths(accessPaths)); } } return null; @@ -366,7 +380,7 @@ public Void visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, StatementCon } Collection accessPaths = allSlotToAccessPaths.get(slot.getExprId().asInt()); if (!accessPaths.isEmpty()) { - scanSlotToAccessPaths.put(slot, new ArrayList<>(accessPaths)); + scanSlotToAccessPaths.put(slot, normalizeDataSkippingOnlyAccessPaths(accessPaths)); } } return null; @@ -393,4 +407,33 @@ private void collectByExpressions(Plan plan, StatementContext context, boolean b exprCollector.collect(expression); } } + + static List normalizeDataSkippingOnlyAccessPaths( + Collection accessPaths) { + List normalizedAccessPaths = new ArrayList<>(); + for (CollectAccessPathResult accessPath : accessPaths) { + List path = accessPath.getPath(); + if (isDataSkippingOnlyAccessPath(path) && path.size() > 1) { + // NULL/OFFSET suffixes are OLAP segment-reader-only optimizations. External + // table and TVF readers use access paths as real nested field paths, so read + // the referenced column/sub-column normally instead of sending a pseudo field. + normalizedAccessPaths.add(new CollectAccessPathResult( + new ArrayList<>(path.subList(0, path.size() - 1)), + accessPath.isPredicate(), + TAccessPathType.DATA)); + } else { + normalizedAccessPaths.add(accessPath); + } + } + return normalizedAccessPaths; + } + + private static boolean isDataSkippingOnlyAccessPath(List path) { + if (path.isEmpty()) { + return false; + } + String lastComponent = path.get(path.size() - 1); + return AccessPathInfo.ACCESS_NULL.equals(lastComponent) + || AccessPathInfo.ACCESS_STRING_OFFSET.equals(lastComponent); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java index 2eac7d353ff77e..8056ebd747c214 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java @@ -24,13 +24,18 @@ import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectAccessPathResult; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Cardinality; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Length; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.MapType; +import org.apache.doris.nereids.types.NestedColumnPrunable; import org.apache.doris.nereids.types.NullType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; @@ -49,6 +54,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Collection; import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; @@ -80,13 +86,16 @@ public Plan rewriteRoot(Plan plan, JobContext jobContext) { StatementContext statementContext = jobContext.getCascadesContext().getStatementContext(); SessionVariable sessionVariable = statementContext.getConnectContext().getSessionVariable(); if (!sessionVariable.enablePruneNestedColumns - || (!statementContext.hasNestedColumns() && !containsVariant(plan))) { + || (!statementContext.hasNestedColumns() + && !containsVariant(plan) + && !containsStringLength(plan) + && !containsNullCheck(plan))) { return plan; } - AccessPathPlanCollector collector = new AccessPathPlanCollector(); Map> slotToAccessPaths = collector.collect(plan, statementContext); - Map slotToResult = pruneDataType(slotToAccessPaths); + Map slotToResult = pruneDataType(slotToAccessPaths, + jobContext.getCascadesContext().isMaterializedViewRewritePlanFragment()); if (!slotToResult.isEmpty()) { Map slotIdToPruneType = Maps.newLinkedHashMap(); @@ -104,6 +113,44 @@ public Plan rewriteRoot(Plan plan, JobContext jobContext) { } } + /** Returns true when the plan tree contains length() applied to a string-type expression. + * Used in the early-exit guard so that string offset optimizations are not skipped even + * when no nested (struct/array/map) or variant columns are present. */ + private static boolean containsStringLength(Plan plan) { + AtomicBoolean found = new AtomicBoolean(false); + plan.foreachUp(node -> { + if (found.get()) { + return; + } + Plan current = (Plan) node; + for (Expression expression : current.getExpressions()) { + if (expressionContainsStringLength(expression)) { + found.set(true); + return; + } + } + }); + return found.get(); + } + + private static boolean expressionContainsStringLength(Expression expr) { + if (expr instanceof Length && expr.child(0).getDataType().isStringLikeType()) { + return true; + } + if (expr instanceof Cardinality) { + DataType argType = expr.child(0).getDataType(); + if (argType.isArrayType() || argType.isMapType()) { + return true; + } + } + for (Expression child : expr.children()) { + if (expressionContainsStringLength(child)) { + return true; + } + } + return false; + } + private static boolean containsVariant(Plan plan) { AtomicBoolean hasVariant = new AtomicBoolean(false); plan.foreachUp(node -> { @@ -123,8 +170,43 @@ private static boolean containsVariant(Plan plan) { return hasVariant.get(); } + /** Returns true when the plan tree contains IS NULL or IS NOT NULL on a nullable slot. */ + private static boolean containsNullCheck(Plan plan) { + AtomicBoolean found = new AtomicBoolean(false); + plan.foreachUp(node -> { + if (found.get()) { + return; + } + Plan current = (Plan) node; + for (Expression expression : current.getExpressions()) { + if (expressionContainsNullCheck(expression)) { + found.set(true); + return; + } + } + }); + return found.get(); + } + + private static boolean expressionContainsNullCheck(Expression expr) { + if (expr instanceof IsNull && expr.child(0).nullable()) { + return true; + } + if (expr instanceof Not && expr.child(0) instanceof IsNull + && expr.child(0).child(0).nullable()) { + return true; + } + for (Expression child : expr.children()) { + if (expressionContainsNullCheck(child)) { + return true; + } + } + return false; + } + private static Map pruneDataType( - Map> slotToAccessPaths) { + Map> slotToAccessPaths, + boolean skipDataSkippingOnlyAccessPath) { Map result = new LinkedHashMap<>(); Map slotIdToAllAccessTree = new LinkedHashMap<>(); Map slotIdToPredicateAccessTree = new LinkedHashMap<>(); @@ -162,6 +244,19 @@ private static Map pruneDataType( } continue; } + if (skipDataSkippingOnlyAccessPath + && containsDataSkippingOnlyAccessPath(collectAccessPathResults)) { + // An MV rewrite child context optimizes a temporary plan fragment rather + // than the final plan. A nested metadata-only path such as + // [s, city, NULL] or [s, city, OFFSET] would otherwise prune the scan slot + // to only that nested field, while the final MV rewritten plan may still + // reuse the same slot as a full complex value or need another child. Drop + // access-info for the whole slot instead of just removing that path: + // predicate expressions inside this fragment still reference the original + // slot shape, so partial pruning after deleting the predicate-only path + // could make the fragment itself inconsistent. + continue; + } for (CollectAccessPathResult collectAccessPathResult : collectAccessPathResults) { List path = collectAccessPathResult.getPath(); TAccessPathType pathType = collectAccessPathResult.getType(); @@ -189,7 +284,101 @@ private static Map pruneDataType( DataTypeAccessTree accessTree = kv.getValue(); DataType prunedDataType = accessTree.pruneDataType().orElse(slot.getDataType()); + if (slot.getDataType().isStringLikeType()) { + if (accessTree.hasStringOffsetOnlyAccess()) { + if (skipDataSkippingOnlyAccessPath) { + continue; + } + // Offset-only access (e.g. length(str_col)): type stays varchar, + // but we must still send the access path to BE so it skips the char data. + stripExactCoveredDataSkippingSuffixPaths(slot, allAccessPaths, allAccessPaths); + stripNullSuffixPaths(slot, allAccessPaths); + List allPaths = buildColumnAccessPaths(slot, allAccessPaths); + result.put(slot.getExprId().asInt(), + new AccessPathInfo(slot.getDataType(), allPaths, new ArrayList<>())); + } else if (accessTree.hasNullCheckOnlyAccess()) { + if (skipDataSkippingOnlyAccessPath) { + continue; + } + // Null-check-only access (e.g. str_col IS NULL): type stays varchar, + // but we send [col, NULL] access path so BE only reads the null flag. + List allPaths = buildColumnAccessPaths(slot, allAccessPaths); + result.put(slot.getExprId().asInt(), + new AccessPathInfo(slot.getDataType(), allPaths, new ArrayList<>())); + } + // direct access (accessAll=true) or other: skip — no type change, no access paths needed. + continue; + } + + if ((slot.getDataType().isArrayType() || slot.getDataType().isMapType()) + && accessTree.hasStringOffsetOnlyAccess()) { + if (skipDataSkippingOnlyAccessPath) { + continue; + } + // Offset-only access (e.g. length(arr_col) / length(map_col)): type stays unchanged, + // but we must send the OFFSET access path to BE so it skips element/key-value data. + List allPaths = buildColumnAccessPaths(slot, allAccessPaths); + result.put(slot.getExprId().asInt(), + new AccessPathInfo(slot.getDataType(), allPaths, new ArrayList<>())); + continue; + } + + // Null-check-only access (e.g. col IS NULL / col IS NOT NULL): type stays unchanged, + // but we must send the [col, NULL] access path to BE so it only reads the null flag. + if (accessTree.hasNullCheckOnlyAccess()) { + if (skipDataSkippingOnlyAccessPath) { + continue; + } + List allPaths = buildColumnAccessPaths(slot, allAccessPaths); + result.put(slot.getExprId().asInt(), + new AccessPathInfo(slot.getDataType(), allPaths, new ArrayList<>())); + continue; + } + + if (slot.getDataType().isMapType() && accessTree.hasMapValueOffsetOnlyAccess()) { + if (skipDataSkippingOnlyAccessPath) { + continue; + } + // length(map_col['key']): keys read in full (element lookup) + values offset-only. + // Emit [col, KEYS] and [col, VALUES, OFFSET] directly instead of the collected + // [col, *, OFFSET] path which the BE cannot interpret for split key/value access. + String colName = slot.getName().toLowerCase(); + TDataAccessPath keysDataPath = new TDataAccessPath(); + keysDataPath.setPath( + new ArrayList<>(ImmutableList.of(colName, AccessPathInfo.ACCESS_MAP_KEYS))); + TColumnAccessPath keysColumnPath = new TColumnAccessPath(TAccessPathType.DATA); + keysColumnPath.setDataAccessPath(keysDataPath); + + TDataAccessPath valsOffsetDataPath = new TDataAccessPath(); + valsOffsetDataPath.setPath(new ArrayList<>(ImmutableList.of( + colName, AccessPathInfo.ACCESS_MAP_VALUES, AccessPathInfo.ACCESS_STRING_OFFSET))); + TColumnAccessPath valsOffsetColumnPath = new TColumnAccessPath(TAccessPathType.DATA); + valsOffsetColumnPath.setDataAccessPath(valsOffsetDataPath); + + result.put(slot.getExprId().asInt(), new AccessPathInfo( + slot.getDataType(), + ImmutableList.of(keysColumnPath, valsOffsetColumnPath), + new ArrayList<>())); + continue; + } + + // If a field is read in full, its metadata-only NULL/OFFSET access is redundant + // for any data type: e.g. [s] covers both [s.NULL] and [s.OFFSET]. + stripExactCoveredDataSkippingSuffixPaths(slot, allAccessPaths, allAccessPaths); + + // Strip OFFSET-suffix paths when a non-OFFSET path covers the same nested field or + // container. The overlapping array/map container may live under the root slot itself + // or under a nested struct field, so compare against the actual nested prefix instead + // of gating this logic on the root slot type. + stripCoveredOffsetSuffixPaths(slot, allAccessPaths, allAccessPaths); + + // Strip NULL-suffix paths when a non-NULL path also exists for the same slot. + // E.g. `SELECT col FROM t WHERE col IS NULL` — full data is needed, NULL path is redundant. + stripNullSuffixPaths(slot, allAccessPaths); List allPaths = buildColumnAccessPaths(slot, allAccessPaths); + if (shouldSkipAccessInfo(slot, prunedDataType, allPaths, predicateAccessPaths)) { + continue; + } result.put(slot.getExprId().asInt(), new AccessPathInfo(prunedDataType, allPaths, new ArrayList<>())); } @@ -204,11 +393,18 @@ private static Map pruneDataType( // third: build predicate access path for (Entry kv : slotIdToPredicateAccessTree.entrySet()) { Slot slot = kv.getKey(); - + stripExactCoveredDataSkippingSuffixPaths(slot, predicateAccessPaths, allAccessPaths); + stripCoveredOffsetSuffixPaths(slot, predicateAccessPaths, allAccessPaths); + stripCoveredArrayNullSuffixPaths(slot, predicateAccessPaths, allAccessPaths); + stripNullSuffixPaths(slot, predicateAccessPaths); List predicatePaths = buildColumnAccessPaths(slot, predicateAccessPaths); AccessPathInfo accessPathInfo = result.get(slot.getExprId().asInt()); - accessPathInfo.getPredicateAccessPaths().addAll(predicatePaths); + if (accessPathInfo != null) { + retainPredicatePathsInFinalAllAccessPaths( + predicatePaths, accessPathInfo.getAllAccessPaths()); + accessPathInfo.getPredicateAccessPaths().addAll(predicatePaths); + } } for (Entry kv : variantSlots.entrySet()) { @@ -216,12 +412,491 @@ private static Map pruneDataType( List predicatePaths = buildColumnAccessPaths(slot, predicateAccessPaths); AccessPathInfo accessPathInfo = result.get(slot.getExprId().asInt()); - accessPathInfo.getPredicateAccessPaths().addAll(predicatePaths); + if (accessPathInfo != null) { + retainPredicatePathsInFinalAllAccessPaths( + predicatePaths, accessPathInfo.getAllAccessPaths()); + accessPathInfo.getPredicateAccessPaths().addAll(predicatePaths); + } } return result; } + private static boolean containsDataSkippingOnlyAccessPath( + List collectAccessPathResults) { + for (CollectAccessPathResult collectAccessPathResult : collectAccessPathResults) { + if (isDataSkippingOnlyAccessPath(collectAccessPathResult.getPath())) { + return true; + } + } + return false; + } + + private static boolean isDataSkippingOnlyAccessPath(List path) { + if (path.isEmpty()) { + return false; + } + String lastComponent = path.get(path.size() - 1); + return AccessPathInfo.ACCESS_NULL.equals(lastComponent) + || AccessPathInfo.ACCESS_STRING_OFFSET.equals(lastComponent); + } + + /** + * Decide whether an OFFSET-suffix path can be removed because another non-OFFSET path + * already covers the same container. + * + *

For map element_at paths, {@code *} means "read keys fully, then follow the rest of + * the path on the value side". So a VALUES path can cover the value-side OFFSET access, + * but it does NOT cover the key lookup requirement. In that case we remove the OFFSET path + * and add a KEYS-only path instead. + */ + private static OffsetPathRewrite analyzeOffsetPathRewrite( + DataType slotType, List path, List> nonOffsetPaths) { + if (path.isEmpty() + || !AccessPathInfo.ACCESS_STRING_OFFSET.equals(path.get(path.size() - 1))) { + return OffsetPathRewrite.keep(); + } + List prefix = path.subList(0, path.size() - 1); + return analyzePrefixCoverage(slotType, prefix, nonOffsetPaths); + } + + private static OffsetPathRewrite analyzePrefixCoverage( + DataType slotType, List prefix, List> nonOffsetPaths) { + List> supplementalPaths = new ArrayList<>(); + for (List nonOffset : nonOffsetPaths) { + OffsetPathRewrite candidate = compareOffsetPrefixCoverage(slotType, prefix, nonOffset); + if (!candidate.shouldRemoveOffsetPath()) { + continue; + } + if (candidate.getSupplementalPaths().isEmpty()) { + return OffsetPathRewrite.remove(); + } + supplementalPaths.addAll(candidate.getSupplementalPaths()); + } + if (supplementalPaths.isEmpty()) { + return OffsetPathRewrite.keep(); + } + return OffsetPathRewrite.rewriteWithSupplementalPaths(supplementalPaths); + } + + /** + * Remove OFFSET-only paths from {@code targetAccessPaths} when data paths in + * {@code coveringAccessPaths} already read the same array/map/string container or a child + * under it. + * + *

Examples: + *

    + *
  • {@code [arr.OFFSET, arr.*.field]} becomes {@code [arr.*.field]} because the array + * child read must keep BE on the normal data iterator path.
    • + *
  • {@code [map.*.OFFSET, map.VALUES]} becomes {@code [map.KEYS, map.VALUES]} because + * {@code map['k']} still needs full keys for lookup, while values cover the offset.
    • + *
+ */ + private static void stripCoveredOffsetSuffixPaths( + Slot slot, Multimap>> targetAccessPaths, + Multimap>> coveringAccessPaths) { + int slotId = slot.getExprId().asInt(); + Collection>> targetPaths = targetAccessPaths.get(slotId); + if (targetPaths.isEmpty()) { + return; + } + + List> nonOffsetPaths = new ArrayList<>(); + for (Pair> p : coveringAccessPaths.get(slotId)) { + List path = p.second; + if (path.isEmpty() + || !AccessPathInfo.ACCESS_STRING_OFFSET.equals(path.get(path.size() - 1))) { + nonOffsetPaths.add(path); + } + } + for (Pair> p : targetPaths) { + List path = p.second; + if (path.isEmpty() + || !AccessPathInfo.ACCESS_STRING_OFFSET.equals(path.get(path.size() - 1))) { + nonOffsetPaths.add(path); + } + } + + List>> pathsToRemove = new ArrayList<>(); + List>> pathsToAdd = new ArrayList<>(); + for (Pair> p : new ArrayList<>(targetPaths)) { + OffsetPathRewrite rewrite = analyzeOffsetPathRewrite( + slot.getDataType(), p.second, nonOffsetPaths); + if (!rewrite.shouldRemoveOffsetPath()) { + continue; + } + pathsToRemove.add(p); + for (List supplementalPath : rewrite.getSupplementalPaths()) { + pathsToAdd.add(Pair.of(p.first, supplementalPath)); + } + } + targetPaths.removeAll(pathsToRemove); + targetPaths.addAll(pathsToAdd); + } + + /** + * Remove array NULL-only paths from {@code targetAccessPaths} when another path already reads + * the same array container or data under it. This mirrors OFFSET coverage because an array + * element/data read must not be combined with an array NULL_MAP_ONLY read for the same prefix. + * + *

Examples: + *

    + *
  • {@code [map.VALUES.NULL, map.VALUES.*.field]} becomes + * {@code [map.VALUES.*.field]}.
    • + *
  • {@code [map.*.NULL, map.VALUES.*.field]} becomes + * {@code [map.KEYS, map.VALUES.*.field]} so map lookup keys are still available.
    • + *
+ */ + private static void stripCoveredArrayNullSuffixPaths( + Slot slot, Multimap>> targetAccessPaths, + Multimap>> coveringAccessPaths) { + int slotId = slot.getExprId().asInt(); + Collection>> targetPaths = targetAccessPaths.get(slotId); + if (targetPaths.isEmpty()) { + return; + } + + List> nonNullPaths = new ArrayList<>(); + for (Pair> p : coveringAccessPaths.get(slotId)) { + List path = p.second; + if (path.isEmpty() || !AccessPathInfo.ACCESS_NULL.equals(path.get(path.size() - 1))) { + nonNullPaths.add(path); + } + } + for (Pair> p : targetPaths) { + List path = p.second; + if (path.isEmpty() || !AccessPathInfo.ACCESS_NULL.equals(path.get(path.size() - 1))) { + nonNullPaths.add(path); + } + } + + List>> pathsToRemove = new ArrayList<>(); + List>> pathsToAdd = new ArrayList<>(); + for (Pair> p : new ArrayList<>(targetPaths)) { + List path = p.second; + if (path.isEmpty() || !AccessPathInfo.ACCESS_NULL.equals(path.get(path.size() - 1))) { + continue; + } + List prefix = path.subList(0, path.size() - 1); + Optional prefixType = dataTypeAtPath(slot.getDataType(), prefix); + if (!prefixType.isPresent() || !prefixType.get().isArrayType()) { + continue; + } + OffsetPathRewrite rewrite = analyzePrefixCoverage(slot.getDataType(), prefix, nonNullPaths); + if (!rewrite.shouldRemoveOffsetPath()) { + continue; + } + pathsToRemove.add(p); + for (List supplementalPath : rewrite.getSupplementalPaths()) { + pathsToAdd.add(Pair.of(p.first, supplementalPath)); + } + } + targetPaths.removeAll(pathsToRemove); + targetPaths.addAll(pathsToAdd); + } + + /** + * Remove exact metadata-only NULL/OFFSET paths when the same field is read in full. + * This rule is type-agnostic: once {@code s} itself is accessed, {@code s.NULL} and + * {@code s.OFFSET} are redundant and unsafe to keep with the full data path. + * + *

Examples: + *

    + *
  • {@code [str_col, str_col.NULL]} becomes {@code [str_col]}.
    • + *
  • {@code [arr, arr.OFFSET]} becomes {@code [arr]}.
    • + *
  • {@code [map.*, map.*.OFFSET]} becomes {@code [map.*]}.
    • + *
+ */ + private static void stripExactCoveredDataSkippingSuffixPaths( + Slot slot, Multimap>> targetAccessPaths, + Multimap>> coveringAccessPaths) { + int slotId = slot.getExprId().asInt(); + Collection>> targetPaths = targetAccessPaths.get(slotId); + if (targetPaths.isEmpty()) { + return; + } + + List> fullAccessPaths = new ArrayList<>(); + for (Pair> p : coveringAccessPaths.get(slotId)) { + if (!isDataSkippingOnlyAccessPath(p.second)) { + fullAccessPaths.add(p.second); + } + } + for (Pair> p : targetPaths) { + if (!isDataSkippingOnlyAccessPath(p.second)) { + fullAccessPaths.add(p.second); + } + } + + List>> pathsToRemove = new ArrayList<>(); + for (Pair> p : targetPaths) { + List path = p.second; + if (!isDataSkippingOnlyAccessPath(path)) { + continue; + } + List prefix = path.subList(0, path.size() - 1); + for (List fullAccessPath : fullAccessPaths) { + if (pathCoversPrefix(fullAccessPath, prefix)) { + pathsToRemove.add(p); + break; + } + } + } + targetPaths.removeAll(pathsToRemove); + } + + private static Optional dataTypeAtPath(DataType slotType, List path) { + if (path.isEmpty()) { + return Optional.empty(); + } + DataType currentType = slotType; + for (int i = 1; i < path.size(); i++) { + String component = path.get(i); + if (currentType.isStructType()) { + StructField field = ((StructType) currentType).getField(component); + if (field == null) { + return Optional.empty(); + } + currentType = field.getDataType(); + } else if (currentType.isArrayType()) { + if (!AccessPathInfo.ACCESS_ALL.equals(component)) { + return Optional.empty(); + } + currentType = ((ArrayType) currentType).getItemType(); + } else if (currentType.isMapType()) { + currentType = descendMapType((MapType) currentType, component); + } else { + return Optional.empty(); + } + } + return Optional.of(currentType); + } + + private static OffsetPathRewrite compareOffsetPrefixCoverage( + DataType slotType, List prefix, List nonOffset) { + if (nonOffset.isEmpty()) { + return OffsetPathRewrite.remove(); + } + int minLen = Math.min(prefix.size(), nonOffset.size()); + List> supplementalPaths = new ArrayList<>(); + DataType currentType = slotType; + for (int i = 0; i < minLen; i++) { + String prefixComponent = prefix.get(i); + String nonOffsetComponent = nonOffset.get(i); + if (i == 0) { + if (!prefixComponent.equals(nonOffsetComponent)) { + return OffsetPathRewrite.keep(); + } + continue; + } + if (currentType.isStructType()) { + if (!prefixComponent.equals(nonOffsetComponent)) { + return OffsetPathRewrite.keep(); + } + StructField field = ((StructType) currentType).getField(prefixComponent); + if (field == null) { + return OffsetPathRewrite.keep(); + } + currentType = field.getDataType(); + continue; + } + if (currentType.isArrayType()) { + if (!prefixComponent.equals(nonOffsetComponent) + || !AccessPathInfo.ACCESS_ALL.equals(prefixComponent)) { + return OffsetPathRewrite.keep(); + } + currentType = ((ArrayType) currentType).getItemType(); + continue; + } + if (currentType.isMapType()) { + MapType mapType = (MapType) currentType; + if (prefixComponent.equals(nonOffsetComponent)) { + currentType = descendMapType(mapType, prefixComponent); + continue; + } + if (AccessPathInfo.ACCESS_ALL.equals(prefixComponent) + && AccessPathInfo.ACCESS_MAP_VALUES.equals(nonOffsetComponent)) { + supplementalPaths.add(buildMapKeysOnlyPath(prefix, i)); + currentType = mapType.getValueType(); + continue; + } + if (AccessPathInfo.ACCESS_MAP_VALUES.equals(prefixComponent) + && AccessPathInfo.ACCESS_ALL.equals(nonOffsetComponent)) { + currentType = mapType.getValueType(); + continue; + } + if (AccessPathInfo.ACCESS_MAP_KEYS.equals(prefixComponent) + && AccessPathInfo.ACCESS_ALL.equals(nonOffsetComponent)) { + currentType = mapType.getKeyType(); + continue; + } + return OffsetPathRewrite.keep(); + } + if (!prefixComponent.equals(nonOffsetComponent)) { + return OffsetPathRewrite.keep(); + } + } + if (supplementalPaths.isEmpty()) { + return OffsetPathRewrite.remove(); + } + return OffsetPathRewrite.rewriteWithSupplementalPaths(supplementalPaths); + } + + private static DataType descendMapType(MapType mapType, String component) { + if (AccessPathInfo.ACCESS_MAP_KEYS.equals(component)) { + return mapType.getKeyType(); + } + return mapType.getValueType(); + } + + private static List buildMapKeysOnlyPath(List prefix, int mapTokenIndex) { + List keyPath = new ArrayList<>(prefix.subList(0, mapTokenIndex)); + keyPath.add(AccessPathInfo.ACCESS_MAP_KEYS); + return keyPath; + } + + private static final class OffsetPathRewrite { + private static final OffsetPathRewrite KEEP = new OffsetPathRewrite(false, ImmutableList.of()); + private static final OffsetPathRewrite REMOVE = new OffsetPathRewrite(true, ImmutableList.of()); + + private final boolean removeOffsetPath; + private final List> supplementalPaths; + + private OffsetPathRewrite(boolean removeOffsetPath, List> supplementalPaths) { + this.removeOffsetPath = removeOffsetPath; + this.supplementalPaths = supplementalPaths; + } + + private static OffsetPathRewrite keep() { + return KEEP; + } + + private static OffsetPathRewrite remove() { + return REMOVE; + } + + private static OffsetPathRewrite rewriteWithSupplementalPaths(List> supplementalPaths) { + return new OffsetPathRewrite(true, ImmutableList.copyOf(supplementalPaths)); + } + + private boolean shouldRemoveOffsetPath() { + return removeOffsetPath; + } + + private List> getSupplementalPaths() { + return supplementalPaths; + } + } + + /** + * Strip NULL-suffix paths that are redundant because a non-NULL path reads child + * data below the same prefix or reads an OFFSET path over the same prefix. + * + *

Examples: + *

    + *
  • {@code [struct_col.NULL, struct_col.city]} becomes {@code [struct_col.city]}.
    • + *
  • {@code [str_col.NULL, str_col.OFFSET]} becomes {@code [str_col.OFFSET]} because + * the offset read can provide nullness for variable-length columns.
    • + *
+ * + *

A parent NULL path must also be removed when any child path is required under the + * same prefix, e.g. [struct_col, NULL] with [struct_col, city]. This looks like the + * parent null map may still be useful for predicates, but it cannot be kept in + * allAccessPaths with the current BE iterator contract: Struct/Array/Map iterators + * treat a leading NULL sub-path as NULL_MAP_ONLY and skip all children. If FE kept + * [struct_col.NULL, struct_col.city] in allAccessPaths, BE would read only the + * struct null map and default-fill city instead of routing the city child iterator. + * When the NULL path is removed from allAccessPaths, it must also be removed from + * predicateAccessPaths so the BE can rely on predicate paths being a subset of all + * paths. The normal nullable container read materializes the parent null map + * together with required children. + */ + private static void stripNullSuffixPaths( + Slot slot, Multimap>> allAccessPaths) { + int slotId = slot.getExprId().asInt(); + Collection>> slotPaths = allAccessPaths.get(slotId); + + List>> toRemove = new ArrayList<>(); + for (Pair> p : slotPaths) { + List path = p.second; + if (path.isEmpty() || !AccessPathInfo.ACCESS_NULL.equals(path.get(path.size() - 1))) { + continue; + } + // Prefix is the column/subcolumn path without the trailing NULL suffix. + // A non-NULL path that equals this prefix means the same column/subcolumn + // is read in full, making the NULL-only path redundant. + // An OFFSET-suffix path over the same prefix is also enough for the BE to + // derive null-ness for variable-length columns, so [col.NULL] is redundant + // when [col.OFFSET] already exists. + List prefix = path.subList(0, path.size() - 1); + boolean covered = false; + for (Pair> q : slotPaths) { + List other = q.second; + if (other.isEmpty() + || AccessPathInfo.ACCESS_NULL.equals(other.get(other.size() - 1))) { + continue; + } + if (other.equals(prefix)) { + covered = true; + break; + } + if (hasStrictPrefix(other, prefix)) { + covered = true; + break; + } + if (other.size() == prefix.size() + 1 + && AccessPathInfo.ACCESS_STRING_OFFSET.equals(other.get(other.size() - 1)) + && other.subList(0, prefix.size()).equals(prefix)) { + covered = true; + break; + } + } + if (covered) { + toRemove.add(p); + } + } + for (Pair> r : toRemove) { + allAccessPaths.remove(slotId, r); + } + } + + /** + * Keep predicate access paths as a subset of final all access paths after NULL/OFFSET cleanup. + * Predicate paths are built from filter expressions first, but later all-path rewrites may drop + * metadata-only paths or collapse paths to whole-column access. Any predicate path not present + * in final all paths must be removed before sending access info to BE. + * + *

Examples: + *

    + *
  • All paths {@code [s]}, predicate paths {@code [s.city.NULL]} becomes no predicate + * paths after parent NULL removal.
    • + *
  • All paths {@code [s.city.NULL, s.zip]}, predicate paths + * {@code [s.NULL, s.city.NULL]} becomes {@code [s.city.NULL]}.
    • + *
+ */ + private static void retainPredicatePathsInFinalAllAccessPaths( + List predicatePaths, List allPaths) { + if (predicatePaths.isEmpty()) { + return; + } + + List toRemove = new ArrayList<>(); + for (TColumnAccessPath predicatePath : predicatePaths) { + if (!allPaths.contains(predicatePath)) { + toRemove.add(predicatePath); + } + } + predicatePaths.removeAll(toRemove); + } + + private static boolean hasStrictPrefix(List path, List prefix) { + return path.size() > prefix.size() && path.subList(0, prefix.size()).equals(prefix); + } + + private static boolean pathCoversPrefix(List path, List prefix) { + return prefix.size() >= path.size() && prefix.subList(0, path.size()).equals(path); + } + private static List buildColumnAccessPaths( Slot slot, Multimap>> accessPaths) { List paths = new ArrayList<>(); @@ -263,7 +938,7 @@ private static List buildColumnAccessPaths( } else { accessPath.setMetaAccessPath(new TMetaAccessPath(ImmutableList.of(wholeColumnName))); } - return ImmutableList.of(accessPath); + return new ArrayList<>(ImmutableList.of(accessPath)); } return paths; } @@ -279,6 +954,32 @@ private static int comparePathSegments(List left, List right) { return Integer.compare(left.size(), right.size()); } + private static boolean shouldSkipAccessInfo( + Slot slot, DataType prunedDataType, List allPaths, + Multimap>> predicateAccessPaths) { + if (!prunedDataType.equals(slot.getDataType())) { + return false; + } + if (slot.getDataType() instanceof NestedColumnPrunable || slot.getDataType().isVariantType()) { + return false; + } + if (!predicateAccessPaths.get(slot.getExprId().asInt()).isEmpty()) { + return false; + } + if (allPaths.size() != 1) { + return false; + } + return getAccessPathList(allPaths.get(0)).size() == 1; + } + + /** The path components of a column access path, whichever of DATA/META it carries. */ + private static List getAccessPathList(TColumnAccessPath accessPath) { + if (accessPath.getType() == TAccessPathType.DATA) { + return accessPath.getDataAccessPath().getPath(); + } + return accessPath.getMetaAccessPath().getPath(); + } + /** DataTypeAccessTree */ public static class DataTypeAccessTree { // type of this level @@ -288,6 +989,15 @@ public static class DataTypeAccessTree { // if access 's.a.b' the node 's' and 'a' has accessPartialChild, and node 'b' has accessAll private boolean accessPartialChild; private boolean accessAll; + // True when this string-typed node is accessed ONLY via the offset array + // (e.g. length(str_col) or length(element_at(c_struct,'f3'))). + // When this flag is set and accessAll is NOT set, pruneDataType() returns BigIntType + // to signal that the BE only needs to read the offset array, not the chars data. + private boolean isStringOffsetOnly; + // True when this column node is accessed ONLY via IS NULL / IS NOT NULL. + // When this flag is set and accessAll is NOT set, the BE only needs to read the null flag, + // not the actual column data. + private boolean isNullCheckOnly; // for the future, only access the meta of the column, // e.g. `is not null` can only access the column's offset, not need to read the data private TAccessPathType pathType; @@ -329,6 +1039,83 @@ public Map getChildren() { return children; } + /** + * True when a MAP column is accessed as {@code length(map_col['key'])}: the keys must + * be read in full (for the element lookup) while the values only need the offset array + * (since only their length, not their content, is used). + * Expected access paths: [col, KEYS] and [col, VALUES, OFFSET]. + */ + public boolean hasMapValueOffsetOnlyAccess() { + if (!isRoot) { + return false; + } + DataTypeAccessTree child = children.values().iterator().next(); + if (!child.type.isMapType() || child.accessAll) { + return false; + } + DataTypeAccessTree keysChild = child.children.get(AccessPathInfo.ACCESS_MAP_KEYS); + DataTypeAccessTree valsChild = child.children.get(AccessPathInfo.ACCESS_MAP_VALUES); + // Keys must be fully accessed (element-at lookup). + if (!keysChild.accessAll) { + return false; + } + // Values must be accessed offset-only (no deeper element reads). + if (!valsChild.isStringOffsetOnly || valsChild.accessAll) { + return false; + } + if (valsChild.type.isStringLikeType()) { + // String value: accessAll check above is sufficient. + return true; + } + if (valsChild.type.isArrayType()) { + // Array value (e.g. MAP>): verify no element was read directly + // (e.g. map_col['k'][0] would set allChild.accessAll=true). + DataTypeAccessTree allChild = valsChild.children.get(AccessPathInfo.ACCESS_ALL); + return !allChild.accessAll && !allChild.accessPartialChild; + } + return true; + } + + /** True when the column is accessed ONLY via the offset array (e.g. length(str_col), + * length(arr_col), length(map_col)), meaning the type must not change but an access + * path still needs to be sent to BE so it can skip the char/element data. */ + public boolean hasStringOffsetOnlyAccess() { + if (isRoot) { + DataTypeAccessTree child = children.values().iterator().next(); + if (!child.isStringOffsetOnly || child.accessAll) { + return false; + } + if (child.type.isStringLikeType()) { + return true; + } + if (child.type.isArrayType()) { + // True only if no element was accessed (element_at / explode etc.) + DataTypeAccessTree allChild = child.children.get(AccessPathInfo.ACCESS_ALL); + return !allChild.accessAll && !allChild.accessPartialChild; + } + if (child.type.isMapType()) { + // True only if neither keys nor values were accessed directly + DataTypeAccessTree keysChild = child.children.get(AccessPathInfo.ACCESS_MAP_KEYS); + DataTypeAccessTree valsChild = child.children.get(AccessPathInfo.ACCESS_MAP_VALUES); + return !keysChild.accessAll && !keysChild.accessPartialChild + && !valsChild.accessAll && !valsChild.accessPartialChild; + } + return false; + } + return type.isStringLikeType() && isStringOffsetOnly && !accessAll; + } + + /** True when the column is accessed ONLY via IS NULL / IS NOT NULL, + * meaning the BE only needs to read the null flag, not the actual data. */ + public boolean hasNullCheckOnlyAccess() { + if (isRoot) { + DataTypeAccessTree child = children.values().iterator().next(); + return child.isNullCheckOnly && !child.accessAll + && !child.isStringOffsetOnly && !child.accessPartialChild; + } + return isNullCheckOnly && !accessAll && !isStringOffsetOnly && !accessPartialChild; + } + /** pruneCastType */ public DataType pruneCastType(DataTypeAccessTree origin, DataTypeAccessTree cast) { if (type instanceof StructType) { @@ -414,14 +1201,22 @@ public void setAccessByPath(List path, int accessIndex, TAccessPathType if (accessIndex >= path.size()) { accessAll = true; return; - } else { - accessPartialChild = true; } if (pathType == TAccessPathType.DATA) { this.pathType = TAccessPathType.DATA; } + // NULL path component: the column is accessed only via IS NULL / IS NOT NULL. + // Mark null-check-only and return without setting accessAll or accessPartialChild, + // so that parent nodes can distinguish "null-only leaf" from "has real sub-access". + if (path.get(accessIndex).equals(AccessPathInfo.ACCESS_NULL)) { + isNullCheckOnly = true; + return; + } + + accessPartialChild = true; + if (this.type.isStructType()) { String fieldName = path.get(accessIndex).toLowerCase(); DataTypeAccessTree child = children.get(fieldName); @@ -433,6 +1228,11 @@ public void setAccessByPath(List path, int accessIndex, TAccessPathType } return; } else if (this.type.isArrayType()) { + if (path.get(accessIndex).equals(AccessPathInfo.ACCESS_STRING_OFFSET)) { + // length(array_col) — only the offset array is needed, not element data. + isStringOffsetOnly = true; + return; + } DataTypeAccessTree child = children.get(AccessPathInfo.ACCESS_ALL); if (path.get(accessIndex).equals(AccessPathInfo.ACCESS_ALL)) { // enter this array and skip next * @@ -441,6 +1241,11 @@ public void setAccessByPath(List path, int accessIndex, TAccessPathType return; } else if (this.type.isMapType()) { String fieldName = path.get(accessIndex); + if (fieldName.equals(AccessPathInfo.ACCESS_STRING_OFFSET)) { + // length(map_col) — only the offset array is needed, not key/value data. + isStringOffsetOnly = true; + return; + } if (fieldName.equals(AccessPathInfo.ACCESS_ALL)) { // access value by the key, so we should access key and access value, then prune the value's type. // e.g. map_column['id'] should access the keys, and access the values @@ -450,10 +1255,12 @@ public void setAccessByPath(List path, int accessIndex, TAccessPathType valuesChild.setAccessByPath(path, accessIndex + 1, pathType); return; } else if (fieldName.equals(AccessPathInfo.ACCESS_MAP_KEYS)) { - // only access the keys and not need enter keys, because it must be primitive type. - // e.g. map_keys(map_column) + // Access the keys sub-column. Delegate to child so that trailing path + // components (e.g. NULL for IS NULL) are processed correctly. + // When no trailing component exists, setAccessByPath reaches end-of-path + // and sets accessAll = true, preserving the original behavior. DataTypeAccessTree keysChild = children.get(AccessPathInfo.ACCESS_MAP_KEYS); - keysChild.accessAll = true; + keysChild.setAccessByPath(path, accessIndex + 1, pathType); return; } else if (fieldName.equals(AccessPathInfo.ACCESS_MAP_VALUES)) { // only access the values without keys, and maybe prune the value's data type. @@ -469,6 +1276,16 @@ public void setAccessByPath(List path, int accessIndex, TAccessPathType // without discarding the path before BE can project a shredded leaf. accessAll = true; return; + } else if (type.isStringLikeType()) { + // String leaf accessed via the offset array (e.g. path ends in "offset"). + // Mark offset-only so pruneDataType() can return BigIntType instead of full data. + if (path.get(accessIndex).equals(AccessPathInfo.ACCESS_STRING_OFFSET)) { + isStringOffsetOnly = true; + return; // do NOT set accessAll — offset-only is distinguishable from full access + } + // Any other sub-path on a string column means full data is needed. + accessAll = true; + return; } else if (isRoot) { children.get(path.get(accessIndex).toLowerCase()).setAccessByPath(path, accessIndex + 1, pathType); return; @@ -506,6 +1323,15 @@ public Optional pruneDataType() { return children.values().iterator().next().pruneDataType(); } else if (accessAll) { return Optional.of(type); + } else if (isStringOffsetOnly && !accessPartialChild) { + // Only the offset array is accessed (e.g. length(str_col)). + // The slot type stays unchanged (varchar); the access path tells BE to skip char data. + return Optional.empty(); + } else if (isNullCheckOnly && !accessPartialChild) { + // Only the null flag is accessed (e.g. col IS NULL / struct_element(s,'f') IS NULL). + // Return the node's type so that parent nodes include this child in their pruned type, + // while the access path (ending in NULL) tells BE to skip actual data reading. + return Optional.of(type); } else if (!accessPartialChild) { return Optional.empty(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OperativeColumnDerive.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OperativeColumnDerive.java index da2127af8d6e49..5345a5d302eb0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OperativeColumnDerive.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OperativeColumnDerive.java @@ -17,8 +17,6 @@ package org.apache.doris.nereids.rules.rewrite; -import org.apache.doris.catalog.KeysType; -import org.apache.doris.catalog.OlapTable; import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.rules.rewrite.OperativeColumnDerive.DeriveContext; import org.apache.doris.nereids.trees.expressions.Expression; @@ -32,6 +30,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalSink; import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink; import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; @@ -66,6 +65,11 @@ public Plan visitLogicalSink(LogicalSink sink, DeriveContext con return visitChildren(this, sink, context); } + @Override + public Plan visitPhysicalResultSink(PhysicalResultSink sink, DeriveContext context) { + return visitChildren(this, sink, context); + } + private Plan deriveUnion(LogicalUnion union, DeriveContext context) { for (int i = 0; i < union.getOutput().size(); i++) { Slot output = union.getOutput().get(i); @@ -140,18 +144,6 @@ public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, DeriveContext context } } - OlapTable table = olapScan.getTable(); - if (KeysType.UNIQUE_KEYS.equals(table.getKeysType()) - && !table.getTableProperty().getEnableUniqueKeyMergeOnWrite() - || KeysType.AGG_KEYS.equals(table.getKeysType()) - || KeysType.PRIMARY_KEYS.equals(table.getKeysType())) { - for (Slot slot : olapScan.getOutput()) { - SlotReference slotReference = (SlotReference) slot; - if (slotReference.getOriginalColumn().isPresent() && slotReference.getOriginalColumn().get().isKey()) { - intersectSlots.add(slotReference); - } - } - } for (NamedExpression virtualColumn : olapScan.getVirtualColumns()) { intersectSlots.add(virtualColumn.toSlot()); intersectSlots.addAll(virtualColumn.getInputSlots()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java index 611d7fc78c8763..b0cfde50962979 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java @@ -97,7 +97,7 @@ public List buildRules() { return rewrittenLogicalRelation; } boolean skipPrunePredicate = ctx.connectContext.getSessionVariable().skipPrunePredicate - || ctx.statementContext.isSkipPrunePredicate(); + || ctx.statementContext.isDelete(); if (!skipPrunePredicate && prunedRes.second.isPresent()) { // Defer the predicate removal to PlanPostProcessor so that materialized-view // rewrite still sees the original predicates. Otherwise, partition predicates diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopN.java new file mode 100644 index 00000000000000..b225bb891e3eae --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopN.java @@ -0,0 +1,541 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction; +import org.apache.doris.nereids.trees.expressions.functions.scalar.L2DistanceApproximate; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Score; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; + +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Pull up non-trivial expressions from Projects below TopN to above TopN, + * exposing their input base columns as lazy materialization candidates. + * + *

The rewriter runs bottom-up. Each LogicalTopN is treated as the current + * target TopN after its child has already been rewritten. The target TopN then + * collects only Projects in its own child subtree, stops at nested TopNs, and + * adds one upper Project to restore the original TopN output. This lets an + * upper TopN pull expressions that were just restored above a lower TopN. + */ +public class PullUpProjectExprUnderTopN implements CustomRewriter { + + @Override + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + ConnectContext ctx = jobContext.getCascadesContext() + .getStatementContext().getConnectContext(); + if (ctx != null && !ctx.getSessionVariable().enableTopnExprPullup) { + return plan; + } + + return plan.accept(new Rewriter(), new RewriteContext()); + } + + // ========================================================================= + // Data structures + // ========================================================================= + + /** Info collected per TopN about which expressions to pull up from which Projects. */ + static class PullUpInfo { + final LogicalTopN topN; + final List originalTopNOutput; + final List allPulledUpExprs = new ArrayList<>(); + final Map, List> projectToPulledUpExprs + = new LinkedHashMap<>(); + final Map pullUpExprReplaceMap = new LinkedHashMap<>(); + + PullUpInfo(LogicalTopN topN) { + this.topN = topN; + this.originalTopNOutput = ImmutableList.copyOf(topN.getOutput()); + } + + void addPulledUpExpr(LogicalProject project, NamedExpression expr) { + allPulledUpExprs.add(expr); + projectToPulledUpExprs.computeIfAbsent(project, k -> new ArrayList<>()).add(expr); + addPullUpExprReplace(expr); + } + + void addPullUpExprReplace(NamedExpression expr) { + if (expr instanceof Alias) { + pullUpExprReplaceMap.putIfAbsent(expr.toSlot(), expr.child(0)); + } + } + } + + /** Context for the bottom-up traversal. */ + static class RewriteContext { + int cteProducerDepth = 0; + } + + // ========================================================================= + // Bottom-up TopN rewriter + // ========================================================================= + + private static boolean qualifiesForLazyMatThreshold(LogicalTopN topN) { + long limit = topN.getLimit(); + if (limit <= 0) { + return false; + } + long threshold = SessionVariable.getTopNLazyMaterializationThreshold(); + return threshold >= limit; + } + + static class Rewriter extends DefaultPlanRewriter { + + @Override + public Plan visitLogicalCTEProducer( + LogicalCTEProducer cteProducer, RewriteContext context) { + context.cteProducerDepth++; + try { + return visit(cteProducer, context); + } finally { + context.cteProducerDepth--; + } + } + + @Override + public Plan visitLogicalTopN(LogicalTopN topN, RewriteContext context) { + LogicalTopN rewritten = (LogicalTopN) visit(topN, context); + if (context.cteProducerDepth > 0 || !qualifiesForLazyMatThreshold(rewritten)) { + return rewritten; + } + PullUpInfo info = new PullUpInfo(rewritten); + // Seed blockedExprIds with this TopN's order key ExprIds so that + // expressions used by order keys are not pulled up past this TopN. + collectFromNode((Plan) rewritten.child(0), info, buildOrderKeyExprIds(rewritten)); + if (info.allPulledUpExprs.isEmpty()) { + return rewritten; + } + + Plan simplifiedChild = ((Plan) rewritten.child(0)).accept(new ProjectSimplifier(), info); + if (simplifiedChild == rewritten.child(0)) { + return rewritten; + } + LogicalTopN topNWithSimplifiedChild = rewritten.withChildren(ImmutableList.of(simplifiedChild)); + return addUpperProject(topNWithSimplifiedChild, info); + } + } + + /** + * Recursively walk down from a TopN's child to find Projects with pull-able expressions. + * + *

{@code blockedExprIds} contains ExprIds of slots that are referenced by operators + * along the path from the TopN to the current node. An expression whose output ExprId + * is in this set cannot be pulled up past the operators that reference it. + */ + private static void collectFromNode(Plan node, PullUpInfo info, Set blockedExprIds) { + if (node instanceof LogicalProject) { + LogicalProject project = (LogicalProject) node; + Set childBlockedExprIds = new HashSet<>(blockedExprIds); + for (NamedExpression ne : project.getProjects()) { + info.addPullUpExprReplace(ne); + boolean canPullUp = canPullUp(ne); + if (canPullUp && !blockedExprIds.contains(ne.getExprId())) { + info.addPulledUpExpr(project, ne); + } + if (shouldBlockProjectInputs(ne, canPullUp, blockedExprIds)) { + childBlockedExprIds.addAll(ne.getInputSlotExprIds()); + } + } + // Continue into the project's child. Chained projects are all visited. + collectFromNode((Plan) project.child(0), info, childBlockedExprIds); + return; + } + + if (node instanceof LogicalTopN) { + // The bottom-up rewriter has already handled this nested TopN. + // The current target TopN only collects Projects above it. + return; + } + + // Stop at boundary nodes that transform the schema or are data sources. + if (node instanceof LogicalRelation || node instanceof LogicalCTEProducer + || isBlockingNode(node)) { + return; + } + + // Set operations are a boundary for the current TopN: do NOT collect + // expressions from below them. UNION ALL children may compute the same + // output column with different expressions (e.g. a+1 vs a+2), and a + // single pull-up Project above the TopN cannot represent branch-specific + // semantics. The normal visitor will still traverse into the children, + // so nested TopNs inside set operations are handled independently. + if (node instanceof LogicalSetOperation) { + return; + } + + // For null-generating outer joins, block all output slots from the + // nullable side(s). Expressions inside a nullable side are protected + // by join null-extension: when there is no match, the entire nullable + // tuple is set to NULL. Pulling such an expression above the join + // would break this, e.g. ifnull(r.b, 0) inside the right side of a + // LEFT JOIN would see individual column NULLs and convert them to 0, + // changing the NULL that null-extension produced. + // Example: SELECT l.id, sub.x FROM l LEFT JOIN ( + // SELECT id, ifnull(b, 0) AS x FROM r) sub ON l.id = sub.id + // ORDER BY l.id LIMIT 3; + // Here x=ifnull(b,0) is in a Project on the nullable (right) side. + // Pulling it above the join turns unmatched-row x from NULL to 0. + if (node instanceof LogicalJoin) { + LogicalJoin join = (LogicalJoin) node; + JoinType joinType = join.getJoinType(); + Set newBlocked = new HashSet<>(blockedExprIds); + // add join expression slots (same as default branch) + for (Expression expr : node.getExpressions()) { + newBlocked.addAll(expr.getInputSlotExprIds()); + if (expr instanceof NamedExpression) { + newBlocked.add(((NamedExpression) expr).getExprId()); + } + } + // block all output slots from the nullable side(s) + if (joinType.isLeftOuterJoin() || joinType.isAsofLeftOuterJoin() + || joinType.isFullOuterJoin()) { + for (Slot s : join.right().getOutput()) { + newBlocked.add(s.getExprId()); + } + } + if (joinType.isRightOuterJoin() || joinType.isAsofRightOuterJoin() + || joinType.isFullOuterJoin()) { + for (Slot s : join.left().getOutput()) { + newBlocked.add(s.getExprId()); + } + } + for (Plan child : node.children()) { + collectFromNode(child, info, newBlocked); + } + return; + } + + // For all other nodes, add their input slot ExprIds to the blocked set. + // Any operator that references a slot in its expressions prevents + // expressions that output that slot from being pulled up past it. + Set newBlocked = new HashSet<>(blockedExprIds); + for (Expression expr : node.getExpressions()) { + newBlocked.addAll(expr.getInputSlotExprIds()); + if (expr instanceof NamedExpression) { + newBlocked.add(((NamedExpression) expr).getExprId()); + } + } + + for (Plan child : node.children()) { + collectFromNode(child, info, newBlocked); + } + } + + // ========================================================================= + // Pull-up eligibility + // ========================================================================= + + /** + * Check if a named expression can be pulled up above TopN. + * Eligible: Alias with non-trivial child, not blocked, no NoneMovableFunction. + */ + static boolean canPullUp(NamedExpression ne) { + if (!(ne instanceof Alias)) { + return false; + } + Expression child = ((Alias) ne).child(); + if (child instanceof Slot || child instanceof Literal) { + return false; + } + if (ne.anyMatch(e -> e instanceof NoneMovableFunction)) { + return false; + } + if (ne.containsVolatileExpression()) { + return false; + } + if (ne.anyMatch(e -> e instanceof Score)) { + return false; + } + if (ne.anyMatch(e -> e instanceof L2DistanceApproximate)) { + return false; + } + return true; + } + + private static boolean shouldBlockProjectInputs( + NamedExpression ne, boolean canPullUp, Set blockedExprIds) { + if (blockedExprIds.contains(ne.getExprId())) { + return true; + } + if (!(ne instanceof Alias)) { + return false; + } + Expression child = ne.child(0); + if (child instanceof Slot || child instanceof Literal) { + return false; + } + // Non-forwarding aliases that cannot be synthesized above TopN must + // keep their inputs below TopN. Otherwise, a lower pull-up can remove + // the input slot and make this alias look unavailable, causing it to + // be reconstructed above TopN through pullUpExprReplaceMap and bypass + // canPullUp(), e.g. z = assert_true(x > 0), x = a + 1. + return !canPullUp; + } + + private static boolean isBlockingNode(Plan node) { + return node instanceof LogicalAggregate + || node instanceof LogicalWindow + || node instanceof LogicalRepeat; + } + + private static Set buildOrderKeyExprIds(LogicalTopN topN) { + Set orderKeyExprIds = new HashSet<>(); + for (OrderKey orderKey : topN.getOrderKeys()) { + Expression keyExpr = orderKey.getExpr(); + orderKeyExprIds.addAll(keyExpr.getInputSlotExprIds()); + if (keyExpr instanceof NamedExpression) { + orderKeyExprIds.add(((NamedExpression) keyExpr).getExprId()); + } + } + return orderKeyExprIds; + } + + static class ProjectSimplifier extends DefaultPlanRewriter { + @Override + public Plan visitLogicalTopN(LogicalTopN topN, PullUpInfo info) { + return topN; + } + + @Override + public Plan visitLogicalProject(LogicalProject project, PullUpInfo info) { + LogicalProject rewritten = (LogicalProject) visit(project, info); + List exprs = info.projectToPulledUpExprs.get(rewritten); + if (exprs != null) { + return simplifyProject(rewritten, exprs, info); + } + if (rewritten != project && rewritten.getProjects().equals(project.getProjects())) { + exprs = info.projectToPulledUpExprs.get(project); + if (exprs != null) { + return simplifyProject(rewritten, exprs, info); + } + } + return simplifyProject(rewritten, ImmutableList.of(), info); + } + } + + /** + * Remove pulled-up expressions from this Project and expose their base input slots. + * + *

For example, pulling up {@code x = a + 1} from cascaded Projects: + * + * 

+     * Before:
+     *   TopN
+     *     Project(id, x)                -- forwards x from child
+     *       Project(id, a + 1 AS x)
+     *         Scan(id, a)
+     *
+     * After simplifyProject (both Projects lose x, gain a):
+     *   TopN
+     *     Project(id, a)                -- x removed because child no longer outputs it
+     *       Project(id, a)              -- a+1 removed, base slot a exposed
+     *         Scan(id, a)
+     *
+     * {@code addUpperProject} then restores the computation above TopN:
+     *   Project(id, a + 1 AS x)         -- new upper Project
+     *     TopN
+     *       Project(id, a)
+     *         Project(id, a)
+     *           Scan(id, a)
+     *
+ */ + private static LogicalProject simplifyProject( + LogicalProject project, + List pulledUpExprs, + PullUpInfo info) { + Set childOutputExprIds = ((Plan) project.child(0)).getOutputExprIdSet(); + List passThroughExprs = collectUnavailablePullUpExprs(project, info, childOutputExprIds); + if (pulledUpExprs.isEmpty() && passThroughExprs.isEmpty()) { + return project; + } + + Set pulledUpExprIds = new HashSet<>(); + for (NamedExpression ne : pulledUpExprs) { + pulledUpExprIds.add(ne.getExprId()); + } + + List simplified = new ArrayList<>(); + Set existingExprIds = new HashSet<>(); + for (NamedExpression ne : project.getProjects()) { + if (!pulledUpExprIds.contains(ne.getExprId())) { + Expression replaceExpr = getPullUpReplaceExpression(ne.toSlot(), info); + if (replaceExpr == null || !isUnavailableExpression(ne, childOutputExprIds)) { + NamedExpression resolved = resolveAliasChildIfNeeded(ne, info, childOutputExprIds); + simplified.add(resolved); + existingExprIds.add(resolved.getExprId()); + } + } + } + + for (NamedExpression pulledUpExpr : pulledUpExprs) { + for (Slot baseSlot : resolveInputSlots(pulledUpExpr.child(0), info, childOutputExprIds)) { + if (!existingExprIds.contains(baseSlot.getExprId())) { + simplified.add(baseSlot); + existingExprIds.add(baseSlot.getExprId()); + } + } + } + for (Expression passThroughExpr : passThroughExprs) { + for (Slot baseSlot : resolveInputSlots(passThroughExpr, info, childOutputExprIds)) { + if (!existingExprIds.contains(baseSlot.getExprId())) { + simplified.add(baseSlot); + existingExprIds.add(baseSlot.getExprId()); + } + } + } + + if (simplified.equals(project.getProjects())) { + return project; + } + return (LogicalProject) project.withProjects(simplified); + } + + private static List collectUnavailablePullUpExprs( + LogicalProject project, PullUpInfo info, Set childOutputExprIds) { + List passThroughExprs = new ArrayList<>(); + for (NamedExpression ne : project.getProjects()) { + Expression replaceExpr = getPullUpReplaceExpression(ne.toSlot(), info); + if (replaceExpr != null && isUnavailableExpression(ne, childOutputExprIds)) { + passThroughExprs.add(replaceExpr); + } + } + return passThroughExprs; + } + + /** Check the non-replaceExpr conditions for unavailability. + * Caller must have already verified {@code getPullUpReplaceExpression(ne.toSlot()) != null}. */ + private static boolean isUnavailableExpression(NamedExpression ne, Set childOutputExprIds) { + if (ne instanceof Slot) { + return !childOutputExprIds.contains(ne.getExprId()); + } + return ne instanceof Alias + && ne.getInputSlots().stream().anyMatch(slot -> !childOutputExprIds.contains(slot.getExprId())); + } + + private static Expression getPullUpReplaceExpression(Slot slot, PullUpInfo info) { + Expression expression = info.pullUpExprReplaceMap.get(slot); + while (expression instanceof Slot) { + Expression next = info.pullUpExprReplaceMap.get((Slot) expression); + if (next == null) { + return expression; + } + expression = next; + } + return expression; + } + + /** Create a new Project above the TopN that restores pulled-up expressions. */ + private static LogicalProject addUpperProject(LogicalTopN topN, PullUpInfo info) { + Map pulledUpBySlotExprId = new HashMap<>(); + Set currentOutputExprIds = topN.getOutputExprIdSet(); + for (NamedExpression e : info.allPulledUpExprs) { + pulledUpBySlotExprId.put(e.toSlot().getExprId(), resolveAliasChildIfNeeded(e, info, currentOutputExprIds)); + } + + List upperOutput = new ArrayList<>(); + Set upperOutputExprIds = new HashSet<>(); + for (int i = 0; i < info.originalTopNOutput.size(); i++) { + Slot origSlot = info.originalTopNOutput.get(i); + NamedExpression pulledUpExpr = pulledUpBySlotExprId.get(origSlot.getExprId()); + if (pulledUpExpr != null) { + upperOutput.add(pulledUpExpr); + upperOutputExprIds.add(pulledUpExpr.getExprId()); + } else if (currentOutputExprIds.contains(origSlot.getExprId())) { + upperOutput.add(origSlot); + upperOutputExprIds.add(origSlot.getExprId()); + } else { + Expression resolvedExpr = resolveExpression(origSlot, info, currentOutputExprIds); + upperOutput.add(new Alias(origSlot.getExprId(), resolvedExpr, origSlot.getName())); + upperOutputExprIds.add(origSlot.getExprId()); + } + } + + return new LogicalProject<>(ImmutableList.copyOf(upperOutput), topN); + } + + private static NamedExpression resolveAliasChildIfNeeded(NamedExpression expr, PullUpInfo info, + Set availableExprIds) { + if (!(expr instanceof Alias)) { + return expr; + } + Expression resolvedChild = resolveExpression(expr.child(0), info, availableExprIds); + if (resolvedChild.equals(expr.child(0))) { + return expr; + } + return new Alias(expr.getExprId(), resolvedChild, expr.getName()); + } + + private static List resolveInputSlots(Expression expr, PullUpInfo info, + Set availableExprIds) { + return ImmutableList.copyOf(resolveExpression(expr, info, availableExprIds).getInputSlots()); + } + + private static Expression resolveExpression(Expression expression, PullUpInfo info, + Set availableExprIds) { + Expression resolved = replaceUnavailableSlots(expression, info, availableExprIds); + while (!resolved.equals(expression)) { + expression = resolved; + resolved = replaceUnavailableSlots(expression, info, availableExprIds); + } + return resolved; + } + + private static Expression replaceUnavailableSlots(Expression expression, PullUpInfo info, + Set availableExprIds) { + Map replaceMap = new LinkedHashMap<>(); + for (Map.Entry entry : info.pullUpExprReplaceMap.entrySet()) { + if (!availableExprIds.contains(entry.getKey().getExprId())) { + replaceMap.put(entry.getKey(), entry.getValue()); + } + } + return ExpressionUtils.replace(expression, replaceMap); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderLimit.java index 7ecb31201c8237..285db5414835b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderLimit.java @@ -30,6 +30,7 @@ import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Pull up Project under Limit. @@ -42,14 +43,20 @@ public Rule build() { .whenNot(p -> p.isAllSlots())) .then(limit -> { LogicalProject> project = limit.child(); - Set allUsedSlots = project.getProjects().stream().flatMap(ne -> ne.getInputSlots().stream()) + Set allUsedSlots = project.getProjects().stream() + .flatMap(ne -> ne instanceof Slot ? Stream.of((Slot) ne) : ne.getInputSlots().stream()) .collect(Collectors.toSet()); Set outputSet = project.child().getOutputSet(); + if (!outputSet.containsAll(allUsedSlots)) { + return null; + } if (outputSet.size() == allUsedSlots.size()) { Preconditions.checkState(outputSet.equals(allUsedSlots)); return project.withChildren(limit.withChildren(project.child())); } else { - Plan columnProject = PlanUtils.projectOrSelf(ImmutableList.copyOf(allUsedSlots), + Plan columnProject = PlanUtils.projectOrSelf(project.child().getOutput().stream() + .filter(allUsedSlots::contains) + .collect(ImmutableList.toImmutableList()), project.child()); return project.withChildren(limit.withChildren(columnProject)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderTopN.java index 5e0dc47af6bf9d..88482d948b6e95 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderTopN.java @@ -37,6 +37,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Pull up Project under TopN for PushDownTopNThroughJoin @@ -66,14 +67,20 @@ public Rule build() { } } - Set allUsedSlots = project.getProjects().stream().flatMap(ne -> ne.getInputSlots().stream()) + Set allUsedSlots = project.getProjects().stream() + .flatMap(ne -> ne instanceof Slot ? Stream.of((Slot) ne) : ne.getInputSlots().stream()) .collect(Collectors.toSet()); + if (!outputSet.containsAll(allUsedSlots)) { + return null; + } LogicalTopN newTopN = topN.withOrderKeys(newOrderKeys); if (outputSet.size() == allUsedSlots.size()) { Preconditions.checkState(outputSet.equals(allUsedSlots)); return project.withChildren(newTopN.withChildren(project.child())); } else { - Plan columnProject = PlanUtils.projectOrSelf(ImmutableList.copyOf(allUsedSlots), + Plan columnProject = PlanUtils.projectOrSelf(project.child().getOutput().stream() + .filter(allUsedSlots::contains) + .collect(ImmutableList.toImmutableList()), project.child()); return project.withChildren(newTopN.withChildren(columnProject)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoin.java index 74b9cb1f8e6f90..14d56ad9172354 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoin.java @@ -20,19 +20,23 @@ import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.TopN; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.nereids.util.PlanUtils; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Push down TopN through Outer Join into left child ..... @@ -53,7 +57,7 @@ public List buildRules() { .allMatch(Slot.class::isInstance)) .then(topN -> { LogicalJoin join = topN.child(); - Plan newJoin = pushLimitThroughJoin(topN, join); + Plan newJoin = pushLimitThroughJoin(topN, join, ImmutableList.of()); if (newJoin == null) { return null; } @@ -80,7 +84,26 @@ public List buildRules() { return null; } - Plan newJoin = pushLimitThroughJoin(topN, join); + List projectInputSlots = project.getProjects().stream() + .flatMap(ne -> ne instanceof Slot + ? Stream.of((Slot) ne) : ne.getInputSlots().stream()) + .collect(Collectors.toList()); + if (!join.getOutputSet().containsAll(projectInputSlots)) { + return null; + } + // Verify the push-down target child has all required + // slots. Intermediate Projects (e.g. from ColumnPruning) + // may restrict output, and push-down would break the plan. + List orderbySlots = topN.getOrderKeys().stream() + .map(OrderKey::getExpr) + .flatMap(e -> e.getInputSlots().stream()) + .collect(Collectors.toList()); + Plan targetChild = join.left().getOutputSet().containsAll(orderbySlots) + ? join.left() : join.right(); + if (!targetChild.getOutputSet().containsAll(projectInputSlots)) { + return null; + } + Plan newJoin = pushLimitThroughJoin(topN, join, projectInputSlots); if (newJoin == null) { return null; } @@ -89,7 +112,8 @@ public List buildRules() { ); } - private Plan pushLimitThroughJoin(LogicalTopN topN, LogicalJoin join) { + private Plan pushLimitThroughJoin(LogicalTopN topN, LogicalJoin join, + List requiredOutputSlots) { List orderbySlots = topN.getOrderKeys().stream().map(OrderKey::getExpr) .flatMap(e -> e.getInputSlots().stream()).collect(Collectors.toList()); switch (join.getJoinType()) { @@ -99,8 +123,10 @@ private Plan pushLimitThroughJoin(LogicalTopN topN, LogicalJoin< return null; } if (join.left().getOutputSet().containsAll(orderbySlots)) { + List childRequiredOutputSlots = buildRequiredOutputSlots( + requiredOutputSlots, join.getLeftConditionSlot()); return join.withChildren( - topN.withLimitChild(topN.getLimit() + topN.getOffset(), 0, join.left()), + pushTopNToChild(topN, join.left(), orderbySlots, childRequiredOutputSlots), join.right()); } return null; @@ -110,9 +136,11 @@ private Plan pushLimitThroughJoin(LogicalTopN topN, LogicalJoin< return null; } if (join.right().getOutputSet().containsAll(orderbySlots)) { + List childRequiredOutputSlots = buildRequiredOutputSlots( + requiredOutputSlots, join.getRightConditionSlot()); return join.withChildren( join.left(), - topN.withLimitChild(topN.getLimit() + topN.getOffset(), 0, join.right())); + pushTopNToChild(topN, join.right(), orderbySlots, childRequiredOutputSlots)); } return null; case CROSS_JOIN: @@ -120,16 +148,20 @@ private Plan pushLimitThroughJoin(LogicalTopN topN, LogicalJoin< if (join.left() instanceof TopN) { return null; } + List childRequiredOutputSlots = buildRequiredOutputSlots( + requiredOutputSlots, join.getLeftConditionSlot()); return join.withChildren( - topN.withLimitChild(topN.getLimit() + topN.getOffset(), 0, join.left()), + pushTopNToChild(topN, join.left(), orderbySlots, childRequiredOutputSlots), join.right()); } else if (join.right().getOutputSet().containsAll(orderbySlots)) { if (join.right() instanceof TopN) { return null; } + List childRequiredOutputSlots = buildRequiredOutputSlots( + requiredOutputSlots, join.getRightConditionSlot()); return join.withChildren( join.left(), - topN.withLimitChild(topN.getLimit() + topN.getOffset(), 0, join.right())); + pushTopNToChild(topN, join.right(), orderbySlots, childRequiredOutputSlots)); } else { return null; } @@ -138,4 +170,40 @@ private Plan pushLimitThroughJoin(LogicalTopN topN, LogicalJoin< return null; } } + + private boolean canPushToChild(Plan child, List requiredOutputSlots) { + return requiredOutputSlots.isEmpty() || child.getOutputSet().containsAll(requiredOutputSlots); + } + + private List buildRequiredOutputSlots(List requiredOutputSlots, Set conditionSlots) { + Set requiredSet = new HashSet<>(requiredOutputSlots); + requiredSet.addAll(conditionSlots); + return ImmutableList.copyOf(requiredSet); + } + + private Plan pushTopNToChild(LogicalTopN topN, Plan child, List orderbySlots, + List requiredOutputSlots) { + // Keep only the slots that this child actually outputs. When pushing to + // one side of an outer join, requiredOutputSlots may include columns from + // the other side which this child cannot provide. + Set childOutputSet = child.getOutputSet(); + List childRequired = requiredOutputSlots.stream() + .filter(childOutputSet::contains) + .collect(Collectors.toList()); + if (childRequired.isEmpty()) { + return topN.withLimitChild(topN.getLimit() + topN.getOffset(), 0, child); + } + // Ensure the child outputs all required columns. If the child already + // outputs everything needed, no extra Project is required. + Set requiredSet = new HashSet<>(childRequired); + requiredSet.addAll(orderbySlots); + if (childOutputSet.containsAll(requiredSet)) { + return topN.withLimitChild(topN.getLimit() + topN.getOffset(), 0, child); + } + List childOutput = child.getOutput().stream() + .filter(requiredSet::contains) + .collect(Collectors.toList()); + Plan topNChild = PlanUtils.projectOrSelf(childOutput, child); + return topN.withLimitChild(topN.getLimit() + topN.getOffset(), 0, topNChild); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java index c90d85d55e667d..1f7d0966062668 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java @@ -730,7 +730,9 @@ private void tryRecordReplaceSlots(Plan plan, Object checkObj, Set shou for (Slot slot : output) { int slotId = slot.getExprId().asInt(); if ((slot.getDataType() instanceof NestedColumnPrunable - || slot.getDataType().isVariantType()) + || slot.getDataType().isVariantType() + || slot.getDataType().isStringLikeType() + || slot.nullable()) && replacedDataTypes.containsKey(slotId)) { shouldReplaceSlots.add(slotId); shouldPrune = true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java index 2aec77b76e76c9..20ccfca4d920e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java @@ -166,8 +166,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalQuery, ctx.getStatementContext()); updateSessionVariableForDelete(ctx.getSessionVariable()); StatementContext statementContext = ctx.getStatementContext(); - // delete not prune predicate after partition prune - statementContext.setSkipPrunePredicate(true); + statementContext.setIsDelete(true); NereidsPlanner planner = new NereidsPlanner(statementContext); boolean originalIsSkipAuth = ctx.isSkipAuth(); // delete not need select priv diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java index dc245ce1b7e732..4faae89e3c36d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java @@ -93,7 +93,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { ctx.getStatementContext().setIsInsert(true); } if (explainable instanceof DeleteFromCommand) { - ctx.getStatementContext().setSkipPrunePredicate(true); + ctx.getStatementContext().setIsDelete(true); } explainPlan = ((LogicalPlan) explainable.getExplainPlan(ctx)); NereidsPlanner planner = explainable.getExplainPlanner(explainPlan, ctx.getStatementContext()).orElseGet(() -> diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java index abcca92ef482a3..95765db284c148 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java @@ -254,6 +254,18 @@ public Set getLeftConditionSlot() { .collect(ImmutableSet.toImmutableSet()); } + /** + * getRightConditionSlot + */ + public Set getRightConditionSlot() { + Set rightOutputSet = this.right().getOutputSet(); + return Stream + .concat(Stream.concat(hashJoinConjuncts.stream(), otherJoinConjuncts.stream()), + markJoinConjuncts.stream()) + .flatMap(expr -> expr.getInputSlots().stream()).filter(rightOutputSet::contains) + .collect(ImmutableSet.toImmutableSet()); + } + /** * getOnClauseCondition */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeFileScan.java index 4c0befccf97005..dee8638ccdf2a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeFileScan.java @@ -56,9 +56,13 @@ public List getQualifier() { @Override public List computeOutput() { if (output == null) { - output = ImmutableList.builder() - .addAll(scan.getOperativeSlots()) - .add(rowId).build(); + ImmutableList.Builder outputBuilder = ImmutableList.builder(); + for (Slot slot : scan.getOutput()) { + if (!lazySlots.contains(slot)) { + outputBuilder.add(slot); + } + } + output = outputBuilder.add(rowId).build(); } return output; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeOlapScan.java index cc5095644f34b7..b0d1a90e619ab4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeOlapScan.java @@ -75,9 +75,13 @@ public R accept(PlanVisitor visitor, C context) { @Override public List computeOutput() { - return ImmutableList.builder() - .addAll(scan.getOperativeSlots()) - .add(rowId).build(); + ImmutableList.Builder output = ImmutableList.builder(); + for (Slot slot : scan.getOutput()) { + if (!lazySlots.contains(slot)) { + output.add(slot); + } + } + return output.add(rowId).build(); } public PhysicalOlapScan getScan() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeTVFScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeTVFScan.java index 1278cfee53bde6..87bcb3f4279953 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeTVFScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeTVFScan.java @@ -57,9 +57,13 @@ public List getLazySlots() { @Override public List computeOutput() { if (output == null) { - output = ImmutableList.builder() - .addAll(scan.getOperativeSlots()) - .add(rowId).build(); + ImmutableList.Builder outputBuilder = ImmutableList.builder(); + for (Slot slot : scan.getOutput()) { + if (!lazySlots.contains(slot)) { + outputBuilder.add(slot); + } + } + output = outputBuilder.add(rowId).build(); } return output; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 6ab9695c1ca59a..905436b9421269 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -68,12 +68,14 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplodeOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.Unnest; import org.apache.doris.nereids.trees.expressions.functions.scalar.If; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Length; import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable; import org.apache.doris.nereids.trees.expressions.functions.scalar.NullIf; import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable; import org.apache.doris.nereids.trees.expressions.functions.scalar.Nvl; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.ComparableLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; @@ -1171,6 +1173,15 @@ public static ImmutableMap extractUniformSlot(Expression expre if (expression instanceof EqualTo) { if (isInjective(expression.child(0)) && expression.child(1).isConstant()) { builder.put((Slot) expression.child(0), expression.child(1)); + } else { + // length(str_col)=0 => str_col='' + if (expression.child(0) instanceof Length + && expression.child(1).equals(new IntegerLiteral(0))) { + Length len = (Length) expression.child(0); + if (len.child() instanceof Slot && len.child().getDataType().isStringLikeType()) { + builder.put((Slot) len.child(), new StringLiteral("")); + } + } } } return builder.build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java index 8674ef85af9451..ed0df7c49cd857 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MaterializationNode.java @@ -171,7 +171,7 @@ public String getNodeExplainString(String detailPrefix, TExplainLevel detailLeve output.append(detailPrefix).append("column_idxs_lists: ").append(columnIdxsLists).append("\n"); output.append(detailPrefix).append("row_ids: ").append(rowIds).append("\n"); output.append(detailPrefix).append("isTopMaterializeNode: ").append(isTopMaterializeNode).append("\n"); - printNestedColumns(output, detailPrefix, outputTupleDesc); + printNestedColumns(output, detailPrefix, materializeTupleDescriptor); return output.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index bd03966d5576a9..3174aed5dc604c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -994,9 +994,11 @@ protected void printNestedColumns(StringBuilder output, String prefix, TupleDesc } + List subColLables = slot.getSubColLables(); if (prunedType == null && displayAllAccessPathsString == null - && displayPredicateAccessPathsString == null) { + && displayPredicateAccessPathsString == null + && (subColLables == null || subColLables.isEmpty())) { continue; } @@ -1009,6 +1011,10 @@ protected void printNestedColumns(StringBuilder output, String prefix, TupleDesc if (prunedType != null) { output.append(prefix).append(" pruned type: ").append(prunedType).append("\n"); } + if (subColLables != null && !subColLables.isEmpty()) { + output.append(prefix).append(" sub path: [") + .append(String.join(".", subColLables)).append("]\n"); + } if (displayAllAccessPathsString != null) { output.append(prefix).append(" all access paths: [") .append(displayAllAccessPathsString).append("]\n"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 4432844120bdaa..271d7e3c3f1252 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1863,6 +1863,15 @@ public enum IgnoreSplitType { varType = VariableAnnotation.EXPERIMENTAL) public boolean topNLazyMaterializationUsingIndex = false; + @VariableMgr.VarAttr(name = "enable_topn_expr_pullup", needForward = true, + fuzzy = false, + varType = VariableAnnotation.EXPERIMENTAL, + description = {"是否将TopN下方Project中的非平凡表达式上拉至TopN之上，" + + "以扩大延迟物化范围", + "Whether to pull up non-trivial expressions from Project below TopN, " + + "to expand lazy materialization scope"}) + public boolean enableTopnExprPullup = true; + @VariableMgr.VarAttr(name = ENABLE_PRUNE_NESTED_COLUMN, needForward = true, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL, diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopNRuntimeFilterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopNRuntimeFilterTest.java index 41c7001f7925e3..f66a284121ca58 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopNRuntimeFilterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopNRuntimeFilterTest.java @@ -18,24 +18,29 @@ package org.apache.doris.nereids.postprocess; import org.apache.doris.nereids.datasets.ssb.SSBTestBase; +import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.nereids.processor.post.PlanPostProcessors; import org.apache.doris.nereids.processor.post.TopnFilterContext; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable; import org.apache.doris.nereids.trees.expressions.functions.scalar.Substring; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.SortPhase; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterializeOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN; import org.apache.doris.nereids.trees.plans.physical.TopnFilter; import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.planner.ScanNode; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import java.util.Map; @@ -60,6 +65,37 @@ public void testUseTopNRf() { Assertions.assertTrue(checker.getCascadesContext().getTopnFilterContext().isTopnFilterSource(localTopN)); } + @Test + public void testTranslateTopNFilterTargetThroughLazyMaterializeOlapScan() { + String sql = "select * from customer order by c_custkey limit 5"; + PlanChecker checker = PlanChecker.from(connectContext).analyze(sql) + .rewrite() + .implement(); + PhysicalPlan plan = checker.getPhysicalPlan(); + new PlanPostProcessors(checker.getCascadesContext()).process(plan); + + TopnFilter filter = checker.getCascadesContext().getTopnFilterContext().getTopnFilters().stream() + .filter(f -> f.targets.keySet().stream().anyMatch(PhysicalLazyMaterializeOlapScan.class::isInstance)) + .findFirst() + .orElseThrow(() -> new AssertionError("topn filter target is not lazy materialize olap scan")); + PhysicalLazyMaterializeOlapScan lazyScan = filter.targets.keySet().stream() + .filter(PhysicalLazyMaterializeOlapScan.class::isInstance) + .map(PhysicalLazyMaterializeOlapScan.class::cast) + .findFirst() + .orElseThrow(() -> new AssertionError("lazy materialize olap scan not found")); + Expression probeExpr = filter.targets.get(lazyScan); + + TopnFilterContext topnFilterContext = new TopnFilterContext(); + topnFilterContext.addTopnFilter(filter.topn, lazyScan.getScan(), probeExpr); + PlanTranslatorContext translatorContext = new PlanTranslatorContext(); + probeExpr.getInputSlots().forEach(slot -> translatorContext + .createSlotDesc(translatorContext.generateTupleDesc(), (SlotReference) slot)); + ScanNode legacyScan = Mockito.mock(ScanNode.class); + topnFilterContext.translateTarget(lazyScan, legacyScan, translatorContext); + + Assertions.assertTrue(topnFilterContext.getTopnFilter(filter.topn).legacyTargets.containsKey(legacyScan)); + } + @Test public void testUseTopNRfForComplexCase() { String sql = "select * from (select 1) tl join (select * from customer order by c_custkey limit 5) tb"; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitorTest.java index f37806b9c6e5ab..3764f4258a6b53 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitorTest.java @@ -20,13 +20,18 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.table.VectorSearch; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Relation; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterialize; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.qe.ConnectContext; @@ -45,11 +50,13 @@ import org.mockito.Mockito; import java.util.BitSet; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; -class MaterializeProbeVisitorTest { +public class MaterializeProbeVisitorTest { @Test void testVectorSearchSupportsLazyMaterialization() { @@ -184,4 +191,71 @@ private PhysicalTVFRelation mockVectorSearchRelation() { Mockito.when(relation.getFunction()).thenReturn(function); return relation; } + + @Test + public void testOlapScanRejectsRequiredMaterializedSlots() { + SlotReference baseSlot = new SlotReference("a", IntegerType.INSTANCE); + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(table.getBaseIndexId()).thenReturn(1L); + Mockito.when(table.getKeysType()).thenReturn(KeysType.DUP_KEYS); + PhysicalOlapScan scan = Mockito.mock(PhysicalOlapScan.class); + Mockito.when(scan.getSelectedIndexId()).thenReturn(1L); + Mockito.when(scan.getTable()).thenReturn(table); + + Set requiredMaterializedSlots = new HashSet<>(); + requiredMaterializedSlots.add(baseSlot); + MaterializeProbeVisitor.ProbeContext context = new MaterializeProbeVisitor.ProbeContext( + baseSlot, requiredMaterializedSlots); + Optional source = new MaterializeProbeVisitor().visitPhysicalOlapScan(scan, context); + + Assertions.assertFalse(source.isPresent()); + } + + @Test + @SuppressWarnings("unchecked") + public void testComplexProjectInputSlotsAreRequiredMaterialized() { + SlotReference baseSlot = new SlotReference("a", IntegerType.INSTANCE); + Alias complexAlias = new Alias(new Add(baseSlot, new IntegerLiteral(1)), "x"); + SlotReference aliasSlot = (SlotReference) complexAlias.toSlot(); + PhysicalProject project = Mockito.mock(PhysicalProject.class); + Mockito.when(project.getOutput()).thenReturn(ImmutableList.of(baseSlot, aliasSlot)); + Mockito.when(project.getProjects()).thenReturn(ImmutableList.of(baseSlot, complexAlias)); + Plan child = Mockito.mock(Plan.class); + Mockito.when(project.child()).thenReturn(child); + Mockito.when(child.accept(Mockito.any(MaterializeProbeVisitor.class), + Mockito.any(MaterializeProbeVisitor.ProbeContext.class))).thenReturn(Optional.empty()); + + Set requiredMaterializedSlots = new HashSet<>(); + MaterializeProbeVisitor.ProbeContext context = new MaterializeProbeVisitor.ProbeContext( + aliasSlot, requiredMaterializedSlots); + Optional source = new MaterializeProbeVisitor().visitPhysicalProject(project, context); + + Assertions.assertFalse(source.isPresent()); + Assertions.assertEquals(ImmutableList.of(baseSlot), ImmutableList.copyOf(requiredMaterializedSlots)); + } + + @Test + @SuppressWarnings("unchecked") + public void testPushedDownProjectSlotInputsAreRequiredMaterialized() { + SlotReference baseSlot = new SlotReference("a", IntegerType.INSTANCE); + SlotReference pushedDownSlot = new SlotReference("pushed", IntegerType.INSTANCE); + Alias pushedDownAlias = new Alias(pushedDownSlot, "x"); + SlotReference aliasSlot = (SlotReference) pushedDownAlias.toSlot(); + PhysicalProject project = Mockito.mock(PhysicalProject.class); + Mockito.when(project.getOutput()).thenReturn(ImmutableList.of(baseSlot, aliasSlot)); + Mockito.when(project.getProjects()).thenReturn(ImmutableList.of(baseSlot, pushedDownAlias)); + Mockito.when(project.getInputSlots()).thenReturn(ImmutableSet.of(baseSlot, pushedDownSlot)); + Plan child = Mockito.mock(Plan.class); + Mockito.when(project.child()).thenReturn(child); + Mockito.when(child.accept(Mockito.any(MaterializeProbeVisitor.class), + Mockito.any(MaterializeProbeVisitor.ProbeContext.class))).thenReturn(Optional.empty()); + + Set requiredMaterializedSlots = new HashSet<>(); + MaterializeProbeVisitor.ProbeContext context = new MaterializeProbeVisitor.ProbeContext( + aliasSlot, requiredMaterializedSlots); + Optional source = new MaterializeProbeVisitor().visitPhysicalProject(project, context); + + Assertions.assertFalse(source.isPresent()); + Assertions.assertEquals(ImmutableSet.of(baseSlot, pushedDownSlot), requiredMaterializedSlots); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/StringEmptyToLengthRuleTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/StringEmptyToLengthRuleTest.java new file mode 100644 index 00000000000000..4c9003e29b240c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/StringEmptyToLengthRuleTest.java @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.expression.rules; + +import org.apache.doris.nereids.rules.expression.ExpressionRewriteTestHelper; +import org.apache.doris.nereids.rules.expression.ExpressionRuleExecutor; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Length; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.VarcharType; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class StringEmptyToLengthRuleTest extends ExpressionRewriteTestHelper { + + @BeforeEach + public void setup() { + executor = new ExpressionRuleExecutor(ImmutableList.of( + bottomUp(StringEmptyToLengthRule.INSTANCE) + )); + } + + // ─── Helper: rewrite without type coercion ─────────────────────────────────── + // We bypass ExpressionRewriteTestHelper.assertRewrite() because that method + // applies typeCoercion() to the input, which may wrap string literals in Cast + // nodes and prevent our pattern from matching. + + private void assertRuleRewrite(Expression before, Expression expected) { + Expression result = executor.rewrite(before, context); + Assertions.assertEquals(expected, result); + } + + private void assertRuleNoRewrite(Expression before) { + Expression result = executor.rewrite(before, context); + Assertions.assertEquals(before, result); + } + + // ─── Rewrite cases ─────────────────────────────────────────────────────────── + + @Test + public void testStringSlotEqualEmptyRewrite() { + // str_col = '' → length(str_col) = 0 + SlotReference slot = new SlotReference("str_col", StringType.INSTANCE, true); + VarcharLiteral empty = new VarcharLiteral(""); + assertRuleRewrite( + new EqualTo(slot, empty), + new EqualTo(new Length(slot), new IntegerLiteral(0)) + ); + } + + @Test + public void testVarcharSlotEqualEmptyRewrite() { + // varchar_col = '' → length(varchar_col) = 0 + SlotReference slot = new SlotReference("vc_col", VarcharType.SYSTEM_DEFAULT, true); + VarcharLiteral empty = new VarcharLiteral(""); + assertRuleRewrite( + new EqualTo(slot, empty), + new EqualTo(new Length(slot), new IntegerLiteral(0)) + ); + } + + @Test + public void testReversedOperandsRewrite() { + // '' = str_col (literal on left) → length(str_col) = 0 + SlotReference slot = new SlotReference("str_col", StringType.INSTANCE, true); + VarcharLiteral empty = new VarcharLiteral(""); + assertRuleRewrite( + new EqualTo(empty, slot), + new EqualTo(new Length(slot), new IntegerLiteral(0)) + ); + } + + @Test + public void testNotEqualToEmptyRewrite() { + // NOT(str_col = '') → NOT(length(str_col) = 0) (i.e. str_col <> '') + SlotReference slot = new SlotReference("str_col", StringType.INSTANCE, true); + VarcharLiteral empty = new VarcharLiteral(""); + assertRuleRewrite( + new Not(new EqualTo(slot, empty)), + new Not(new EqualTo(new Length(slot), new IntegerLiteral(0))) + ); + } + + @Test + public void testNotEqualToEmptyReversedRewrite() { + // NOT('' = str_col) → NOT(length(str_col) = 0) + SlotReference slot = new SlotReference("str_col", StringType.INSTANCE, true); + VarcharLiteral empty = new VarcharLiteral(""); + assertRuleRewrite( + new Not(new EqualTo(empty, slot)), + new Not(new EqualTo(new Length(slot), new IntegerLiteral(0))) + ); + } + + // ─── No-op cases ───────────────────────────────────────────────────────────── + + @Test + public void testNonEmptyLiteralNotRewritten() { + // str_col = 'abc' — literal is non-empty, must not be rewritten + SlotReference slot = new SlotReference("str_col", StringType.INSTANCE, true); + assertRuleNoRewrite(new EqualTo(slot, new VarcharLiteral("abc"))); + } + + @Test + public void testNonStringSlotNotRewritten() { + // int_col = 0 — slot is not string-like, must not be rewritten + SlotReference intSlot = new SlotReference("int_col", IntegerType.INSTANCE, true); + assertRuleNoRewrite(new EqualTo(intSlot, new IntegerLiteral(0))); + } + + @Test + public void testNotWithNonEmptyLiteralNotRewritten() { + // NOT(str_col = 'abc') — non-empty literal, must not be rewritten + SlotReference slot = new SlotReference("str_col", StringType.INSTANCE, true); + assertRuleNoRewrite(new Not(new EqualTo(slot, new VarcharLiteral("abc")))); + } + + // ─── Struct element (non-SlotReference expression) cases ───────────────────── + + @Test + public void testStructElementEqualEmptyRewrite() { + // struct_element(struct_col, 'f3') = '' → length(struct_element(struct_col, 'f3')) = 0 + SlotReference structSlot = new SlotReference("struct_col", + new StructType(ImmutableList.of( + new StructField("f1", IntegerType.INSTANCE, true, ""), + new StructField("f3", StringType.INSTANCE, true, "") + )), true); + ElementAt elementAt = new ElementAt(structSlot, new VarcharLiteral("f3")); + VarcharLiteral empty = new VarcharLiteral(""); + assertRuleRewrite( + new EqualTo(elementAt, empty), + new EqualTo(new Length(elementAt), new IntegerLiteral(0)) + ); + } + + @Test + public void testStructElementReversedOperandsRewrite() { + // '' = struct_element(struct_col, 'f3') → length(struct_element(struct_col, 'f3')) = 0 + SlotReference structSlot = new SlotReference("struct_col", + new StructType(ImmutableList.of( + new StructField("f1", IntegerType.INSTANCE, true, ""), + new StructField("f3", StringType.INSTANCE, true, "") + )), true); + ElementAt elementAt = new ElementAt(structSlot, new VarcharLiteral("f3")); + VarcharLiteral empty = new VarcharLiteral(""); + assertRuleRewrite( + new EqualTo(empty, elementAt), + new EqualTo(new Length(elementAt), new IntegerLiteral(0)) + ); + } + + @Test + public void testNotStructElementEqualEmptyRewrite() { + // NOT(struct_element(struct_col, 'f3') = '') → NOT(length(struct_element(struct_col, 'f3')) = 0) + SlotReference structSlot = new SlotReference("struct_col", + new StructType(ImmutableList.of( + new StructField("f1", IntegerType.INSTANCE, true, ""), + new StructField("f3", StringType.INSTANCE, true, "") + )), true); + ElementAt elementAt = new ElementAt(structSlot, new VarcharLiteral("f3")); + VarcharLiteral empty = new VarcharLiteral(""); + assertRuleRewrite( + new Not(new EqualTo(elementAt, empty)), + new Not(new EqualTo(new Length(elementAt), new IntegerLiteral(0))) + ); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OperativeColumnDeriveTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OperativeColumnDeriveTest.java index 8be0c7da7385e4..c258d4ae1fd993 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OperativeColumnDeriveTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OperativeColumnDeriveTest.java @@ -17,12 +17,21 @@ package org.apache.doris.nereids.rules.rewrite; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink; import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + public class OperativeColumnDeriveTest extends TestWithFeService implements MemoPatternMatchSupported { @Override protected void runBeforeAll() throws Exception { @@ -48,9 +57,8 @@ public void testProject() { .analyze("select sid+1, grade as x from score") .customRewrite(new OperativeColumnDerive()) .matches(logicalProject( - logicalOlapScan().when(scan -> - scan.getOperativeSlots().size() == 1 - && scan.getOperativeSlots().get(0).getName().equals("sid")) + logicalOlapScan().when(scan -> scan.getOperativeSlots().size() == 1 + && scan.getOperativeSlots().get(0).getName().equals("sid")) )); } @@ -89,4 +97,25 @@ public void testUnionAllBackPropagate() { ) )); } + + @Test + public void testPhysicalResultSinkDoesNotMarkOutputAsOperativeSlots() { + Plan analyzedPlan = PlanChecker.from(connectContext) + .analyze("select * from score") + .getCascadesContext() + .getRewritePlan(); + LogicalOlapScan scan = analyzedPlan.collect(LogicalOlapScan.class::isInstance) + .iterator() + .next(); + + List outputExprs = new ArrayList<>(scan.getOutput()); + PhysicalResultSink sink = new PhysicalResultSink<>(outputExprs, Optional.empty(), + scan.getLogicalProperties(), scan); + Plan rewritten = new OperativeColumnDerive().rewriteRoot(sink, null); + LogicalOlapScan rewrittenScan = rewritten.collect(LogicalOlapScan.class::isInstance) + .iterator() + .next(); + + Assertions.assertTrue(rewrittenScan.getOperativeSlots().isEmpty()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java index b94b21de7a0ee0..cd0e08327b29cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java @@ -23,6 +23,7 @@ import org.apache.doris.common.Triple; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectAccessPathResult; import org.apache.doris.nereids.rules.rewrite.NestedColumnPruning.DataTypeAccessTree; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.ArrayItemReference; @@ -38,6 +39,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion; +import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.NestedColumnPrunable; import org.apache.doris.nereids.types.NullType; @@ -65,6 +67,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.TreeSet; import java.util.function.Consumer; @@ -103,6 +106,39 @@ public void createTable() throws Exception { + " v variant\n" + ") properties ('replication_num'='1')"); + // Table for string-length offset-only optimization tests + createTable("create table str_tbl(\n" + + " id int,\n" + + " str_col string,\n" + + " c_struct struct,\n" + + " map_col map\n" + + ") properties ('replication_num'='1')"); + + createTable("create table nested_container_tbl(\n" + + " id int,\n" + + " s struct<\n" + + " arr: array>,\n" + + " m: map\n" + + " >\n" + + ") properties ('replication_num'='1')"); + + createTable("create table nested_array_tbl(\n" + + " id int,\n" + + " a array>\n" + + ") properties ('replication_num'='1')"); + + createTable("create table map_array_tbl(\n" + + " id int,\n" + + " map_arr_col map>\n" + + ") properties ('replication_num'='1')"); + + createTable("create table map_array_value_tbl(\n" + + " id int,\n" + + " s struct<\n" + + " m: map>>\n" + + " >\n" + + ") properties ('replication_num'='1')"); + connectContext.getSessionVariable().setDisableNereidsRules(RuleType.PRUNE_EMPTY_PARTITION.name()); connectContext.getSessionVariable().enableNereidsTimeout = false; } @@ -131,6 +167,109 @@ public void testMap() throws Exception { ); } + @Test + public void testMapElementLengthWithMapValuesKeepsKeysPath() throws Exception { + assertColumn("select length(map_col['a']), map_values(map_col)[1] from str_tbl", + "map", + ImmutableList.of(path("map_col", "KEYS"), path("map_col", "VALUES")), + ImmutableList.of() + ); + } + + @Test + public void testStructRootArrayMixedAccessSuppressesOffsetPath() throws Exception { + assertAllAccessPathsContain( + "select cardinality(struct_element(s, 'arr')), " + + "struct_element(element_at(struct_element(s, 'arr'), 1), 'int_field') " + + "from nested_container_tbl", + ImmutableList.of(path("s", "arr", "*", "int_field")), + ImmutableList.of(path("s", "arr", "OFFSET"))); + } + + @Test + public void testStructRootMapMixedAccessKeepsKeysPath() throws Exception { + assertAllAccessPathsContain( + "select length(element_at(struct_element(s, 'm'), 'a')), " + + "element_at(map_values(struct_element(s, 'm')), 1) " + + "from nested_container_tbl", + ImmutableList.of(path("s", "m", "KEYS"), path("s", "m", "VALUES")), + ImmutableList.of(path("s", "m", "*", "OFFSET"), path("s", "m", "VALUES", "OFFSET"))); + } + + @Test + public void testCardinalityArrayElementKeepsOffsetPath() throws Exception { + assertAllAccessPathsContain( + "select cardinality(element_at(a, 1)) from nested_array_tbl", + ImmutableList.of(path("a", "*", "OFFSET")), + ImmutableList.of(path("a", "*"))); + } + + @Test + public void testCardinalityMapElementKeepsValueOffsetPath() throws Exception { + assertColumn("select cardinality(map_arr_col['a']) from map_array_tbl", + "map>", + ImmutableList.of(path("map_arr_col", "KEYS"), path("map_arr_col", "VALUES", "OFFSET")), + ImmutableList.of()); + } + + @Test + public void testFullFieldAccessStripsExactDataSkippingPath() throws Exception { + assertColumn("select struct_element(s, 'city') from tbl " + + "where struct_element(s, 'city') is null", + "struct", + ImmutableList.of(path("s", "city")), + ImmutableList.of()); + + assertColumn("select cardinality(struct_element(s, 'data')), struct_element(s, 'data') from tbl", + "struct>>>", + ImmutableList.of(path("s", "data")), + ImmutableList.of()); + + assertColumn("select cardinality(a), a from nested_array_tbl", + "array>", + ImmutableList.of(path("a")), + ImmutableList.of()); + + assertColumn("select cardinality(map_arr_col['a']), map_arr_col['a'] from map_array_tbl", + "map>", + ImmutableList.of(path("map_arr_col", "*")), + ImmutableList.of()); + } + + @Test + public void testCardinalityMapElementOffsetCoveredByValueFieldAccess() throws Exception { + Pair> result = collectComplexSlots( + "select struct_element(element_at(element_at(struct_element(s, 'm'), 'null'), 1), 'verified') " + + "from map_array_value_tbl " + + "where cardinality(element_at(struct_element(s, 'm'), 'null')) > 0"); + TreeSet allAccessPaths = new TreeSet<>(); + TreeSet predicateAccessPaths = new TreeSet<>(); + for (SlotDescriptor slotDescriptor : result.second) { + allAccessPaths.addAll(slotDescriptor.getAllAccessPaths()); + predicateAccessPaths.addAll(slotDescriptor.getPredicateAccessPaths()); + } + Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "*", "*", "verified"))); + Assertions.assertFalse(allAccessPaths.contains(path("s", "m", "*", "OFFSET"))); + Assertions.assertFalse(predicateAccessPaths.contains(path("s", "m", "*", "OFFSET"))); + } + + @Test + public void testMapElementArrayNullPathCoveredByValueFieldAccess() throws Exception { + Pair> result = collectComplexSlots( + "select struct_element(element_at(element_at(struct_element(s, 'm'), 'null'), 1), 'verified') " + + "from map_array_value_tbl " + + "where element_at(struct_element(s, 'm'), 'null') is null"); + TreeSet allAccessPaths = new TreeSet<>(); + TreeSet predicateAccessPaths = new TreeSet<>(); + for (SlotDescriptor slotDescriptor : result.second) { + allAccessPaths.addAll(slotDescriptor.getAllAccessPaths()); + predicateAccessPaths.addAll(slotDescriptor.getPredicateAccessPaths()); + } + Assertions.assertTrue(allAccessPaths.contains(path("s", "m", "*", "*", "verified"))); + Assertions.assertFalse(allAccessPaths.contains(path("s", "m", "*", "NULL"))); + Assertions.assertFalse(predicateAccessPaths.contains(path("s", "m", "*", "NULL"))); + } + @Test public void testVariantAccessPath() throws Exception { assertColumn("select v['a']['B'] from variant_tbl", @@ -507,61 +646,91 @@ public void testProject() throws Exception { public void testFilter() throws Throwable { assertColumn("select 100 from tbl where s is not null", "struct>>>", - ImmutableList.of(path("s")), - ImmutableList.of(path("s")) - ); - - assertColumn("select 100 from tbl where if(id = 1, null, s) is not null or element_at(s, 'city') = 'beijing'", - "struct>>>", - ImmutableList.of(path("s")), - ImmutableList.of(path("s")) + ImmutableList.of(path("s", "NULL")), + ImmutableList.of(path("s", "NULL")) ); - assertColumn("select 100 from tbl where element_at(s, 'city') is not null", + // The IF expression itself is not collected as a null-only parent access here; the + // struct_element predicate still lets NCP prune the scan slot to the city field. + assertColumn("select 100 from tbl where if(id = 1, null, s) is not null or struct_element(s, 'city') = 'beijing'", "struct", ImmutableList.of(path("s", "city")), ImmutableList.of(path("s", "city")) ); - assertColumn("select 100 from tbl where element_at(s, 'data') is not null", + assertColumn("select 100 from tbl where struct_element(s, 'city') is not null", + "struct", + ImmutableList.of(path("s", "city", "NULL")), + ImmutableList.of(path("s", "city", "NULL")) + ); + + assertColumn("select 100 from tbl where struct_element(s, 'data') is not null", "struct>>>", - ImmutableList.of(path("s", "data")), - ImmutableList.of(path("s", "data")) + ImmutableList.of(path("s", "data", "NULL")), + ImmutableList.of(path("s", "data", "NULL")) ); assertColumn("select 100 from tbl where element_at(s, 'data')[1] is not null", "struct>>>", - ImmutableList.of(path("s", "data", "*")), - ImmutableList.of(path("s", "data", "*")) + ImmutableList.of(path("s", "data", "*", "NULL")), + ImmutableList.of(path("s", "data", "*", "NULL")) ); assertColumn("select 100 from tbl where map_keys(element_at(s, 'data')[1]) is not null", "struct>>>", - ImmutableList.of(path("s", "data", "*", "KEYS")), - ImmutableList.of(path("s", "data", "*", "KEYS")) + ImmutableList.of(path("s", "data", "*", "NULL")), + ImmutableList.of(path("s", "data", "*", "NULL")) ); assertColumn("select 100 from tbl where map_values(element_at(s, 'data')[1]) is not null", "struct>>>", - ImmutableList.of(path("s", "data", "*", "VALUES")), - ImmutableList.of(path("s", "data", "*", "VALUES")) + ImmutableList.of(path("s", "data", "*", "NULL")), + ImmutableList.of(path("s", "data", "*", "NULL")) ); assertColumn("select 100 from tbl where element_at(map_values(element_at(s, 'data')[1])[1], 'a') is not null", "struct>>>", - ImmutableList.of(path("s", "data", "*", "VALUES", "a")), - ImmutableList.of(path("s", "data", "*", "VALUES", "a")) + ImmutableList.of(path("s", "data", "*", "VALUES", "a", "NULL")), + ImmutableList.of(path("s", "data", "*", "VALUES", "a", "NULL")) ); assertColumn("select 100 from tbl where element_at(s, 'data')[1][1] is not null", "struct>>>", - ImmutableList.of(path("s", "data", "*", "*")), - ImmutableList.of(path("s", "data", "*", "*")) + ImmutableList.of(path("s", "data", "*", "*", "NULL")), + ImmutableList.of(path("s", "data", "*", "*", "NULL")) ); assertColumn("select 100 from tbl where element_at(element_at(s, 'data')[1][1], 'a') is not null", "struct>>>", - ImmutableList.of(path("s", "data", "*", "*", "a")), - ImmutableList.of(path("s", "data", "*", "*", "a")) + ImmutableList.of(path("s", "data", "*", "*", "a", "NULL")), + ImmutableList.of(path("s", "data", "*", "*", "a", "NULL")) ); assertColumn("select 100 from tbl where element_at(element_at(s, 'data')[1][1], 'b') is not null", "struct>>>", - ImmutableList.of(path("s", "data", "*", "*", "b")), - ImmutableList.of(path("s", "data", "*", "*", "b")) + ImmutableList.of(path("s", "data", "*", "*", "b", "NULL")), + ImmutableList.of(path("s", "data", "*", "*", "b", "NULL")) + ); + } + + @Test + public void testMapKeysAndValuesFunctionNullCheckUseParentMapNullPath() throws Exception { + // map_keys/map_values are PropagateNullable functions: the returned array is NULL only + // when the input map is NULL. Their function-level IS NULL predicates must therefore + // read the parent map null map, not the KEYS/VALUES child null maps. + assertColumn("select 100 from str_tbl where map_keys(map_col) is null", + "map", + ImmutableList.of(path("map_col", "NULL")), + ImmutableList.of(path("map_col", "NULL")) + ); + assertColumn("select 100 from str_tbl where map_values(map_col) is null", + "map", + ImmutableList.of(path("map_col", "NULL")), + ImmutableList.of(path("map_col", "NULL")) + ); + + assertColumn("select map_keys(map_col) from str_tbl where map_keys(map_col) is null", + "map", + ImmutableList.of(path("map_col", "KEYS")), + ImmutableList.of() + ); + assertColumn("select map_values(map_col) from str_tbl where map_values(map_col) is null", + "map", + ImmutableList.of(path("map_col", "VALUES")), + ImmutableList.of() ); } @@ -570,19 +739,24 @@ public void testProjectFilter() throws Throwable { assertColumn("select s from tbl where element_at(s, 'city') is not null", "struct>>>", ImmutableList.of(path("s")), - ImmutableList.of(path("s", "city")) + ImmutableList.of() + ); + assertColumn("select s from tbl where struct_element(s, 'city') is null", + "struct>>>", + ImmutableList.of(path("s")), + ImmutableList.of() ); assertColumn("select element_at(s, 'data') from tbl where element_at(s, 'city') is not null", "struct>>>", - ImmutableList.of(path("s", "data"), path("s", "city")), - ImmutableList.of(path("s", "city")) + ImmutableList.of(path("s", "city", "NULL"), path("s", "data")), + ImmutableList.of(path("s", "city", "NULL")) ); assertColumn("select element_at(s, 'data') from tbl where element_at(s, 'city') is not null and element_at(s, 'data') is not null", "struct>>>", - ImmutableList.of(path("s", "data"), path("s", "city")), - ImmutableList.of(path("s", "data"), path("s", "city")) + ImmutableList.of(path("s", "city", "NULL"), path("s", "data")), + ImmutableList.of(path("s", "city", "NULL")) ); } @@ -1234,6 +1408,7 @@ private void assertColumns(String sql, TreeSet actualPredicateAccessPaths = new TreeSet<>(slotDescriptor.getPredicateAccessPaths()); Assertions.assertEquals(expectPredicateAccessPathSet, actualPredicateAccessPaths); + Assertions.assertTrue(actualAllAccessPaths.containsAll(actualPredicateAccessPaths)); Map slotIdToDataTypes = new LinkedHashMap<>(); Consumer assertHasSameType = e -> { @@ -1282,6 +1457,244 @@ private void assertColumns(String sql, } } + @Test + public void testStructIsNullPruning() throws Exception { + // struct column IS NULL → null-only access, emit [s, NULL] path, type stays struct + assertColumn("select 1 from tbl where s is null", + "struct>>>", + ImmutableList.of(path("s", "NULL")), + ImmutableList.of(path("s", "NULL"))); + } + + @Test + public void testStructIsNotNullPruning() throws Exception { + // struct column IS NOT NULL → same null-only access pattern + assertColumn("select 1 from tbl where s is not null", + "struct>>>", + ImmutableList.of(path("s", "NULL")), + ImmutableList.of(path("s", "NULL"))); + } + + @Test + public void testStructIsNullMixedAccess() throws Exception { + // Parent NULL path must be stripped from allPaths when a child path is also required. + // Otherwise BE StructFileColumnIterator sees the parent NULL sub-path first, switches + // the whole struct iterator to NULL_MAP_ONLY, and skips the child iterator. + // predicateAccessPaths drops [s, NULL] too, keeping it a subset of allAccessPaths. + assertColumn("select struct_element(s, 'city') from tbl where s is null", + "struct", + ImmutableList.of(path("s", "city")), + ImmutableList.of()); + + // This shape is closer to the production bug: one predicate needs the parent + // null map, another predicate needs a child null map, and the projection needs + // a different child data path. The parent [s.NULL] cannot remain in allPaths + // with [s.data], so it is also removed from predicate paths; [s.city.NULL] stays + // because it is still present in allPaths. + assertColumn("select struct_element(s, 'data') from tbl " + + "where s is null or struct_element(s, 'city') is null", + "struct>>>", + ImmutableList.of(path("s", "city", "NULL"), path("s", "data")), + ImmutableList.of(path("s", "city", "NULL"))); + } + + @Test + public void testStringLengthPruning() throws Exception { + // ── Case 1: length(str_col) only ─ offset-only optimization applied ────────── + assertStringColumn( + "select length(str_col) from str_tbl", + "str_col", + true, + ImmutableList.of(path("str_col", "OFFSET"))); + + // ── Case 2: length(str_col) + direct projection of str_col ─ suppressed ───── + assertStringColumn( + "select length(str_col), str_col from str_tbl", + "str_col", + false, + ImmutableList.of()); + + // ── Case 3: length(str_col) + substr(str_col, …) ─ suppressed ─────────────── + assertStringColumn( + "select length(str_col), substr(str_col, 2) from str_tbl", + "str_col", + false, + ImmutableList.of()); + + // ── Case 4: length applied to a struct field ─ struct pruned to bigint field ─ + // c_struct has {f1:int, f3:string}; only f3 accessed offset-only → + // pruned type is struct, access path is DATA(["c_struct","f3","offset"]) + assertColumn( + "select length(struct_element(c_struct, 'f3')) from str_tbl", + "struct", + ImmutableList.of(path("c_struct", "f3", "OFFSET")), + ImmutableList.of()); + + // ── Case 5: length(struct field) + direct read of same field ─ suppressed ─── + // Both the full-data path ["c_struct","f3"] and offset path ["c_struct","f3","offset"] + // are recorded; f3 pruneDataType() sees accessAll=true → returns text (not bigint). + assertColumn( + "select length(struct_element(c_struct, 'f3')), struct_element(c_struct, 'f3') from str_tbl", + "struct", + ImmutableList.of(path("c_struct", "f3")), + ImmutableList.of()); + } + + @Test + public void testNonOlapDataSkippingOnlyAccessPathFallback() { + List normalizedAccessPaths = + AccessPathPlanCollector.normalizeDataSkippingOnlyAccessPaths(ImmutableList.of( + new CollectAccessPathResult( + ImmutableList.of("s", "city", "NULL"), true, TAccessPathType.DATA), + new CollectAccessPathResult( + ImmutableList.of("array_column", "OFFSET"), false, TAccessPathType.DATA), + new CollectAccessPathResult( + ImmutableList.of("s", "city"), false, TAccessPathType.DATA))); + + Assertions.assertEquals(3, normalizedAccessPaths.size()); + Assertions.assertEquals(ImmutableList.of("s", "city"), normalizedAccessPaths.get(0).getPath()); + Assertions.assertTrue(normalizedAccessPaths.get(0).isPredicate()); + Assertions.assertEquals(TAccessPathType.DATA, normalizedAccessPaths.get(0).getType()); + Assertions.assertEquals(ImmutableList.of("array_column"), normalizedAccessPaths.get(1).getPath()); + Assertions.assertFalse(normalizedAccessPaths.get(1).isPredicate()); + Assertions.assertEquals(TAccessPathType.DATA, normalizedAccessPaths.get(1).getType()); + Assertions.assertEquals(ImmutableList.of("s", "city"), normalizedAccessPaths.get(2).getPath()); + } + + @Test + public void testMvRewritePlanFragmentSkipsNullOnlyAccessPath() { + SlotReference normalSlot = rewriteAndFindScanSlot( + "select 1 from str_tbl where str_col is not null", "str_col", false); + Assertions.assertEquals( + new TreeSet<>(ImmutableList.of(path("str_col", "NULL"))), + new TreeSet<>(normalSlot.getAllAccessPaths().get())); + Assertions.assertEquals( + new TreeSet<>(ImmutableList.of(path("str_col", "NULL"))), + new TreeSet<>(normalSlot.getPredicateAccessPaths().get())); + + SlotReference fragmentSlot = rewriteAndFindScanSlot( + "select 1 from str_tbl where str_col is not null", "str_col", true); + assertNoAccessPaths(fragmentSlot); + + SlotReference nestedNormalSlot = rewriteAndFindScanSlot( + "select 1 from tbl where struct_element(s, 'city') is not null", "s", false); + Assertions.assertEquals( + new TreeSet<>(ImmutableList.of(path("s", "city", "NULL"))), + new TreeSet<>(nestedNormalSlot.getAllAccessPaths().get())); + Assertions.assertEquals( + new TreeSet<>(ImmutableList.of(path("s", "city", "NULL"))), + new TreeSet<>(nestedNormalSlot.getPredicateAccessPaths().get())); + + // MV rewrite optimizes temporary fragments whose later consumers are not visible. + // If the fragment only needs nested null metadata, e.g. [s.city.NULL], pruning the + // scan slot to struct can break the final rewritten MV plan when it still + // needs the full struct or another child. The fragment marker therefore suppresses + // nested null-only access info too, not just top-level [col.NULL]. + SlotReference nestedFragmentSlot = rewriteAndFindScanSlot( + "select 1 from tbl where struct_element(s, 'city') is not null", "s", true); + assertNoAccessPaths(nestedFragmentSlot); + } + + @Test + public void testMvRewritePlanFragmentSkipsOffsetOnlyAccessPath() { + SlotReference normalSlot = rewriteAndFindScanSlot( + "select 1 from str_tbl where length(str_col) > 0", "str_col", false); + Assertions.assertEquals( + new TreeSet<>(ImmutableList.of(path("str_col", "OFFSET"))), + new TreeSet<>(normalSlot.getAllAccessPaths().get())); + Assertions.assertEquals( + new TreeSet<>(ImmutableList.of(path("str_col", "OFFSET"))), + new TreeSet<>(normalSlot.getPredicateAccessPaths().get())); + + SlotReference fragmentSlot = rewriteAndFindScanSlot( + "select 1 from str_tbl where length(str_col) > 0", "str_col", true); + assertNoAccessPaths(fragmentSlot); + + SlotReference nestedNormalSlot = rewriteAndFindScanSlot( + "select 1 from str_tbl where length(struct_element(c_struct, 'f3')) > 0", + "c_struct", false); + Assertions.assertEquals( + new TreeSet<>(ImmutableList.of(path("c_struct", "f3", "OFFSET"))), + new TreeSet<>(nestedNormalSlot.getAllAccessPaths().get())); + Assertions.assertEquals( + new TreeSet<>(ImmutableList.of(path("c_struct", "f3", "OFFSET"))), + new TreeSet<>(nestedNormalSlot.getPredicateAccessPaths().get())); + + SlotReference nestedFragmentSlot = rewriteAndFindScanSlot( + "select 1 from str_tbl where length(struct_element(c_struct, 'f3')) > 0", + "c_struct", true); + assertNoAccessPaths(nestedFragmentSlot); + } + + /** + * Verify that a specific string-typed column in the rewritten LogicalOlapScan either has + * BigIntType (offset-only optimization applied) or retains its original string type (suppressed). + * + * @param sql query to analyze and rewrite + * @param columnName name of the string column to inspect + * @param expectOptimized true → expect BigIntType + access paths; false → expect string type + * @param expectAllPaths expected access paths when {@code expectOptimized} is true + */ + private void assertStringColumn(String sql, String columnName, + boolean expectOptimized, List expectAllPaths) { + Plan rewritePlan = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .getCascadesContext() + .getRewritePlan(); + + LogicalOlapScan scan = rewritePlan.collect(LogicalOlapScan.class::isInstance) + .stream() + .map(p -> (LogicalOlapScan) p) + .findFirst() + .orElseThrow(() -> new AssertionError("No LogicalOlapScan in plan for: " + sql)); + + for (Slot slot : scan.getOutput()) { + if (!slot.getName().equalsIgnoreCase(columnName)) { + continue; + } + SlotReference slotRef = (SlotReference) slot; + if (expectOptimized) { + Optional> allPaths = slotRef.getAllAccessPaths(); + Assertions.assertTrue(allPaths.isPresent() && !allPaths.get().isEmpty(), + "Slot '" + columnName + "' should have access paths set"); + Assertions.assertEquals( + new TreeSet<>(expectAllPaths), + new TreeSet<>(allPaths.get()), + "Unexpected access paths for slot '" + columnName + "'"); + } else { + Assertions.assertNotEquals(BigIntType.INSTANCE, slotRef.getDataType(), + "Slot '" + columnName + "' should NOT be BigIntType (optimization suppressed)"); + } + return; + } + Assertions.fail("Column '" + columnName + "' not found in LogicalOlapScan output for: " + sql); + } + + private SlotReference rewriteAndFindScanSlot(String sql, String columnName, + boolean materializedViewRewritePlanFragment) { + PlanChecker planChecker = PlanChecker.from(connectContext).analyze(sql); + planChecker.getCascadesContext().setMaterializedViewRewritePlanFragment(materializedViewRewritePlanFragment); + Plan rewritePlan = planChecker.rewrite().getCascadesContext().getRewritePlan(); + LogicalOlapScan scan = rewritePlan.collect(LogicalOlapScan.class::isInstance) + .stream() + .map(p -> (LogicalOlapScan) p) + .findFirst() + .orElseThrow(() -> new AssertionError("No LogicalOlapScan in plan for: " + sql)); + return scan.getOutput().stream() + .filter(slot -> slot.getName().equalsIgnoreCase(columnName)) + .map(slot -> (SlotReference) slot) + .findFirst() + .orElseThrow(() -> new AssertionError("Column '" + columnName + + "' not found in LogicalOlapScan output for: " + sql)); + } + + private void assertNoAccessPaths(SlotReference slot) { + Assertions.assertTrue(!slot.getAllAccessPaths().isPresent() || slot.getAllAccessPaths().get().isEmpty()); + Assertions.assertTrue(!slot.getPredicateAccessPaths().isPresent() + || slot.getPredicateAccessPaths().get().isEmpty()); + } + private Pair> collectComplexSlots(String sql) throws Exception { NereidsPlanner planner = (NereidsPlanner) executeNereidsSql(sql).planner(); List complexSlots = new ArrayList<>(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopNTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopNTest.java new file mode 100644 index 00000000000000..692f978e9f0c11 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectExprUnderTopNTest.java @@ -0,0 +1,1219 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.functions.scalar.AssertTrue; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Score; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalSort; +import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; +import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; +import org.apache.doris.nereids.util.MemoTestUtils; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.nereids.util.PlanConstructor; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +class PullUpProjectExprUnderTopNTest implements MemoPatternMatchSupported { + private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); + + @Test + void testPullUpAddExpression() { + List exprs = ImmutableList.of( + scan1.getOutput().get(0), + new Alias(new Add(scan1.getOutput().get(1), new IntegerLiteral((byte) 1)), "b") + ); + LogicalPlan plan = new LogicalPlanBuilder(scan1) + .projectExprs(exprs) + .topN(3, 0, ImmutableList.of(0)) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .matches( + logicalProject( + logicalTopN( + logicalProject( + logicalOlapScan() + ) + ) + ) + ) + .getPlan(); + } + + @Test + void testNotPullUpSimpleAlias() { + List exprs = ImmutableList.of( + scan1.getOutput().get(0).alias("a"), + scan1.getOutput().get(1) + ); + LogicalPlan plan = new LogicalPlanBuilder(scan1) + .projectExprs(exprs) + .topN(3, 0, ImmutableList.of(0)) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .matches( + logicalTopN( + logicalProject( + logicalOlapScan() + ) + ) + ) + .getPlan(); + } + + @Test + void testNotPullUpScoreExpression() { + List exprs = ImmutableList.of( + new Alias(new Score(), "score"), + scan1.getOutput().get(0) + ); + LogicalPlan plan = new LogicalPlanBuilder(scan1) + .projectExprs(exprs) + .topN(3, 0, ImmutableList.of(1)) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .matchesFromRoot( + logicalTopN( + logicalProject( + logicalOlapScan() + ) + ) + ) + .getPlan(); + } + + @Test + void testBlockedByFilterGoesToInnerTopN() { + // project(y) -> topn1 -> filter(x>1) -> topn2 -> project(x, y) -> scan + // x should stay below topn1 (blocked by filter and topn2 order key), + // y should be pulled up past both topn1 and topn2. + Slot a = scan1.getOutput().get(1); + Slot b = scan1.getOutput().get(0); + + Alias x = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "x"); + Alias y = new Alias(new Add(b, new IntegerLiteral((byte) 1)), "y"); + GreaterThan filter = new GreaterThan(x.toSlot(), new IntegerLiteral((byte) 1)); + + LogicalPlan plan = new LogicalPlanBuilder(scan1) + .projectExprs(ImmutableList.of(x, y)) + .topN(10, 0, ImmutableList.of(0)) + .filter(filter) + .topN(3, 0, ImmutableList.of(0)) + .projectExprs(ImmutableList.of(y.toSlot())) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + // outer pull-up exists (project above topn1) + .matches( + logicalProject( + logicalTopN( + logicalFilter( + logicalProject( + logicalTopN( + logicalProject( + logicalOlapScan() + ) + ) + ) + ) + ) + ) + ) + // inner pull-up also exists (project above topn2) + .matches( + logicalProject( + logicalTopN( + logicalProject( + logicalOlapScan() + ) + ) + ) + ) + .getPlan(); + } + + @Test + void testOuterTopNPullsUnblockedExpressionRestoredByInnerTopN() { + // topn1(order by id) -> filter(x>1) -> topn2(order by id) -> project(id, x, y) -> scan + // The bottom-up rule lets topn2 pull up x and y first. Then topn1 can + // continue pulling y through the Project restored above topn2. x is + // still blocked by the Filter between topn1 and topn2. + Slot id = scan1.getOutput().get(0); + Slot a = scan1.getOutput().get(1); + Slot b = scan1.getOutput().get(0); + + Alias x = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "x"); + Alias y = new Alias(new Add(b, new IntegerLiteral((byte) 1)), "y"); + GreaterThan filter = new GreaterThan(x.toSlot(), new IntegerLiteral((byte) 1)); + + LogicalPlan plan = new LogicalPlanBuilder(scan1) + .projectExprs(ImmutableList.of(id, x, y)) + .topN(10, 0, ImmutableList.of(0)) + .filter(filter) + .topN(3, 0, ImmutableList.of(0)) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .matchesFromRoot( + logicalProject( + logicalTopN( + logicalFilter( + logicalProject( + logicalTopN( + logicalProject( + logicalOlapScan() + ) + ) + ) + ) + ) + ).when(proj -> proj.getProjects().stream() + .anyMatch(e -> "y".equals(e.getName()))) + ) + // x remains restored above topn(10), because the Filter above + // topn(10) consumes x and blocks it from being pulled to topn(3). + .matches( + logicalProject( + logicalTopN( + logicalProject( + logicalOlapScan() + ) + ) + ).when(proj -> proj.getProjects().stream() + .anyMatch(e -> "x".equals(e.getName()))) + ) + .getPlan(); + } + + @Test + void testPullUpThroughJoin() { + // topn -> project(x, y) -> join -> [scan1, scan2] + // x and y should be pulled up above topn. + Slot a = scan1.getOutput().get(1); + Slot b = scan1.getOutput().get(0); + Alias x = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "x"); + Alias y = new Alias(new Add(b, new IntegerLiteral((byte) 1)), "y"); + + LogicalPlan join = new LogicalPlanBuilder(scan1) + .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0)) + .build(); + + LogicalPlan plan = new LogicalPlanBuilder(join) + .projectExprs(ImmutableList.of(x, y)) + .topN(3, 0, ImmutableList.of(0)) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .matches( + logicalProject( + logicalTopN( + logicalProject( + logicalJoin( + logicalOlapScan(), + logicalOlapScan() + ) + ) + ) + ) + ) + .getPlan(); + } + + @Test + void testPullUpThroughForwardedSlotFromLowerProject() { + // This is the minimal trigger pattern: + // TopN -> Project(id, x) -> Project(id, a + 1 as x) -> Scan. + // The lower Project can remove x after pull-up, so the forwarding Project + // must pass through x's input slot a instead of keeping the unavailable x. + Slot id = scan1.getOutput().get(0); + Slot a = scan1.getOutput().get(1); + Alias x = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "x"); + + LogicalProject lowerProject = new LogicalProject<>(ImmutableList.of(id, x), scan1); + LogicalProject> upperProject = new LogicalProject<>( + ImmutableList.of(id, x.toSlot()), lowerProject); + LogicalPlan plan = new LogicalPlanBuilder(upperProject) + .topN(3, 0, ImmutableList.of(0)) + .build(); + + LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .matchesFromRoot( + logicalProject( + logicalTopN( + logicalProject( + logicalProject( + logicalOlapScan() + ) + ) + ) + ) + ) + .getPlan(); + + LogicalProject topProject = (LogicalProject) rewritten; + Assertions.assertEquals(x.getExprId(), topProject.getProjects().get(1).getExprId()); + + LogicalTopN topN = (LogicalTopN) topProject.child(0); + LogicalProject rewrittenForwardingProject = (LogicalProject) topN.child(0); + Assertions.assertTrue(rewrittenForwardingProject.getProjects().stream() + .anyMatch(expr -> expr.getExprId().equals(a.getExprId()))); + Assertions.assertFalse(rewrittenForwardingProject.getProjects().stream() + .anyMatch(expr -> expr.getExprId().equals(x.getExprId()))); + } + + @Test + void testPullUpThroughProjectSlotAboveJoinProject() { + Slot id1 = scan1.getOutput().get(0); + Slot a = scan1.getOutput().get(1); + Slot id2 = scan2.getOutput().get(0); + Alias x = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "x"); + + LogicalPlan join = new LogicalPlanBuilder(scan1) + .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0)) + .build(); + LogicalProject lowerProject = new LogicalProject<>(ImmutableList.of(id1, x, id2), join); + LogicalProject> upperProject = new LogicalProject<>( + ImmutableList.of(id1, x.toSlot(), id2), lowerProject); + LogicalPlan plan = new LogicalPlanBuilder(upperProject) + .topN(3, 0, ImmutableList.of(0)) + .build(); + + LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .matchesFromRoot( + logicalProject( + logicalTopN( + logicalProject( + logicalProject( + logicalJoin( + logicalOlapScan(), + logicalOlapScan() + ) + ) + ) + ) + ) + ) + .getPlan(); + + LogicalProject topProject = (LogicalProject) rewritten; + Assertions.assertEquals(x.getExprId(), topProject.getProjects().get(1).getExprId()); + + LogicalTopN topN = (LogicalTopN) topProject.child(0); + LogicalProject rewrittenUpperProject = (LogicalProject) topN.child(0); + Assertions.assertTrue(rewrittenUpperProject.getProjects().stream() + .anyMatch(expr -> expr.getExprId().equals(a.getExprId()))); + Assertions.assertFalse(rewrittenUpperProject.getProjects().stream() + .anyMatch(expr -> expr.getExprId().equals(x.getExprId()))); + } + + @Test + void testBlockedByJoinCondition() { + // topn -> project(id, x, y) -> join on x = scan2.id -> project(id, x, y) -> scan + // x is referenced by join condition, so it should be blocked. + Slot id = scan1.getOutput().get(0); + Slot a = scan1.getOutput().get(1); + Alias x = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "x"); + Alias y = new Alias(new Add(id, new IntegerLiteral((byte) 1)), "y"); + + EqualTo joinCond = new EqualTo(x.toSlot(), scan2.getOutput().get(0)); + LogicalProject lowerProject = new LogicalProject<>(ImmutableList.of(id, x, y), scan1); + LogicalJoin join = new LogicalJoin<>( + JoinType.INNER_JOIN, + ImmutableList.of(joinCond), + lowerProject, (LogicalPlan) scan2, null); + + LogicalPlan plan = new LogicalPlanBuilder(join) + .projectExprs(ImmutableList.of(id, x.toSlot(), y.toSlot())) + .topN(3, 0, ImmutableList.of(0)) + .build(); + + LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + // y is pulled up, x stays in the bottom project because it's blocked by join condition + .matchesFromRoot( + logicalProject( + logicalTopN( + logicalProject( + logicalJoin( + logicalProject(logicalOlapScan()), + logicalOlapScan() + ) + ) + ) + ) + ) + .getPlan(); + + LogicalProject topProject = (LogicalProject) rewritten; + Assertions.assertEquals(y.getExprId(), topProject.getProjects().get(2).getExprId()); + + LogicalTopN topN = (LogicalTopN) topProject.child(0); + LogicalProject rewrittenProject = (LogicalProject) topN.child(0); + Assertions.assertTrue(rewrittenProject.getProjects().stream() + .anyMatch(expr -> expr.getExprId().equals(x.getExprId()))); + Assertions.assertFalse(rewrittenProject.getProjects().stream() + .anyMatch(expr -> expr.getExprId().equals(y.getExprId()))); + } + + @Test + void testPullUpFromChainedProjects() { + // topn -> project1(y) -> project2(x, y) -> scan + // Both projects should be visited and both x and y pulled up. + Slot a = scan1.getOutput().get(1); + Slot b = scan1.getOutput().get(0); + Alias x = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "x"); + Alias y = new Alias(new Add(b, new IntegerLiteral((byte) 1)), "y"); + + LogicalPlan innerProject = new LogicalProject<>(ImmutableList.of(x, y), scan1); + LogicalPlan outerProject = new LogicalProject<>(ImmutableList.of(x.toSlot(), y.toSlot()), innerProject); + LogicalPlan plan = new LogicalTopN<>( + ImmutableList.of(new OrderKey(scan1.getOutput().get(0), false, false)), + 3, 0, outerProject); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .matches( + logicalProject( + logicalTopN( + logicalProject( + logicalProject( + logicalOlapScan() + ) + ) + ) + ) + ); + } + + @Test + void testBlockedByAggregate() { + // topn -> project(x, y) -> agg -> scan + // Aggregate is a boundary node, so nothing below it is pulled up. + Slot a = scan1.getOutput().get(1); + Slot b = scan1.getOutput().get(0); + Alias x = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "x"); + Alias y = new Alias(new Add(b, new IntegerLiteral((byte) 1)), "y"); + + LogicalPlan plan = new LogicalPlanBuilder(scan1) + .aggGroupUsingIndex(ImmutableList.of(0), ImmutableList.of(x, y)) + .topN(3, 0, ImmutableList.of(0)) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .matches( + logicalTopN( + logicalAggregate( + logicalOlapScan() + ) + ) + ); + } + + @Test + void testBlockedByWindow() { + // topn -> project(x, y) -> window -> scan + // Window is a boundary node, so nothing below it is pulled up. + Slot a = scan1.getOutput().get(1); + Slot b = scan1.getOutput().get(0); + Alias x = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "x"); + Alias y = new Alias(new Add(b, new IntegerLiteral((byte) 1)), "y"); + + List windowExprs = ImmutableList.of( + new Alias(new WindowExpression( + new org.apache.doris.nereids.trees.expressions.functions.window.RowNumber(), + ImmutableList.of(), + ImmutableList.of(new OrderExpression( + new OrderKey(scan1.getOutput().get(0), false, false))) + ), "rn") + ); + LogicalWindow window = new LogicalWindow<>(windowExprs, scan1); + LogicalPlan plan = new LogicalPlanBuilder(window) + .projectExprs(ImmutableList.of(x, y)) + .topN(3, 0, ImmutableList.of(0)) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .matches( + logicalTopN( + logicalProject( + logicalWindow( + logicalOlapScan() + ) + ) + ) + ); + } + + @Test + void testPullUpMultipleExpressions() { + // topn -> project(x, y) -> scan + // Both x and y should be pulled up. + Slot a = scan1.getOutput().get(1); + Slot b = scan1.getOutput().get(0); + Alias x = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "x"); + Alias y = new Alias(new Add(b, new IntegerLiteral((byte) 1)), "y"); + + LogicalPlan plan = new LogicalPlanBuilder(scan1) + .projectExprs(ImmutableList.of(x, y)) + .topN(3, 0, ImmutableList.of(0)) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .matches( + logicalProject( + logicalTopN( + logicalProject( + logicalOlapScan() + ) + ) + ) + ); + } + + @Test + void testPullUpDoesNotExposeInternalPassThroughSlotInUpperProject() { + // This checks the schema boundary restored by the upper Project above TopN. + // The same invariant is required even when this Project is not the query root: + // + // Project(y = x + 1, id) + // TopN(order by id) + // Project(x = a + 1, b, id) + // Scan(a, b, id) + // + // After pulling up x = a + 1, TopN still needs to carry a internally, but the + // upper Project output must stay as the original TopN output [x, b, id] rather + // than leaking internal pass-through slot a as [x, a, b, id]. + LogicalOlapScan scan = new LogicalOlapScan( + PlanConstructor.getNextRelationId(), PlanConstructor.student, ImmutableList.of("db")); + Slot id = scan.getOutput().get(0); + Slot a = scan.getOutput().get(1); + Slot b = scan.getOutput().get(3); + Alias x = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "x"); + + LogicalPlan plan = new LogicalPlanBuilder(scan) + .projectExprs(ImmutableList.of(x, b, id)) + .topN(10, 0, ImmutableList.of(2)) + .build(); + + LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .getPlan(); + + LogicalProject upperProject = (LogicalProject) rewritten; + Assertions.assertEquals(3, upperProject.getProjects().size()); + Assertions.assertEquals(x.getExprId(), upperProject.getProjects().get(0).getExprId()); + Assertions.assertEquals(b.getExprId(), upperProject.getProjects().get(1).getExprId()); + Assertions.assertEquals(id.getExprId(), upperProject.getProjects().get(2).getExprId()); + Assertions.assertFalse(upperProject.getProjects().stream() + .anyMatch(expr -> expr.getExprId().equals(a.getExprId()))); + + LogicalTopN topN = (LogicalTopN) upperProject.child(0); + Assertions.assertTrue(topN.getOutput().stream() + .anyMatch(slot -> slot.getExprId().equals(a.getExprId()))); + } + + @Test + void testRestoreNonPulledSlotsByExprIdAfterPullUp() { + LogicalOlapScan scan = new LogicalOlapScan( + PlanConstructor.getNextRelationId(), PlanConstructor.student, ImmutableList.of("db")); + Slot id = scan.getOutput().get(0); + Slot a = scan.getOutput().get(1); + Slot c = scan.getOutput().get(3); + Alias x = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "x"); + + LogicalPlan plan = new LogicalPlanBuilder(scan) + .projectExprs(ImmutableList.of(id, x, c)) + .topN(3, 0, ImmutableList.of(0)) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .matchesFromRoot( + logicalProject( + logicalTopN( + logicalProject( + logicalOlapScan() + ) + ) + ).when(project -> project.getProjects().size() == 3 + && project.getProjects().get(0).getExprId().equals(id.getExprId()) + && project.getProjects().get(1).getExprId().equals(x.getExprId()) + && project.getProjects().get(2).getExprId().equals(c.getExprId())) + ); + } + + @Test + void testOuterTopNPullUpDoesNotExposePassThroughInputSlots() { + // topn(3) -> filter(x>1) -> topn(10) -> project(x=a+1, y=b+1, id) -> scan + // topn(10) pulls up x and y first. Then topn(3) can pull y farther up, + // while x remains blocked by the Filter. The root Project must still + // expose only the original TopN output, not y's internal input slot b. + LogicalOlapScan scan = new LogicalOlapScan( + PlanConstructor.getNextRelationId(), PlanConstructor.student, ImmutableList.of("db")); + Slot id = scan.getOutput().get(0); + Slot a = scan.getOutput().get(1); + Slot b = scan.getOutput().get(3); + Alias x = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "x"); + Alias y = new Alias(new Add(b, new IntegerLiteral((byte) 1)), "y"); + GreaterThan filter = new GreaterThan(x.toSlot(), new IntegerLiteral((byte) 1)); + + LogicalPlan plan = new LogicalPlanBuilder(scan) + .projectExprs(ImmutableList.of(x, y, id)) + .topN(10, 0, ImmutableList.of(2)) + .filter(filter) + .topN(3, 0, ImmutableList.of(2)) + .build(); + + LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .getPlan(); + + LogicalProject rootProject = (LogicalProject) rewritten; + Assertions.assertEquals(3, rootProject.getProjects().size()); + Assertions.assertEquals(x.getExprId(), rootProject.getProjects().get(0).getExprId()); + Assertions.assertEquals(y.getExprId(), rootProject.getProjects().get(1).getExprId()); + Assertions.assertEquals(id.getExprId(), rootProject.getProjects().get(2).getExprId()); + + LogicalTopN rootTopN = (LogicalTopN) rootProject.child(0); + LogicalFilter midFilter = (LogicalFilter) rootTopN.child(0); + LogicalProject topN10UpperProject = (LogicalProject) midFilter.child(0); + Assertions.assertEquals(3, topN10UpperProject.getProjects().size()); + Assertions.assertEquals(x.getExprId(), topN10UpperProject.getProjects().get(0).getExprId()); + Assertions.assertEquals(id.getExprId(), topN10UpperProject.getProjects().get(1).getExprId()); + Assertions.assertEquals(b.getExprId(), topN10UpperProject.getProjects().get(2).getExprId()); + + LogicalTopN topN10 = (LogicalTopN) topN10UpperProject.child(0); + // b is needed only to restore y above topn(3); it must not leak from + // the root Project output. + Assertions.assertTrue(topN10.getOutput().stream() + .anyMatch(slot -> slot.getExprId().equals(b.getExprId()))); + Assertions.assertFalse(rootProject.getProjects().stream() + .anyMatch(expr -> expr.getExprId().equals(b.getExprId()))); + } + + @Test + void testNestedTopNPullUpPassesThroughTransitiveInputSlots() { + // topn(10) -> topn(20) -> project(y, id) -> topn(30) -> project(x, id) -> scan + // Each TopN handles its own child subtree after lower TopNs have been + // rewritten, so y can be pulled all the way above topn(10). + LogicalOlapScan scan = new LogicalOlapScan( + PlanConstructor.getNextRelationId(), PlanConstructor.student, ImmutableList.of("db")); + Slot id = scan.getOutput().get(0); + Slot a = scan.getOutput().get(1); + Slot b = scan.getOutput().get(3); + Alias x = new Alias(new Add(a, b), "x"); + Alias y = new Alias(new Add(x.toSlot(), new IntegerLiteral((byte) 1)), "y"); + + LogicalPlan plan = new LogicalPlanBuilder(scan) + .projectExprs(ImmutableList.of(x, id)) + .topN(30, 0, ImmutableList.of(1)) + .projectExprs(ImmutableList.of(y, id)) + .topN(20, 0, ImmutableList.of(1)) + .topN(10, 0, ImmutableList.of(1)) + .build(); + + LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .getPlan(); + + LogicalProject rootProject = (LogicalProject) rewritten; + Assertions.assertTrue(rootProject.getProjects().stream() + .anyMatch(e -> "y".equals(e.getName()))); + LogicalTopN topN10 = (LogicalTopN) rootProject.child(0); + Assertions.assertTrue(topN10.getOutput().stream() + .anyMatch(slot -> slot.getExprId().equals(a.getExprId()))); + Assertions.assertTrue(topN10.getOutput().stream() + .anyMatch(slot -> slot.getExprId().equals(b.getExprId()))); + Assertions.assertFalse(topN10.getOutput().stream() + .anyMatch(slot -> slot.getExprId().equals(y.getExprId()))); + } + + @Test + void testNestedTopNPullUpKeepsInputSlotRestoredByLowerTopN() { + // topn(10) -> topn(20) -> project(y, id, x) -> topn(30) -> project(x, id) -> scan + // topn(20) orders by x, so x remains below topn(20). y is not blocked + // by topn(10), so the bottom-up rule can pull y above topn(10). + LogicalOlapScan scan = new LogicalOlapScan( + PlanConstructor.getNextRelationId(), PlanConstructor.student, ImmutableList.of("db")); + Slot id = scan.getOutput().get(0); + Slot a = scan.getOutput().get(1); + Slot b = scan.getOutput().get(3); + Alias x = new Alias(new Add(a, b), "x"); + Alias y = new Alias(new Add(x.toSlot(), new IntegerLiteral((byte) 1)), "y"); + + LogicalPlan plan = new LogicalPlanBuilder(scan) + .projectExprs(ImmutableList.of(x, id)) + .topN(30, 0, ImmutableList.of(1)) + .projectExprs(ImmutableList.of(y, id, x.toSlot())) + .topN(20, 0, ImmutableList.of(2)) + .topN(10, 0, ImmutableList.of(1)) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .matchesFromRoot(logicalProject(logicalTopN().when(t -> t.getLimit() == 10)) + .when(proj -> proj.getProjects().stream() + .anyMatch(e -> "y".equals(e.getName())))) + // topn(30)'s upper project contains x + .matches( + logicalProject( + logicalTopN(logicalProject(logicalOlapScan())) + ).when(proj -> proj.getProjects().stream() + .anyMatch(e -> "x".equals(e.getName()))) + ); + } + + @Test + void testOuterTopNPullsExpressionRestoredByInnerTopNThroughRename() { + // TopN1(order by id) + // Project(id, b AS c) + // TopN2(order by id) + // Project(a + 1 AS b, id) + // Scan(id, a) + // + // The bottom-up rule first pulls b above TopN2. Then TopN1 sees the + // restored Project(a + 1 AS b) and can continue pulling the expression + // through the rename chain c -> b -> a + 1. + Slot id = scan1.getOutput().get(0); + Slot a = scan1.getOutput().get(1); + Alias b = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "b"); + Alias c = new Alias(b.toSlot(), "c"); + + LogicalProject lowerProject = new LogicalProject<>(ImmutableList.of(b, id), scan1); + LogicalTopN> innerTopN = new LogicalTopN<>( + ImmutableList.of(new OrderKey(id, false, false)), + 20, 0, lowerProject); + LogicalProject>> renameProject + = new LogicalProject<>(ImmutableList.of(id, c), innerTopN); + LogicalTopN>>> outerTopN = new LogicalTopN<>( + ImmutableList.of(new OrderKey(id, false, false)), + 10, 0, renameProject); + + LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(MemoTestUtils.createConnectContext(), outerTopN) + .applyCustom(new PullUpProjectExprUnderTopN()) + .getPlan(); + + LogicalProject outerUpperProject = (LogicalProject) rewritten; + Assertions.assertEquals(2, outerUpperProject.getProjects().size()); + Assertions.assertEquals(id.getExprId(), outerUpperProject.getProjects().get(0).getExprId()); + Assertions.assertEquals(c.getExprId(), outerUpperProject.getProjects().get(1).getExprId()); + Assertions.assertTrue(outerUpperProject.getProjects().get(1).getInputSlots().stream() + .anyMatch(slot -> slot.getExprId().equals(a.getExprId()))); + + LogicalTopN rewrittenOuterTopN = (LogicalTopN) outerUpperProject.child(0); + Assertions.assertTrue(rewrittenOuterTopN.getOutput().stream() + .anyMatch(slot -> slot.getExprId().equals(a.getExprId()))); + Assertions.assertFalse(rewrittenOuterTopN.getOutput().stream() + .anyMatch(slot -> slot.getExprId().equals(b.getExprId()))); + Assertions.assertFalse(rewrittenOuterTopN.getOutput().stream() + .anyMatch(slot -> slot.getExprId().equals(c.getExprId()))); + } + + @Test + void testNotPullUpNoneMovableFunction() { + // topn -> project(assert_true(a+1, "msg") as x) -> scan + // NoneMovableFunction should not be pulled up. + Slot a = scan1.getOutput().get(1); + Alias x = new Alias( + new AssertTrue( + new GreaterThan(new Add(a, new IntegerLiteral((byte) 1)), new IntegerLiteral((byte) 0)), + new StringLiteral("msg") + ), + "x" + ); + + LogicalPlan plan = new LogicalPlanBuilder(scan1) + .projectExprs(ImmutableList.of(x)) + .topN(3, 0, ImmutableList.of(0)) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .matches( + logicalTopN( + logicalProject( + logicalOlapScan() + ) + ) + ); + } + + @Test + void testNoneMovableAliasBlocksDependentPullUp() { + // TopN + // Project(z = assert_true(x > 0), id) + // Project(x = a + 1, id) + // Scan + // + // z cannot be synthesized above TopN. Therefore z must block x, so the + // lower Project must keep x instead of pulling x up and making z + // unavailable below TopN. + Slot id = scan1.getOutput().get(0); + Slot a = scan1.getOutput().get(1); + Alias x = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "x"); + Alias z = new Alias( + new AssertTrue( + new GreaterThan(x.toSlot(), new IntegerLiteral((byte) 0)), + new StringLiteral("msg") + ), + "z" + ); + + LogicalProject lowerProject = new LogicalProject<>(ImmutableList.of(x, id), scan1); + LogicalProject> upperProject = new LogicalProject<>( + ImmutableList.of(z, id), lowerProject); + LogicalTopN>> plan = new LogicalTopN<>( + ImmutableList.of(new OrderKey(id, false, false)), 3, 0, upperProject); + + LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .matchesFromRoot( + logicalTopN( + logicalProject( + logicalProject( + logicalOlapScan() + ) + ) + ) + ) + .getPlan(); + + LogicalTopN topN = (LogicalTopN) rewritten; + LogicalProject rewrittenUpperProject = (LogicalProject) topN.child(0); + Assertions.assertTrue(rewrittenUpperProject.getProjects().stream() + .anyMatch(expr -> expr.getExprId().equals(z.getExprId()))); + + LogicalProject rewrittenLowerProject = (LogicalProject) rewrittenUpperProject.child(0); + Assertions.assertTrue(rewrittenLowerProject.getProjects().stream() + .anyMatch(expr -> expr.getExprId().equals(x.getExprId()))); + } + + @Test + void testBlockedBySort() { + // topn -> project(id, x, y) -> sort(by x) -> project(id, x, y) -> scan + // x is used by sort order key, so x is blocked. + // y is not blocked and should be pulled up. + Slot id = scan1.getOutput().get(0); + Slot a = scan1.getOutput().get(1); + Alias x = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "x"); + Alias y = new Alias(new Add(id, new IntegerLiteral((byte) 1)), "y"); + + LogicalProject lowerProject = new LogicalProject<>(ImmutableList.of(id, x, y), scan1); + LogicalSort sort = new LogicalSort<>( + ImmutableList.of(new OrderKey(x.toSlot(), false, false)), + lowerProject); + LogicalPlan plan = new LogicalPlanBuilder(sort) + .projectExprs(ImmutableList.of(id, x.toSlot(), y.toSlot())) + .topN(3, 0, ImmutableList.of(0)) + .build(); + + LogicalPlan rewritten = (LogicalPlan) PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + // y is pulled up, x stays in bottom project due to sort order key + .matchesFromRoot( + logicalProject( + logicalTopN( + logicalProject( + logicalSort( + logicalProject(logicalOlapScan()) + ) + ) + ) + ) + ) + .getPlan(); + + LogicalProject topProject = (LogicalProject) rewritten; + Assertions.assertEquals(y.getExprId(), topProject.getProjects().get(2).getExprId()); + + LogicalTopN topN = (LogicalTopN) topProject.child(0); + LogicalProject rewrittenProject = (LogicalProject) topN.child(0); + Assertions.assertTrue(rewrittenProject.getProjects().stream() + .anyMatch(expr -> expr.getExprId().equals(x.getExprId()))); + Assertions.assertFalse(rewrittenProject.getProjects().stream() + .anyMatch(expr -> expr.getExprId().equals(y.getExprId()))); + } + + @Test + void testSetOperationIsBoundary() { + // topn -> union all -> [project(a+1 as x, a+1 as y) -> scan1, + // project(a+1 as x, a+1 as y) -> scan2] + // Set operations are a boundary: expressions below them are NOT + // collected for the current TopN because UNION ALL children may + // compute the same output column differently. + Slot a = scan1.getOutput().get(1); + Slot b = scan1.getOutput().get(0); + Alias x = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "x"); + Alias y = new Alias(new Add(b, new IntegerLiteral((byte) 1)), "y"); + + LogicalProject project1 = new LogicalProject<>(ImmutableList.of(x, y), scan1); + LogicalProject project2 = new LogicalProject<>(ImmutableList.of(x, y), scan2); + + List outputs = ImmutableList.of(x, y); + List> regularChildrenOutputs + = ImmutableList.of( + ImmutableList.of((org.apache.doris.nereids.trees.expressions.SlotReference) x.toSlot(), + (org.apache.doris.nereids.trees.expressions.SlotReference) y.toSlot()), + ImmutableList.of((org.apache.doris.nereids.trees.expressions.SlotReference) x.toSlot(), + (org.apache.doris.nereids.trees.expressions.SlotReference) y.toSlot()) + ); + LogicalUnion union = new LogicalUnion( + Qualifier.ALL, outputs, regularChildrenOutputs, + ImmutableList.of(), false, + ImmutableList.of(project1, project2)); + + LogicalPlan plan = new LogicalTopN<>( + ImmutableList.of(new OrderKey(scan1.getOutput().get(0), false, false)), + 3, 0, union); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + // Set operation is a boundary: no pull-up, plan unchanged + .matchesFromRoot( + logicalTopN( + logicalUnion( + logicalProject( + logicalOlapScan() + ), + logicalProject( + logicalOlapScan() + ) + ) + ) + ); + } + + @Test + void testUnionAllBoundaryWithDifferentChildExpressions() { + // Reproduce the exact scenario from the review: + // SELECT x, id FROM ( + // SELECT a + 1 AS x, id FROM t1 + // UNION ALL + // SELECT a + 2 AS x, id FROM t2 + // ) u ORDER BY id LIMIT 10 + // + // UNION ALL children compute x differently (a+1 vs a+2). + // A single pull-up Project above TopN cannot represent both + // branch-specific expressions, so the union must be a boundary. + Slot id1 = scan1.getOutput().get(0); + Slot a1 = scan1.getOutput().get(1); + Slot id2 = scan2.getOutput().get(0); + Slot a2 = scan2.getOutput().get(1); + + Alias x1 = new Alias(new Add(a1, new IntegerLiteral((byte) 1)), "x"); + Alias x2 = new Alias(new Add(a2, new IntegerLiteral((byte) 2)), "x"); + Alias y1 = new Alias(new Add(id1, new IntegerLiteral((byte) 1)), "y"); + Alias y2 = new Alias(new Add(id2, new IntegerLiteral((byte) 1)), "y"); + + LogicalProject project1 = new LogicalProject<>(ImmutableList.of(x1, y1), scan1); + LogicalProject project2 = new LogicalProject<>(ImmutableList.of(x2, y2), scan2); + + // Union outputs use x1, y1 as the representative output schema + List outputs = ImmutableList.of(x1, y1); + List> regularChildrenOutputs = ImmutableList.of( + ImmutableList.of((SlotReference) x1.toSlot(), (SlotReference) y1.toSlot()), + ImmutableList.of((SlotReference) x2.toSlot(), (SlotReference) y2.toSlot()) + ); + LogicalUnion union = new LogicalUnion( + Qualifier.ALL, outputs, regularChildrenOutputs, + ImmutableList.of(), false, + ImmutableList.of(project1, project2)); + + LogicalPlan plan = new LogicalTopN<>( + ImmutableList.of(new OrderKey(id1, false, false)), + 10, 0, union); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + // Union is a boundary even though x, y are not order keys: + // plan should be unchanged — no project above topn + .matchesFromRoot( + logicalTopN( + logicalUnion( + logicalProject(logicalOlapScan()), + logicalProject(logicalOlapScan()) + ) + ) + ); + } + + @Test + void testNestedTopNInsideUnionAllIsHandledIndependently() { + // topn(outer, order by id limit 10) -> union all + // -> topn(inner, order by id limit 3) -> project(a+1 as z, id) -> scan1 + // -> project(a+2 as x, id) -> scan2 + // + // The outer TopN stops at the union boundary — it does NOT pull up + // expressions from inside union children. + // The inner TopN independently pulls up z above itself. + Slot id1 = scan1.getOutput().get(0); + Slot a1 = scan1.getOutput().get(1); + Slot id2 = scan2.getOutput().get(0); + Slot a2 = scan2.getOutput().get(1); + + // Below inner TopN: project(a1+1 as z, id1) + Alias z = new Alias(new Add(a1, new IntegerLiteral((byte) 1)), "z"); + LogicalProject projectBelow + = new LogicalProject<>(ImmutableList.of(z, id1), scan1); + + // Inner TopN: order by id1 limit 3 + LogicalTopN> innerTopN = new LogicalTopN<>( + ImmutableList.of(new OrderKey(id1, false, false)), + 3, 0, projectBelow); + + // child1: project(a2+2 as x, id2) + Alias x = new Alias(new Add(a2, new IntegerLiteral((byte) 2)), "x"); + LogicalProject project2 + = new LogicalProject<>(ImmutableList.of(x, id2), scan2); + + // Union outputs: use z and id1 as representative output schema + List outputs = ImmutableList.of(z, id1); + List> regularChildrenOutputs = ImmutableList.of( + ImmutableList.of((SlotReference) z.toSlot(), (SlotReference) id1), + ImmutableList.of((SlotReference) x.toSlot(), (SlotReference) id2) + ); + LogicalUnion union = new LogicalUnion( + Qualifier.ALL, outputs, regularChildrenOutputs, + ImmutableList.of(), false, + ImmutableList.of(innerTopN, project2)); + + // Outer TopN: order by id1 limit 10 + LogicalTopN outerTopN = new LogicalTopN<>( + ImmutableList.of(new OrderKey(id1, false, false)), + 10, 0, union); + + PlanChecker.from(MemoTestUtils.createConnectContext(), outerTopN) + .applyCustom(new PullUpProjectExprUnderTopN()) + // Outer TopN: no pull-up (union is boundary) + .matchesFromRoot( + logicalTopN( + logicalUnion( + logicalProject(logicalTopN(logicalProject(logicalOlapScan()))), + logicalProject(logicalOlapScan()) + ) + ) + ) + // Inner TopN: independently pulls up z above itself + .matches( + logicalProject( + logicalTopN( + logicalProject(logicalOlapScan()) + ) + ) + ); + } + + @Test + void testNestedTopNPullUpWithFilterBlocking() { + // Each TopN handles its own child subtree after lower TopNs have been + // rewritten. topn(10) restores x and y first; then topn(3) can pull y + // farther up, while x remains blocked by the Filter. + // + // Plan: topn(3) -> filter(x>1) -> topn(10) -> project(id, x, y) -> scan + Slot id = scan1.getOutput().get(0); + Slot a = scan1.getOutput().get(1); + Slot b = scan1.getOutput().get(0); // b == id, so y = id + 1 + + Alias x = new Alias(new Add(a, new IntegerLiteral((byte) 1)), "x"); + Alias y = new Alias(new Add(b, new IntegerLiteral((byte) 1)), "y"); + GreaterThan filter = new GreaterThan(x.toSlot(), new IntegerLiteral((byte) 1)); + + LogicalPlan plan = new LogicalPlanBuilder(scan1) + .projectExprs(ImmutableList.of(id, x, y)) + .topN(10, 0, ImmutableList.of(0)) + .filter(filter) + .topN(3, 0, ImmutableList.of(0)) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .matchesFromRoot( + logicalProject( + logicalTopN( + logicalFilter( + logicalProject( + logicalTopN( + logicalProject( + logicalOlapScan() + ) + ) + ) + ) + ) + ).when(proj -> proj.getProjects().stream() + .anyMatch(e -> "y".equals(e.getName()))) + ) + // Inner project above topn(10) must still contain x because the + // Filter consumes it. + .matches( + logicalProject( + logicalTopN( + logicalProject(logicalOlapScan()) + ) + ).when(proj -> proj.getProjects().stream() + .anyMatch(e -> "x".equals(e.getName()))) + ); + } + + /** + * Regression test for correlated scalar subquery + LEFT OUTER JOIN + nested TopN. + * When a LIMIT is pushed down to the left side of a LEFT JOIN, a nested TopN is + * created. The inner TopN's collector must inherit the outer blocked slots + * (from join conditions) so that pass-through aggregate slots like AVG are not + * incorrectly removed during project simplification. + */ + @Test + void testCorrelatedSubqueryWithNestedTopN() { + // Simulate: OuterTopN → Project[C1] → LeftOuterJoin → [ + // InnerTopN → Project[AVG, elem_at, C1] → Join → [Project[elem_at] → Scan1, Scan3], + // Scan2] + // (Simplified: use Scan3 instead of Aggregate to avoid memo duplication) + Slot id1 = scan1.getOutput().get(0); + Slot col1 = scan1.getOutput().get(1); + // x = pull-up eligible expression (simulates element_at(var, 'col')) + Alias x = new Alias(new Add(col1, new IntegerLiteral((byte) 1)), "x"); + + LogicalOlapScan scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); + Slot avgSlot = scan3.getOutput().get(0); // simulates AVG result from correlated subquery + + // Inner left: Project[elem_at=x, C1=id1] → Scan1 + LogicalPlan innerLeft = new LogicalPlanBuilder(scan1) + .projectExprs(ImmutableList.of(x, id1)) + .build(); + + // Inner Join (simulates Apply decomposition): [Project] JOIN Scan3 + LogicalPlan innerJoin = new LogicalPlanBuilder(innerLeft) + .join(scan3, JoinType.INNER_JOIN, Pair.of(1, 0)) + .build(); + + // Project[AVG=avgSlot, elem_at=x, C1=id1] above inner join, then InnerTopN + LogicalPlan innerTopN = new LogicalPlanBuilder(innerJoin) + .projectExprs(ImmutableList.of(avgSlot, x, id1)) + .topN(1, 0, ImmutableList.of(2)) + .build(); + + // Outer: LeftOuterJoin between [InnerTopN side] and [Scan2] + LogicalPlan outerJoin = new LogicalPlanBuilder(innerTopN) + .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(2, 0)) + .build(); + + // OuterTopN → Project[C1] + LogicalPlan plan = new LogicalPlanBuilder(outerJoin) + .projectExprs(ImmutableList.of(id1.alias("C1"))) + .topN(1, 0, ImmutableList.of(0)) + .build(); + + // Must not crash with "Original slot ... should be restored or passed through" + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .getPlan(); + } + + /** + * Expressions on the nullable side of an outer join should NOT be pulled up. + * The nullable side is protected by join null-extension: unmatched rows get + * NULL for all nullable-side columns. Pulling an expression above the join + * would break this — e.g. ifnull(r.b, 0) would turn NULLs into 0s. + * + * Plan: TopN → Project[l.id, x] → LEFT JOIN → [Scan(l), Project[x = r.b+1] → Scan(r)] + * x is on the nullable (right) side → must stay below TopN. + */ + @Test + void testBlockedByNullableSideOfOuterJoin() { + LogicalOlapScan scanL = PlanConstructor.newLogicalOlapScan(0, "l", 0); + LogicalOlapScan scanR = PlanConstructor.newLogicalOlapScan(1, "r", 0); + Slot lId = scanL.getOutput().get(0); + Slot rId = scanR.getOutput().get(0); + Slot rB = scanR.getOutput().get(1); + Alias x = new Alias(new Add(rB, new IntegerLiteral((byte) 1)), "x"); + + // Right side Project [x = r.b+1, r.id] → Scan(r) + LogicalPlan rightSide = new LogicalPlanBuilder(scanR) + .projectExprs(ImmutableList.of(x, rId)) + .build(); + + // LEFT JOIN between Scan(l) and the right-side Project + LogicalPlan join = new LogicalPlanBuilder(scanL) + .join(rightSide, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 1)) + .build(); + + // Project [l.id, x] above the join, then TopN + LogicalPlan plan = new LogicalPlanBuilder(join) + .projectExprs(ImmutableList.of(lId, x.toSlot())) + .topN(3, 0, ImmutableList.of(0)) + .build(); + + // x is on the nullable side — must NOT be pulled up above TopN + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyCustom(new PullUpProjectExprUnderTopN()) + .matches( + logicalTopN( + logicalProject( + logicalJoin() + ) + ) + ) + .nonMatch( + logicalProject( + logicalTopN() + ).when(proj -> proj.getProjects().stream() + .anyMatch(e -> "x".equals(e.getName()))) + ); + } +} diff --git a/regression-test/data/nereids_p0/eliminate_outer_join/eliminate_outer_join.out b/regression-test/data/nereids_p0/eliminate_outer_join/eliminate_outer_join.out index 235247efe8494d..2449cf55b94d5f 100644 --- a/regression-test/data/nereids_p0/eliminate_outer_join/eliminate_outer_join.out +++ b/regression-test/data/nereids_p0/eliminate_outer_join/eliminate_outer_join.out @@ -54,7 +54,7 @@ SyntaxError: -- !4 -- PhysicalResultSink --hashJoin[LEFT_OUTER_JOIN] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=() -----filter(( not r_name IS NULL) and (region.r_name = '')) +----filter(( not r_name IS NULL) and (length(r_name) = 0)) ------PhysicalOlapScan[region] ----PhysicalOlapScan[nation] @@ -67,7 +67,7 @@ SyntaxError: PhysicalResultSink --hashJoin[LEFT_OUTER_JOIN] hashCondition=((nation.n_nationkey = supplier.s_suppkey)) otherCondition=() ----hashJoin[LEFT_OUTER_JOIN] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=() -------filter(( not r_name IS NULL) and (region.r_name = '')) +------filter(( not r_name IS NULL) and (length(r_name) = 0)) --------PhysicalOlapScan[region] ------PhysicalOlapScan[nation] ----PhysicalOlapScan[supplier] @@ -82,7 +82,7 @@ PhysicalResultSink --hashJoin[LEFT_OUTER_JOIN] hashCondition=((partsupp.ps_suppkey = supplier.s_suppkey)) otherCondition=() ----hashJoin[LEFT_OUTER_JOIN] hashCondition=((nation.n_nationkey = supplier.s_suppkey)) otherCondition=() ------hashJoin[LEFT_OUTER_JOIN] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=() ---------filter(( not r_name IS NULL) and (region.r_name = '')) +--------filter(( not r_name IS NULL) and (length(r_name) = 0)) ----------PhysicalOlapScan[region] --------PhysicalOlapScan[nation] ------PhysicalOlapScan[supplier] @@ -115,7 +115,7 @@ PhysicalResultSink --hashJoin[LEFT_OUTER_JOIN] hashCondition=((partsupp.ps_suppkey = supplier.s_suppkey)) otherCondition=() ----hashJoin[LEFT_OUTER_JOIN] hashCondition=((nation.n_nationkey = supplier.s_suppkey)) otherCondition=() ------hashJoin[INNER_JOIN] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=() ---------filter(( not r_name IS NULL) and ( not r_regionkey IS NULL) and (region.r_name = '')) +--------filter(( not r_name IS NULL) and ( not r_regionkey IS NULL) and (length(r_name) = 0)) ----------PhysicalOlapScan[region] --------filter(( not n_regionkey IS NULL)) ----------PhysicalOlapScan[nation] @@ -132,7 +132,7 @@ PhysicalResultSink --hashJoin[LEFT_OUTER_JOIN] hashCondition=((partsupp.ps_suppkey = supplier.s_suppkey)) otherCondition=() ----hashJoin[INNER_JOIN] hashCondition=((nation.n_nationkey = supplier.s_suppkey)) otherCondition=() ------hashJoin[INNER_JOIN] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=() ---------filter(( not r_name IS NULL) and ( not r_regionkey IS NULL) and (region.r_name = '')) +--------filter(( not r_name IS NULL) and ( not r_regionkey IS NULL) and (length(r_name) = 0)) ----------PhysicalOlapScan[region] --------filter(( not n_regionkey IS NULL)) ----------PhysicalOlapScan[nation] diff --git a/regression-test/data/nereids_rules_p0/column_pruning/nested_container_offset_pruning.out b/regression-test/data/nereids_rules_p0/column_pruning/nested_container_offset_pruning.out new file mode 100644 index 00000000000000..8b2d33fd6e3e32 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/column_pruning/nested_container_offset_pruning.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !struct_root_arr_mixed -- +1 2 10 + +-- !struct_root_map_mixed -- +1 1 x + diff --git a/regression-test/data/nereids_rules_p0/column_pruning/null_column_pruning.out b/regression-test/data/nereids_rules_p0/column_pruning/null_column_pruning.out new file mode 100644 index 00000000000000..5e18f989b1a4f7 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/column_pruning/null_column_pruning.out @@ -0,0 +1,107 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !1 -- + +-- !2 -- + +-- !string_full_access_strips_null -- + +-- !3 -- +1 + +-- !4 -- +1 + +-- !5 -- +0 + +-- !6 -- + +-- !array_full_access_strips_null -- + +-- !7 -- + +-- !map_full_access_strips_null -- + +-- !8 -- + +-- !9 -- +1 + +-- !10 -- + +-- !11 -- + +-- !parent_null_with_child_data -- +10001 + +-- !12 -- + +-- !13 -- +\N + +-- !14 -- +0 + +-- !15 -- +1 + +-- !16 -- +0 + +-- !17 -- +1 + +-- !map_values_parent_null_semantics -- +1 + +-- !map_values_parent_not_null_semantics -- +2 +3 + +-- !18 -- +0 + +-- !19 -- +1 + +-- !20 -- +0 + +-- !21 -- +1 + +-- !22 -- +1 + +-- !23 -- +0 + +-- !24 -- +0 + +-- !25 -- + +-- !26 -- + +-- !27 -- +0 + +-- !28 -- +0 + +-- !29 -- +0 + +-- !30 -- +0 + +-- !31 -- +0 + +-- !32 -- +0 + +-- !33 -- + +-- !34 -- + diff --git a/regression-test/data/nereids_rules_p0/column_pruning/string_length_column_pruning.out b/regression-test/data/nereids_rules_p0/column_pruning/string_length_column_pruning.out new file mode 100644 index 00000000000000..c1c825e9619e3a --- /dev/null +++ b/regression-test/data/nereids_rules_p0/column_pruning/string_length_column_pruning.out @@ -0,0 +1,18 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !array_full_access_strips_offset -- +1 3 [1, 2, 3] + +-- !map_element_full_access_strips_offset -- +1 2 [1, 2] + +-- !map_value_array_predicate_offset_covered -- +true + +-- !arr_struct_mixed -- +1 true 10 +2 true 30 +3 \N \N + +-- !map_element_with_map_values -- +1 x + diff --git a/regression-test/data/nereids_rules_p0/column_pruning/topn_expr_pullup.out b/regression-test/data/nereids_rules_p0/column_pruning/topn_expr_pullup.out new file mode 100644 index 00000000000000..d419f9a2b3013f --- /dev/null +++ b/regression-test/data/nereids_rules_p0/column_pruning/topn_expr_pullup.out @@ -0,0 +1,76 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !struct_ppd -- +1 NYC +2 LA +3 SF + +-- !struct_col_order -- +NYC 1 +LA 2 +SF 3 + +-- !upper_str -- +1 HELLO +2 WORLD +3 DORIS + +-- !math_expr -- +1 11 +2 21 +3 31 + +-- !concat_expr -- +1 hello-suffix +2 world-suffix +3 doris-suffix + +-- !map_subscript -- +1 1 +2 \N +3 \N + +-- !array_subscript -- +1 1 +2 4 +3 6 + +-- !simple_alias -- +1 NYC +2 LA +3 SF + +-- !order_by_expr -- +2 LA +1 NYC +3 SF + +-- !select_star -- +1 hello {"city":"NYC", "zip":10001} [1, 2, 3] {"a":1, "b":2} 10 NYC +2 world {"city":"LA", "zip":90001} [4, 5] {"x":3} 20 LA +3 doris {"city":"SF", "zip":94101} [6] {"y":5} 30 SF + +-- !switch_off -- +1 NYC +2 LA +3 SF + +-- !join_ppd -- +1 NYC 10 +2 LA 20 +3 SF 30 + +-- !join_nonppd -- +1 HELLO +2 WORLD +3 DORIS + +-- !join_both -- +1 NYC HELLO +2 LA WORLD +3 SF DORIS + +-- !join_base_slot -- +1 NYC +2 LA +3 SF + diff --git a/regression-test/data/nereids_rules_p0/column_pruning/topn_lazy_nested_column_pruning.out b/regression-test/data/nereids_rules_p0/column_pruning/topn_lazy_nested_column_pruning.out new file mode 100644 index 00000000000000..ef64dd77ea22d7 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/column_pruning/topn_lazy_nested_column_pruning.out @@ -0,0 +1,39 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !struct_result -- +1 \N + +-- !variant_result -- +1 aaa 张三 +2 bbb 李四 +3 ccc 王五 + +-- !map_result -- +1 1 + +-- !array_result -- +1 1 + +-- !variant_nested_result -- +4 上海 +5 北京 + +-- !map_using_index_result -- +1 1 + +-- !array_using_index_result -- +1 1 + +-- !struct_col_order_result -- +\N 1 + +-- !variant_col_order_result -- +张三 1 aaa +李四 2 bbb +王五 3 ccc + +-- !map_col_order_result -- +1 1 + +-- !project_under_topn_consumed_slot -- +1 hello {"city":null, "zip":10001} [1, 2, 3] {"a":1, "b":2} 1 \N + diff --git a/regression-test/data/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.out b/regression-test/data/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.out new file mode 100644 index 00000000000000..ddc91a442d2c39 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.out @@ -0,0 +1,6 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !two_phase_sort -- +1 10 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +2 20 cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd +3 30 eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff + diff --git a/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out b/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out index 9b1274a93a4b7d..f1acddd25757b5 100644 --- a/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out +++ b/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out @@ -49,7 +49,7 @@ PhysicalResultSink -- !filter_join_inner -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() -----filter((t1.msg = '')) +----filter((length(msg) = 0)) ------PhysicalOlapScan[t1] ----PhysicalOlapScan[t2] @@ -319,9 +319,9 @@ PhysicalResultSink PhysicalResultSink --PhysicalExcept ----filter((id = 2)) -------filter((t1.msg = '')) +------filter((length(msg) = 0)) --------PhysicalOlapScan[t1] -----filter((t2.id = 2) and (t2.msg = '')) +----filter((length(msg) = 0) and (t2.id = 2)) ------PhysicalOlapScan[t2] -- !push_filter_except -- @@ -366,6 +366,6 @@ PhysicalResultSink ----PhysicalQuickSort[LOCAL_SORT] ------PhysicalWindow --------PhysicalQuickSort[LOCAL_SORT] -----------filter(OR[(t1.msg = ''),(t1.id = 2)]) +----------filter(OR[(length(msg) = 0),(t1.id = 2)]) ------------PhysicalOlapScan[t1] diff --git a/regression-test/data/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.out b/regression-test/data/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.out index 874bfea0ea0b1a..439a81599ce7dd 100644 --- a/regression-test/data/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.out +++ b/regression-test/data/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.out @@ -4,8 +4,20 @@ 2 o mi 4 o yy +-- !query2_0_before -- +1 \N yy +1 o mm +2 o mi +4 o yy + -- !query1_0_after -- 1 o mm 2 o mi 4 o yy +-- !query2_0_after -- +1 \N yy +1 o mm +2 o mi +4 o yy + diff --git a/regression-test/data/shape_check/clickbench/query13.out b/regression-test/data/shape_check/clickbench/query13.out index ce6675dc3bb26e..0856fc73a84030 100644 --- a/regression-test/data/shape_check/clickbench/query13.out +++ b/regression-test/data/shape_check/clickbench/query13.out @@ -8,6 +8,6 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(( not (SearchPhrase = ''))) +----------------filter(( not (length(SearchPhrase) = 0))) ------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query15.out b/regression-test/data/shape_check/clickbench/query15.out index bf7f267f0e47be..a2eb22e6f2b532 100644 --- a/regression-test/data/shape_check/clickbench/query15.out +++ b/regression-test/data/shape_check/clickbench/query15.out @@ -8,6 +8,6 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(( not (SearchPhrase = ''))) +----------------filter(( not (length(SearchPhrase) = 0))) ------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query22.out b/regression-test/data/shape_check/clickbench/query22.out index d5274c3548eb28..02b2d803778be1 100644 --- a/regression-test/data/shape_check/clickbench/query22.out +++ b/regression-test/data/shape_check/clickbench/query22.out @@ -8,6 +8,6 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(( not (SearchPhrase = '')) and (URL like '%google%')) +----------------filter(( not (length(SearchPhrase) = 0)) and (URL like '%google%')) ------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query25.out b/regression-test/data/shape_check/clickbench/query25.out index 271149db672442..436a325ed91d88 100644 --- a/regression-test/data/shape_check/clickbench/query25.out +++ b/regression-test/data/shape_check/clickbench/query25.out @@ -6,6 +6,6 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------filter(( not (SearchPhrase = ''))) +------------filter(( not (length(SearchPhrase) = 0))) --------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query26.out b/regression-test/data/shape_check/clickbench/query26.out index 7317f810a3bb23..373df683ec473e 100644 --- a/regression-test/data/shape_check/clickbench/query26.out +++ b/regression-test/data/shape_check/clickbench/query26.out @@ -5,6 +5,6 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------filter(( not (SearchPhrase = ''))) +----------filter(( not (length(SearchPhrase) = 0))) ------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query27.out b/regression-test/data/shape_check/clickbench/query27.out index 1dbae1e0dc1a8c..e551d03fc48b41 100644 --- a/regression-test/data/shape_check/clickbench/query27.out +++ b/regression-test/data/shape_check/clickbench/query27.out @@ -6,6 +6,6 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------filter(( not (SearchPhrase = ''))) +------------filter(( not (length(SearchPhrase) = 0))) --------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query28.out b/regression-test/data/shape_check/clickbench/query28.out index e5cb28eab7aa2b..7b07ad4defb38d 100644 --- a/regression-test/data/shape_check/clickbench/query28.out +++ b/regression-test/data/shape_check/clickbench/query28.out @@ -9,6 +9,6 @@ PhysicalResultSink ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------filter(( not (URL = ''))) +------------------filter(( not (length(URL) = 0))) --------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query29.out b/regression-test/data/shape_check/clickbench/query29.out index 01e642b5b4339f..d1ed207fcdb070 100644 --- a/regression-test/data/shape_check/clickbench/query29.out +++ b/regression-test/data/shape_check/clickbench/query29.out @@ -9,6 +9,6 @@ PhysicalResultSink ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------filter(( not (Referer = ''))) +------------------filter(( not (length(Referer) = 0))) --------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query31.out b/regression-test/data/shape_check/clickbench/query31.out index a662fac4ef4581..fc6a57d2e1ec1e 100644 --- a/regression-test/data/shape_check/clickbench/query31.out +++ b/regression-test/data/shape_check/clickbench/query31.out @@ -8,6 +8,6 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(( not (SearchPhrase = ''))) +----------------filter(( not (length(SearchPhrase) = 0))) ------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query32.out b/regression-test/data/shape_check/clickbench/query32.out index 29828472ccab40..60bbde67814354 100644 --- a/regression-test/data/shape_check/clickbench/query32.out +++ b/regression-test/data/shape_check/clickbench/query32.out @@ -8,6 +8,6 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(( not (SearchPhrase = ''))) +----------------filter(( not (length(SearchPhrase) = 0))) ------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query36.out b/regression-test/data/shape_check/clickbench/query36.out index 2d49c7645c7528..5000e553f81e34 100644 --- a/regression-test/data/shape_check/clickbench/query36.out +++ b/regression-test/data/shape_check/clickbench/query36.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ckbench_shape_36 -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/clickbench/query37.out b/regression-test/data/shape_check/clickbench/query37.out index 757b4f64df4e98..58405571e1386a 100644 --- a/regression-test/data/shape_check/clickbench/query37.out +++ b/regression-test/data/shape_check/clickbench/query37.out @@ -8,6 +8,6 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(( not (URL = '')) and (hits.CounterID = 62) and (hits.DontCountHits = 0) and (hits.EventDate <= '2013-07-31') and (hits.EventDate >= '2013-07-01') and (hits.IsRefresh = 0)) +----------------filter(( not (length(URL) = 0)) and (hits.CounterID = 62) and (hits.DontCountHits = 0) and (hits.EventDate <= '2013-07-31') and (hits.EventDate >= '2013-07-01') and (hits.IsRefresh = 0)) ------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query38.out b/regression-test/data/shape_check/clickbench/query38.out index 37d0392a7b2490..a7bb448d79997c 100644 --- a/regression-test/data/shape_check/clickbench/query38.out +++ b/regression-test/data/shape_check/clickbench/query38.out @@ -8,6 +8,6 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(( not (Title = '')) and (hits.CounterID = 62) and (hits.DontCountHits = 0) and (hits.EventDate <= '2013-07-31') and (hits.EventDate >= '2013-07-01') and (hits.IsRefresh = 0)) +----------------filter(( not (length(Title) = 0)) and (hits.CounterID = 62) and (hits.DontCountHits = 0) and (hits.EventDate <= '2013-07-31') and (hits.EventDate >= '2013-07-01') and (hits.IsRefresh = 0)) ------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query17.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query17.out index c10cc616923d3c..8d000d6a01fe54 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query17.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query17.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_17 -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query54.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query54.out index 84b5754b67aedb..5d5db9eea808ff 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query54.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query54.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_54 -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query61.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query61.out index 62da8c9cb21a0f..c2704b1cdaeb04 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query61.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query61.out @@ -1,8 +1,8 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_61 -- PhysicalResultSink ---PhysicalTopN[GATHER_SORT] -----PhysicalProject +--PhysicalProject +----PhysicalTopN[GATHER_SORT] ------NestedLoopJoin[CROSS_JOIN] --------hashAgg[GLOBAL] ----------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query17.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query17.out index 52da90d84ff3a8..1a5abfa9ed9bc1 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query17.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query17.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_17 -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query54.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query54.out index a2a326baf8f39a..26c56d73b844a8 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query54.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query54.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_54 -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query61.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query61.out index 62da8c9cb21a0f..c2704b1cdaeb04 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query61.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query61.out @@ -1,8 +1,8 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_61 -- PhysicalResultSink ---PhysicalTopN[GATHER_SORT] -----PhysicalProject +--PhysicalProject +----PhysicalTopN[GATHER_SORT] ------NestedLoopJoin[CROSS_JOIN] --------hashAgg[GLOBAL] ----------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query17.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query17.out index 5342955a97aae2..7440bd6c7cdea1 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query17.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query17.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_17 -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query54.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query54.out index ad8a262d79400b..d9dcfc5a922c23 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query54.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query54.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_54 -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query61.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query61.out index ec1819093f180e..162749d3f1cfe9 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query61.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query61.out @@ -1,8 +1,8 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_61 -- PhysicalResultSink ---PhysicalTopN[GATHER_SORT] -----PhysicalProject +--PhysicalProject +----PhysicalTopN[GATHER_SORT] ------NestedLoopJoin[CROSS_JOIN] --------hashAgg[GLOBAL] ----------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query17.out b/regression-test/data/shape_check/tpcds_sf100/shape/query17.out index 7cc4a196c206c3..c7ff6c3f8c19e4 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query17.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query17.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_17 -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query54.out b/regression-test/data/shape_check/tpcds_sf100/shape/query54.out index ad8a262d79400b..d9dcfc5a922c23 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query54.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query54.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_54 -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query61.out b/regression-test/data/shape_check/tpcds_sf100/shape/query61.out index ec1819093f180e..162749d3f1cfe9 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query61.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query61.out @@ -1,8 +1,8 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_61 -- PhysicalResultSink ---PhysicalTopN[GATHER_SORT] -----PhysicalProject +--PhysicalProject +----PhysicalTopN[GATHER_SORT] ------NestedLoopJoin[CROSS_JOIN] --------hashAgg[GLOBAL] ----------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.out b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.out index b7c2fe2adcd5b6..bb40006f559770 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_54 -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query61.out b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query61.out index 654f58923a6f2b..583e9c456e0b02 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query61.out +++ b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query61.out @@ -1,8 +1,8 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_61 -- PhysicalResultSink ---PhysicalTopN[GATHER_SORT] -----PhysicalProject +--PhysicalProject +----PhysicalTopN[GATHER_SORT] ------NestedLoopJoin[CROSS_JOIN] --------hashAgg[GLOBAL] ----------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query17.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query17.out index 72003f9609ac64..8e2041bc77ce64 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query17.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query17.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_17 -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query54.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query54.out index ddc3b69f35f222..7c843f5fbe9a40 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query54.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_54 -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query61.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query61.out index 9dac6305cf3b8a..1e05e576021561 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query61.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query61.out @@ -1,8 +1,8 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_61 -- PhysicalResultSink ---PhysicalTopN[GATHER_SORT] -----PhysicalProject +--PhysicalProject +----PhysicalTopN[GATHER_SORT] ------NestedLoopJoin[CROSS_JOIN] --------PhysicalProject ----------hashAgg[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query17.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query17.out index 12fa11701b619f..ab7309cd69eb4d 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query17.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query17.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_17 -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query54.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query54.out index 62d473efd02e89..3fea1ccba3bd58 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query54.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_54 -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query61.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query61.out index 654f58923a6f2b..583e9c456e0b02 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query61.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query61.out @@ -1,8 +1,8 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_61 -- PhysicalResultSink ---PhysicalTopN[GATHER_SORT] -----PhysicalProject +--PhysicalProject +----PhysicalTopN[GATHER_SORT] ------NestedLoopJoin[CROSS_JOIN] --------hashAgg[GLOBAL] ----------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/shape_check/tpcds_sf1000_constraints/shape/query17.out b/regression-test/data/shape_check/tpcds_sf1000_constraints/shape/query17.out index 22d29b59b5be57..23fda8700a4301 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_constraints/shape/query17.out +++ b/regression-test/data/shape_check/tpcds_sf1000_constraints/shape/query17.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_17_constraints -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000_constraints/shape/query54.out b/regression-test/data/shape_check/tpcds_sf1000_constraints/shape/query54.out index 65a0c29e5ab7f5..539d7293531f0f 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_constraints/shape/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000_constraints/shape/query54.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_54_constraints -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000_constraints/shape/query61.out b/regression-test/data/shape_check/tpcds_sf1000_constraints/shape/query61.out index cabf732464f880..6a7aaee126b384 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_constraints/shape/query61.out +++ b/regression-test/data/shape_check/tpcds_sf1000_constraints/shape/query61.out @@ -1,8 +1,8 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_61_constraints -- PhysicalResultSink ---PhysicalTopN[GATHER_SORT] -----PhysicalProject +--PhysicalProject +----PhysicalTopN[GATHER_SORT] ------NestedLoopJoin[CROSS_JOIN] --------hashAgg[GLOBAL] ----------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query17.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query17.out index e0b281146ad099..22eac9a0189c00 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query17.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query17.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_17 -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query54.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query54.out index 1a47463dfde84e..c50da13cf82a94 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query54.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query54.out @@ -1,10 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_54 -- PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------PhysicalProject +--PhysicalProject +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query61.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query61.out index 98825fe6cf8c4a..3f72ae2dd8a5b1 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query61.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query61.out @@ -1,8 +1,8 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_61 -- PhysicalResultSink ---PhysicalTopN[GATHER_SORT] -----PhysicalProject +--PhysicalProject +----PhysicalTopN[GATHER_SORT] ------NestedLoopJoin[CROSS_JOIN] --------hashAgg[GLOBAL] ----------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy index 222d3a393b215e..8254657acfd8e8 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy @@ -235,11 +235,11 @@ suite("test_hive_topn_lazy_mat", "p0,external,hive,external_docker,external_dock explain { sql """ select a.name,length(a.name),a.value,b.*,a.* from parquet_topn_lazy_mat_table as a join orc_topn_lazy_mat_table as b on a.id = b.id order by a.name limit 10 """ - contains("projectList:[name, length(a.name), value, id, name, value, active, score, file_id, id, name, value, active, score, file_id]") - contains("column_descs_lists[[`name` text NULL, `value` double NULL, `active` boolean NULL, `score` double NULL, `file_id` int NULL], [`value` double NULL, `active` boolean NULL, `score` double NULL, `file_id` int NULL]]") - contains("locations: [[5, 6, 7, 8, 9], [10, 11, 12, 13]]") - contains("column_idxs_lists: [[1, 2, 3, 4, 5], [2, 3, 4, 5]]") - contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__orc_topn_lazy_mat_table, __DORIS_GLOBAL_ROWID_COL__parquet_topn_lazy_mat_table]") + contains("projectList:[name, FunctionCallExpr{type=int}, value, id, name, value, active, score, file_id, id, name, value, active, score, file_id]") + contains("column_descs_lists[[`value` double NULL, `active` boolean NULL, `score` double NULL, `file_id` int NULL], [`name` text NULL, `value` double NULL, `active` boolean NULL, `score` double NULL, `file_id` int NULL]]") + contains("locations: [[3, 4, 5, 6], [7, 8, 9, 10, 11]]") + contains("column_idxs_lists: [[2, 3, 4, 5], [1, 2, 3, 4, 5]]") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__parquet_topn_lazy_mat_table, __DORIS_GLOBAL_ROWID_COL__orc_topn_lazy_mat_table]") } runTopNLazyMatTests() diff --git a/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy b/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy index 6eef30eb939cf2..abc920df35931f 100644 --- a/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy @@ -151,8 +151,8 @@ suite("test_tvf_topn_lazy_mat","external,hive,tvf,external_docker") { contains("column_idxs_lists: [[1, 2, 3, 4]]") contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__hdfs]") contains("isTopMaterializeNode: true") - contains("SlotDescriptor{id=0, col=id, colUniqueId=-1, type=bigint, nullable=true") - contains("SlotDescriptor{id=1, col=__DORIS_GLOBAL_ROWID_COL__hdfs, colUniqueId=2147483647, type=text, nullable=false,") + contains("col=id, colUniqueId=-1, type=bigint, nullable=true") + contains("col=__DORIS_GLOBAL_ROWID_COL__hdfs, colUniqueId=2147483647, type=text, nullable=false,") } @@ -168,8 +168,8 @@ suite("test_tvf_topn_lazy_mat","external,hive,tvf,external_docker") { contains("isTopMaterializeNode: true") - contains("SlotDescriptor{id=0, col=id, colUniqueId=-1, type=int, nullable=true") - contains("SlotDescriptor{id=1, col=__DORIS_GLOBAL_ROWID_COL__hdfs, colUniqueId=2147483647, type=text, nullable=false,") + contains("col=id, colUniqueId=-1, type=int, nullable=true") + contains("col=__DORIS_GLOBAL_ROWID_COL__hdfs, colUniqueId=2147483647, type=text, nullable=false,") } diff --git a/regression-test/suites/nereids_p0/compress_materialize/pushdown_encode.groovy b/regression-test/suites/nereids_p0/compress_materialize/pushdown_encode.groovy index 1172177bc5f435..b5e9be8dad6a4f 100644 --- a/regression-test/suites/nereids_p0/compress_materialize/pushdown_encode.groovy +++ b/regression-test/suites/nereids_p0/compress_materialize/pushdown_encode.groovy @@ -265,7 +265,21 @@ suite("pushdown_encode") { select v1 from (select v1 from t1 where k1 > 0 order by v1 limit 10) t group by v1 """ - contains("projects=[decode_as_varchar(encode_as_bigint(v1#1)) AS `decode_as_varchar(encode_as_bigint(v1))`#1, encode_as_bigint(v1#1) AS `encode_as_bigint(v1)`#3]") + contains("projects=[decode_as_varchar(encode_as_bigint(v1)#2) AS `v1`#1]") + /** + PhysicalResultSink[78] ( outputExprs=[v1#1] ) ++--PhysicalProject[77]@6 ( stats=0.5, projects=[decode_as_varchar(encode_as_bigint(v1)#2) AS `v1`#1] ) + +--PhysicalHashAggregate[76]@5 ( stats=0.5, aggPhase=GLOBAL, aggMode=BUFFER_TO_RESULT, maybeUseStreaming=false, groupByExpr=[encode_as_bigint(v1)#2], outputExpr=[encode_as_bigint(v1)#2], partitionExpr=Optional.empty, topnFilter=false, topnPushDown=false ) + +--PhysicalDistribute[75]@8 ( stats=0.5, distributionSpec=DistributionSpecHash ( orderedShuffledColumns=[2], shuffleType=EXECUTION_BUCKETED, tableId=-1, selectedIndexId=-1, partitionIds=[], equivalenceExprIds=[[2]], exprIdToEquivalenceSet={2=0} ) ) + +--PhysicalHashAggregate[74]@8 ( stats=0.5, aggPhase=LOCAL, aggMode=INPUT_TO_BUFFER, maybeUseStreaming=false, groupByExpr=[encode_as_bigint(v1)#2], outputExpr=[encode_as_bigint(v1)#2], partitionExpr=Optional.empty, topnFilter=false, topnPushDown=false ) + +--PhysicalProject[73]@4 ( stats=0.5, projects=[encode_as_bigint(decode_as_varchar(encode_as_bigint(v1#1))) AS `encode_as_bigint(v1)`#2] ) + +--PhysicalTopN[72]@3 ( stats=0.5, limit=10, offset=0, orderKeys=[encode_as_bigint(v1)#3 asc null first], phase=MERGE_SORT ) + +--PhysicalDistribute[71]@9 ( stats=0.5, distributionSpec=DistributionSpecGather ) + +--PhysicalTopN[70]@9 ( stats=0.5, limit=10, offset=0, orderKeys=[encode_as_bigint(v1)#3 asc null first], phase=LOCAL_SORT ) + +--PhysicalProject[69]@2 ( stats=0.5, projects=[encode_as_bigint(v1#1) AS `encode_as_bigint(v1)`#3, v1#1] ) + +--PhysicalFilter[68]@1 ( stats=0.5, predicates=(k1#0 > 0) ) + +--PhysicalOlapScan[58]@0 ( table=t1, stats=1, operativeSlots=[k1#0, v1#1], virtualColumns=[] ) + **/ } // if encodeBody is used in windowExpression, do not push encode down diff --git a/regression-test/suites/nereids_rules_p0/column_pruning/nested_container_offset_pruning.groovy b/regression-test/suites/nereids_rules_p0/column_pruning/nested_container_offset_pruning.groovy new file mode 100644 index 00000000000000..327925754e62d7 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/column_pruning/nested_container_offset_pruning.groovy @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("nested_container_offset_pruning") { + sql """ DROP TABLE IF EXISTS nested_container_offset_pruning_tbl """ + sql """ + CREATE TABLE nested_container_offset_pruning_tbl ( + id INT, + s STRUCT< + arr: ARRAY>, + m: MAP + > + ) ENGINE = OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + sql """ + INSERT INTO nested_container_offset_pruning_tbl VALUES ( + 1, + named_struct( + 'arr', array( + named_struct('str_field', 'hello', 'int_field', 10), + named_struct('str_field', 'world', 'int_field', 20) + ), + 'm', {'a': 'x', 'b': 'y'} + ) + ) + """ + + // cardinality(s.arr) only needs array offsets, but element_at(...).int_field also needs + // array item data. The redundant s.arr.OFFSET path must be removed even though the root slot + // itself is STRUCT. + order_qt_struct_root_arr_mixed """ + SELECT id, + cardinality(struct_element(s, 'arr')), + struct_element(element_at(struct_element(s, 'arr'), 1), 'int_field') + FROM nested_container_offset_pruning_tbl ORDER BY id + """ + + // Same issue for nested maps: length(element_at(s.m, 'a')) needs the key lookup path, + // while map_values(s.m)[1] needs full value data. Dedup must therefore keep KEYS + VALUES + // and drop only the redundant value-side OFFSET path under the nested map container. + order_qt_struct_root_map_mixed """ + SELECT id, + length(element_at(struct_element(s, 'm'), 'a')), + element_at(map_values(struct_element(s, 'm')), 1) + FROM nested_container_offset_pruning_tbl ORDER BY id + """ + + explain { + sql """ + SELECT cardinality(struct_element(s, 'arr')), + struct_element(element_at(struct_element(s, 'arr'), 1), 'int_field') + FROM nested_container_offset_pruning_tbl + """ + contains "s.arr.*.int_field" + notContains "s.arr.OFFSET" + } + + explain { + sql """ + SELECT length(element_at(struct_element(s, 'm'), 'a')), + element_at(map_values(struct_element(s, 'm')), 1) + FROM nested_container_offset_pruning_tbl + """ + contains "s.m.KEYS" + contains "s.m.VALUES" + notContains "OFFSET" + } +} diff --git a/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy b/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy new file mode 100644 index 00000000000000..cbbeac2935701f --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/column_pruning/null_column_pruning.groovy @@ -0,0 +1,532 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Regression tests for the IS NULL / IS NOT NULL column pruning optimization. +// +// When IS NULL (or IS NOT NULL) is the *only* use of a nullable column, the FE +// should emit a DATA access path with a "NULL" component so that the BE can +// satisfy the query by reading only the null flag instead of the full column data. +// The EXPLAIN plan should show: +// nested columns: : all access paths: [.NULL] +// +// When the same column is also accessed for data (e.g., projected or used in +// struct_element), the NULL-only path must be stripped from allAccessPaths and +// predicateAccessPaths unless the same path is still present in allAccessPaths. + +suite("null_column_pruning") { + sql """ DROP TABLE IF EXISTS ncp_tbl """ + sql """ + CREATE TABLE ncp_tbl ( + id INT, + str_col STRING NULL, + struct_col STRUCT NULL, + arr_col ARRAY NULL, + map_col MAP NULL, + int_col INT NULL + ) ENGINE = OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + + sql """ + INSERT INTO ncp_tbl VALUES + (1, 'hello', named_struct('city', null, 'zip', 10001), [1, 2, 3], {'a': 1, 'b': 2 }, 1) + """ + // ─── Struct IS NULL only ──────────────────────────────────────────────────── + // Only null check on struct_col → emit [struct_col, NULL] access path, + // type stays full struct (no pruning needed). + explain { + sql "select 1 from ncp_tbl where struct_col is null" + contains "nested columns" + contains "struct_col.NULL" + } + + order_qt_1 "select 1 from ncp_tbl where struct_col is null"; + + // ─── String IS NULL only ──────────────────────────────────────────────────── + // Only null check on str_col → emit [str_col, NULL] access path, + // type stays string (no pruning needed). + explain { + sql "select 1 from ncp_tbl where str_col is null" + contains "nested columns" + contains "str_col.NULL" + } + + order_qt_2 "select 1 from ncp_tbl where str_col is null"; + + // Direct full access to the same field covers its null flag for any data type. + // The exact [str_col.NULL] metadata path must be removed. + explain { + sql "select id, str_col from ncp_tbl where str_col is null" + notContains "str_col.NULL" + notContains "predicate access paths:" + } + + order_qt_string_full_access_strips_null """ + select id, str_col from ncp_tbl where str_col is null + order by id + """ + + // ─── String IS NOT NULL only ──────────────────────────────────────────────── + explain { + sql "select 1 from ncp_tbl where str_col is not null" + contains "nested columns" + contains "str_col.NULL" + } + + order_qt_3 "select 1 from ncp_tbl where str_col is not null"; + + // ─── Struct IS NOT NULL only ──────────────────────────────────────────────── + // IS NOT NULL is the same optimization (only null flag needed). + explain { + sql "select 1 from ncp_tbl where struct_col is not null" + contains "nested columns" + contains "struct_col.NULL" + } + + order_qt_4 "select 1 from ncp_tbl where struct_col is not null"; + + // ─── Struct IS NULL in aggregate ──────────────────────────────────────────── + explain { + sql "select count(*) from ncp_tbl where struct_col is null" + contains "nested columns" + contains "struct_col.NULL" + } + + order_qt_5 "select count(*) from ncp_tbl where struct_col is null"; + + // ─── Array IS NULL only ───────────────────────────────────────────────────── + explain { + sql "select 1 from ncp_tbl where arr_col is null" + contains "nested columns" + contains "arr_col.NULL" + } + + order_qt_6 "select 1 from ncp_tbl where arr_col is null"; + + explain { + sql "select id, arr_col from ncp_tbl where arr_col is null" + contains "nested columns" + contains "all access paths: [arr_col]" + notContains "arr_col.NULL" + notContains "predicate access paths:" + } + + order_qt_array_full_access_strips_null """ + select id, arr_col from ncp_tbl where arr_col is null + order by id + """ + + // ─── Map IS NULL only ─────────────────────────────────────────────────────── + explain { + sql "select 1 from ncp_tbl where map_col is null" + contains "nested columns" + contains "map_col.NULL" + } + + order_qt_7 "select 1 from ncp_tbl where map_col is null"; + + explain { + sql "select id, map_col from ncp_tbl where map_col is null" + contains "nested columns" + contains "all access paths: [map_col]" + notContains "map_col.NULL" + notContains "predicate access paths:" + } + + order_qt_map_full_access_strips_null """ + select id, map_col from ncp_tbl where map_col is null + order by id + """ + + // ─── Int IS NULL only ─────────────────────────────────────────────────────── + // Nullable primitive type (INT) accessed only via IS NULL → emit [int_col, NULL] + // access path so BE only reads the null flag. + explain { + sql "select 1 from ncp_tbl where int_col is null" + contains "nested columns" + contains "int_col.NULL" + } + + order_qt_8 "select 1 from ncp_tbl where int_col is null"; + + // ─── Int IS NOT NULL only ─────────────────────────────────────────────────── + explain { + sql "select 1 from ncp_tbl where int_col is not null" + contains "nested columns" + contains "int_col.NULL" + } + + order_qt_9 "select 1 from ncp_tbl where int_col is not null"; + + // ─── Mixed: int IS NULL + projected ──────────────────────────────────────── + // int_col IS NULL in WHERE + int_col in SELECT → data is also needed. + // [int_col, NULL] stripped from allAccessPaths because [int_col] (full data) + // covers the same prefix and inherently includes the null flag. + explain { + sql "select int_col from ncp_tbl where int_col is null" + contains "nested columns" + contains "all access paths: [int_col]" + notContains "predicate access paths:" + } + + order_qt_10 "select int_col from ncp_tbl where int_col is null"; + + // ─── Mixed: struct IS NULL + partial field access ─────────────────────────── + // struct_col IS NULL in WHERE + struct_element in SELECT → child data is also needed. + // The parent struct_col.NULL path must NOT stay in allAccessPaths with child paths. + // BE StructFileColumnIterator treats a leading NULL sub-path as NULL_MAP_ONLY; if + // allAccessPaths were [struct_col.NULL, struct_col.city], BE would skip the city + // child iterator and default-fill the projected value. The normal nullable struct + // read materializes the parent null map together with child data, and + // predicateAccessPaths is filtered so it remains a subset of allAccessPaths. + explain { + sql "select struct_element(struct_col, 'city') from ncp_tbl where struct_col is null" + contains "nested columns" + contains "all access paths: [struct_col.city]" + notContains "predicate access paths:" + } + + order_qt_11 "select struct_element(struct_col, 'city') from ncp_tbl where struct_col is null"; + + // This query verifies the real correctness risk: one branch needs the parent null + // map, another branch needs a child null map, and the projection needs another + // child data path. Keeping struct_col.NULL in allAccessPaths would put BE in + // NULL_MAP_ONLY mode for the whole struct and return the default zip value instead + // of reading the zip child column. + explain { + sql "select struct_element(struct_col, 'zip') from ncp_tbl where struct_col is null or struct_element(struct_col, 'city') is null" + contains "nested columns" + contains "all access paths: [struct_col.city.NULL, struct_col.zip]" + contains "predicate access paths: [struct_col.city.NULL]" + } + + order_qt_parent_null_with_child_data "select struct_element(struct_col, 'zip') from ncp_tbl where struct_col is null or struct_element(struct_col, 'city') is null"; + + // ─── Non-optimizable: struct IS NULL + full struct projected ──────────────── + // Full struct access covers its own null flag, so [struct_col.NULL] is stripped + // from allAccessPaths but kept in predicateAccessPaths. + explain { + sql "select struct_col from ncp_tbl where struct_col is null" + contains "nested columns" + contains "all access paths: [struct_col]" + notContains "predicate access paths:" + } + + order_qt_12 "select struct_col from ncp_tbl where struct_col is null"; + + // ─── Nested struct field IS NULL ──────────────────────────────────────────── + // struct_element(struct_col, 'city') IS NULL should produce a null-flag-only + // predicate path [struct_col.city.NULL] while the projection reads city data. + // [struct_col.city.NULL] is stripped from allAccessPaths because [struct_col.city] + // covers the same prefix (full city data includes its null flag). + explain { + sql "select struct_element(struct_col, 'city') from ncp_tbl where struct_element(struct_col, 'city') is null" + contains "nested columns" + contains "all access paths: [struct_col.city]" + notContains "predicate access paths:" + } + + order_qt_13 "select struct_element(struct_col, 'city') from ncp_tbl where struct_element(struct_col, 'city') is null"; + + // ========================================================================= + // IS NULL on nested-type extraction functions (map_keys, map_values, + // element_at, struct_element, and nested combinations) + // ========================================================================= + + // ─── map_keys(map_col) IS NULL ───────────────────────────────────────────── + // map_keys(nullable_map) returns a NULL array only when the parent map itself + // is NULL, so the null-only path must be the parent map null map. Emitting + // map_col.KEYS.NULL would ask BE to inspect the key child null map instead. + explain { + sql "select count(1) from ncp_tbl where map_keys(map_col) is null" + contains "nested columns" + contains "map_col.NULL" + notContains "map_col.KEYS.NULL" + } + + order_qt_14 "select count(1) from ncp_tbl where map_keys(map_col) is null"; + + // ─── map_keys(map_col) IS NOT NULL ────────────────────────────────────────── + explain { + sql "select count(1) from ncp_tbl where map_keys(map_col) is not null" + contains "nested columns" + contains "map_col.NULL" + notContains "map_col.KEYS.NULL" + } + + order_qt_15 "select count(1) from ncp_tbl where map_keys(map_col) is not null"; + + // ─── map_values(map_col) IS NULL ──────────────────────────────────────────── + // A non-NULL map containing a NULL value, e.g. {'b': NULL}, still produces a + // non-NULL values array [NULL]. Therefore map_values(map_col) IS NULL is a + // parent-map null check, not a VALUES-child null check. + explain { + sql "select count(1) from ncp_tbl where map_values(map_col) is null" + contains "nested columns" + contains "map_col.NULL" + notContains "map_col.VALUES.NULL" + } + + order_qt_16 "select count(1) from ncp_tbl where map_values(map_col) is null"; + + // ─── map_values(map_col) IS NOT NULL ──────────────────────────────────────── + explain { + sql "select count(1) from ncp_tbl where map_values(map_col) is not null" + contains "nested columns" + contains "map_col.NULL" + notContains "map_col.VALUES.NULL" + } + + order_qt_17 "select count(1) from ncp_tbl where map_values(map_col) is not null"; + + sql """ DROP TABLE IF EXISTS ncp_map_null_semantics_tbl """ + sql """ + CREATE TABLE ncp_map_null_semantics_tbl ( + id INT, + map_col MAP NULL + ) ENGINE = OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + sql """ + INSERT INTO ncp_map_null_semantics_tbl VALUES + (1, NULL), + (2, {'a': 1}), + (3, {'b': NULL}) + """ + + explain { + sql "select id from ncp_map_null_semantics_tbl where map_values(map_col) is null order by id" + contains "nested columns" + contains "map_col.NULL" + notContains "map_col.VALUES.NULL" + } + + // Only the NULL map row should match. Row 3 has a NULL value element, but + // map_values({'b': NULL}) is the non-NULL array [NULL]. + order_qt_map_values_parent_null_semantics """ + select id from ncp_map_null_semantics_tbl where map_values(map_col) is null order by id + """ + + order_qt_map_values_parent_not_null_semantics """ + select id from ncp_map_null_semantics_tbl where map_values(map_col) is not null order by id + """ + + // ─── element_at(arr_col, 1) IS NULL ───────────────────────────────────────── + explain { + sql "select count(1) from ncp_tbl where arr_col[1] is null" + contains "nested columns" + contains "arr_col.*.NULL" + } + + order_qt_18 "select count(1) from ncp_tbl where arr_col[1] is null"; + + // ─── element_at(arr_col, 1) IS NOT NULL ───────────────────────────────────── + explain { + sql "select count(1) from ncp_tbl where arr_col[1] is not null" + contains "nested columns" + contains "arr_col.*.NULL" + } + + order_qt_19 "select count(1) from ncp_tbl where arr_col[1] is not null"; + + // ─── element_at(map_col, 'a') IS NULL ─────────────────────────────────────── + explain { + sql "select count(1) from ncp_tbl where map_col['a'] is null" + contains "nested columns" + contains "map_col.*.NULL" + } + + order_qt_20 "select count(1) from ncp_tbl where map_col['a'] is null"; + + // ─── element_at(map_col, 'a') IS NOT NULL ─────────────────────────────────── + explain { + sql "select count(1) from ncp_tbl where map_col['a'] is not null" + contains "nested columns" + contains "map_col.*.NULL" + } + + order_qt_21 "select count(1) from ncp_tbl where map_col['a'] is not null"; + + // ─── struct_element(struct_col, 'city') IS NULL only (no projection) ──────── + explain { + sql "select count(1) from ncp_tbl where struct_element(struct_col, 'city') is null" + contains "nested columns" + contains "struct_col.city.NULL" + } + + order_qt_22 "select count(1) from ncp_tbl where struct_element(struct_col, 'city') is null"; + + // ─── struct_element(struct_col, 'zip') IS NULL ────────────────────────────── + explain { + sql "select count(1) from ncp_tbl where struct_element(struct_col, 'zip') is null" + contains "nested columns" + contains "struct_col.zip.NULL" + } + + order_qt_23 "select count(1) from ncp_tbl where struct_element(struct_col, 'zip') is null"; + + // ─── struct_element IS NOT NULL ───────────────────────────────────────────── + explain { + sql "select count(1) from ncp_tbl where struct_element(struct_col, 'city') is not null" + contains "nested columns" + contains "struct_col.city.NULL" + } + + order_qt_24 "select count(1) from ncp_tbl where struct_element(struct_col, 'city') is not null"; + + // ─── Mixed: map_keys IS NULL + map_keys projected ────────────────────────── + // Projection needs key data, while the predicate checks whether the parent map + // is NULL. The parent NULL path must not stay in either access path list, so BE + // does not switch the whole map iterator to NULL_MAP_ONLY and skip the keys child. + explain { + sql "select map_keys(map_col) from ncp_tbl where map_keys(map_col) is null" + contains "nested columns" + contains "all access paths: [map_col.KEYS]" + notContains "predicate access paths:" + } + + order_qt_25 "select map_keys(map_col) from ncp_tbl where map_keys(map_col) is null"; + + // ─── Mixed: map_values IS NULL + map_values projected ────────────────────── + // Projection needs value data, while the predicate checks whether the parent + // map is NULL. A NULL value element does not make map_values(map_col) NULL. + explain { + sql "select map_values(map_col) from ncp_tbl where map_values(map_col) is null" + contains "nested columns" + contains "all access paths: [map_col.VALUES]" + notContains "predicate access paths:" + } + + order_qt_26 "select map_values(map_col) from ncp_tbl where map_values(map_col) is null"; + + // ─── Nested types: struct containing map and array ───────────────────────── + sql """ DROP TABLE IF EXISTS ncp_nested_tbl """ + sql """ + CREATE TABLE ncp_nested_tbl ( + id INT, + nested_struct STRUCT, inner_arr: ARRAY> NULL, + arr_of_structs ARRAY> NULL, + map_of_arrs MAP> NULL + ) ENGINE = OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + sql """ + INSERT INTO ncp_nested_tbl SELECT + 1, + named_struct('inner_map', map('x', 10), 'inner_arr', array('a', 'b')), + array(named_struct('name', 'Alice', 'age', 30)), + map('k', array(1, 2)) + """ + + // ─── struct_element → map field IS NULL ───────────────────────────────────── + explain { + sql "select count(1) from ncp_nested_tbl where struct_element(nested_struct, 'inner_map') is null" + contains "nested columns" + contains "nested_struct.inner_map.NULL" + } + + order_qt_27 "select count(1) from ncp_nested_tbl where struct_element(nested_struct, 'inner_map') is null"; + + // ─── struct_element → array field IS NULL ─────────────────────────────────── + explain { + sql "select count(1) from ncp_nested_tbl where struct_element(nested_struct, 'inner_arr') is null" + contains "nested columns" + contains "nested_struct.inner_arr.NULL" + } + + order_qt_28 "select count(1) from ncp_nested_tbl where struct_element(nested_struct, 'inner_arr') is null"; + + // ─── map_keys through struct_element IS NULL ──────────────────────────────── + explain { + sql "select count(1) from ncp_nested_tbl where map_keys(struct_element(nested_struct, 'inner_map')) is null" + contains "nested columns" + contains "nested_struct.inner_map.NULL" + notContains "nested_struct.inner_map.KEYS.NULL" + } + + order_qt_29 "select count(1) from ncp_nested_tbl where map_keys(struct_element(nested_struct, 'inner_map')) is null"; + + // ─── map_values through struct_element IS NULL ────────────────────────────── + explain { + sql "select count(1) from ncp_nested_tbl where map_values(struct_element(nested_struct, 'inner_map')) is null" + contains "nested columns" + contains "nested_struct.inner_map.NULL" + notContains "nested_struct.inner_map.VALUES.NULL" + } + + order_qt_30 "select count(1) from ncp_nested_tbl where map_values(struct_element(nested_struct, 'inner_map')) is null"; + + // ─── map_values(map_of_arrs) IS NULL ──────────────────────────────────────── + explain { + sql "select count(1) from ncp_nested_tbl where map_values(map_of_arrs) is null" + contains "nested columns" + contains "map_of_arrs.NULL" + notContains "map_of_arrs.VALUES.NULL" + } + + order_qt_31 "select count(1) from ncp_nested_tbl where map_values(map_of_arrs) is null"; + + // ─── map_keys(map_of_arrs) IS NULL ────────────────────────────────────────── + explain { + sql "select count(1) from ncp_nested_tbl where map_keys(map_of_arrs) is null" + contains "nested columns" + contains "map_of_arrs.NULL" + notContains "map_of_arrs.KEYS.NULL" + } + + order_qt_32 "select count(1) from ncp_nested_tbl where map_keys(map_of_arrs) is null"; + + // ─── Non-nullable column IS NULL → no nested column pruning ───────────────── + // A NOT NULL column has no null flags; IS NULL is always false and the optimizer + // must NOT generate a .NULL access path for it. + sql """ DROP TABLE IF EXISTS ncp_tbl_nn """ + sql """ + CREATE TABLE ncp_tbl_nn ( + id INT NOT NULL, + str_col STRING NULL + ) ENGINE = OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + sql """ INSERT INTO ncp_tbl_nn VALUES (1, 'hello') """ + + explain { + sql "select 1 from ncp_tbl_nn where id is null" + notContains "nested columns" + } + + order_qt_33 "select 1 from ncp_tbl_nn where id is null"; + + // ─── length(str_col) = 0 OR str_col IS NULL ──────────────────────────────── + // length(str_col) already uses the OFFSET path, and BE can derive null-ness + // from that layout, so the extra NULL-only path is redundant. + explain { + sql "select 1 from ncp_tbl where length(str_col) = 0 or str_col is null" + contains "nested columns" + contains "str_col.OFFSET" + notContains "str_col.NULL" + } + + order_qt_34 "select 1 from ncp_tbl where length(str_col) = 0 or str_col is null"; +} diff --git a/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy b/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy new file mode 100644 index 00000000000000..8549d128ccd95f --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy @@ -0,0 +1,565 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Regression tests for the string-length OFFSET-only optimization. +// +// When length() is the *only* use of a string column (or a string field inside a +// struct), the FE should emit a DATA access path with an extra "OFFSET" component so +// that the BE can satisfy the query by reading only the OFFSET array instead of the +// full chars data. The EXPLAIN plan should show: +// nested columns: :[DATA(.OFFSET)] +// +// Crucially, the slot type must remain varchar (not bigint), and any predicate +// using length() must be preserved as-is (e.g. "length(str_col) > 1"), never +// rewritten to "CAST(str_col AS int) > 1". +// +// When the same string column is also read directly (e.g. projected, passed to +// substr(), …) the optimization must be suppressed: no nested-columns entry for +// the plain string column should appear. + +suite("string_length_column_pruning") { + // fe_debug performs strict nullability-change assertions in AdjustNullable. + // The IF/ORDER BY shape below can hit that pre-existing debug assertion before + // NestedColumnPruning runs, so keep this suite independent of global fe_debug. + sql "set fe_debug=false" + + sql """ DROP TABLE IF EXISTS slcp_str_tbl """ + sql """ + CREATE TABLE slcp_str_tbl ( + id INT, + str_col STRING, + struct_col STRUCT, + arr_col ARRAY, + map_col MAP, + map_arr_col MAP>, + map_arr_struct_col MAP>> + ) ENGINE = OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + sql """ + INSERT INTO slcp_str_tbl VALUES + (1, 'hello', named_struct('f1', 10, 'f3', 'world'), [1, 2, 3], {'a': 'x', 'b': 'y'}, + {'a': [1, 2], 'b': [3]}, + map('a', array(named_struct('verified', true, 'value', 10)))) + """ + + // ─── Optimizable cases ────────────────────────────────────────────────────── + + // Plain string column: length() is the only use → OFFSET access path emitted, + // slot type stays varchar (not bigint). + explain { + sql "select length(str_col) from slcp_str_tbl" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + sql "select length(str_col) from slcp_str_tbl" + + // length(str_col) in IF plus ORDER BY on a plain primitive column: + // only str_col should appear in nested columns, and NULL is redundant when OFFSET exists. + explain { + sql "select if(length(str_col) >= 5, true, false) a from slcp_str_tbl order by id" + contains "nested columns" + contains "str_col.OFFSET" + notContains "str_col.NULL" + notContains "all access paths: [id]" + } + sql "select if(length(str_col) >= 5, true, false) a from slcp_str_tbl order by id" + + // Struct string field: length(struct_element) is the only use + explain { + sql "select length(struct_element(struct_col, 'f3')) from slcp_str_tbl" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + //sql "select length(struct_element(struct_col, 'f3')) from slcp_str_tbl" + // length() in both SELECT and WHERE: predicate must remain length(str_col) > 1, + // never be rewritten to CAST(str_col AS int) > 1. Slot type must stay varchar. + explain { + sql "select length(str_col) from slcp_str_tbl where length(str_col) > 1" + contains "nested columns" + contains "OFFSET" + contains "length(str_col" + notContains "CAST(str_col" + notContains "type=bigint" + } + sql "select length(str_col) from slcp_str_tbl where length(str_col) > 1" + + // ─── Aggregate cases ───────────────────────────────────────────────────────── + + // sum(length(str_col)): length() is still the only consumer of str_col → OFFSET path applies. + explain { + sql "select sum(length(str_col)) from slcp_str_tbl" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + sql "select sum(length(str_col)) from slcp_str_tbl" + // count(length(str_col)) + explain { + sql "select count(length(str_col)) from slcp_str_tbl" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + + // max(length(str_col)) + explain { + sql "select max(length(str_col)) from slcp_str_tbl" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + sql "select max(length(str_col)) from slcp_str_tbl" + // ─── Array column cases ────────────────────────────────────────────────────── + + // cardinality(arr_col): only the offset array is needed → OFFSET access path emitted, + // slot type stays ARRAY (not bigint). + explain { + sql "select cardinality(arr_col) from slcp_str_tbl" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + sql "select cardinality(arr_col) from slcp_str_tbl" + // cardinality(arr_col) in aggregate: OFFSET still applies. + explain { + sql "select sum(cardinality(arr_col)) from slcp_str_tbl" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + sql "select sum(cardinality(arr_col)) from slcp_str_tbl" + // arr_col also accessed via element_at → full element data needed, OFFSET suppressed. + explain { + sql "select cardinality(arr_col), arr_col[1] from slcp_str_tbl" + notContains "OFFSET" + notContains "type=bigint" + } + + // Full access to the same array field covers its OFFSET metadata for any data type. + explain { + sql "select id, cardinality(arr_col), arr_col from slcp_str_tbl" + contains "nested columns" + contains "all access paths: [arr_col]" + notContains "arr_col.OFFSET" + notContains "predicate access paths:" + notContains "type=bigint" + } + + order_qt_array_full_access_strips_offset """ + select id, cardinality(arr_col), arr_col from slcp_str_tbl + order by id + """ + + // ─── Map column cases ──────────────────────────────────────────────────────── + + // cardinality(map_col): only the offset array is needed → OFFSET access path emitted, + // slot type stays MAP (not bigint). + explain { + sql "select cardinality(map_col) from slcp_str_tbl" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + sql "select cardinality(map_col) from slcp_str_tbl" + + // cardinality(map_col) in aggregate: OFFSET still applies. + explain { + sql "select sum(cardinality(map_col)) from slcp_str_tbl" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + + explain { + sql "select sum(map_size(map_col)) from slcp_str_tbl" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + + // cardinality(map_keys(map_col)): only the keys offset array is needed → OFFSET access path emitted. + explain { + sql "select cardinality(map_keys(map_col)) from slcp_str_tbl" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + + explain { + sql "select cardinality(map_values(map_col)) from slcp_str_tbl" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + + // cardinality(map_keys(map_col)) in aggregate: OFFSET still applies. + explain { + sql "select sum(cardinality(map_keys(map_col))) from slcp_str_tbl" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + + explain { + sql "select sum(cardinality(map_values(map_col))) from slcp_str_tbl" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + + // Both map_keys and map_values sizes in the same query: both equal cardinality(map), + // so only a single [map_col, OFFSET] path is needed. + explain { + sql "select sum(cardinality(map_keys(map_col))), sum(cardinality(map_values(map_col))) from slcp_str_tbl" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + + // map_col also accessed via map_keys → full key data needed, OFFSET suppressed. + explain { + sql "select cardinality(map_col), map_keys(map_col) from slcp_str_tbl" + notContains "type=bigint" + } + + // ─── Map with complex value cases ──────────────────────────────────────────── + + // cardinality(map_arr_col['a']): value is ARRAY. + // Keys read in full (element lookup); values need only the OFFSET array (array size). + // Expected paths: map_arr_col.KEYS + map_arr_col.VALUES.OFFSET + explain { + sql "select cardinality(map_arr_col['a']) from slcp_str_tbl" + contains "nested columns" + contains "KEYS" + contains "VALUES" + contains "OFFSET" + notContains "type=bigint" + } + + // same in aggregate + explain { + sql "select sum(cardinality(map_arr_col['a'])) from slcp_str_tbl" + contains "nested columns" + contains "KEYS" + contains "VALUES" + contains "OFFSET" + notContains "type=bigint" + } + + // value also accessed directly (arr[0]) → full VALUES needed, OFFSET suppressed + explain { + sql "select cardinality(map_arr_col['a']), map_arr_col['b'][0] from slcp_str_tbl" + notContains "OFFSET" + notContains "type=bigint" + } + + // value array item also accessed directly → full VALUES item path covers value OFFSET. + explain { + sql "select cardinality(map_arr_struct_col['a']), map_arr_struct_col['a'][1].verified from slcp_str_tbl" + contains "nested columns" + contains "map_arr_struct_col.*.*.verified" + notContains "map_arr_struct_col.*.OFFSET" + notContains "type=bigint" + } + + explain { + sql "select id, cardinality(map_arr_col['a']), map_arr_col['a'] from slcp_str_tbl" + contains "nested columns" + contains "all access paths: [map_arr_col.*]" + notContains "map_arr_col.*.OFFSET" + notContains "predicate access paths:" + notContains "type=bigint" + } + + order_qt_map_element_full_access_strips_offset """ + select id, cardinality(map_arr_col['a']), map_arr_col['a'] from slcp_str_tbl + order by id + """ + + // Predicate OFFSET path must also be removed when the projected value field already + // makes the corresponding array data path available. predicateAccessPaths remains a + // subset of allAccessPaths. + explain { + sql "select map_arr_struct_col['a'][1].verified from slcp_str_tbl where cardinality(map_arr_struct_col['a']) > 0" + contains "nested columns" + contains "all access paths: [map_arr_struct_col.*.*.verified]" + notContains "map_arr_struct_col.*.OFFSET" + notContains "predicate access paths:" + notContains "type=bigint" + } + + order_qt_map_value_array_predicate_offset_covered """ + select map_arr_struct_col['a'][1].verified from slcp_str_tbl + where cardinality(map_arr_struct_col['a']) > 0 + order by 1 + """ + + // value array item also accessed directly → full VALUES item path covers value NULL. + explain { + sql "select map_arr_struct_col['a'][1].verified from slcp_str_tbl where map_arr_struct_col['a'] is null" + contains "nested columns" + contains "map_arr_struct_col.*.*.verified" + notContains "map_arr_struct_col.*.NULL" + notContains "predicate access paths:" + } + + // ─── Non-optimizable cases ────────────────────────────────────────────────── + + // str_col also projected directly → full chars data needed, OFFSET path suppressed. + // No nested-columns entry for str_col, slot type stays varchar. + explain { + sql "select length(str_col), str_col from slcp_str_tbl" + notContains "nested columns" + notContains "type=bigint" + notContains "CAST(str_col" + } + + // str_col also used in substr() → full chars data needed + explain { + sql "select length(str_col), substr(str_col, 2) from slcp_str_tbl" + notContains "nested columns" + notContains "type=bigint" + notContains "CAST(str_col" + } + + // ─── StringEmptyToLength rule cases ───────────────────────────────────────── + + // str_col <> '' rewrites to length(str_col) != 0 → OFFSET optimization applies + explain { + sql "select length(str_col) from slcp_str_tbl where str_col <> ''" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + + // str_col = '' also rewrites to length(str_col) = 0 → OFFSET applies + explain { + sql "select length(str_col) from slcp_str_tbl where str_col = ''" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + + // aggregate + predicate rewrite: sum(length(str_col)) where str_col <> '' + explain { + sql "select sum(length(str_col)) from slcp_str_tbl where str_col <> ''" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + + // str_col is also projected directly alongside the predicate → OFFSET suppressed + // (str_col <> '' rewrites to length(str_col) != 0, but str_col is also projected as-is) + explain { + sql "select str_col, length(str_col) from slcp_str_tbl where str_col <> ''" + notContains "nested columns" + notContains "type=bigint" + notContains "CAST(str_col" + } + + // reversed operand: '' <> str_col → same rewrite + explain { + sql "select length(str_col) from slcp_str_tbl where '' <> str_col" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + + // ─── Struct field empty-string rewrite cases ──────────────────────────────── + + // element_at(struct_col, 'f3') = '' rewrites to length(element_at(struct_col, 'f3')) = 0 + // → OFFSET optimization applies to the struct string field + explain { + sql "select 1 from slcp_str_tbl where element_at(struct_col, 'f3') = ''" + contains "length" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + + // element_at(struct_col, 'f3') <> '' rewrites to length(element_at(struct_col, 'f3')) != 0 + explain { + sql "select 1 from slcp_str_tbl where element_at(struct_col, 'f3') <> ''" + contains "length" + contains "nested columns" + contains "OFFSET" + notContains "type=bigint" + } + + // Struct field also projected directly → field access is full, not OFFSET-only + // The struct's nested-columns entry still appears (partial struct pruning), + // but the pruned field type must remain text (not bigint). + explain { + sql "select length(struct_element(struct_col, 'f3')), struct_element(struct_col, 'f3') from slcp_str_tbl" + contains "nested columns" + notContains "bigint" + } + + // length(map_col['a']): keys read fully for element lookup, values accessed offset-only. + // Expect access paths: map_col.KEYS (full) + map_col.VALUES.OFFSET + explain { + sql "select length(map_col['a']) from slcp_str_tbl" + contains "nested columns" + contains "KEYS" + contains "VALUES" + contains "OFFSET" + notContains "bigint" + } + + // sum(length(map_col['a'])): same optimization in aggregate context + explain { + sql "select sum(length(map_col['a'])) from slcp_str_tbl" + contains "nested columns" + contains "KEYS" + contains "VALUES" + contains "OFFSET" + notContains "bigint" + } + + // length(map_col['a']) + direct map access → OFFSET suppressed, full VALUES needed + explain { + sql "select length(map_col['a']), map_col['b'] from slcp_str_tbl" + notContains "OFFSET" + notContains "bigint" + } + + // ─── Array mixed field access (Bug fix: OFFSET dedup per field, not per slot) ── + + sql """ DROP TABLE IF EXISTS slcp_arr_struct_tbl """ + sql """ + CREATE TABLE slcp_arr_struct_tbl ( + id INT, + arr_struct ARRAY> + ) ENGINE = OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + sql """ + INSERT INTO slcp_arr_struct_tbl SELECT 1, + array(named_struct('str_field', 'hello', 'int_field', 10), + named_struct('str_field', 'world', 'int_field', 20)) + """ + sql """ + INSERT INTO slcp_arr_struct_tbl SELECT 2, + array(named_struct('str_field', 'foo', 'int_field', 30)) + """ + sql """ + INSERT INTO slcp_arr_struct_tbl SELECT 3, NULL + """ + + // LENGTH on one struct field + direct access on a sibling field. + // The OFFSET path for str_field must NOT be stripped by the non-OFFSET path for int_field. + order_qt_arr_struct_mixed """ + SELECT id, + array_match_all(x -> length(struct_element(x, 'str_field')) > 0, arr_struct), + struct_element(element_at(arr_struct, 1), 'int_field') + FROM slcp_arr_struct_tbl ORDER BY id + """ + + // Verify the OFFSET path is preserved in the explain plan + explain { + sql """ + SELECT id, + array_match_all(x -> length(struct_element(x, 'str_field')) > 0, arr_struct), + struct_element(element_at(arr_struct, 1), 'int_field') + FROM slcp_arr_struct_tbl + """ + contains "OFFSET" + } + + // ─── Nested array/map under STRUCT root slot ────────────────────────────────── + + sql """ DROP TABLE IF EXISTS slcp_struct_root_tbl """ + sql """ + CREATE TABLE slcp_struct_root_tbl ( + id INT, + s STRUCT< + arr: ARRAY>, + m: MAP + > + ) ENGINE = OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + sql """ + INSERT INTO slcp_struct_root_tbl VALUES ( + 1, + named_struct( + 'arr', array( + named_struct('str_field', 'hello', 'int_field', 10), + named_struct('str_field', 'world', 'int_field', 20) + ), + 'm', {'a': 'x', 'b': 'y'} + ) + ) + """ + + explain { + sql """ + SELECT cardinality(struct_element(s, 'arr')), + struct_element(element_at(struct_element(s, 'arr'), 1), 'int_field') + FROM slcp_struct_root_tbl + """ + contains "s.arr.*.int_field" + notContains "s.arr.OFFSET" + } + + explain { + sql """ + SELECT length(element_at(struct_element(s, 'm'), 'a')), + element_at(map_values(struct_element(s, 'm')), 1) + FROM slcp_struct_root_tbl + """ + contains "s.m.KEYS" + contains "s.m.VALUES" + notContains "OFFSET" + } + + // ─── Map element_at + map_values mixed access ───────────────────────────────── + + // length(map_col['a']) needs keys for the element_at lookup and value offsets for length(). + // map_values(map_col)[1] needs full value data. The mixed query must therefore keep a KEYS + // path for element_at lookup while dropping the redundant value-side OFFSET path. + order_qt_map_element_with_map_values """ + select length(map_col['a']), map_values(map_col)[1] from slcp_str_tbl + """ + + explain { + sql "select length(map_col['a']), map_values(map_col)[1] from slcp_str_tbl" + contains "nested columns" + contains "KEYS" + contains "VALUES" + notContains "OFFSET" + notContains "bigint" + } + + // Reverse direction: length(map_values(map_col)[1]) produces [map_col, VALUES, OFFSET] + // while map_col['a'] produces [map_col, *]. The * path reads full values, so OFFSET + // must be suppressed here as well. + explain { + sql "select length(map_values(map_col)[1]), map_col['a'] from slcp_str_tbl" + notContains "OFFSET" + notContains "bigint" + } +} diff --git a/regression-test/suites/nereids_rules_p0/column_pruning/topn_expr_pullup.groovy b/regression-test/suites/nereids_rules_p0/column_pruning/topn_expr_pullup.groovy new file mode 100644 index 00000000000000..a9d72879ffc010 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/column_pruning/topn_expr_pullup.groovy @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("topn_expr_pullup") { + sql """ set topn_lazy_materialization_threshold=1024; """ + sql """ DROP TABLE IF EXISTS tep_tbl """ + sql """ + CREATE TABLE tep_tbl ( + id INT, + str_col STRING NULL, + struct_col STRUCT NULL, + arr_col ARRAY NULL, + map_col MAP NULL, + int_col INT NULL + ) ENGINE = OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + sql """ INSERT INTO tep_tbl VALUES + (1, 'hello', named_struct('city', 'NYC', 'zip', 10001), [1, 2, 3], {'a': 1, 'b': 2}, 10), + (2, 'world', named_struct('city', 'LA', 'zip', 90001), [4, 5], {'x': 3}, 20), + (3, 'doris', named_struct('city', 'SF', 'zip', 94101), [6], {'y': 5}, 30) + """ + + // ============================================= + // Test 1: STRUCT PPD — expression pulled above TopN + lazy mat + // ============================================= + explain { + sql """ select id, struct_element(struct_col, 'city') as city + from tep_tbl order by id limit 3 """ + contains("VMaterializeNode") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__tep_tbl]") + } + qt_struct_ppd """ select id, struct_element(struct_col, 'city') as city + from tep_tbl order by id limit 3 """ + + // ============================================= + // Test 2: STRUCT PPD — column order preserved (expr before id) + // ============================================= + qt_struct_col_order """ select struct_element(struct_col, 'city') as city, id + from tep_tbl order by id limit 3 """ + + // ============================================= + // Test 3: Non-PPD — upper(str_col) pulled up + // ============================================= + explain { + sql """ select id, upper(str_col) as name + from tep_tbl order by id limit 3 """ + contains("VMaterializeNode") + } + qt_upper_str """ select id, upper(str_col) as name + from tep_tbl order by id limit 3 """ + + // ============================================= + // Test 4: Non-PPD — math expr pulled up + // ============================================= + qt_math_expr """ select id, int_col + 1 as next_val + from tep_tbl order by id limit 3 """ + + // ============================================= + // Test 5: Non-PPD — concat pulled up + // ============================================= + qt_concat_expr """ select id, concat(str_col, '-suffix') as label + from tep_tbl order by id limit 3 """ + + // ============================================= + // Test 6: MAP subscript pulled up + // ============================================= + explain { + sql """ select id, element_at(map_col, 'a') as val + from tep_tbl order by id limit 3 """ + contains("VMaterializeNode") + } + qt_map_subscript """ select id, element_at(map_col, 'a') as val + from tep_tbl order by id limit 3 """ + + // ============================================= + // Test 7: ARRAY subscript pulled up + // ============================================= + qt_array_subscript """ select id, element_at(arr_col, 1) as val + from tep_tbl order by id limit 3 """ + + // ============================================= + // Test 8: Simple alias NOT pulled up (child is Slot) + // ============================================= + qt_simple_alias """ select id as a, struct_element(struct_col, 'city') as city + from tep_tbl order by a limit 3 """ + + // ============================================= + // Test 9: Negative — order by depends on expression, stays below TopN + // ============================================= + qt_order_by_expr """ select id, struct_element(struct_col, 'city') as city + from tep_tbl order by city limit 3 """ + + // ============================================= + // Test 10: select * with expression pulled up + // ============================================= + explain { + sql """ select *, struct_element(struct_col, 'city') as city + from tep_tbl order by id limit 3 """ + contains("VMaterializeNode") + } + qt_select_star """ select *, struct_element(struct_col, 'city') as city + from tep_tbl order by id limit 3 """ + + // ============================================= + // Test 11: Switch disabled — same results + // ============================================= + sql """ set enable_topn_expr_pullup = false; """ + qt_switch_off """ select id, struct_element(struct_col, 'city') as city + from tep_tbl order by id limit 3 """ + sql """ set enable_topn_expr_pullup = true; """ + + // ============================================= + // Test 12: Join — PPD on left table + // ============================================= + qt_join_ppd """ select t1.id, struct_element(t1.struct_col, 'city') as city, t2.int_col + from tep_tbl t1 join tep_tbl t2 on t1.id = t2.id + order by t1.id limit 3 """ + + // ============================================= + // Test 13: Join — non-PPD on right table + // ============================================= + qt_join_nonppd """ select t1.id, upper(t2.str_col) as name + from tep_tbl t1 join tep_tbl t2 on t1.id = t2.id + order by t1.id limit 3 """ + + // ============================================= + // Test 14: Join — both sides have pull-up expressions + // ============================================= + qt_join_both """ select t1.id, struct_element(t1.struct_col, 'city') as c1, upper(t2.str_col) as c2 + from tep_tbl t1 join tep_tbl t2 on t1.id = t2.id + order by t1.id limit 3 """ + + // ============================================= + // Test 15: Join — condition references baseSlot (should still pull up) + // ============================================= + qt_join_base_slot """ select t1.id, struct_element(t1.struct_col, 'city') as c1 + from tep_tbl t1 join tep_tbl t2 on t1.id = t2.id and t1.str_col = t2.str_col + order by t1.id limit 3 """ + + // ============================================= + // Cleanup + // ============================================= + sql """ DROP TABLE IF EXISTS tep_tbl """ +} diff --git a/regression-test/suites/nereids_rules_p0/column_pruning/topn_lazy_nested_column_pruning.groovy b/regression-test/suites/nereids_rules_p0/column_pruning/topn_lazy_nested_column_pruning.groovy new file mode 100644 index 00000000000000..9d954f0048ca4f --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/column_pruning/topn_lazy_nested_column_pruning.groovy @@ -0,0 +1,365 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("topn_lazy_nested_column_pruning") { + sql """ set topn_lazy_materialization_threshold=1024; """ + sql """ DROP TABLE IF EXISTS tlncp_tbl """ + sql """ + CREATE TABLE tlncp_tbl ( + id INT, + str_col STRING NULL, + struct_col STRUCT NULL, + arr_col ARRAY NULL, + map_col MAP NULL, + int_col INT NULL + ) ENGINE = OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + + sql """ + INSERT INTO tlncp_tbl VALUES + (1, 'hello', named_struct('city', null, 'zip', 10001), [1, 2, 3], {'a': 1, 'b': 2 }, 1) + """ + + sql """ + drop table if exists vt; + CREATE TABLE IF NOT EXISTS vt ( + id BIGINT NOT NULL, + s varchar(100) null, + payload VARIANT< + 'name' : STRING, + 'age' : INT + > NULL + ) + ENGINE = OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "storage_format" = "V3" + ); + + + INSERT INTO vt VALUES + (1, 'aaa', '{"name": "张三", "age": 25}'), + (2, 'bbb', '{"name": "李四", "age": 30}'), + (3, 'ccc', '{"name": "王五", "age": 28, "city": "北京"}'); + """ + + // ============================================= + // Test 1: STRUCT type - lazy mat + nested column pruning + // ============================================= + explain { + sql """ + select id, substring(struct_element(struct_col, 'city'), 1) as city + from tlncp_tbl + order by id + limit 3 + """ + contains("VMaterializeNode") + // struct_col lazy, scan only outputs id + rowId + contains("final projections: id[#0], __DORIS_GLOBAL_ROWID_COL__tlncp_tbl[#6]") + // nested column pruning: struct_col pruned to city only + contains("nested columns:") + contains("pruned type: struct") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__tlncp_tbl]") + } + + // ============================================= + // Test 2: STRUCT with select * - struct_col explicit in output + // ============================================= + explain { + sql """ + select *, substring(struct_element(struct_col, 'city'), 1) as city + from tlncp_tbl + order by id + limit 3 + """ + contains("VMaterializeNode") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__tlncp_tbl]") + } + + // ============================================= + // Test 3: VARIANT type - lazy mat + sub path pruning + // ============================================= + explain { + sql """ + select id, s, substring(element_at(payload, 'name'), 1) as name + from vt + order by id + limit 3 + """ + contains("VMaterializeNode") + // payload lazy, scan only outputs id + rowId + contains("final projections: id[#0], __DORIS_GLOBAL_ROWID_COL__vt[#4]") + // sub path pruning for variant + contains("nested columns:") + contains("sub path: [name]") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__vt]") + } + + // ============================================= + // Test 4: STRUCT - verify actual query results + // ============================================= + qt_struct_result """ + select id, substring(struct_element(struct_col, 'city'), 1) as city + from tlncp_tbl + order by id + limit 3 + """ + + // ============================================= + // Test 5: VARIANT - verify actual query results + // ============================================= + qt_variant_result """ + select id, s, substring(element_at(payload, 'name'), 1) as name + from vt + order by id + limit 3 + """ + + // ============================================= + // Test 6: MAP subscript - lazy mat + nested column pruning + // ============================================= + explain { + sql """ + select id, element_at(map_col, 'a') as val + from tlncp_tbl + order by id + limit 3 + """ + contains("VMaterializeNode") + // map_col lazy, scan only outputs id + rowId + contains("final projections: id[#0], __DORIS_GLOBAL_ROWID_COL__tlncp_tbl[#6]") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__tlncp_tbl]") + } + + // ============================================= + // Test 7: MAP subscript - verify actual query results + // ============================================= + qt_map_result """ + select id, element_at(map_col, 'a') as val + from tlncp_tbl + order by id + limit 3 + """ + + // ============================================= + // Test 8: ARRAY subscript - lazy mat + nested column pruning + // ============================================= + explain { + sql """ + select id, element_at(arr_col, 1) as val + from tlncp_tbl + order by id + limit 3 + """ + contains("VMaterializeNode") + // arr_col lazy, scan only outputs id + rowId + contains("final projections: id[#0], __DORIS_GLOBAL_ROWID_COL__tlncp_tbl[#6]") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__tlncp_tbl]") + } + + // ============================================= + // Test 9: ARRAY subscript - verify actual query results + // ============================================= + qt_array_result """ + select id, element_at(arr_col, 1) as val + from tlncp_tbl + order by id + limit 3 + """ + + // ============================================= + // Test 10: Multi-level VARIANT nested - insert nested data + // ============================================= + sql """ + INSERT INTO vt VALUES + (4, 'ddd', '{"address": {"city": "上海", "zip": "200000"}}'), + (5, 'eee', '{"address": {"city": "北京", "zip": "100000"}}') + """ + + // ============================================= + // Test 11: Multi-level VARIANT nested - explain + // Access payload['address']['city'] via two levels: + // inner payload['address'] → variant with subColPath [address] + // outer element_at(..., 'city') → final value + // ============================================= + explain { + sql """ + select id, element_at(payload['address'], 'city') as city + from vt + where id >= 4 + order by id + limit 3 + """ + contains("VMaterializeNode") + // payload lazy, scan only outputs id + rowId + contains("final projections: id[#0], __DORIS_GLOBAL_ROWID_COL__vt[#4]") + // sub path pruning for variant: only read address sub-path during materialization + contains("nested columns:") + contains("sub path: [address.city]") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__vt]") + } + + // ============================================= + // Test 12: Multi-level VARIANT nested - verify query results + // ============================================= + qt_variant_nested_result """ + select id, element_at(payload['address'], 'city') as city + from vt + where id >= 4 + order by id + limit 3 + """ + + // ============================================= + // Test 13: using_index=true with MAP subscript + // Verify that map/array lazy mat still works when + // topn_lazy_materialization_using_index is enabled. + // Regression test for the risk that MaterializeProbeVisitor + // .visitPhysicalProject skips alias→child slot tracing + // when using_index=true, which could prevent base columns + // from being probed as lazy candidates. + // ============================================= + sql """ set topn_lazy_materialization_using_index = true; """ + explain { + sql """ + select id, element_at(map_col, 'a') as val + from tlncp_tbl + order by id + limit 3 + """ + contains("VMaterializeNode") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__tlncp_tbl]") + } + qt_map_using_index_result """ + select id, element_at(map_col, 'a') as val + from tlncp_tbl + order by id + limit 3 + """ + sql """ set topn_lazy_materialization_using_index = false; """ + + // ============================================= + // Test 14: using_index=true with ARRAY subscript + // ============================================= + sql """ set topn_lazy_materialization_using_index = true; """ + explain { + sql """ + select id, element_at(arr_col, 1) as val + from tlncp_tbl + order by id + limit 3 + """ + contains("VMaterializeNode") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__tlncp_tbl]") + } + qt_array_using_index_result """ + select id, element_at(arr_col, 1) as val + from tlncp_tbl + order by id + limit 3 + """ + sql """ set topn_lazy_materialization_using_index = false; """ + + // ============================================= + // Test 15: STRUCT nested expr BEFORE id — verify column order preserved + // SELECT city_expr, id should produce [city, id] not [id, city] + // ============================================= + explain { + sql """ + select substring(struct_element(struct_col, 'city'), 1) as city, id + from tlncp_tbl + order by id + limit 3 + """ + contains("VMaterializeNode") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__tlncp_tbl]") + } + qt_struct_col_order_result """ + select substring(struct_element(struct_col, 'city'), 1) as city, id + from tlncp_tbl + order by id + limit 3 + """ + + // ============================================= + // Test 16: VARIANT nested expr BEFORE id — verify column order preserved + // ============================================= + explain { + sql """ + select substring(element_at(payload, 'name'), 1) as name, id, s + from vt + order by id + limit 3 + """ + contains("VMaterializeNode") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__vt]") + } + qt_variant_col_order_result """ + select substring(element_at(payload, 'name'), 1) as name, id, s + from vt + order by id + limit 3 + """ + + // ============================================= + // Test 17: MAP nested expr BEFORE id — verify column order preserved + // ============================================= + explain { + sql """ + select element_at(map_col, 'a') as val, id + from tlncp_tbl + order by id + limit 3 + """ + contains("VMaterializeNode") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__tlncp_tbl]") + } + qt_map_col_order_result """ + select element_at(map_col, 'a') as val, id + from tlncp_tbl + order by id + limit 3 + """ + + // ============================================= + // Test 18: Project expression kept below TopN still needs its input slot + // ============================================= + sql """ set enable_topn_expr_pullup = false; """ + explain { + sql """ + select *, substring(struct_element(struct_col, 'city'), 1) as city + from tlncp_tbl + order by id + limit 3 + """ + contains("VMaterializeNode") + contains("final projections: id[#0], struct_col[#2], substring(element_at(struct_col[#2]") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__tlncp_tbl]") + } + qt_project_under_topn_consumed_slot """ + select *, substring(struct_element(struct_col, 'city'), 1) as city + from tlncp_tbl + order by id + limit 3 + """ + sql """ set enable_topn_expr_pullup = true; """ +} diff --git a/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy b/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy index 16c6c4d1d1e2bf..8c652503f8e348 100644 --- a/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy @@ -16,7 +16,7 @@ package mv.unsafe_equals // specific language governing permissions and limitations // under the License. -suite("null_unsafe_equals") { +suite("null_un_safe_equals") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" sql "set runtime_filter_mode=OFF"; @@ -75,8 +75,21 @@ suite("null_unsafe_equals") { group by o_orderstatus, o_comment; """ + // query contains length(o_comment), which only needs string offsets in a standalone filter, + // but the final output still needs full o_comment data after mv rewrite. + def query2_0 = + """ + select count(*), o_orderstatus, o_comment + from orders + where length(o_comment) > 0 + group by + o_orderstatus, o_comment; + """ order_qt_query1_0_before "${query1_0}" + order_qt_query2_0_before "${query2_0}" async_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0") + mv_rewrite_success(query2_0, "mv1_0") order_qt_query1_0_after "${query1_0}" - sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" + order_qt_query2_0_after "${query2_0}" + // sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" } diff --git a/regression-test/suites/variant_p0/test_sub_path_pruning.groovy b/regression-test/suites/variant_p0/test_sub_path_pruning.groovy index 52bd03156517e8..b2b6f0cc09a304 100644 --- a/regression-test/suites/variant_p0/test_sub_path_pruning.groovy +++ b/regression-test/suites/variant_p0/test_sub_path_pruning.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("variant_sub_path_pruning", "variant_type"){ +suite("test_sub_path_pruning", "variant_type"){ def enableVariantV2 = getFeConfig("enable_variant_v2").toBoolean() def variantV2Function = enableVariantV2 ? "parse_to_variant" : ""