diff --git a/be/src/core/block/block.cpp b/be/src/core/block/block.cpp index a0598dcc8121d5..e0fca38b2bb84d 100644 --- a/be/src/core/block/block.cpp +++ b/be/src/core/block/block.cpp @@ -464,8 +464,7 @@ std::string Block::dump_data_json(size_t begin, size_t row_limit, bool allow_nul // This value-extraction logic is preserved from your original function // to maintain consistency, especially for handling nullability mismatches. - if (data[i].column && data[i].type->is_nullable() && - !data[i].column->is_concrete_nullable()) { + if (data[i].column && data[i].type->is_nullable() && !data[i].column->is_nullable()) { // This branch handles a specific internal representation of nullable columns. // The original code would assert here if allow_null_mismatch is false. assert(allow_null_mismatch); @@ -530,7 +529,7 @@ std::string Block::dump_data(size_t begin, size_t row_limit, bool allow_null_mis std::string s; if (data[i].column) { // column may be const // for code inside `default_implementation_for_nulls`, there's could have: type = null, col != null - if (data[i].type->is_nullable() && !data[i].column->is_concrete_nullable()) { + if (data[i].type->is_nullable() && !data[i].column->is_nullable()) { assert(allow_null_mismatch); s = assert_cast(data[i].type.get()) ->get_nested_type() diff --git a/be/src/core/column/column.h b/be/src/core/column/column.h index 06934c4c712d8b..ecd0245d718c18 100644 --- a/be/src/core/column/column.h +++ b/be/src/core/column/column.h @@ -611,10 +611,8 @@ class IColumn : public COW { /// Various properties on behaviour of column type. - /// It's true for ColumnNullable only. + /// It's true for ColumnNullable and Const(ColumnNullable). virtual bool is_nullable() const { return false; } - /// It's true for ColumnNullable, can be true or false for ColumnConst, etc. - virtual bool is_concrete_nullable() const { return false; } // true if column has null element virtual bool has_null() const { return false; } diff --git a/be/src/core/column/column_const.h b/be/src/core/column/column_const.h index bda7ea13fb1aa2..238d0f971b5a03 100644 --- a/be/src/core/column/column_const.h +++ b/be/src/core/column/column_const.h @@ -270,8 +270,7 @@ class ColumnConst final : public COWHelper { return false; } - // ColumnConst is not nullable, but may be concrete nullable. - bool is_concrete_nullable() const override { return is_column_nullable(*data); } + bool is_nullable() const override { return is_column_nullable(*data); } bool only_null() const override { return data->is_null_at(0); } StringRef get_raw_data() const override { return data->get_raw_data(); } diff --git a/be/src/core/column/column_nullable.h b/be/src/core/column/column_nullable.h index bf924fcb2ec8dc..563d5e7011ddbf 100644 --- a/be/src/core/column/column_nullable.h +++ b/be/src/core/column/column_nullable.h @@ -268,7 +268,6 @@ class ColumnNullable final : public COWHelper { } bool is_nullable() const override { return true; } - bool is_concrete_nullable() const override { return true; } bool is_column_string() const override { return get_nested_column().is_column_string(); } bool is_exclusive() const override { diff --git a/be/src/exec/common/hash_table/hash_map_context.h b/be/src/exec/common/hash_table/hash_map_context.h index 5e590ac7789109..1dc33319dfec95 100644 --- a/be/src/exec/common/hash_table/hash_map_context.h +++ b/be/src/exec/common/hash_table/hash_map_context.h @@ -378,7 +378,7 @@ struct MethodStringNoCache : public MethodBase { DorisVector& stored_keys) { const IColumn& column = *key_columns[0]; const auto& nested_column = - column.is_nullable() + is_column_nullable(column) ? assert_cast(column).get_nested_column() : column; auto serialized_str = [](const auto& column_string, DorisVector& stored_keys) { @@ -744,7 +744,7 @@ struct MethodOneNumber : public MethodBase { void init_serialized_keys(const ColumnRawPtrs& key_columns, uint32_t num_rows, const uint8_t* null_map = nullptr, bool is_join = false, bool is_build = false, uint32_t bucket_size = 0) override { - Base::keys = (FieldType*)(key_columns[0]->is_nullable() + Base::keys = (FieldType*)(is_column_nullable(*key_columns[0]) ? assert_cast(key_columns[0]) ->get_nested_column_ptr() ->get_raw_data() @@ -782,7 +782,7 @@ struct MethodOneNumberDirect : public MethodOneNumber { void init_serialized_keys(const ColumnRawPtrs& key_columns, uint32_t num_rows, const uint8_t* null_map = nullptr, bool is_join = false, bool is_build = false, uint32_t bucket_size = 0) override { - Base::keys = (FieldType*)(key_columns[0]->is_nullable() + Base::keys = (FieldType*)(is_column_nullable(*key_columns[0]) ? assert_cast(key_columns[0]) ->get_nested_column_ptr() ->get_raw_data() @@ -1174,4 +1174,4 @@ struct MethodSingleNullableColumn : public SingleColumnMethod { } } }; -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/exec/common/join_utils.h b/be/src/exec/common/join_utils.h index 5b48c4e140e956..7482c6fbdae611 100644 --- a/be/src/exec/common/join_utils.h +++ b/be/src/exec/common/join_utils.h @@ -163,7 +163,7 @@ void primary_to_direct_mapping(Method* context, const ColumnRawPtrs& key_columns FieldType min_key = std::numeric_limits::max(); size_t num_rows = key_columns[0]->size(); - if (key_columns[0]->is_nullable()) { + if (is_column_nullable(*key_columns[0])) { const FieldType* input_keys = (FieldType*)assert_cast(key_columns[0]) ->get_nested_column_ptr() ->get_raw_data() diff --git a/be/src/exec/common/util.hpp b/be/src/exec/common/util.hpp index 2b6eb69c7bf0a6..013d0a4f0362a1 100644 --- a/be/src/exec/common/util.hpp +++ b/be/src/exec/common/util.hpp @@ -105,12 +105,12 @@ class VectorizedUtils { // Helper function to extract null map from column (including ColumnConst cases) static const NullMap* get_null_map(const ColumnPtr& col) { - if (col->is_nullable()) { - return &static_cast(*col).get_null_map_data(); + if (const auto* nullable = check_and_get_column(col.get())) { + return &nullable->get_null_map_data(); } // Handle Const(Nullable) case if (const auto* const_col = check_and_get_column(col.get()); - const_col != nullptr && const_col->is_concrete_nullable()) { + const_col != nullptr && const_col->is_nullable()) { return &static_cast(const_col->get_data_column()) .get_null_map_data(); } @@ -274,7 +274,7 @@ inline size_t calculate_false_number(ColumnPtr column) { return calculate_false_number( assert_cast(column.get())->get_data_column_ptr()) * rows; - } else if (column->is_nullable()) { + } else if (is_column_nullable(*column)) { const auto* nullable = assert_cast(column.get()); const auto* data = assert_cast(nullable->get_nested_column_ptr().get()) ->get_data() diff --git a/be/src/exec/common/variant_util.cpp b/be/src/exec/common/variant_util.cpp index 0aa1172b5fc9ca..5314b38e480c74 100644 --- a/be/src/exec/common/variant_util.cpp +++ b/be/src/exec/common/variant_util.cpp @@ -17,7 +17,6 @@ #include "exec/common/variant_util.h" -#include #include #include #include @@ -349,13 +348,13 @@ Status cast_column(const ColumnWithTypeAndName& arg, const DataTypePtr& type, Co return Status::OK(); } // set variant root column/type to from column/type - CHECK(arg.column->is_nullable()); + CHECK(is_column_nullable(*arg.column)); auto to_type = remove_nullable(type); const auto& data_type_object = assert_cast(*to_type); auto variant = ColumnVariant::create(data_type_object.variant_max_subcolumns_count(), data_type_object.enable_doc_mode()); - variant->create_root(arg.type, std::move(*arg.column).mutate()); + variant->create_root(arg.type, IColumn::mutate(arg.column)); ColumnPtr nullable = ColumnNullable::create( variant->get_ptr(), assert_cast(arg.column.get())->get_null_map_column_ptr()); @@ -2152,14 +2151,14 @@ Status _parse_and_materialize_variant_columns(Block& block, const std::vector& configs) { for (size_t i = 0; i < variant_pos.size(); ++i) { auto column_ref = block.get_by_position(variant_pos[i]).column; - bool is_nullable = column_ref->is_nullable(); - MutableColumnPtr owner_column = std::move(*column_ref).mutate(); + bool is_nullable = is_column_nullable(*column_ref); + MutableColumnPtr owner_column = IColumn::mutate(std::move(column_ref)); ColumnPtr nullable_null_map; MutableColumnPtr var_column; if (is_nullable) { const auto& nullable = assert_cast(*owner_column); nullable_null_map = nullable.get_null_map_column_ptr(); - var_column = std::move(*nullable.get_nested_column_ptr()).mutate(); + var_column = IColumn::mutate(nullable.get_nested_column_ptr()); } else { var_column = std::move(owner_column); } @@ -2182,14 +2181,14 @@ Status _parse_and_materialize_variant_columns(Block& block, ? make_nullable(std::make_shared()) : std::make_shared(), &scalar_root_column)); - if (scalar_root_column->is_nullable()) { + if (is_column_nullable(*scalar_root_column)) { scalar_root_column = assert_cast(scalar_root_column.get()) ->get_nested_column_ptr(); } } else { const auto& root = *var.get_root(); scalar_root_column = - root.is_nullable() + is_column_nullable(root) ? assert_cast(root).get_nested_column_ptr() : var.get_root(); } diff --git a/be/src/exec/operator/hashjoin_build_sink.cpp b/be/src/exec/operator/hashjoin_build_sink.cpp index 14772cf366263a..ebd90a462ece46 100644 --- a/be/src/exec/operator/hashjoin_build_sink.cpp +++ b/be/src/exec/operator/hashjoin_build_sink.cpp @@ -22,6 +22,7 @@ #include #include "core/block/block.h" +#include "core/column/column_const.h" #include "core/column/column_nullable.h" #include "core/data_type/data_type_nullable.h" #include "exec/common/template_helpers.hpp" @@ -388,8 +389,8 @@ Status HashJoinBuildSinkLocalState::build_asof_index(Block& block) { // Handle nullable: extract nested column for value access, keep nullable for null checks const ColumnNullable* nullable_col = nullptr; ColumnPtr build_col_nested = asof_build_col; - if (asof_build_col->is_nullable()) { - nullable_col = assert_cast(asof_build_col.get()); + if (const auto* nullable = check_and_get_column(asof_build_col.get())) { + nullable_col = nullable; build_col_nested = nullable_col->get_nested_column_ptr(); } @@ -514,7 +515,9 @@ Status HashJoinBuildSinkLocalState::_do_evaluate(Block& block, VExprContextSPtrs RETURN_IF_ERROR(exprs[i]->execute(&block, &result_col_id)); } - // TODO: opt the column is const + // _extract_join_column() handles physical ColumnNullable only, so build-key const + // columns, including Const(Nullable), must be materialized before they are merged. + // TODO: if const-key optimization is added, update _extract_join_column() together. block.get_by_position(result_col_id).column = block.get_by_position(result_col_id).column->convert_to_full_column_if_const(); res_col_ids[i] = result_col_id; @@ -544,15 +547,21 @@ Status HashJoinBuildSinkLocalState::_extract_join_column(Block& block, DCHECK(_should_build_hash_table); auto& shared_state = *_shared_state; for (size_t i = 0; i < shared_state.build_exprs_size; ++i) { - const auto* column = block.get_by_position(res_col_ids[i]).column.get(); - if (!column->is_nullable() && - _parent->cast()._serialize_null_into_key[i]) { + const auto& column_ptr = block.get_by_position(res_col_ids[i]).column; + const auto* column = column_ptr.get(); + const bool serialize_null_into_key = + _parent->cast()._serialize_null_into_key[i]; + // _do_evaluate() must have materialized Const(Nullable) build keys. If this check fails, + // is_nullable() no longer implies a physical ColumnNullable for the logic below. + const auto* const_column = check_and_get_column(*column); + DORIS_CHECK(const_column == nullptr || + !is_column_nullable(const_column->get_data_column())); + if (!column->is_nullable() && serialize_null_into_key) { _key_columns_holder.emplace_back( make_nullable(block.get_by_position(res_col_ids[i]).column)); raw_ptrs[i] = _key_columns_holder.back().get(); } else if (const auto* nullable = check_and_get_column(*column); - !_parent->cast()._serialize_null_into_key[i] && - nullable) { + !serialize_null_into_key && nullable) { // update nulllmap and split nested out of ColumnNullable when serialize_null_into_key is false and column is nullable const auto& col_nested = nullable->get_nested_column(); const auto& col_nullmap = nullable->get_null_map_data(); @@ -574,7 +583,7 @@ Status HashJoinBuildSinkLocalState::process_build_block(RuntimeState* state, Blo // 1. Dispose the overflow of ColumnString // 2. Finalize the ColumnVariant to speed up for (auto& data : block) { - data.column = std::move(*data.column).mutate()->convert_column_if_overflow(); + data.column = IColumn::mutate(std::move(data.column))->convert_column_if_overflow(); if (p._need_finalize_variant_column) { auto mutable_column = IColumn::mutate(std::move(data.column)); mutable_column->finalize(); diff --git a/be/src/exec/operator/hashjoin_probe_operator.cpp b/be/src/exec/operator/hashjoin_probe_operator.cpp index ea4b812323a9e6..80543c55e4ec4a 100644 --- a/be/src/exec/operator/hashjoin_probe_operator.cpp +++ b/be/src/exec/operator/hashjoin_probe_operator.cpp @@ -25,6 +25,8 @@ #include "common/cast_set.h" #include "common/logging.h" #include "core/assert_cast.h" +#include "core/column/column_const.h" +#include "core/column/column_nullable.h" #include "core/data_type/data_type_nullable.h" #include "exec/operator/operator.h" #include "runtime/descriptors.h" @@ -349,15 +351,21 @@ Status HashJoinProbeLocalState::_extract_join_column(Block& block, auto& shared_state = *_shared_state; for (size_t i = 0; i < shared_state.build_exprs_size; ++i) { - const auto* column = block.get_by_position(res_col_ids[i]).column.get(); - if (!column->is_nullable() && - _parent->cast()._serialize_null_into_key[i]) { + const auto& column_ptr = block.get_by_position(res_col_ids[i]).column; + const auto* column = column_ptr.get(); + const bool serialize_null_into_key = + _parent->cast()._serialize_null_into_key[i]; + // _do_evaluate() must have materialized Const(Nullable) probe keys. If this check fails, + // is_nullable() no longer implies a physical ColumnNullable for the logic below. + const auto* const_column = check_and_get_column(*column); + DORIS_CHECK(const_column == nullptr || + !is_column_nullable(const_column->get_data_column())); + if (!column->is_nullable() && serialize_null_into_key) { _key_columns_holder.emplace_back( make_nullable(block.get_by_position(res_col_ids[i]).column)); _probe_columns[i] = _key_columns_holder.back().get(); } else if (const auto* nullable = check_and_get_column(*column); - nullable && - !_parent->cast()._serialize_null_into_key[i]) { + nullable && !serialize_null_into_key) { // update nulllmap and split nested out of ColumnNullable when serialize_null_into_key is false and column is nullable const auto& col_nested = nullable->get_nested_column(); const auto& col_nullmap = nullable->get_null_map_data(); @@ -420,7 +428,9 @@ Status HashJoinProbeOperatorX::_do_evaluate(Block& block, VExprContextSPtrs& exp RETURN_IF_ERROR(exprs[i]->execute(&block, &result_col_id)); } - // TODO: opt the column is const + // _extract_join_column() handles physical ColumnNullable only, so probe-key const + // columns, including Const(Nullable), must be materialized before probing. + // TODO: if const-key optimization is added, update _extract_join_column() together. block.get_by_position(result_col_id).column = block.get_by_position(result_col_id).column->convert_to_full_column_if_const(); res_col_ids[i] = result_col_id; diff --git a/be/src/exec/operator/join/process_hash_table_probe_impl.h b/be/src/exec/operator/join/process_hash_table_probe_impl.h index 8e4e0131ffe12f..277da08c01ec0a 100644 --- a/be/src/exec/operator/join/process_hash_table_probe_impl.h +++ b/be/src/exec/operator/join/process_hash_table_probe_impl.h @@ -114,7 +114,7 @@ void ProcessHashTableProbe::build_side_output_column(MutableColumns& for (int i = 0; i < _right_col_len; i++) { const auto& column = *_build_block->safe_get_by_position(i).column; _build_column_has_null[i] = false; - if (_right_output_slot_flags[i] && column.is_nullable()) { + if (_right_output_slot_flags[i] && is_column_nullable(column)) { const auto& nullable = assert_cast(column); _build_column_has_null[i] = !simd::contain_one( nullable.get_null_map_data().data() + 1, nullable.size() - 1); @@ -264,7 +264,7 @@ uint32_t ProcessHashTableProbe:: // Remove nullable wrapper for comparison - keep original for null check ColumnPtr probe_col_for_compare = probe_col_ptr; const uint8_t* asof_probe_null_map = nullptr; - if (probe_col_ptr->is_nullable()) { + if (is_column_nullable(*probe_col_ptr)) { const auto* nullable_probe_col = assert_cast(probe_col_ptr.get()); asof_probe_null_map = nullable_probe_col->get_null_map_data().data(); probe_col_for_compare = nullable_probe_col->get_nested_column_ptr(); diff --git a/be/src/exec/operator/nested_loop_join_probe_operator.cpp b/be/src/exec/operator/nested_loop_join_probe_operator.cpp index 9444a822863948..fd80d5471cea73 100644 --- a/be/src/exec/operator/nested_loop_join_probe_operator.cpp +++ b/be/src/exec/operator/nested_loop_join_probe_operator.cpp @@ -49,15 +49,61 @@ ColumnPtr align_eval_column_nullable(const ColumnWithTypeAndName& target, const return column; } +void append_many_from_column(MutableColumnPtr& dst_column, const IColumn& src_column, size_t row, + size_t rows) { + if (!is_column_nullable(src_column) && is_column_nullable(*dst_column)) { + if (src_column.is_nullable()) { + auto full_src_column = src_column.convert_to_full_column_if_const(); + dst_column->insert_many_from(*full_src_column, row, rows); + return; + } + const auto origin_size = dst_column->size(); + auto* nullable_column = assert_cast(dst_column.get()); + nullable_column->get_nested_column_ptr()->insert_many_from(src_column, row, rows); + nullable_column->get_null_map_column().get_data().resize_fill(origin_size + rows, 0); + } else { + dst_column->insert_many_from(src_column, row, rows); + } +} + void append_many_from_source(MutableColumnPtr& dst_column, const ColumnWithTypeAndName& src_column, size_t row, size_t rows) { - if (!src_column.column->is_nullable() && dst_column->is_nullable()) { + append_many_from_column(dst_column, *src_column.column, row, rows); +} + +void append_range_from_column(MutableColumnPtr& dst_column, const IColumn& src_column, size_t start, + size_t length) { + if (!is_column_nullable(src_column) && is_column_nullable(*dst_column)) { + if (src_column.is_nullable()) { + auto full_src_column = src_column.convert_to_full_column_if_const(); + dst_column->insert_range_from(*full_src_column, start, length); + return; + } const auto origin_size = dst_column->size(); auto* nullable_column = assert_cast(dst_column.get()); - nullable_column->get_nested_column_ptr()->insert_many_from(*src_column.column, row, rows); - nullable_column->get_null_map_column().get_data().resize_fill(origin_size + rows, 0); + nullable_column->get_nested_column_ptr()->insert_range_from(src_column, start, length); + nullable_column->get_null_map_column().get_data().resize_fill(origin_size + length, 0); + } else { + dst_column->insert_range_from(src_column, start, length); + } +} + +void append_indices_from_column(MutableColumnPtr& dst_column, const IColumn& src_column, + const uint32_t* indices_begin, const uint32_t* indices_end) { + if (!is_column_nullable(src_column) && is_column_nullable(*dst_column)) { + if (src_column.is_nullable()) { + auto full_src_column = src_column.convert_to_full_column_if_const(); + dst_column->insert_indices_from(*full_src_column, indices_begin, indices_end); + return; + } + const auto origin_size = dst_column->size(); + auto* nullable_column = assert_cast(dst_column.get()); + nullable_column->get_nested_column_ptr()->insert_indices_from(src_column, indices_begin, + indices_end); + nullable_column->get_null_map_column().get_data().resize_fill( + origin_size + (indices_end - indices_begin), 0); } else { - dst_column->insert_many_from(*src_column.column, row, rows); + dst_column->insert_indices_from(src_column, indices_begin, indices_end); } } @@ -68,16 +114,7 @@ void append_filtered_from_source(MutableColumnPtr& dst_column, return; } auto filtered_column = src_column.column->filter(filter, selected_rows); - if (!src_column.column->is_nullable() && dst_column->is_nullable()) { - const auto origin_size = dst_column->size(); - auto* nullable_column = assert_cast(dst_column.get()); - nullable_column->get_nested_column_ptr()->insert_range_from(*filtered_column, 0, - selected_rows); - nullable_column->get_null_map_column().get_data().resize_fill(origin_size + selected_rows, - 0); - } else { - dst_column->insert_range_from(*filtered_column, 0, selected_rows); - } + append_range_from_column(dst_column, *filtered_column, 0, selected_rows); } void append_mark_value(MutableColumnPtr& dst_column, int8_t mark_value) { @@ -167,36 +204,13 @@ void process_probe_block(int64_t probe_block_pos, Block& block, const Block& pro const size_t max_added_rows = build_block.rows(); for (size_t i = 0; i < probe_side_columns; ++i) { const ColumnWithTypeAndName& src_column = probe_block.get_by_position(i); - if (!src_column.column->is_nullable() && dst_columns[i]->is_nullable()) { - auto origin_sz = dst_columns[i]->size(); - assert_cast(dst_columns[i].get()) - ->get_nested_column_ptr() - ->insert_many_from(*src_column.column, probe_block_pos, max_added_rows); - assert_cast(dst_columns[i].get()) - ->get_null_map_column() - .get_data() - .resize_fill(origin_sz + max_added_rows, 0); - } else { - // TODO: for cross join, maybe could insert one row, and wrap for a const column - dst_columns[i]->insert_many_from(*src_column.column, probe_block_pos, max_added_rows); - } + // TODO: for cross join, maybe could insert one row, and wrap for a const column + append_many_from_source(dst_columns[i], src_column, probe_block_pos, max_added_rows); } for (size_t i = 0; i < build_side_columns; ++i) { const ColumnWithTypeAndName& src_column = build_block.get_by_position(i); - if (!src_column.column->is_nullable() && - dst_columns[probe_side_columns + i]->is_nullable()) { - auto origin_sz = dst_columns[probe_side_columns + i]->size(); - assert_cast(dst_columns[probe_side_columns + i].get()) - ->get_nested_column_ptr() - ->insert_range_from(*src_column.column.get(), 0, max_added_rows); - assert_cast(dst_columns[probe_side_columns + i].get()) - ->get_null_map_column() - .get_data() - .resize_fill(origin_sz + max_added_rows, 0); - } else { - dst_columns[probe_side_columns + i]->insert_range_from(*src_column.column.get(), 0, - max_added_rows); - } + append_range_from_column(dst_columns[probe_side_columns + i], *src_column.column, 0, + max_added_rows); } } @@ -208,35 +222,12 @@ void process_build_block(int64_t build_block_pos, Block& block, const Block& bui const size_t max_added_rows = probe_block.rows(); for (size_t i = 0; i < probe_side_columns; ++i) { const ColumnWithTypeAndName& src_column = probe_block.get_by_position(i); - if (!src_column.column->is_nullable() && dst_columns[i]->is_nullable()) { - auto origin_sz = dst_columns[i]->size(); - assert_cast(dst_columns[i].get()) - ->get_nested_column_ptr() - ->insert_range_from(*src_column.column.get(), 0, max_added_rows); - assert_cast(dst_columns[i].get()) - ->get_null_map_column() - .get_data() - .resize_fill(origin_sz + max_added_rows, 0); - } else { - dst_columns[i]->insert_range_from(*src_column.column.get(), 0, max_added_rows); - } + append_range_from_column(dst_columns[i], *src_column.column, 0, max_added_rows); } for (size_t i = 0; i < build_side_columns; ++i) { const ColumnWithTypeAndName& src_column = build_block.get_by_position(i); - if (!src_column.column->is_nullable() && - dst_columns[probe_side_columns + i]->is_nullable()) { - auto origin_sz = dst_columns[probe_side_columns + i]->size(); - assert_cast(dst_columns[probe_side_columns + i].get()) - ->get_nested_column_ptr() - ->insert_many_from(*src_column.column, build_block_pos, max_added_rows); - assert_cast(dst_columns[probe_side_columns + i].get()) - ->get_null_map_column() - .get_data() - .resize_fill(origin_sz + max_added_rows, 0); - } else { - dst_columns[probe_side_columns + i]->insert_many_from(*src_column.column, - build_block_pos, max_added_rows); - } + append_many_from_source(dst_columns[probe_side_columns + i], src_column, build_block_pos, + max_added_rows); } } @@ -1032,22 +1023,9 @@ void NestedLoopJoinProbeLocalState::_finalize_current_phase(Block& block, size_t } for (size_t j = 0; j < p._num_build_side_columns; ++j) { auto src_column = cur_block.get_by_position(j); - if (!src_column.column->is_nullable() && - dst_columns[p._num_probe_side_columns + j]->is_nullable()) { - DCHECK(p._join_op == TJoinOp::FULL_OUTER_JOIN); - assert_cast(dst_columns[p._num_probe_side_columns + j].get()) - ->get_nested_column_ptr() - ->insert_indices_from(*src_column.column, selector.data(), - selector.data() + selector_idx); - assert_cast(dst_columns[p._num_probe_side_columns + j].get()) - ->get_null_map_column() - .get_data() - .resize_fill(column_size, 0); - } else { - dst_columns[p._num_probe_side_columns + j]->insert_indices_from( - *src_column.column.get(), selector.data(), - selector.data() + selector_idx); - } + append_indices_from_column(dst_columns[p._num_probe_side_columns + j], + *src_column.column, selector.data(), + selector.data() + selector_idx); } } _output_null_idx_build_side = i; @@ -1061,18 +1039,7 @@ void NestedLoopJoinProbeLocalState::_finalize_current_phase(Block& block, size_t new_size++; for (size_t i = 0; i < p._num_probe_side_columns; ++i) { const ColumnWithTypeAndName src_column = _child_block->get_by_position(i); - if (!src_column.column->is_nullable() && dst_columns[i]->is_nullable()) { - DCHECK(p._join_op == TJoinOp::FULL_OUTER_JOIN); - assert_cast(dst_columns[i].get()) - ->get_nested_column_ptr() - ->insert_many_from(*src_column.column, j, 1); - assert_cast(dst_columns[i].get()) - ->get_null_map_column() - .get_data() - .resize_fill(new_size, 0); - } else { - dst_columns[i]->insert_many_from(*src_column.column, j, 1); - } + append_many_from_source(dst_columns[i], src_column, j, 1); } } } @@ -1093,8 +1060,8 @@ void NestedLoopJoinProbeLocalState::_finalize_current_phase(Block& block, size_t for (size_t i = 0; i < p._num_probe_side_columns; ++i) { const ColumnWithTypeAndName src_column = _child_block->get_by_position(i); DCHECK(p._join_op != TJoinOp::FULL_OUTER_JOIN); - dst_columns[i]->insert_range_from(*src_column.column, _probe_block_start_pos, - _probe_side_process_count); + append_range_from_column(dst_columns[i], *src_column.column, _probe_block_start_pos, + _probe_side_process_count); } for (size_t i = 0; i < p._num_build_side_columns; ++i) { dst_columns[p._num_probe_side_columns + i]->insert_many_defaults( @@ -1111,22 +1078,8 @@ void NestedLoopJoinProbeLocalState::_append_probe_data_with_null(Block& block) c DCHECK(p._is_mark_join); for (size_t i = 0; i < p._num_probe_side_columns; ++i) { const ColumnWithTypeAndName& src_column = _child_block->get_by_position(i); - if (!src_column.column->is_nullable() && dst_columns[i]->is_nullable()) { - auto origin_sz = dst_columns[i]->size(); - DCHECK(p._join_op == TJoinOp::RIGHT_OUTER_JOIN || - p._join_op == TJoinOp::FULL_OUTER_JOIN); - assert_cast(dst_columns[i].get()) - ->get_nested_column_ptr() - ->insert_range_from(*src_column.column, _probe_block_start_pos, - _probe_side_process_count); - assert_cast(dst_columns[i].get()) - ->get_null_map_column() - .get_data() - .resize_fill(origin_sz + _probe_side_process_count, 0); - } else { - dst_columns[i]->insert_range_from(*src_column.column, _probe_block_start_pos, - _probe_side_process_count); - } + append_range_from_column(dst_columns[i], *src_column.column, _probe_block_start_pos, + _probe_side_process_count); } for (size_t i = 0; i < p._num_build_side_columns; ++i) { dst_columns[p._num_probe_side_columns + i]->insert_many_defaults(_probe_side_process_count); diff --git a/be/src/exec/operator/operator.cpp b/be/src/exec/operator/operator.cpp index b833ca83ff3aae..60f199b2b49779 100644 --- a/be/src/exec/operator/operator.cpp +++ b/be/src/exec/operator/operator.cpp @@ -358,7 +358,7 @@ Status OperatorXBase::do_projections(RuntimeState* state, Block* origin_block, input_block.rows(), rows, input_block.dump_structure()); } auto insert_column_datas = [&](auto& to, ColumnPtr& from, size_t rows) { - if (to->is_nullable() && !from->is_nullable()) { + if (is_column_nullable(*to) && !is_column_nullable(*from)) { if (_keep_origin || !from->is_exclusive()) { auto& null_column = reinterpret_cast(*to); null_column.get_nested_column().insert_range_from(*from, 0, rows); diff --git a/be/src/exec/operator/schema_scan_operator.cpp b/be/src/exec/operator/schema_scan_operator.cpp index f50d6a003bbf9e..95ee374824c38a 100644 --- a/be/src/exec/operator/schema_scan_operator.cpp +++ b/be/src/exec/operator/schema_scan_operator.cpp @@ -261,10 +261,10 @@ Status SchemaScanOperatorX::get_block_impl(RuntimeState* state, Block* block, bo IColumn::mutate(std::move(block->get_by_position(i).column)); ColumnPtr src_column = src_block.safe_get_by_position(_slot_offsets[i]) .column->convert_to_full_column_if_const(); - if (column_ptr->is_nullable() && !src_column->is_nullable()) { + if (is_column_nullable(*column_ptr) && !is_column_nullable(*src_column)) { src_column = make_nullable(src_column); } - DORIS_CHECK(column_ptr->is_nullable() == src_column->is_nullable()); + DORIS_CHECK(is_column_nullable(*column_ptr) == is_column_nullable(*src_column)); column_ptr->insert_range_from(*src_column, 0, src_block.rows()); block->replace_by_position(i, std::move(column_ptr)); } diff --git a/be/src/exec/operator/table_function_operator.cpp b/be/src/exec/operator/table_function_operator.cpp index bb4f8945359c42..4cf3a3642c5008 100644 --- a/be/src/exec/operator/table_function_operator.cpp +++ b/be/src/exec/operator/table_function_operator.cpp @@ -279,7 +279,7 @@ Status TableFunctionLocalState::_get_expanded_block_block_fast_path( IColumn* value_col_ptr = nullptr; ColumnInt32* pos_col_ptr = nullptr; if (is_posexplode) { - if (out_col->is_nullable()) { + if (is_column_nullable(*out_col)) { auto* nullable = assert_cast(out_col.get()); struct_col_ptr = assert_cast(nullable->get_nested_column_ptr().get()); outer_struct_nullmap_ptr = nullable->get_null_map_column_ptr().get(); @@ -336,7 +336,7 @@ Status TableFunctionLocalState::_get_expanded_block_block_fast_path( reinterpret_cast(segment_ctx.seg_positions.data()), segment_ctx.seg_positions.size()); // Write nested values to the struct's value sub-column - DCHECK(value_col_ptr->is_nullable()) + DCHECK(is_column_nullable(*value_col_ptr)) << "posexplode fast path requires nullable value column"; auto* val_nullable = assert_cast(value_col_ptr); val_nullable->get_nested_column_ptr()->insert_range_from( @@ -358,7 +358,7 @@ Status TableFunctionLocalState::_get_expanded_block_block_fast_path( if (outer_struct_nullmap_ptr) { outer_struct_nullmap_ptr->insert_many_defaults(segment_ctx.seg_nested_count); } - } else if (out_col->is_nullable()) { + } else if (is_column_nullable(*out_col)) { auto* out_nullable = assert_cast(out_col.get()); out_nullable->get_nested_column_ptr()->insert_range_from( *_block_fast_path_ctx.nested_col, segment_ctx.seg_nested_start, diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index 5ab1579be636c1..d5c18a38b851c0 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -795,7 +795,7 @@ Status FileScanner::_convert_to_output_block(Block* block) { // because of src_slot_desc is always be nullable, so the column_ptr after do dest_expr // is likely to be nullable - if (LIKELY(column_ptr->is_nullable())) { + if (LIKELY(is_column_nullable(*column_ptr))) { const auto* nullable_column = reinterpret_cast(column_ptr.get()); for (int i = 0; i < rows; ++i) { if (filter_map[i] && nullable_column->is_null_at(i)) { diff --git a/be/src/exec/sink/vrow_distribution.cpp b/be/src/exec/sink/vrow_distribution.cpp index 96cf8966fe0ce1..4a9dd959ebb5b4 100644 --- a/be/src/exec/sink/vrow_distribution.cpp +++ b/be/src/exec/sink/vrow_distribution.cpp @@ -62,6 +62,7 @@ Status VRowDistribution::_save_missing_values( cur_row_values.clear(); for (int col = 0; col < col_size; ++col) { TNullableStringLiteral node; + // OlapTableBlockConvertor::_validate_data() materializes destination slots so won't be const. const auto* null_map = col_null_maps[col]; // null map for this col node.__set_is_null((null_map && (*null_map)[filter[row]]) ? true diff --git a/be/src/exec/sink/vtablet_block_convertor.cpp b/be/src/exec/sink/vtablet_block_convertor.cpp index e59fa923375998..134aa2a214049a 100644 --- a/be/src/exec/sink/vtablet_block_convertor.cpp +++ b/be/src/exec/sink/vtablet_block_convertor.cpp @@ -592,6 +592,7 @@ Status OlapTableBlockConvertor::_validate_data(RuntimeState* state, Block* block }}; for (int i = 0; i < _output_tuple_desc->slots().size(); ++i) { SlotDescriptor* desc = _output_tuple_desc->slots()[i]; + // VRowDistribution's _deal_missing_map() relies on these slots being full columns. block->get_by_position(i).column = block->get_by_position(i).column->convert_to_full_column_if_const(); const auto& column = block->get_by_position(i).column; diff --git a/be/src/exec/sink/writer/iceberg/partition_transformers.h b/be/src/exec/sink/writer/iceberg/partition_transformers.h index 3b1f6adbf4f271..b9dcceb3e8088a 100644 --- a/be/src/exec/sink/writer/iceberg/partition_transformers.h +++ b/be/src/exec/sink/writer/iceberg/partition_transformers.h @@ -17,6 +17,8 @@ #pragma once +#include "core/column/column.h" +#include "core/column/column_nullable.h" #include "core/data_type/data_type_factory.hpp" #include "exec/common/stringop_substring.h" #include "exprs/function/cast/cast_to_datetimev2_impl.hpp" @@ -215,7 +217,7 @@ class IntegerTruncatePartitionColumnTransform : public PartitionColumnTransform //2) get the input data from block ColumnPtr null_map_column_ptr; bool is_nullable = false; - if (column_ptr->is_nullable()) { + if (is_column_nullable(*column_ptr)) { const ColumnNullable* nullable_column = reinterpret_cast(column_ptr.get()); is_nullable = true; @@ -272,7 +274,7 @@ class BigintTruncatePartitionColumnTransform : public PartitionColumnTransform { //2) get the input data from block ColumnPtr null_map_column_ptr; bool is_nullable = false; - if (column_ptr->is_nullable()) { + if (is_column_nullable(*column_ptr)) { const ColumnNullable* nullable_column = reinterpret_cast(column_ptr.get()); is_nullable = true; @@ -390,7 +392,7 @@ class IntBucketPartitionColumnTransform : public PartitionColumnTransform { //2) get the input data from block ColumnPtr null_map_column_ptr; bool is_nullable = false; - if (column_ptr->is_nullable()) { + if (is_column_nullable(*column_ptr)) { const ColumnNullable* nullable_column = reinterpret_cast(column_ptr.get()); is_nullable = true; @@ -451,7 +453,7 @@ class BigintBucketPartitionColumnTransform : public PartitionColumnTransform { //2) get the input data from block ColumnPtr null_map_column_ptr; bool is_nullable = false; - if (column_ptr->is_nullable()) { + if (is_column_nullable(*column_ptr)) { const ColumnNullable* nullable_column = reinterpret_cast(column_ptr.get()); is_nullable = true; @@ -514,7 +516,7 @@ class DecimalBucketPartitionColumnTransform : public PartitionColumnTransform { //2) get the input data from block ColumnPtr null_map_column_ptr; bool is_nullable = false; - if (column_ptr->is_nullable()) { + if (is_column_nullable(*column_ptr)) { const ColumnNullable* nullable_column = reinterpret_cast(column_ptr.get()); is_nullable = true; @@ -588,7 +590,7 @@ class DateBucketPartitionColumnTransform : public PartitionColumnTransform { //2) get the input data from block ColumnPtr null_map_column_ptr; bool is_nullable = false; - if (column_ptr->is_nullable()) { + if (is_column_nullable(*column_ptr)) { const ColumnNullable* nullable_column = reinterpret_cast(column_ptr.get()); is_nullable = true; @@ -654,7 +656,7 @@ class TimestampBucketPartitionColumnTransform : public PartitionColumnTransform //2) get the input data from block ColumnPtr null_map_column_ptr; bool is_nullable = false; - if (column_ptr->is_nullable()) { + if (is_column_nullable(*column_ptr)) { const ColumnNullable* nullable_column = reinterpret_cast(column_ptr.get()); is_nullable = true; @@ -732,7 +734,7 @@ class StringBucketPartitionColumnTransform : public PartitionColumnTransform { //2) get the input data from block ColumnPtr null_map_column_ptr; bool is_nullable = false; - if (column_ptr->is_nullable()) { + if (is_column_nullable(*column_ptr)) { const ColumnNullable* nullable_column = reinterpret_cast(column_ptr.get()); is_nullable = true; @@ -794,7 +796,7 @@ class DateYearPartitionColumnTransform : public PartitionColumnTransform { //2) get the input data from block ColumnPtr null_map_column_ptr; bool is_nullable = false; - if (column_ptr->is_nullable()) { + if (is_column_nullable(*column_ptr)) { const ColumnNullable* nullable_column = reinterpret_cast(column_ptr.get()); is_nullable = true; @@ -863,7 +865,7 @@ class TimestampYearPartitionColumnTransform : public PartitionColumnTransform { //2) get the input data from block ColumnPtr null_map_column_ptr; bool is_nullable = false; - if (column_ptr->is_nullable()) { + if (is_column_nullable(*column_ptr)) { const ColumnNullable* nullable_column = reinterpret_cast(column_ptr.get()); is_nullable = true; @@ -932,7 +934,7 @@ class DateMonthPartitionColumnTransform : public PartitionColumnTransform { //2) get the input data from block ColumnPtr null_map_column_ptr; bool is_nullable = false; - if (column_ptr->is_nullable()) { + if (is_column_nullable(*column_ptr)) { const ColumnNullable* nullable_column = reinterpret_cast(column_ptr.get()); is_nullable = true; @@ -1001,7 +1003,7 @@ class TimestampMonthPartitionColumnTransform : public PartitionColumnTransform { //2) get the input data from block ColumnPtr null_map_column_ptr; bool is_nullable = false; - if (column_ptr->is_nullable()) { + if (is_column_nullable(*column_ptr)) { const ColumnNullable* nullable_column = reinterpret_cast(column_ptr.get()); is_nullable = true; @@ -1070,7 +1072,7 @@ class DateDayPartitionColumnTransform : public PartitionColumnTransform { //2) get the input data from block ColumnPtr null_map_column_ptr; bool is_nullable = false; - if (column_ptr->is_nullable()) { + if (is_column_nullable(*column_ptr)) { const ColumnNullable* nullable_column = reinterpret_cast(column_ptr.get()); is_nullable = true; @@ -1144,7 +1146,7 @@ class TimestampDayPartitionColumnTransform : public PartitionColumnTransform { //2) get the input data from block ColumnPtr null_map_column_ptr; bool is_nullable = false; - if (column_ptr->is_nullable()) { + if (is_column_nullable(*column_ptr)) { const ColumnNullable* nullable_column = reinterpret_cast(column_ptr.get()); is_nullable = true; @@ -1217,7 +1219,7 @@ class TimestampHourPartitionColumnTransform : public PartitionColumnTransform { //2) get the input data from block ColumnPtr null_map_column_ptr; bool is_nullable = false; - if (column_ptr->is_nullable()) { + if (is_column_nullable(*column_ptr)) { const ColumnNullable* nullable_column = reinterpret_cast(column_ptr.get()); is_nullable = true; diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp index 5584491e99c452..ce980e44dd188d 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp @@ -651,7 +651,7 @@ std::any VIcebergTableWriter::_get_iceberg_partition_value( //1) get the partition column ptr ColumnPtr col_ptr = partition_column.column->convert_to_full_column_if_const(); CHECK(col_ptr); - if (col_ptr->is_nullable()) { + if (is_column_nullable(*col_ptr)) { const auto* nullable_column = reinterpret_cast(col_ptr.get()); const auto* __restrict null_map_data = nullable_column->get_null_map_data().data(); if (null_map_data[position]) { diff --git a/be/src/exprs/aggregate/aggregate_function_count.h b/be/src/exprs/aggregate/aggregate_function_count.h index 2ae2692ccb529f..1a5b25a18e9752 100644 --- a/be/src/exprs/aggregate/aggregate_function_count.h +++ b/be/src/exprs/aggregate/aggregate_function_count.h @@ -238,7 +238,7 @@ class AggregateFunctionCountNotNullUnary final } void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override { - if (to.is_nullable()) { + if (is_column_nullable(to)) { auto& null_column = assert_cast(to); null_column.get_null_map_data().push_back(0); assert_cast(null_column.get_nested_column()) diff --git a/be/src/exprs/aggregate/aggregate_function_foreachv2.cpp b/be/src/exprs/aggregate/aggregate_function_foreachv2.cpp index d8f0eac327a398..5558c230549338 100644 --- a/be/src/exprs/aggregate/aggregate_function_foreachv2.cpp +++ b/be/src/exprs/aggregate/aggregate_function_foreachv2.cpp @@ -52,7 +52,7 @@ class AggregateFunctionForEachV2 final : public AggregateFunctionForEach { auto& offsets_to = arr_to.get_offsets(); IColumn& elems_nullable = arr_to.get_data(); - DCHECK(elems_nullable.is_nullable()); + DCHECK(is_column_nullable(elems_nullable)); auto& elems_to = assert_cast(elems_nullable).get_nested_column(); auto& elements_null_map = assert_cast(elems_nullable).get_null_map_column(); diff --git a/be/src/exprs/aggregate/aggregate_function_group_array_set_op.h b/be/src/exprs/aggregate/aggregate_function_group_array_set_op.h index 2eafcc337ab7eb..65413be8efcc1b 100644 --- a/be/src/exprs/aggregate/aggregate_function_group_array_set_op.h +++ b/be/src/exprs/aggregate/aggregate_function_group_array_set_op.h @@ -432,7 +432,7 @@ class AggregateFunctionGroupArraySetOp final void add(AggregateDataPtr __restrict place, const IColumn** columns, ssize_t row_num, Arena& arena) const override { - const bool col_is_nullable = (*columns[0]).is_nullable(); + const bool col_is_nullable = is_column_nullable(*columns[0]); const ColumnArray& column = col_is_nullable ? assert_cast( diff --git a/be/src/exprs/aggregate/aggregate_function_map.h b/be/src/exprs/aggregate/aggregate_function_map.h index f8a0b402d485c6..e6e85842f5a901 100644 --- a/be/src/exprs/aggregate/aggregate_function_map.h +++ b/be/src/exprs/aggregate/aggregate_function_map.h @@ -197,17 +197,15 @@ class AggregateFunctionMapAgg final void add(AggregateDataPtr __restrict place, const IColumn** columns, ssize_t row_num, Arena&) const override { - if (columns[0]->is_nullable()) { - const auto& nullable_col = - assert_cast(*columns[0]); - const auto& nullable_map = nullable_col.get_null_map_data(); + if (const auto* nullable_col = check_and_get_column(columns[0])) { + const auto& nullable_map = nullable_col->get_null_map_data(); if (nullable_map[row_num]) { return; } Field value; columns[1]->get(row_num, value); this->data(place).add(assert_cast( - nullable_col.get_nested_column()) + nullable_col->get_nested_column()) .get_data_at(row_num), value); } else { diff --git a/be/src/exprs/aggregate/aggregate_function_null.h b/be/src/exprs/aggregate/aggregate_function_null.h index 38c6b92e502937..d9aac91d95c2b8 100644 --- a/be/src/exprs/aggregate/aggregate_function_null.h +++ b/be/src/exprs/aggregate/aggregate_function_null.h @@ -465,7 +465,7 @@ class AggregateFunctionNullUnaryInline final return; } - DCHECK(columns[0]->is_nullable()) << columns[0]->get_name(); + DCHECK(is_column_nullable(*columns[0])) << columns[0]->get_name(); const auto* column = assert_cast(columns[0]); const IColumn* nested_column = &column->get_nested_column(); diff --git a/be/src/exprs/aggregate/aggregate_function_null_v2.h b/be/src/exprs/aggregate/aggregate_function_null_v2.h index 881478e28121f0..ef508db89fefde 100644 --- a/be/src/exprs/aggregate/aggregate_function_null_v2.h +++ b/be/src/exprs/aggregate/aggregate_function_null_v2.h @@ -532,7 +532,7 @@ class AggregateFunctionNullUnaryInlineV2 final return; } - DCHECK(columns[0]->is_nullable()) << columns[0]->get_name(); + DCHECK(is_column_nullable(*columns[0])) << columns[0]->get_name(); const auto* column = assert_cast(columns[0]); const IColumn* nested_column = &column->get_nested_column(); diff --git a/be/src/exprs/aggregate/aggregate_function_percentile.h b/be/src/exprs/aggregate/aggregate_function_percentile.h index 007c236da73d0f..439016aa54da09 100644 --- a/be/src/exprs/aggregate/aggregate_function_percentile.h +++ b/be/src/exprs/aggregate/aggregate_function_percentile.h @@ -736,7 +736,7 @@ class AggregateFunctionPercentileArray final void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override { auto& to_arr = assert_cast(to); auto& to_nested_col = to_arr.get_data(); - if (to_nested_col.is_nullable()) { + if (is_column_nullable(to_nested_col)) { auto col_null = reinterpret_cast(&to_nested_col); AggregateFunctionPercentileArray::data(place).insert_result_into( col_null->get_nested_column()); @@ -926,7 +926,7 @@ class AggregateFunctionPercentileArrayV2 final void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override { auto& to_arr = assert_cast(to); auto& to_nested_col = to_arr.get_data(); - if (to_nested_col.is_nullable()) { + if (is_column_nullable(to_nested_col)) { auto* col_null = reinterpret_cast(&to_nested_col); AggregateFunctionPercentileArrayV2::data(place).insert_result_into( col_null->get_nested_column()); diff --git a/be/src/exprs/aggregate/aggregate_function_retention.h b/be/src/exprs/aggregate/aggregate_function_retention.h index ad6181d5ddb03f..189a31b441a80a 100644 --- a/be/src/exprs/aggregate/aggregate_function_retention.h +++ b/be/src/exprs/aggregate/aggregate_function_retention.h @@ -149,7 +149,7 @@ class AggregateFunctionRetention final void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override { auto& to_arr = assert_cast(to); auto& to_nested_col = to_arr.get_data(); - if (to_nested_col.is_nullable()) { + if (is_column_nullable(to_nested_col)) { auto col_null = reinterpret_cast(&to_nested_col); this->data(place).insert_result_into(col_null->get_nested_column(), get_argument_types().size(), diff --git a/be/src/exprs/aggregate/aggregate_function_topn.h b/be/src/exprs/aggregate/aggregate_function_topn.h index 0787a936e872fc..410ed8c0bc896a 100644 --- a/be/src/exprs/aggregate/aggregate_function_topn.h +++ b/be/src/exprs/aggregate/aggregate_function_topn.h @@ -365,7 +365,7 @@ class AggregateFunctionTopNArray final void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override { auto& to_arr = assert_cast(to); auto& to_nested_col = to_arr.get_data(); - if (to_nested_col.is_nullable()) { + if (is_column_nullable(to_nested_col)) { auto* col_null = assert_cast(&to_nested_col); this->data(place).insert_result_into(col_null->get_nested_column()); col_null->get_null_map_data().resize_fill(col_null->get_nested_column().size(), 0); diff --git a/be/src/exprs/aggregate/aggregate_function_window.h b/be/src/exprs/aggregate/aggregate_function_window.h index 2d9732969a812f..d491a1db99e81f 100644 --- a/be/src/exprs/aggregate/aggregate_function_window.h +++ b/be/src/exprs/aggregate/aggregate_function_window.h @@ -550,7 +550,7 @@ struct WindowFunctionFirstImpl : Data { if constexpr (arg_ignore_null) { frame_end = std::min(frame_end, partition_end); - if (columns[0]->is_nullable()) { + if (is_column_nullable(*columns[0])) { const auto& arg_nullable = assert_cast(*columns[0]); // the valid range is: [frame_start, frame_end) while (frame_start < frame_end - 1 && arg_nullable.is_null_at(frame_start)) { @@ -584,7 +584,7 @@ struct WindowFunctionLastImpl : Data { if constexpr (arg_ignore_null) { frame_start = std::max(frame_start, partition_start); - if (columns[0]->is_nullable()) { + if (is_column_nullable(*columns[0])) { const auto& arg_nullable = assert_cast(*columns[0]); // wants find a not null value in [frame_start, frame_end) // iff has find: set_value and return directly diff --git a/be/src/exprs/bloom_filter_func_adaptor.h b/be/src/exprs/bloom_filter_func_adaptor.h index 119e7c2c93917a..8f2b3eb512f172 100644 --- a/be/src/exprs/bloom_filter_func_adaptor.h +++ b/be/src/exprs/bloom_filter_func_adaptor.h @@ -105,7 +105,7 @@ struct CommonFindOp { static void insert_batch(BloomFilterAdaptor& bloom_filter, const ColumnPtr& column, size_t start) { const auto size = column->size(); - if (column->is_nullable()) { + if (is_column_nullable(*column)) { const auto* nullable = assert_cast(column.get()); const auto& col = nullable->get_nested_column(); const auto& nullmap = nullable->get_null_map_column().get_data(); @@ -138,7 +138,7 @@ struct CommonFindOp { uint8_t* results, const uint8_t* __restrict filter) { const T* __restrict data = nullptr; const uint8_t* __restrict nullmap = nullptr; - if (column->is_nullable()) { + if (is_column_nullable(*column)) { const auto* nullable = assert_cast(column.get()); if (nullable->has_null()) { nullmap = nullable->get_null_map_column().get_data().data(); @@ -184,7 +184,7 @@ struct StringFindOp : CommonFindOp { } }; - if (column->is_nullable()) { + if (is_column_nullable(*column)) { const auto* nullable = assert_cast(column.get()); const auto& nullmap = nullable->get_null_map_column().get_data(); if (nullable->get_nested_column().is_column_string64()) { @@ -209,7 +209,7 @@ struct StringFindOp : CommonFindOp { static void find_batch(const BloomFilterAdaptor& bloom_filter, const ColumnPtr& column, uint8_t* results, const uint8_t* __restrict filter) { - if (column->is_nullable()) { + if (is_column_nullable(*column)) { const auto* nullable = assert_cast(column.get()); const auto& col = assert_cast(nullable->get_nested_column()); const auto& nullmap = nullable->get_null_map_column().get_data(); diff --git a/be/src/exprs/function/array/function_array_aggregation.cpp b/be/src/exprs/function/array/function_array_aggregation.cpp index 9bed157223adb1..e9a20652ed21d4 100644 --- a/be/src/exprs/function/array/function_array_aggregation.cpp +++ b/be/src/exprs/function/array/function_array_aggregation.cpp @@ -206,7 +206,7 @@ struct ArrayAggregateImpl { ArrayAggregateFunctionCreator>; const ColumnType* column = - data->is_nullable() + is_column_nullable(*data) ? check_and_get_column( static_cast(data)->get_nested_column()) : check_and_get_column(&*data); @@ -233,7 +233,7 @@ struct ArrayAggregateImpl { } function->reset(guard.data()); function->add_batch_range(start, end - 1, guard.data(), columns, arena, - data->is_nullable()); + is_column_nullable(*data)); function->insert_result_into(guard.data(), res_column->assert_mutable_ref()); } res_ptr = std::move(res_column); @@ -430,7 +430,7 @@ struct ArrayAggregateImplDecimalV3 { AggregateFunctionTraitsWithResultType>; const ColumnType* column = - data->is_nullable() + is_column_nullable(*data) ? check_and_get_column( static_cast(data)->get_nested_column()) : check_and_get_column(&*data); @@ -458,7 +458,7 @@ struct ArrayAggregateImplDecimalV3 { } function->reset(guard.data()); function->add_batch_range(start, end - 1, guard.data(), columns, arena, - data->is_nullable()); + is_column_nullable(*data)); function->insert_result_into(guard.data(), res_column->assert_mutable_ref()); } res_ptr = std::move(res_column); @@ -494,7 +494,7 @@ class FunctionArrayAggDecimalV3 : public IFunction { const auto& typed_column = block.get_by_position(arguments[0]); auto ptr = typed_column.column->convert_to_full_column_if_const(); const typename Impl::column_type* column_array; - if (ptr->is_nullable()) { + if (is_column_nullable(*ptr)) { column_array = assert_cast( assert_cast(ptr.get())->get_nested_column_ptr().get()); } else { diff --git a/be/src/exprs/function/array/function_array_apply.cpp b/be/src/exprs/function/array/function_array_apply.cpp index d7fec529318e4f..ddbaa3b1c6495e 100644 --- a/be/src/exprs/function/array/function_array_apply.cpp +++ b/be/src/exprs/function/array/function_array_apply.cpp @@ -143,7 +143,7 @@ class FunctionArrayApply : public IFunction { auto& column_filter_data = column_filter->get_data(); const char* src_column_data_ptr = nullptr; const uint8_t* null_map_data = nullptr; - if (!src_column.is_nullable()) { + if (!is_column_nullable(src_column)) { src_column_data_ptr = src_column.get_raw_data().data; } else { const auto* nullable_col = assert_cast(&src_column); diff --git a/be/src/exprs/function/array/function_array_constructor.cpp b/be/src/exprs/function/array/function_array_constructor.cpp index d5f3a6d602908f..57d3c69e24aaa2 100644 --- a/be/src/exprs/function/array/function_array_constructor.cpp +++ b/be/src/exprs/function/array/function_array_constructor.cpp @@ -86,7 +86,7 @@ class FunctionArrayConstructor : public IFunction { auto& col = block.get_by_position(arguments[i]).column; col = col->convert_to_full_column_if_const(); arg[i] = col; - if (result_nested_col.is_nullable() && !col->is_nullable()) { + if (is_column_nullable(result_nested_col) && !is_column_nullable(*col)) { arg[i] = ColumnNullable::create(col, ColumnUInt8::create(col->size(), 0)); } } diff --git a/be/src/exprs/function/array/function_array_count.cpp b/be/src/exprs/function/array/function_array_count.cpp index d8e0130a5b066e..98d65d000c05e4 100644 --- a/be/src/exprs/function/array/function_array_count.cpp +++ b/be/src/exprs/function/array/function_array_count.cpp @@ -52,8 +52,7 @@ class FunctionArrayCount : public IFunction { unpack_if_const(block.get_by_position(arguments[0]).column); const ColumnArray* array_column = nullptr; const UInt8* array_null_map = nullptr; - if (src_column->is_nullable()) { - auto nullable_array = assert_cast(src_column.get()); + if (const auto* nullable_array = check_and_get_column(src_column.get())) { array_column = assert_cast(&nullable_array->get_nested_column()); array_null_map = nullable_array->get_null_map_column().get_data().data(); } else { @@ -68,7 +67,7 @@ class FunctionArrayCount : public IFunction { const auto& offsets = array_column->get_offsets(); ColumnPtr nested_column = nullptr; const UInt8* nested_null_map = nullptr; - if (array_column->get_data().is_nullable()) { + if (is_column_nullable(array_column->get_data())) { const auto& nested_null_column = assert_cast(array_column->get_data()); nested_null_map = nested_null_column.get_null_map_column().get_data().data(); diff --git a/be/src/exprs/function/array/function_array_distance.h b/be/src/exprs/function/array/function_array_distance.h index 12969c23e3481f..d601b8c2988e02 100644 --- a/be/src/exprs/function/array/function_array_distance.h +++ b/be/src/exprs/function/array/function_array_distance.h @@ -128,17 +128,17 @@ class FunctionArrayDistance : public IFunction { } // Check outer nullable - if (raw->is_nullable()) { + if (const auto* nullable_raw = check_and_get_column(raw)) { if (raw->has_null()) { throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "{} for function {} cannot be null", arg_name, func_name); } - raw = assert_cast(raw)->get_nested_column_ptr().get(); + raw = nullable_raw->get_nested_column_ptr().get(); } // Check inner nullable (array elements) const auto& array_col = assert_cast(*raw); - if (array_col.get_data_ptr()->is_nullable() && array_col.get_data_ptr()->has_null()) { + if (is_column_nullable(*array_col.get_data_ptr()) && array_col.get_data_ptr()->has_null()) { throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "{} for function {} cannot have null", arg_name, func_name); } diff --git a/be/src/exprs/function/array/function_array_distinct.h b/be/src/exprs/function/array/function_array_distinct.h index 885677d7b884b2..01babcda365adc 100644 --- a/be/src/exprs/function/array/function_array_distinct.h +++ b/be/src/exprs/function/array/function_array_distinct.h @@ -102,16 +102,15 @@ class FunctionArrayDistinct : public IFunction { dest_offsets.reserve(input_rows_count); const NullMapType* src_null_map = nullptr; - if (src_nested_column->is_nullable()) { - const auto* src_nested_nullable_col = - assert_cast(src_nested_column); + if (const auto* src_nested_nullable_col = + check_and_get_column(src_nested_column)) { src_nested_column = src_nested_nullable_col->get_nested_column_ptr().get(); src_null_map = &src_nested_nullable_col->get_null_map_column().get_data(); } NullMapType* dest_null_map = nullptr; - if (dest_nested_column->is_nullable()) { - auto* dest_nested_nullable_col = reinterpret_cast(dest_nested_column); + if (auto* dest_nested_nullable_col = + check_and_get_column(dest_nested_column)) { dest_nested_column = dest_nested_nullable_col->get_nested_column_ptr().get(); dest_null_map = &dest_nested_nullable_col->get_null_map_column().get_data(); } diff --git a/be/src/exprs/function/array/function_array_element.h b/be/src/exprs/function/array/function_array_element.h index 65070d0d17d1ad..1ea9f6227af7be 100644 --- a/be/src/exprs/function/array/function_array_element.h +++ b/be/src/exprs/function/array/function_array_element.h @@ -144,8 +144,8 @@ class FunctionArrayElement : public IFunction { block.replace_by_position(arguments[0], col_left_raw.column->convert_to_full_column_if_const()); auto col_left = block.get_by_position(arguments[0]); - if (col_left.column->is_nullable()) { - const auto* null_col = assert_cast(col_left.column.get()); + if (const auto* null_col = + check_and_get_column(col_left.column.get())) { src_null_map = null_col->get_null_map_column().get_data().data(); args = {{null_col->get_nested_column_ptr(), remove_nullable(col_left.type), col_left.name}, @@ -160,8 +160,7 @@ class FunctionArrayElement : public IFunction { // unpack_if_const() gives us that inner column plus a constancy flag so inner // loops can use index_check_const() instead of expanding N copies. auto [unpacked_col, is_const_array] = unpack_if_const(col_left_raw.column); - if (unpacked_col->is_nullable()) { - const auto* null_col = assert_cast(unpacked_col.get()); + if (const auto* null_col = check_and_get_column(unpacked_col.get())) { src_null_map = null_col->get_null_map_column().get_data().data(); args = {{null_col->get_nested_column_ptr(), remove_nullable(col_left_raw.type), col_left_raw.name}, @@ -231,8 +230,7 @@ class FunctionArrayElement : public IFunction { // element_at manages nulls itself (use_default_implementation_for_nulls() == false), so a // null struct row must be merged into the result null map manually. const ColumnUInt8* outer_null_map = nullptr; - if (struct_col_ptr->is_nullable()) { - const auto* nullable = assert_cast(struct_col_ptr.get()); + if (const auto* nullable = check_and_get_column(struct_col_ptr.get())) { outer_null_map = &nullable->get_null_map_column(); struct_col_ptr = nullable->get_nested_column_ptr(); } @@ -253,8 +251,7 @@ class FunctionArrayElement : public IFunction { auto res_null_column = ColumnUInt8::create(input_rows_count, 0); auto& res_null_map = res_null_column->get_data(); ColumnPtr res_nested = field_col; - if (field_col->is_nullable()) { - const auto* field_nullable = assert_cast(field_col.get()); + if (const auto* field_nullable = check_and_get_column(field_col.get())) { const auto& field_null_map = field_nullable->get_null_map_column().get_data(); memcpy(res_null_map.data(), field_null_map.data(), input_rows_count); res_nested = field_nullable->get_nested_column_ptr(); @@ -485,10 +482,9 @@ class FunctionArrayElement : public IFunction { .get_nested_type()); const UInt8* idx_null_map = nullptr; auto idx_col_with_const = unpack_if_const(arguments[1].column); - if (idx_col_with_const.first->is_nullable()) { - const auto& idx_null_column = - reinterpret_cast(*idx_col_with_const.first); - idx_null_map = idx_null_column.get_null_map_column().get_data().data(); + if (const auto* idx_null_column = + check_and_get_column(idx_col_with_const.first.get())) { + idx_null_map = idx_null_column->get_null_map_column().get_data().data(); } auto idx_col_raw = remove_nullable(idx_col_with_const.first); bool is_const_index = idx_col_with_const.second; diff --git a/be/src/exprs/function/array/function_array_enumerate.cpp b/be/src/exprs/function/array/function_array_enumerate.cpp index ccfca1f1a3a5ac..7d82da40ddf71c 100644 --- a/be/src/exprs/function/array/function_array_enumerate.cpp +++ b/be/src/exprs/function/array/function_array_enumerate.cpp @@ -105,8 +105,7 @@ class FunctionArrayEnumerate : public IFunction { } ColumnPtr res_column = ColumnArray::create(std::move(nested_column), array->get_offsets_ptr()); - if (block.get_by_position(arguments[0]).column->is_nullable()) { - const auto* nullable = assert_cast(left_column.get()); + if (const auto* nullable = check_and_get_column(left_column.get())) { res_column = ColumnNullable::create( res_column, nullable->get_null_map_column().clone_resized(nullable->size())); } diff --git a/be/src/exprs/function/array/function_array_enumerate_uniq.cpp b/be/src/exprs/function/array/function_array_enumerate_uniq.cpp index e2cc6f2ff371d6..bff8758352e768 100644 --- a/be/src/exprs/function/array/function_array_enumerate_uniq.cpp +++ b/be/src/exprs/function/array/function_array_enumerate_uniq.cpp @@ -149,8 +149,9 @@ class FunctionArrayEnumerateUniq : public IFunction { } const NullMapType* null_map = nullptr; - if (arguments.size() == 1 && data_columns[0]->is_nullable()) { - const auto* nullable = assert_cast(data_columns[0]); + if (arguments.size() == 1 && + (nullptr != check_and_get_column(data_columns[0]))) { + const auto* nullable = check_and_get_column(data_columns[0]); data_columns[0] = nullable->get_nested_column_ptr().get(); null_map = &nullable->get_null_map_column().get_data(); } @@ -188,17 +189,19 @@ class FunctionArrayEnumerateUniq : public IFunction { } ColumnPtr nested_column = dst_nested_column->get_ptr(); - if (first_column_array->get_data().is_nullable()) { + if (is_column_nullable(first_column_array->get_data())) { nested_column = ColumnNullable::create(nested_column, ColumnUInt8::create(nested_column->size(), 0)); } ColumnPtr res_column = ColumnArray::create(std::move(nested_column), src_offsets); - if (arguments.size() == 1 && block.get_by_position(arguments[0]).column->is_nullable()) { + if (arguments.size() == 1) { auto left_column = block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); - const auto* nullable = assert_cast(left_column.get()); - res_column = ColumnNullable::create( - res_column, nullable->get_null_map_column().clone_resized(nullable->size())); + if (const auto* nullable = check_and_get_column(left_column.get())) { + res_column = ColumnNullable::create( + res_column, + nullable->get_null_map_column().clone_resized(nullable->size())); + } } block.replace_by_position(result, std::move(res_column)); diff --git a/be/src/exprs/function/array/function_array_first_or_last_index.cpp b/be/src/exprs/function/array/function_array_first_or_last_index.cpp index 56e1b61954bd13..753b716870284e 100644 --- a/be/src/exprs/function/array/function_array_first_or_last_index.cpp +++ b/be/src/exprs/function/array/function_array_first_or_last_index.cpp @@ -71,8 +71,7 @@ class FunctionArrayFirstOrLastIndex : public IFunction { block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); const ColumnArray* array_column = nullptr; const UInt8* array_null_map = nullptr; - if (src_column->is_nullable()) { - const auto* nullable_array = assert_cast(src_column.get()); + if (const auto* nullable_array = check_and_get_column(src_column.get())) { array_column = assert_cast(&nullable_array->get_nested_column()); array_null_map = nullable_array->get_null_map_column().get_data().data(); } else { diff --git a/be/src/exprs/function/array/function_array_index.h b/be/src/exprs/function/array/function_array_index.h index 3f39fc627b4887..742bb944494510 100644 --- a/be/src/exprs/function/array/function_array_index.h +++ b/be/src/exprs/function/array/function_array_index.h @@ -373,8 +373,7 @@ class FunctionArrayIndex : public IFunction { } const ColumnArray* array_column = nullptr; const UInt8* array_null_map = nullptr; - if (left_column->is_nullable()) { - auto nullable_array = reinterpret_cast(left_column.get()); + if (const auto* nullable_array = check_and_get_column(left_column.get())) { array_column = reinterpret_cast(&nullable_array->get_nested_column()); array_null_map = nullable_array->get_null_map_column().get_data().data(); @@ -384,11 +383,10 @@ class FunctionArrayIndex : public IFunction { const auto& offsets = array_column->get_offsets(); const UInt8* nested_null_map = nullptr; ColumnPtr nested_column = nullptr; - if (array_column->get_data().is_nullable()) { - const auto& nested_null_column = - reinterpret_cast(array_column->get_data()); - nested_null_map = nested_null_column.get_null_map_column().get_data().data(); - nested_column = nested_null_column.get_nested_column_ptr(); + if (const auto* nested_null_column = + check_and_get_column(&array_column->get_data())) { + nested_null_map = nested_null_column->get_null_map_column().get_data().data(); + nested_column = nested_null_column->get_nested_column_ptr(); } else { nested_column = array_column->get_data_ptr(); } @@ -398,10 +396,10 @@ class FunctionArrayIndex : public IFunction { block.get_by_position(arguments[1]).column->convert_to_full_column_if_const(); ColumnPtr right_column = right_full_column; const UInt8* right_nested_null_map = nullptr; - if (right_column->is_nullable()) { - const auto& nested_null_column = assert_cast(*right_full_column); - right_column = nested_null_column.get_nested_column_ptr(); - right_nested_null_map = nested_null_column.get_null_map_column().get_data().data(); + if (const auto* nested_null_column = + check_and_get_column(right_column.get())) { + right_column = nested_null_column->get_nested_column_ptr(); + right_nested_null_map = nested_null_column->get_null_map_column().get_data().data(); } // execute auto array_type = remove_nullable(block.get_by_position(arguments[0]).type); diff --git a/be/src/exprs/function/array/function_array_mapped.h b/be/src/exprs/function/array/function_array_mapped.h index c86b6e204926a1..83296f2eba2e50 100644 --- a/be/src/exprs/function/array/function_array_mapped.h +++ b/be/src/exprs/function/array/function_array_mapped.h @@ -50,9 +50,9 @@ class FunctionArrayMapped : public IFunction { const auto& typed_column = block.get_by_position(arguments[0]); auto ptr = typed_column.column->convert_to_full_column_if_const(); const typename Impl::column_type* column_array; - if (ptr->is_nullable()) { + if (const auto* nullable_ptr = check_and_get_column(ptr.get())) { column_array = assert_cast( - assert_cast(ptr.get())->get_nested_column_ptr().get()); + nullable_ptr->get_nested_column_ptr().get()); } else { column_array = assert_cast(ptr.get()); } diff --git a/be/src/exprs/function/array/function_array_pushback.cpp b/be/src/exprs/function/array/function_array_pushback.cpp index 53e6983d373b5e..898c4d296902a6 100644 --- a/be/src/exprs/function/array/function_array_pushback.cpp +++ b/be/src/exprs/function/array/function_array_pushback.cpp @@ -69,8 +69,7 @@ class FunctionArrayPushback : public IFunction { // extract src array column const ColumnArray* array_column = nullptr; const UInt8* array_null_map = nullptr; - if (src_column->is_nullable()) { - auto nullable_array = static_cast(src_column.get()); + if (const auto* nullable_array = check_and_get_column(src_column.get())) { array_column = assert_cast(&nullable_array->get_nested_column()); array_null_map = nullable_array->get_null_map_column().get_data().data(); } else { @@ -118,4 +117,4 @@ void register_function_array_pushback(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_alias("array_pushback", "array_append"); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/exprs/function/array/function_array_pushfront.cpp b/be/src/exprs/function/array/function_array_pushfront.cpp index 6f69e57cb0b03e..06cb20d504af3b 100644 --- a/be/src/exprs/function/array/function_array_pushfront.cpp +++ b/be/src/exprs/function/array/function_array_pushfront.cpp @@ -70,8 +70,7 @@ class FunctionArrayPushfront : public IFunction { // extract src array column const ColumnArray* array_column = nullptr; const UInt8* array_null_map = nullptr; - if (src_column->is_nullable()) { - auto nullable_array = static_cast(src_column.get()); + if (const auto* nullable_array = check_and_get_column(src_column.get())) { array_column = static_cast(&nullable_array->get_nested_column()); array_null_map = nullable_array->get_null_map_column().get_data().data(); } else { diff --git a/be/src/exprs/function/array/function_array_remove.h b/be/src/exprs/function/array/function_array_remove.h index 09a43e7651c3c9..fe05b64d0b2222 100644 --- a/be/src/exprs/function/array/function_array_remove.h +++ b/be/src/exprs/function/array/function_array_remove.h @@ -331,8 +331,7 @@ class FunctionArrayRemove : public IFunction { auto left_column = arguments[0].column->convert_to_full_column_if_const(); const ColumnArray* array_column = nullptr; const ColumnUInt8* array_null_map = nullptr; - if (left_column->is_nullable()) { - auto nullable_array = reinterpret_cast(left_column.get()); + if (const auto* nullable_array = check_and_get_column(left_column.get())) { array_column = reinterpret_cast(&nullable_array->get_nested_column()); array_null_map = &nullable_array->get_null_map_column(); @@ -343,12 +342,11 @@ class FunctionArrayRemove : public IFunction { const auto& offsets = array_column->get_offsets(); const UInt8* nested_null_map = nullptr; ColumnPtr nested_column = nullptr; - if (array_column->get_data().is_nullable()) { - const auto& nested_null_column = - reinterpret_cast(array_column->get_data()); - nested_null_column.sanity_check(); - nested_null_map = nested_null_column.get_null_map_column().get_data().data(); - nested_column = nested_null_column.get_nested_column_ptr(); + if (const auto* nested_null_column = + check_and_get_column(&array_column->get_data())) { + nested_null_column->sanity_check(); + nested_null_map = nested_null_column->get_null_map_column().get_data().data(); + nested_column = nested_null_column->get_nested_column_ptr(); } else { nested_column = array_column->get_data_ptr(); } @@ -357,10 +355,10 @@ class FunctionArrayRemove : public IFunction { ColumnPtr right_full_column = arguments[1].column->convert_to_full_column_if_const(); ColumnPtr right_column = right_full_column; const UInt8* right_nested_null_map = nullptr; - if (right_column->is_nullable()) { - const auto& nested_null_column = assert_cast(*right_full_column); - right_column = nested_null_column.get_nested_column_ptr(); - right_nested_null_map = nested_null_column.get_null_map_column().get_data().data(); + if (const auto* nested_null_column = + check_and_get_column(right_column.get())) { + right_column = nested_null_column->get_nested_column_ptr(); + right_nested_null_map = nested_null_column->get_null_map_column().get_data().data(); } // execute auto array_type = remove_nullable(arguments[0].type); diff --git a/be/src/exprs/function/array/function_array_split.cpp b/be/src/exprs/function/array/function_array_split.cpp index 370b163e19296a..4311982dacb67f 100644 --- a/be/src/exprs/function/array/function_array_split.cpp +++ b/be/src/exprs/function/array/function_array_split.cpp @@ -75,13 +75,12 @@ class FunctionArraySplit : public IFunction { .get_offsets(); // for check uneven array const NullMap* null_map = nullptr; - if (split_col->is_nullable()) { + if (const auto* nullable_split_col = + check_and_get_column(split_col.get())) { if (split_col->has_null()) { - null_map = - &assert_cast(split_col.get())->get_null_map_data(); + null_map = &nullable_split_col->get_null_map_data(); } - split_col = - assert_cast(split_col.get())->get_nested_column_ptr(); + split_col = nullable_split_col->get_nested_column_ptr(); } const IColumn::Filter& cut = assert_cast(split_col.get())->get_data(); diff --git a/be/src/exprs/function/array/function_array_utils.cpp b/be/src/exprs/function/array/function_array_utils.cpp index 29776153d17af5..b7fc1ba18f12d5 100644 --- a/be/src/exprs/function/array/function_array_utils.cpp +++ b/be/src/exprs/function/array/function_array_utils.cpp @@ -34,10 +34,9 @@ namespace doris { bool extract_column_array_info(const IColumn& src, ColumnArrayExecutionData& data) { const IColumn* array_col = &src; // extract array nullable info - if (src.is_nullable()) { - const auto& null_col = reinterpret_cast(src); - data.array_nullmap_data = null_col.get_null_map_data().data(); - array_col = null_col.get_nested_column_ptr().get(); + if (const auto* nullable = check_and_get_column(&src)) { + data.array_nullmap_data = nullable->get_null_map_data().data(); + array_col = nullable->get_nested_column_ptr().get(); } // check and get array column @@ -50,10 +49,9 @@ bool extract_column_array_info(const IColumn& src, ColumnArrayExecutionData& dat data.offsets_ptr = &data.array_col->get_offsets(); data.nested_col = data.array_col->get_data_ptr(); // extract nested column is nullable - if (data.nested_col->is_nullable()) { - const auto& nested_null_col = reinterpret_cast(*data.nested_col); - data.nested_nullmap_data = nested_null_col.get_null_map_data().data(); - data.nested_col = nested_null_col.get_nested_column_ptr(); + if (const auto* nullable = check_and_get_column(data.nested_col.get())) { + data.nested_nullmap_data = nullable->get_null_map_data().data(); + data.nested_col = nullable->get_nested_column_ptr(); } if (data.output_as_variant && data.nested_type->get_primitive_type() != PrimitiveType::TYPE_VARIANT) { diff --git a/be/src/exprs/function/array/function_array_with_constant.cpp b/be/src/exprs/function/array/function_array_with_constant.cpp index b1e513fbb155b5..f7c689929aa222 100644 --- a/be/src/exprs/function/array/function_array_with_constant.cpp +++ b/be/src/exprs/function/array/function_array_with_constant.cpp @@ -75,7 +75,7 @@ class FunctionArrayWithConstant : public IFunction { uint32_t result, size_t input_rows_count) const override { auto num = block.get_by_position(arguments[FunctionType::param_num_idx]) .column->convert_to_full_column_if_const(); - num = num->is_nullable() + num = is_column_nullable(*num) ? assert_cast(num.get())->get_nested_column_ptr() : num; auto value = block.get_by_position(arguments[FunctionType::param_val_idx]) diff --git a/be/src/exprs/function/array/varray_match_function.cpp b/be/src/exprs/function/array/varray_match_function.cpp index a72705bb2a8b5e..fed2c2d20b7685 100644 --- a/be/src/exprs/function/array/varray_match_function.cpp +++ b/be/src/exprs/function/array/varray_match_function.cpp @@ -62,8 +62,7 @@ class ArrayMatchFunction : public IFunction { unpack_if_const(block.get_by_position(arguments[0]).column); const ColumnArray* array_column = nullptr; const UInt8* array_null_map = nullptr; - if (src_column->is_nullable()) { - auto nullable_array = assert_cast(src_column.get()); + if (const auto* nullable_array = check_and_get_column(src_column.get())) { array_column = assert_cast(&nullable_array->get_nested_column()); array_null_map = nullable_array->get_null_map_column().get_data().data(); } else { @@ -78,11 +77,10 @@ class ArrayMatchFunction : public IFunction { const auto& offsets = array_column->get_offsets(); ColumnPtr nested_column = nullptr; const UInt8* nested_null_map = nullptr; - if (array_column->get_data().is_nullable()) { - const auto& nested_null_column = - assert_cast(array_column->get_data()); - nested_null_map = nested_null_column.get_null_map_column().get_data().data(); - nested_column = nested_null_column.get_nested_column_ptr(); + if (const auto* nested_null_column = + check_and_get_column(&array_column->get_data())) { + nested_null_map = nested_null_column->get_null_map_column().get_data().data(); + nested_column = nested_null_column->get_nested_column_ptr(); } else { nested_column = array_column->get_data_ptr(); } diff --git a/be/src/exprs/function/complex_hash_map_dictionary.cpp b/be/src/exprs/function/complex_hash_map_dictionary.cpp index c17ded51de483f..9af7660a2cede8 100644 --- a/be/src/exprs/function/complex_hash_map_dictionary.cpp +++ b/be/src/exprs/function/complex_hash_map_dictionary.cpp @@ -146,7 +146,7 @@ ColumnPtrs ComplexHashMapDictionary::get_tuple_columns( std::vector nullable_key_raw_columns; for (const auto& column : key_columns) { key_raw_columns.push_back(remove_nullable(column).get()); - if (column->is_nullable()) { + if (is_column_nullable(*column)) { key_hash_nullable = true; nullable_key_raw_columns.push_back(assert_cast(column.get())); } diff --git a/be/src/exprs/function/complex_hash_map_dictionary.h b/be/src/exprs/function/complex_hash_map_dictionary.h index c3ffec802b6cd8..bc1c3bf15983bb 100644 --- a/be/src/exprs/function/complex_hash_map_dictionary.h +++ b/be/src/exprs/function/complex_hash_map_dictionary.h @@ -90,7 +90,7 @@ inline DictionaryPtr create_complex_hash_map_dict_from_column( ColumnPtrs key_columns; DataTypes key_types; for (const auto& key : key_data) { - if (key.column->is_nullable()) { + if (is_column_nullable(*key.column)) { throw doris::Exception(ErrorCode::INTERNAL_ERROR, "ComplexHashMapDictionary key column should not be nullable"); } diff --git a/be/src/exprs/function/dictionary.cpp b/be/src/exprs/function/dictionary.cpp index 7d2fad96139a0a..8ef2b7fbac00f7 100644 --- a/be/src/exprs/function/dictionary.cpp +++ b/be/src/exprs/function/dictionary.cpp @@ -120,7 +120,7 @@ void IDictionary::load_values(const std::vector& values_column) { column_with_type.column = cast_to_column(value_column_without_nullable); // if original value is nullable, the null_map must be not null - if (values_column[i]->is_nullable()) { + if (is_column_nullable(*values_column[i])) { column_with_type.null_map = assert_cast( values_column[i].get()) diff --git a/be/src/exprs/function/dictionary_util.h b/be/src/exprs/function/dictionary_util.h index 87e6603a8663e5..035c9ec1105b9c 100644 --- a/be/src/exprs/function/dictionary_util.h +++ b/be/src/exprs/function/dictionary_util.h @@ -43,9 +43,8 @@ Status inline check_dict_input_data(ColumnsWithTypeAndName& key_data, IColumn::Filter filter(key_data.front().column->size(), 1); for (auto& key : key_data) { - if (key.column->is_nullable()) { - const auto& null_map = assert_cast(key.column.get()) - ->get_null_map_data(); // Get the null_map in the key + if (const auto* nullable_key = check_and_get_column(key.column.get())) { + const auto& null_map = nullable_key->get_null_map_data(); // Get the null_map in the key for (size_t i = 0; i < key.column->size(); ++i) { if (null_map[i] == 1) { // If the value in the null_map of the key is 0, filter it out diff --git a/be/src/exprs/function/function_collection_in.h b/be/src/exprs/function/function_collection_in.h index ef69347a0684cd..ade5681ba48a91 100644 --- a/be/src/exprs/function/function_collection_in.h +++ b/be/src/exprs/function/function_collection_in.h @@ -113,8 +113,7 @@ class FunctionCollectionIn : public IFunction { // Types like struct, array, and map only support constant expressions. DCHECK(const_column_ptr != nullptr); const auto& [col, _] = unpack_if_const(const_column_ptr->column_ptr); - if (col->is_nullable()) { - const auto* null_col = assert_cast(col.get()); + if (const auto* null_col = check_and_get_column(col.get())) { if (null_col->has_null()) { state->null_in_set = true; } else { @@ -136,7 +135,7 @@ class FunctionCollectionIn : public IFunction { Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, uint32_t result, size_t input_rows_count) const override { - auto in_state = reinterpret_cast( + auto* in_state = reinterpret_cast( context->get_function_state(FunctionContext::FRAGMENT_LOCAL)); if (!in_state) { return Status::RuntimeError("function context for function '{}' must have Set;", @@ -155,14 +154,15 @@ class FunctionCollectionIn : public IFunction { const ColumnWithTypeAndName& left_arg = block.get_by_position(arguments[0]); const auto& [materialized_column, col_const] = unpack_if_const(left_arg.column); auto materialized_column_not_null = materialized_column; - if (materialized_column_not_null->is_nullable()) { + if (is_column_nullable(*materialized_column_not_null)) { materialized_column_not_null = assert_cast(materialized_column_not_null.get()) ->get_nested_column_ptr(); } for (size_t i = 0; i < input_rows_count; ++i) { - bool find = args_set.find({materialized_column_not_null, i}) != args_set.end(); + bool find = args_set.contains( + {materialized_column_not_null, index_check_const(i, col_const)}); if constexpr (negative) { vec_res[i] = !find; diff --git a/be/src/exprs/function/function_ip.h b/be/src/exprs/function/function_ip.h index 5c284412848da0..2600041547bee1 100644 --- a/be/src/exprs/function/function_ip.h +++ b/be/src/exprs/function/function_ip.h @@ -231,11 +231,11 @@ class FunctionIPv4StringToNum : public IFunction { Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, uint32_t result, size_t input_rows_count) const override { - ColumnPtr column = block.get_by_position(arguments[0]).column; + ColumnPtr column = + block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); ColumnPtr null_map_column; const NullMap* null_map = nullptr; - if (column->is_nullable()) { - const auto* column_nullable = assert_cast(column.get()); + if (const auto* column_nullable = check_and_get_column(column.get())) { column = column_nullable->get_nested_column_ptr(); null_map_column = column_nullable->get_null_map_column_ptr(); null_map = &column_nullable->get_null_map_data(); @@ -526,12 +526,12 @@ class FunctionIPv6StringToNum : public IFunction { Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, uint32_t result, size_t input_rows_count) const override { - ColumnPtr column = block.get_by_position(arguments[0]).column; + ColumnPtr column = + block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); ColumnPtr null_map_column; const NullMap* null_map = nullptr; - if (column->is_nullable()) { - const auto* column_nullable = assert_cast(column.get()); + if (const auto* column_nullable = check_and_get_column(column.get())) { column = column_nullable->get_nested_column_ptr(); null_map_column = column_nullable->get_null_map_column_ptr(); null_map = &column_nullable->get_null_map_data(); diff --git a/be/src/exprs/function/function_jsonb.cpp b/be/src/exprs/function/function_jsonb.cpp index 83c9279f5adcf0..b557a346aa00ea 100644 --- a/be/src/exprs/function/function_jsonb.cpp +++ b/be/src/exprs/function/function_jsonb.cpp @@ -216,13 +216,11 @@ class FunctionJsonbParseBase : public IFunction { } const ColumnString* col_from_string = nullptr; - if (col_from->is_nullable()) { - const auto& nullable_col = assert_cast(*col_from); - + if (const auto* nullable_col = check_and_get_column(col_from.get())) { VectorizedUtils::update_null_map(null_map->get_data(), - nullable_col.get_null_map_data()); + nullable_col->get_null_map_data(), col_from_is_const); col_from_string = - assert_cast(nullable_col.get_nested_column_ptr().get()); + assert_cast(nullable_col->get_nested_column_ptr().get()); } else { col_from_string = assert_cast(col_from.get()); } @@ -261,12 +259,11 @@ class FunctionJsonbParseBase : public IFunction { constant_default_value = data; } } else { - if (default_value_col->is_nullable()) { - const auto& nullable_col = - assert_cast(*default_value_col); + if (const auto* nullable_col = + check_and_get_column(default_value_col.get())) { default_value_str_col = assert_cast( - nullable_col.get_nested_column_ptr().get()); - default_value_nullmap = &(nullable_col.get_null_map_data()); + nullable_col->get_nested_column_ptr().get()); + default_value_nullmap = &(nullable_col->get_null_map_data()); } else { default_value_str_col = assert_cast(default_value_col.get()); @@ -390,10 +387,10 @@ class FunctionJsonbExtract : public IFunction { // prepare jsonb data column std::tie(jsonb_data_column, jsonb_data_const) = unpack_if_const(block.get_by_position(arguments[0]).column); - if (jsonb_data_column->is_nullable()) { - const auto& nullable_column = assert_cast(*jsonb_data_column); - jsonb_data_column = nullable_column.get_nested_column_ptr(); - data_null_map = &nullable_column.get_null_map_data(); + if (const auto* nullable_column = + check_and_get_column(jsonb_data_column.get())) { + jsonb_data_column = nullable_column->get_nested_column_ptr(); + data_null_map = &nullable_column->get_null_map_data(); } const auto& ldata = assert_cast(jsonb_data_column.get())->get_chars(); const auto& loffsets = @@ -409,10 +406,10 @@ class FunctionJsonbExtract : public IFunction { std::tie(path_column, is_const) = unpack_if_const(block.get_by_position(arguments[i + 1]).column); path_const[i] = is_const; - if (path_column->is_nullable()) { - const auto& nullable_column = assert_cast(*path_column); - path_column = nullable_column.get_nested_column_ptr(); - path_null_maps[i] = &nullable_column.get_null_map_data(); + if (const auto* nullable_column = + check_and_get_column(path_column.get())) { + path_column = nullable_column->get_nested_column_ptr(); + path_null_maps[i] = &nullable_column->get_null_map_data(); } jsonb_path_columns.push_back(assert_cast(path_column.get())); } @@ -500,8 +497,7 @@ class FunctionJsonbKeys : public IFunction { // prepare jsonb data column auto&& [jsonb_data_column, json_data_const] = unpack_if_const(block.get_by_position(arguments[0]).column); - if (jsonb_data_column->is_nullable()) { - const auto* nullable = assert_cast(jsonb_data_column.get()); + if (const auto* nullable = check_and_get_column(jsonb_data_column.get())) { col_from_string = assert_cast(nullable->get_nested_column_ptr().get()); data_null_map = &nullable->get_null_map_data(); @@ -518,8 +514,8 @@ class FunctionJsonbKeys : public IFunction { // we have should have a ColumnString for path std::tie(jsonb_path_column, path_const) = unpack_if_const(block.get_by_position(arguments[1]).column); - if (jsonb_path_column->is_nullable()) { - const auto* nullable = assert_cast(jsonb_path_column.get()); + if (const auto* nullable = + check_and_get_column(jsonb_path_column.get())) { jsonb_path_column = nullable->get_nested_column_ptr(); path_null_map = &nullable->get_null_map_data(); } @@ -672,8 +668,7 @@ class FunctionJsonbExtractPath : public IFunction { const NullMap* data_null_map = nullptr; const ColumnString* data_col = nullptr; - if (jsonb_data_column->is_nullable()) { - const auto* nullable = assert_cast(jsonb_data_column.get()); + if (const auto* nullable = check_and_get_column(jsonb_data_column.get())) { data_col = assert_cast(nullable->get_nested_column_ptr().get()); data_null_map = &nullable->get_null_map_data(); } else { @@ -688,8 +683,7 @@ class FunctionJsonbExtractPath : public IFunction { unpack_if_const(block.get_by_position(arguments[1]).column); const ColumnString* path_col = nullptr; const NullMap* path_null_map = nullptr; - if (path_column->is_nullable()) { - const auto* nullable = assert_cast(path_column.get()); + if (const auto* nullable = check_and_get_column(path_column.get())) { path_col = assert_cast(nullable->get_nested_column_ptr().get()); path_null_map = &nullable->get_null_map_data(); } else { @@ -1598,12 +1592,10 @@ class FunctionJsonbArray : public IFunction { for (auto argument : arguments) { auto&& [arg_column, is_const] = unpack_if_const(block.get_by_position(argument).column); - if (arg_column->is_nullable()) { - const auto& nullable_column = - assert_cast( - *arg_column); - const auto& null_map = nullable_column.get_null_map_data(); - const auto& nested_column = nullable_column.get_nested_column(); + if (const auto* nullable_column = + check_and_get_column(arg_column.get())) { + const auto& null_map = nullable_column->get_null_map_data(); + const auto& nested_column = nullable_column->get_nested_column(); const auto& jsonb_column = assert_cast( nested_column); @@ -1763,12 +1755,10 @@ class FunctionJsonbObject : public IFunction { auto&& [value_column, value_const] = unpack_if_const(block.get_by_position(value_argument).column); - if (key_column->is_nullable()) { - const auto& nullable_column = - assert_cast( - *key_column); - const auto& null_map = nullable_column.get_null_map_data(); - const auto& nested_column = nullable_column.get_nested_column(); + if (const auto* nullable_column = + check_and_get_column(key_column.get())) { + const auto& null_map = nullable_column->get_null_map_data(); + const auto& nested_column = nullable_column->get_nested_column(); const auto& key_arg_column = assert_cast( nested_column); @@ -1783,12 +1773,10 @@ class FunctionJsonbObject : public IFunction { write_key(writer, key_arg_column, key_const, nullptr, arg_idx, i)); } - if (value_column->is_nullable()) { - const auto& nullable_column = - assert_cast( - *value_column); - const auto& null_map = nullable_column.get_null_map_data(); - const auto& nested_column = nullable_column.get_nested_column(); + if (const auto* nullable_column = + check_and_get_column(value_column.get())) { + const auto& null_map = nullable_column->get_null_map_data(); + const auto& nested_column = nullable_column->get_nested_column(); const auto& value_arg_column = assert_cast( nested_column); @@ -1908,10 +1896,10 @@ class FunctionJsonbModify : public IFunction { const NullMap* json_data_null_map = nullptr; const ColumnString* json_data_column; - if (json_data_arg_column->is_nullable()) { - const auto& nullable_column = assert_cast(*json_data_arg_column); - json_data_null_map = &nullable_column.get_null_map_data(); - const auto& nested_column = nullable_column.get_nested_column(); + if (const auto* nullable_column = + check_and_get_column(json_data_arg_column.get())) { + json_data_null_map = &nullable_column->get_null_map_data(); + const auto& nested_column = nullable_column->get_nested_column(); json_data_column = assert_cast(&nested_column); } else { json_data_column = assert_cast(json_data_arg_column.get()); @@ -1931,20 +1919,20 @@ class FunctionJsonbModify : public IFunction { } json_path_constant[i / 2] = path_const; - if (path_column->is_nullable()) { - const auto& nullable_column = assert_cast(*path_column); - json_path_null_maps[i / 2] = &nullable_column.get_null_map_data(); - const auto& nested_column = nullable_column.get_nested_column(); + if (const auto* nullable_column = + check_and_get_column(path_column.get())) { + json_path_null_maps[i / 2] = &nullable_column->get_null_map_data(); + const auto& nested_column = nullable_column->get_nested_column(); json_path_columns[i / 2] = assert_cast(&nested_column); } else { json_path_columns[i / 2] = assert_cast(path_column.get()); } json_value_constant[i / 2] = value_const; - if (value_column->is_nullable()) { - const auto& nullable_column = assert_cast(*value_column); - json_value_null_maps[i / 2] = &nullable_column.get_null_map_data(); - const auto& nested_column = nullable_column.get_nested_column(); + if (const auto* nullable_column = + check_and_get_column(value_column.get())) { + json_value_null_maps[i / 2] = &nullable_column->get_null_map_data(); + const auto& nested_column = nullable_column->get_nested_column(); json_value_columns[i / 2] = assert_cast(&nested_column); } else { json_value_columns[i / 2] = assert_cast(value_column.get()); @@ -3040,13 +3028,13 @@ class FunctionStripNullValue : public IFunction { Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, uint32_t result, size_t input_rows_count) const override { - const auto& arg_column = block.get_by_position(arguments[0]).column; + const auto arg_column = + block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); const ColumnString* json_column = nullptr; const NullMap* json_null_map = nullptr; - if (arg_column->is_nullable()) { - const auto& nullable_col = assert_cast(*arg_column); - json_column = assert_cast(&nullable_col.get_nested_column()); - json_null_map = &nullable_col.get_null_map_data(); + if (const auto* nullable_col = check_and_get_column(arg_column.get())) { + json_column = assert_cast(&nullable_col->get_nested_column()); + json_null_map = &nullable_col->get_null_map_data(); } else { json_column = assert_cast(arg_column.get()); } diff --git a/be/src/exprs/function/function_map.cpp b/be/src/exprs/function/function_map.cpp index cff3c1ba0f8fdc..e0b06e7450dfb3 100644 --- a/be/src/exprs/function/function_map.cpp +++ b/be/src/exprs/function/function_map.cpp @@ -180,8 +180,7 @@ class FunctionMapContains : public IFunction { block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); const ColumnMap* map_column = nullptr; ColumnPtr nullmap_column = nullptr; - if (left_column->is_nullable()) { - auto nullable_column = reinterpret_cast(left_column.get()); + if (const auto* nullable_column = check_and_get_column(left_column.get())) { map_column = check_and_get_column(nullable_column->get_nested_column()); nullmap_column = nullable_column->get_null_map_column_ptr(); } else { @@ -269,8 +268,7 @@ class FunctionMapKeysOrValues : public IFunction { auto left_column = block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); const ColumnMap* map_column = nullptr; - if (left_column->is_nullable()) { - auto nullable_column = reinterpret_cast(left_column.get()); + if (const auto* nullable_column = check_and_get_column(left_column.get())) { map_column = check_and_get_column(nullable_column->get_nested_column()); } else { map_column = check_and_get_column(*left_column.get()); @@ -678,9 +676,8 @@ class FunctionMapContainsEntry : public IFunction { block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); const ColumnMap* map_column = nullptr; ColumnPtr map_row_nullmap_col = nullptr; - if (map_column_ptr->is_nullable()) { - const auto* nullable_column = - reinterpret_cast(map_column_ptr.get()); + if (const auto* nullable_column = + check_and_get_column(map_column_ptr.get())) { map_column = check_and_get_column(nullable_column->get_nested_column()); map_row_nullmap_col = nullable_column->get_null_map_column_ptr(); } else { @@ -702,17 +699,16 @@ class FunctionMapContainsEntry : public IFunction { const UInt8* key_nullmap = nullptr; const UInt8* value_nullmap = nullptr; - if (key_column_ptr->is_nullable()) { - const auto* nullable_column = assert_cast(key_column_ptr.get()); + if (const auto* nullable_column = + check_and_get_column(key_column_ptr.get())) { key_column = &nullable_column->get_nested_column(); key_nullmap = nullable_column->get_null_map_column().get_data().data(); } else { key_column = key_column_ptr.get(); } - if (value_column_ptr->is_nullable()) { - const auto* nullable_column = - assert_cast(value_column_ptr.get()); + if (const auto* nullable_column = + check_and_get_column(value_column_ptr.get())) { value_column = &nullable_column->get_nested_column(); value_nullmap = nullable_column->get_null_map_column().get_data().data(); } else { diff --git a/be/src/exprs/function/function_variant_element.cpp b/be/src/exprs/function/function_variant_element.cpp index c2d984885de00a..012f11e80c9920 100644 --- a/be/src/exprs/function/function_variant_element.cpp +++ b/be/src/exprs/function/function_variant_element.cpp @@ -90,7 +90,7 @@ class FunctionVariantElement : public IFunction { if (var.is_null_root()) { return make_nullable(col, true); } - if (var.is_scalar_variant() && var.get_root()->is_nullable()) { + if (var.is_scalar_variant() && is_column_nullable(*var.get_root())) { const auto* nullable = assert_cast(var.get_root().get()); return ColumnNullable::create( col, nullable->get_null_map_column_ptr()->clone_resized(col->size())); diff --git a/be/src/exprs/function/functions_multi_string_position.cpp b/be/src/exprs/function/functions_multi_string_position.cpp index d345521e067237..f4649dfef27b6d 100644 --- a/be/src/exprs/function/functions_multi_string_position.cpp +++ b/be/src/exprs/function/functions_multi_string_position.cpp @@ -81,27 +81,14 @@ class FunctionMultiStringPosition : public IFunction { auto haystack_column = block.get_by_position(arguments[0]).column; auto needles_column = block.get_by_position(arguments[1]).column; - bool haystack_nullable = false; - bool needles_nullable = false; - - if (haystack_column->is_nullable()) { - haystack_nullable = true; - } - - if (needles_column->is_nullable()) { - needles_nullable = true; - } - auto haystack_ptr = remove_nullable(haystack_column); auto needles_ptr = remove_nullable(needles_column); - const ColumnString* col_haystack_vector = - check_and_get_column(&*haystack_ptr); + const auto* col_haystack_vector = check_and_get_column(&*haystack_ptr); const ColumnConst* col_haystack_const = check_and_get_column_const(&*haystack_ptr); - const ColumnArray* col_needles_vector = - check_and_get_column(needles_ptr.get()); + const auto* col_needles_vector = check_and_get_column(needles_ptr.get()); const ColumnConst* col_needles_const = check_and_get_column_const(needles_ptr.get()); @@ -140,29 +127,8 @@ class FunctionMultiStringPosition : public IFunction { return status; } - if (haystack_nullable) { - const auto* column_nullable = assert_cast(haystack_column.get()); - auto& null_map = column_nullable->get_null_map_data(); - for (size_t i = 0; i != input_rows_count; ++i) { - if (null_map[i] == 1) { - for (size_t offset = offsets_res[i - 1]; offset != offsets_res[i]; ++offset) { - vec_res[offset] = 0; - } - } - } - } - - if (needles_nullable) { - const auto* column_nullable = assert_cast(needles_column.get()); - auto& null_map = column_nullable->get_null_map_data(); - for (size_t i = 0; i != input_rows_count; ++i) { - if (null_map[i] == 1) { - for (size_t offset = offsets_res[i - 1]; offset != offsets_res[i]; ++offset) { - vec_res[offset] = 0; - } - } - } - } + handle_nullable_column(haystack_column, vec_res, offsets_res, input_rows_count); + handle_nullable_column(needles_column, vec_res, offsets_res, input_rows_count); auto nullable_col = ColumnNullable::create(std::move(col_res), ColumnUInt8::create(col_res->size(), 0)); @@ -170,6 +136,36 @@ class FunctionMultiStringPosition : public IFunction { ColumnArray::create(std::move(nullable_col), std::move(col_offsets)); return status; } + +private: + using ResultContainer = typename ColumnVector::Container; + + void fill_result_row_with_zero(ResultContainer& vec_res, + const PaddedPODArray& offsets_res, size_t row) const { + for (size_t offset = offsets_res[row - 1]; offset != offsets_res[row]; ++offset) { + vec_res[offset] = 0; + } + } + + void handle_nullable_column(const ColumnPtr& column, ResultContainer& vec_res, + const PaddedPODArray& offsets_res, + size_t input_rows_count) const { + if (const auto* nullable = check_and_get_column(column.get())) { + const auto& null_map = nullable->get_null_map_data(); + for (size_t i = 0; i != input_rows_count; ++i) { + if (null_map[i]) { + fill_result_row_with_zero(vec_res, offsets_res, i); + } + } + } else if (const auto* const_column = check_and_get_column(column.get()); + const_column && is_column_nullable(const_column->get_data_column())) { + const auto& const_nullable = + assert_cast(const_column->get_data_column()); + if (const_nullable.get_null_map_data()[0]) { + std::fill(vec_res.begin(), vec_res.end(), 0); + } + } + } }; struct FunctionMultiSearchAllPositionsImpl { diff --git a/be/src/exprs/function/functions_multi_string_search.cpp b/be/src/exprs/function/functions_multi_string_search.cpp index 2dcd28d0ecb10b..99a92e6ca11d8e 100644 --- a/be/src/exprs/function/functions_multi_string_search.cpp +++ b/be/src/exprs/function/functions_multi_string_search.cpp @@ -147,14 +147,20 @@ class FunctionsMultiStringSearch : public IFunction { /// Handles nullable column by setting result to 0 if the input is null void handle_nullable_column(const ColumnPtr& column, PaddedPODArray& vec_res, size_t input_rows_count) const { - if (column->is_nullable()) { - const auto* column_nullable = assert_cast(column.get()); - const auto& null_map = column_nullable->get_null_map_data(); + if (const auto* nullable = check_and_get_column(column.get())) { + const auto& null_map = nullable->get_null_map_data(); for (size_t i = 0; i != input_rows_count; ++i) { - if (null_map[i] == 1) { + if (null_map[i]) { vec_res[i] = 0; } } + } else if (const auto* const_column = check_and_get_column(column.get()); + const_column && is_column_nullable(const_column->get_data_column())) { + const auto& const_nullable = + assert_cast(const_column->get_data_column()); + if (const_nullable.get_null_map_data()[0]) { + std::fill(vec_res.begin(), vec_res.begin() + input_rows_count, 0); + } } } }; diff --git a/be/src/exprs/function/if.cpp b/be/src/exprs/function/if.cpp index 7e543499ada6a1..0ff43b30d6b9df 100644 --- a/be/src/exprs/function/if.cpp +++ b/be/src/exprs/function/if.cpp @@ -72,14 +72,13 @@ size_t count_true_with_notnull(const ColumnPtr& col) { return 0; } - if (const auto* const_col = check_and_get_column_const(col.get())) { + if (const auto* const_col = check_and_get_column(col.get())) { bool is_true = const_col->get_bool(0); return is_true ? col->size() : 0; } auto count = col->size(); - if (col->is_nullable()) { - const auto* nullable = assert_cast(col.get()); + if (const auto* nullable = check_and_get_column(col.get())) { const auto* __restrict null_data = nullable->get_null_map_data().data(); const auto* __restrict bool_data = ((const ColumnUInt8&)(nullable->get_nested_column())).get_data().data(); diff --git a/be/src/exprs/function/in.h b/be/src/exprs/function/in.h index 55c709a2a58bb2..9075415e6365cb 100644 --- a/be/src/exprs/function/in.h +++ b/be/src/exprs/function/in.h @@ -210,12 +210,15 @@ class FunctionIn : public IFunction { auto& vec_null_map_to = col_null_map_to->get_data(); const ColumnWithTypeAndName& left_arg = block.get_by_position(arguments[0]); - const auto& [materialized_column, col_const] = unpack_if_const(left_arg.column); + const auto& [unpacked_column, col_const] = unpack_if_const(left_arg.column); + ColumnPtr materialized_column = unpacked_column; + if (in_state->use_set && col_const) { + materialized_column = left_arg.column->convert_to_full_column_if_const(); + } if (in_state->use_set) { - if (materialized_column->is_nullable()) { - const auto* null_col_ptr = - assert_cast(materialized_column.get()); + if (const auto* null_col_ptr = + check_and_get_column(materialized_column.get())) { const auto& null_map = null_col_ptr->get_null_map_column().get_data(); const auto* nested_col_ptr = null_col_ptr->get_nested_column_ptr().get(); diff --git a/be/src/exprs/function/ip_address_dictionary.cpp b/be/src/exprs/function/ip_address_dictionary.cpp index 86b4e744b47f8b..68446550b4461c 100644 --- a/be/src/exprs/function/ip_address_dictionary.cpp +++ b/be/src/exprs/function/ip_address_dictionary.cpp @@ -84,9 +84,7 @@ ColumnPtr IPAddressDictionary::get_column(const std::string& attribute_name, // input key column without nullable const auto* ipv6_column = assert_cast(remove_nullable(key_column).get()); // if input key column is nullable, will not be null - const ColumnNullable* null_key = - key_column->is_nullable() ? assert_cast(key_column.get()) - : nullptr; + const auto* null_key = check_and_get_column(key_column.get()); std::visit( [&](auto&& arg, auto key_is_nullable, auto value_is_nullable) { using ValueDataType = std::decay_t; @@ -121,9 +119,7 @@ ColumnPtr IPAddressDictionary::get_column(const std::string& attribute_name, // input key column without nullable const auto* ipv4_column = assert_cast(remove_nullable(key_column).get()); // if input key column is nullable, will not be null - const ColumnNullable* null_key = - key_column->is_nullable() ? assert_cast(key_column.get()) - : nullptr; + const auto* null_key = check_and_get_column(key_column.get()); std::visit( [&](auto&& arg, auto key_is_nullable, auto value_is_nullable) { using ValueDataType = std::decay_t; diff --git a/be/src/exprs/hybrid_set.h b/be/src/exprs/hybrid_set.h index e2624b359221c8..18b59338b26d04 100644 --- a/be/src/exprs/hybrid_set.h +++ b/be/src/exprs/hybrid_set.h @@ -268,7 +268,7 @@ class HybridSet : public HybridSetBase { "HybridSet::insert_range_from method (data.size() = {}).", start, end, column->size()); } - if (column->is_nullable()) { + if (is_column_nullable(*column)) { const auto* nullable = assert_cast(column.get()); const auto& col = nullable->get_nested_column(); const auto& nullmap = nullable->get_null_map_column().get_data(); @@ -460,7 +460,7 @@ class StringSet : public HybridSetBase { "StringSet::insert_range_from method (data.size() = {}).", start, end, column->size()); } - if (column->is_nullable()) { + if (is_column_nullable(*column)) { const auto* nullable = assert_cast(column.get()); const auto& nullmap = nullable->get_null_map_column().get_data(); if (nullable->get_nested_column().is_column_string64()) { @@ -663,7 +663,7 @@ class StringValueSet : public HybridSetBase { "StringSet::insert_range_from method (data.size() = {}).", start, end, column->size()); } - if (column->is_nullable()) { + if (is_column_nullable(*column)) { const auto* nullable = assert_cast(column.get()); const auto& nullmap = nullable->get_null_map_column().get_data(); if (nullable->get_nested_column().is_column_string64()) { diff --git a/be/src/exprs/lambda_function/varray_filter_function.cpp b/be/src/exprs/lambda_function/varray_filter_function.cpp index f8fa7d0f8a8040..e4bb3897c1b26d 100644 --- a/be/src/exprs/lambda_function/varray_filter_function.cpp +++ b/be/src/exprs/lambda_function/varray_filter_function.cpp @@ -75,7 +75,7 @@ class ArrayFilterFunction : public LambdaFunction { auto input_rows = first_column->size(); auto first_outside_null_map = ColumnUInt8::create(input_rows, 0); auto first_arg_column = first_column; - if (first_arg_column->is_nullable()) { + if (is_column_nullable(*first_arg_column)) { first_arg_column = assert_cast(first_column.get())->get_nested_column_ptr(); const auto& column_array_nullmap = @@ -97,7 +97,7 @@ class ArrayFilterFunction : public LambdaFunction { auto second_arg_column = second_column; auto second_outside_null_map = ColumnUInt8::create(input_rows, 0); - if (second_arg_column->is_nullable()) { + if (is_column_nullable(*second_arg_column)) { second_arg_column = assert_cast(second_column.get()) ->get_nested_column_ptr(); const auto& column_array_nullmap = diff --git a/be/src/exprs/lambda_function/varray_map_function.cpp b/be/src/exprs/lambda_function/varray_map_function.cpp index c21da9d09d841c..db82c46500062c 100644 --- a/be/src/exprs/lambda_function/varray_map_function.cpp +++ b/be/src/exprs/lambda_function/varray_map_function.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#include #include #include #include @@ -85,8 +86,7 @@ class ArrayMapFunction : public LambdaFunction { int gap = 0; if (!output_slot_ref_indexs.empty()) { - auto max_id = - std::max_element(output_slot_ref_indexs.begin(), output_slot_ref_indexs.end()); + auto max_id = std::ranges::max_element(output_slot_ref_indexs); gap = *max_id + 1; _set_column_ref_column_id(children[0], gap); } @@ -134,6 +134,7 @@ class ArrayMapFunction : public LambdaFunction { if (type_array->is_nullable()) { // get the nullmap of nullable column // hold the null column instead of a reference 'cause `column_array` will be assigned and freed below. + DORIS_CHECK(is_column_nullable(*column_array)); auto column_array_nullmap = assert_cast(*column_array).get_null_map_column_ptr(); diff --git a/be/src/exprs/lambda_function/varray_sort_function.cpp b/be/src/exprs/lambda_function/varray_sort_function.cpp index 9a78617625f6a9..6d14609400365c 100644 --- a/be/src/exprs/lambda_function/varray_sort_function.cpp +++ b/be/src/exprs/lambda_function/varray_sort_function.cpp @@ -88,7 +88,7 @@ class ArraySortFunction : public LambdaFunction { ColumnPtr outside_null_map = nullptr; - if (arg_column->is_nullable()) { + if (is_column_nullable(*arg_column)) { arg_column = assert_cast(column.get())->get_nested_column_ptr(); outside_null_map = assert_cast(column.get())->get_null_map_column_ptr(); diff --git a/be/src/exprs/minmax_predicate.h b/be/src/exprs/minmax_predicate.h index 3855da4a8635bd..b0f1b27e7a864f 100644 --- a/be/src/exprs/minmax_predicate.h +++ b/be/src/exprs/minmax_predicate.h @@ -50,7 +50,7 @@ class MinMaxNumFunc : public MinMaxFuncBase { ~MinMaxNumFunc() override = default; void insert_fixed_len(const ColumnPtr& column, size_t start) override { - if (column->is_nullable()) { + if (is_column_nullable(*column)) { const auto* nullable = assert_cast(column.get()); const auto& col = nullable->get_nested_column_ptr(); const auto& nullmap = nullable->get_null_map_data(); diff --git a/be/src/exprs/table_function/vexplode_map.cpp b/be/src/exprs/table_function/vexplode_map.cpp index cff60eaa3d55db..890f39369912f0 100644 --- a/be/src/exprs/table_function/vexplode_map.cpp +++ b/be/src/exprs/table_function/vexplode_map.cpp @@ -37,7 +37,7 @@ VExplodeMapTableFunction::VExplodeMapTableFunction() { bool extract_column_map_info(const IColumn& src, ColumnMapExecutionData& data) { const IColumn* map_col = &src; // extract array nullable info - if (src.is_nullable()) { + if (is_column_nullable(src)) { const auto& null_col = reinterpret_cast(src); // map column's nullmap data.map_nullmap_data = null_col.get_null_map_data().data(); diff --git a/be/src/exprs/table_function/vexplode_numbers.cpp b/be/src/exprs/table_function/vexplode_numbers.cpp index fe90119cefeff3..b3a2dfce3987db 100644 --- a/be/src/exprs/table_function/vexplode_numbers.cpp +++ b/be/src/exprs/table_function/vexplode_numbers.cpp @@ -52,7 +52,7 @@ Status VExplodeNumbersTableFunction::process_init(Block* block, RuntimeState* st // the argument columns -> Int32 const auto& column_nested = assert_cast(*_value_column).get_data_column_ptr(); - if (column_nested->is_nullable()) { + if (is_column_nullable(*column_nested)) { if (!column_nested->is_null_at(0)) { _cur_size = assert_cast( assert_cast(column_nested.get()) diff --git a/be/src/exprs/vcase_expr.h b/be/src/exprs/vcase_expr.h index fef3a3462f90b3..97b2551091d100 100644 --- a/be/src/exprs/vcase_expr.h +++ b/be/src/exprs/vcase_expr.h @@ -156,7 +156,7 @@ class VCaseExpr final : public VExpr { continue; } std::tie(raw_then_columns[i], is_consts[i]) = unpack_if_const(then_columns[i]); - is_nullable[i] = raw_then_columns[i]->is_nullable(); + is_nullable[i] = is_column_nullable(*raw_then_columns[i]); } auto* raw_result_column = result_column_ptr.get(); @@ -260,7 +260,7 @@ class VCaseExpr final : public VExpr { continue; } - if (raw_when_column->is_nullable()) { + if (is_column_nullable(*raw_when_column)) { const auto* column_nullable_ptr = assert_cast( raw_when_column.get()); diff --git a/be/src/exprs/vcondition_expr.cpp b/be/src/exprs/vcondition_expr.cpp index 7ef8ae98ae25bf..ab8f694c2c5899 100644 --- a/be/src/exprs/vcondition_expr.cpp +++ b/be/src/exprs/vcondition_expr.cpp @@ -73,8 +73,7 @@ size_t VConditionExpr::count_true_with_notnull(const ColumnPtr& col) { } auto count = col->size(); - if (col->is_nullable()) { - const auto* nullable = assert_cast(col.get()); + if (const auto* nullable = check_and_get_column(col.get())) { const auto* __restrict null_data = nullable->get_null_map_data().data(); const auto* __restrict bool_data = ((const ColumnUInt8&)(nullable->get_nested_column())).get_data().data(); @@ -502,7 +501,7 @@ Status VectorizedIfNullExpr::execute_column_impl(VExprContext* context, const Bl RETURN_IF_ERROR(_children[0]->execute_column(context, block, selector, count, first_column)); first_column = first_column->convert_to_full_column_if_const(); - if (!first_column->is_nullable()) { + if (!is_column_nullable(*first_column)) { result_column = first_column; DCHECK(_data_type->is_nullable() == false); return Status::OK(); diff --git a/be/src/exprs/vdirect_in_predicate.h b/be/src/exprs/vdirect_in_predicate.h index 16ac0949ccdd72..882c8d4de69030 100644 --- a/be/src/exprs/vdirect_in_predicate.h +++ b/be/src/exprs/vdirect_in_predicate.h @@ -134,11 +134,9 @@ class VDirectInPredicate final : public VExpr { auto res_data_column = ColumnUInt8::create(sz); res_data_column->resize(sz); - if (argument_column->is_nullable()) { - auto column_nested = static_cast(argument_column.get()) - ->get_nested_column_ptr(); - const auto& null_map = - static_cast(argument_column.get())->get_null_map_data(); + if (const auto* nullable = check_and_get_column(argument_column.get())) { + auto column_nested = nullable->get_nested_column_ptr(); + const auto& null_map = nullable->get_null_map_data(); _filter->find_batch_nullable(*column_nested, sz, null_map, res_data_column->get_data(), filter); } else { diff --git a/be/src/format/column_type_convert.h b/be/src/format/column_type_convert.h index 700fdd2ac1cff4..a9105e4ec99284 100644 --- a/be/src/format/column_type_convert.h +++ b/be/src/format/column_type_convert.h @@ -51,7 +51,7 @@ enum FileFormat { COMMON, ORC, PARQUET }; // Returns IColumn* (raw pointer) to avoid creating a second owning MutableColumnPtr, // which would violate COW invariant (use_count > 1). inline IColumn* get_mutable_inner_col(MutableColumnPtr& dst_col) { - if (dst_col->is_nullable()) { + if (is_column_nullable(*dst_col)) { return static_cast(dst_col.get())->get_nested_column_ptr().get(); } else { return dst_col.get(); @@ -230,7 +230,7 @@ class NumericToFloatPointConverter : public ColumnTypeConverter { IColumn* to_col = get_mutable_inner_col(dst_col); NullMap* null_map = nullptr; - if (dst_col->is_nullable()) { + if (is_column_nullable(*dst_col)) { null_map = &static_cast(dst_col.get())->get_null_map_data(); } @@ -287,7 +287,7 @@ class NumericToStringConverter : public ColumnTypeConverter { IColumn* to_col = get_mutable_inner_col(dst_col); NullMap* null_map = nullptr; - if (dst_col->is_nullable()) { + if (is_column_nullable(*dst_col)) { null_map = &reinterpret_cast(dst_col.get())->get_null_map_data(); } @@ -587,7 +587,7 @@ class CastStringConverter : public ColumnTypeConverter { NullMap* null_map = nullptr; IColumn* to_col = nullptr; - if (dst_col->is_nullable()) { + if (is_column_nullable(*dst_col)) { auto* nullable = assert_cast(dst_col.get()); to_col = nullable->get_nested_column_ptr().get(); null_map = &nullable->get_null_map_data(); @@ -646,7 +646,7 @@ class DateTimeToNumericConverter : public ColumnTypeConverter { IColumn* to_col = get_mutable_inner_col(dst_col); NullMap* null_map = nullptr; - if (dst_col->is_nullable()) { + if (is_column_nullable(*dst_col)) { null_map = &reinterpret_cast(dst_col.get())->get_null_map_data(); } @@ -736,7 +736,7 @@ class NumericToDecimalConverter : public ColumnTypeConverter { IColumn* to_col = get_mutable_inner_col(dst_col); NullMap* null_map = nullptr; - if (dst_col->is_nullable()) { + if (is_column_nullable(*dst_col)) { null_map = &reinterpret_cast(dst_col.get())->get_null_map_data(); } @@ -828,7 +828,7 @@ class DecimalToNumericConverter : public ColumnTypeConverter { auto& data = static_cast(*to_col).get_data(); NullMap* null_map = nullptr; - if (dst_col->is_nullable()) { + if (is_column_nullable(*dst_col)) { null_map = &reinterpret_cast(dst_col.get())->get_null_map_data(); } @@ -1000,7 +1000,7 @@ class VarBinaryConverter : public ColumnTypeConverter { IColumn* to_col = nullptr; // nullmap flag seems have been handled in upper level - if (dst_col->is_nullable()) { + if (is_column_nullable(*dst_col)) { const auto* nullable = assert_cast(dst_col.get()); to_col = const_cast(nullable)->get_nested_column_ptr().get(); } else { diff --git a/be/src/format/jni/jni_data_bridge.cpp b/be/src/format/jni/jni_data_bridge.cpp index 4d42574075a662..1a88f4c317b28a 100644 --- a/be/src/format/jni/jni_data_bridge.cpp +++ b/be/src/format/jni/jni_data_bridge.cpp @@ -107,8 +107,7 @@ Status JniDataBridge::fill_column(TableMetaAddress& address, ColumnPtr& doris_co } auto mutable_doris_column = IColumn::mutate(std::move(doris_column)); MutableColumnPtr data_column; - if (mutable_doris_column->is_nullable()) { - auto* nullable_column = assert_cast(mutable_doris_column.get()); + if (auto* nullable_column = check_and_get_column(mutable_doris_column.get())) { data_column = nullable_column->get_nested_column_ptr(); NullMap& null_map = nullable_column->get_null_map_data(); size_t origin_size = null_map.size(); @@ -507,10 +506,9 @@ Status JniDataBridge::_fill_column_meta(const ColumnPtr& doris_column, const Dat // insert null map address const IColumn* data_column = nullptr; - if (column->is_nullable()) { - const auto& nullable_column = assert_cast(*column); - data_column = &(nullable_column.get_nested_column()); - const auto& null_map = nullable_column.get_null_map_data(); + if (const auto* nullable_column = check_and_get_column(column)) { + data_column = &(nullable_column->get_nested_column()); + const auto& null_map = nullable_column->get_null_map_data(); meta_data.emplace_back((long)null_map.data()); } else { meta_data.emplace_back(0); diff --git a/be/src/format/json/new_json_reader.cpp b/be/src/format/json/new_json_reader.cpp index 81ceab2f6b879a..8d53b6009e6bef 100644 --- a/be/src/format/json/new_json_reader.cpp +++ b/be/src/format/json/new_json_reader.cpp @@ -455,7 +455,7 @@ Status NewJsonReader::_get_range_params() { Status json_reader_detail::append_null_for_malformed_json(Block& block) { for (int i = 0; i < block.columns(); ++i) { auto& column_with_type = block.get_by_position(i); - if (!column_with_type.column->is_nullable()) [[unlikely]] { + if (!is_column_nullable(*column_with_type.column)) [[unlikely]] { return Status::DataQualityError("malformed json, but the column `{}` is not nullable.", column_with_type.column->get_name()); } @@ -1119,7 +1119,7 @@ Status NewJsonReader::_simdjson_write_data_to_column(simdjson::ondemand::value& IColumn* data_column_ptr = column_ptr; DataTypeSerDeSPtr data_serde = serde; - if (column_ptr->is_nullable()) { + if (is_column_nullable(*column_ptr)) { nullable_column = reinterpret_cast(column_ptr); data_column_ptr = nullable_column->get_nested_column().get_ptr().get(); @@ -1225,14 +1225,14 @@ Status NewJsonReader::_simdjson_write_data_to_column(simdjson::ondemand::value& sub_serdes[sub_column_idx], valid)); } - //fill missing subcolumn + // fill missing subcolumn for (size_t sub_col_idx = 0; sub_col_idx < sub_col_size; sub_col_idx++) { if (has_value[sub_col_idx]) { continue; } auto sub_column_ptr = struct_column_ptr->get_column(sub_col_idx).get_ptr(); - if (sub_column_ptr->is_nullable()) { + if (is_column_nullable(*sub_column_ptr)) { sub_column_ptr->insert_default(); continue; } else [[unlikely]] { @@ -1258,7 +1258,7 @@ Status NewJsonReader::_simdjson_write_data_to_column(simdjson::ondemand::value& DataTypeSerDe::FormatOptions serde_options, bool* valid) { auto* data_column_ptr = column_ptr; auto data_serde = serde; - if (column_ptr->is_nullable()) { + if (is_column_nullable(*column_ptr)) { auto* nullable_column = static_cast(column_ptr); nullable_column->get_null_map_data().push_back(0); diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index d651ac112452da..48a1894f02e276 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -142,7 +142,7 @@ static void align_orc_null_map(const ColumnPtr& src_column, ColumnNullable* dst_ return; } DCHECK_EQ(dst_null_map.size(), old_rows); - if (src_column->is_nullable()) { + if (is_column_nullable(*src_column)) { const auto* src_nullable = assert_cast(src_column.get()); DCHECK_GE(src_nullable->get_null_map_column().size(), src_null_map_start + new_rows); dst_null_map.insert_range_from(src_nullable->get_null_map_column(), src_null_map_start, @@ -2056,7 +2056,7 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name, if (key_is_missing) { // Fill key column with default values (nulls or empty values) auto mutable_key_column = IColumn::mutate(std::move(doris_key_column)); - if (mutable_key_column->is_nullable()) { + if (is_column_nullable(*mutable_key_column)) { auto* nullable_column = static_cast(mutable_key_column.get()); nullable_column->insert_many_defaults(element_size); } else { @@ -2074,7 +2074,7 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name, if (value_is_missing) { // Fill value column with default values (nulls or empty values) auto mutable_value_column = IColumn::mutate(std::move(doris_value_column)); - if (mutable_value_column->is_nullable()) { + if (is_column_nullable(*mutable_value_column)) { auto* nullable_column = static_cast(mutable_value_column.get()); nullable_column->insert_many_defaults(element_size); } else { @@ -2222,7 +2222,7 @@ Status OrcReader::_orc_column_to_doris_column( } size_t src_null_map_start = 0; - if (mutable_resolved_column->is_nullable()) { + if (is_column_nullable(*mutable_resolved_column)) { SCOPED_RAW_TIMER(&_statistics.decode_null_map_time); auto* nullable_column = reinterpret_cast(mutable_resolved_column.get()); @@ -2255,7 +2255,7 @@ Status OrcReader::_orc_column_to_doris_column( doris_column = IColumn::mutate(std::move(doris_column)); auto converted_column = doris_column->assert_mutable(); - if (converted_column->is_nullable()) { + if (is_column_nullable(*converted_column)) { const size_t new_rows = remove_nullable(resolved_column)->size(); align_orc_null_map(resolved_column, reinterpret_cast(converted_column.get()), @@ -2264,7 +2264,7 @@ Status OrcReader::_orc_column_to_doris_column( return converter->convert(resolved_column, converted_column); } else { auto mutable_column = IColumn::mutate(std::move(doris_column)); - if (mutable_column->is_nullable()) { + if (is_column_nullable(*mutable_column)) { auto* nullable_column = static_cast(mutable_column.get()); nullable_column->insert_many_defaults(num_values); } else { diff --git a/be/src/format/parquet/parquet_column_convert.h b/be/src/format/parquet/parquet_column_convert.h index 6a4d8f39835f80..5ec05d450d8b31 100644 --- a/be/src/format/parquet/parquet_column_convert.h +++ b/be/src/format/parquet/parquet_column_convert.h @@ -199,14 +199,14 @@ struct ConvertParams { inline IColumn* get_mutable_inner_column(ColumnPtr& column) { column = IColumn::mutate(std::move(column)); auto mutable_column = column->assert_mutable(); - if (mutable_column->is_nullable()) { + if (is_column_nullable(*mutable_column)) { return &assert_cast(mutable_column.get())->get_nested_column(); } return mutable_column.get(); } inline size_t get_mutable_inner_column_size(const ColumnPtr& column) { - if (column->is_nullable()) { + if (is_column_nullable(*column)) { const auto* nullable = assert_cast(column.get()); return nullable->get_nested_column().size(); } @@ -214,7 +214,7 @@ inline size_t get_mutable_inner_column_size(const ColumnPtr& column) { } inline size_t get_null_map_size_or_inner_column_size(const ColumnPtr& column) { - if (column->is_nullable()) { + if (is_column_nullable(*column)) { const auto* nullable = assert_cast(column.get()); return nullable->get_null_map_column().size(); } @@ -222,7 +222,7 @@ inline size_t get_null_map_size_or_inner_column_size(const ColumnPtr& column) { } inline size_t get_appended_null_map_start(const ColumnPtr& column, size_t new_rows) { - if (!column->is_nullable()) { + if (!is_column_nullable(*column)) { return 0; } const auto* nullable = assert_cast(column.get()); @@ -233,7 +233,7 @@ inline size_t get_appended_null_map_start(const ColumnPtr& column, size_t new_ro inline void align_null_map(ColumnPtr& src_column, ColumnPtr& dst_column, size_t old_null_map_size, size_t new_rows, size_t src_null_map_start = 0) { - if (!dst_column->is_nullable()) { + if (!is_column_nullable(*dst_column)) { return; } @@ -245,7 +245,7 @@ inline void align_null_map(ColumnPtr& src_column, ColumnPtr& dst_column, size_t return; } DCHECK_EQ(dst_null_map.size(), old_null_map_size); - if (src_column->is_nullable()) { + if (is_column_nullable(*src_column)) { const auto* src_nullable = assert_cast(src_column.get()); DCHECK_GE(src_nullable->get_null_map_column().size(), src_null_map_start + new_rows); dst_null_map.insert_range_from(src_nullable->get_null_map_column(), src_null_map_start, diff --git a/be/src/format/parquet/vparquet_column_reader.cpp b/be/src/format/parquet/vparquet_column_reader.cpp index d5861a231d49e1..2726044c193b4c 100644 --- a/be/src/format/parquet/vparquet_column_reader.cpp +++ b/be/src/format/parquet/vparquet_column_reader.cpp @@ -329,7 +329,7 @@ Status ScalarColumnReader::_read_values(size_t num_ std::vector null_map; NullMap* map_data_column = nullptr; doris_column = IColumn::mutate(std::move(doris_column)); - if (doris_column->is_nullable()) { + if (is_column_nullable(*doris_column)) { SCOPED_RAW_TIMER(&_decode_null_map_time); auto mutable_column = doris_column->assert_mutable(); auto* nullable_column = assert_cast(mutable_column.get()); @@ -411,7 +411,7 @@ Status ScalarColumnReader::_read_nested_column( MutableColumnPtr data_column; NullMap* map_data_column = nullptr; doris_column = IColumn::mutate(std::move(doris_column)); - if (doris_column->is_nullable()) { + if (is_column_nullable(*doris_column)) { SCOPED_RAW_TIMER(&_decode_null_map_time); auto mutable_column = doris_column->assert_mutable(); auto* nullable_column = assert_cast(mutable_column.get()); @@ -661,7 +661,7 @@ Status ArrayColumnReader::read_column_data( MutableColumnPtr data_column; NullMap* null_map_ptr = nullptr; doris_column = IColumn::mutate(std::move(doris_column)); - if (doris_column->is_nullable()) { + if (is_column_nullable(*doris_column)) { auto mutable_column = doris_column->assert_mutable(); auto* nullable_column = assert_cast(mutable_column.get()); null_map_ptr = &nullable_column->get_null_map_data(); @@ -717,7 +717,7 @@ Status MapColumnReader::read_column_data( MutableColumnPtr data_column; NullMap* null_map_ptr = nullptr; doris_column = IColumn::mutate(std::move(doris_column)); - if (doris_column->is_nullable()) { + if (is_column_nullable(*doris_column)) { auto mutable_column = doris_column->assert_mutable(); auto* nullable_column = assert_cast(mutable_column.get()); null_map_ptr = &nullable_column->get_null_map_data(); @@ -794,7 +794,7 @@ Status StructColumnReader::read_column_data( MutableColumnPtr data_column; NullMap* null_map_ptr = nullptr; doris_column = IColumn::mutate(std::move(doris_column)); - if (doris_column->is_nullable()) { + if (is_column_nullable(*doris_column)) { auto mutable_column = doris_column->assert_mutable(); auto* nullable_column = assert_cast(mutable_column.get()); null_map_ptr = &nullable_column->get_null_map_data(); diff --git a/be/src/format/parquet/vparquet_column_reader.h b/be/src/format/parquet/vparquet_column_reader.h index 9f01f4f63243b0..d6c7d4802479c8 100644 --- a/be/src/format/parquet/vparquet_column_reader.h +++ b/be/src/format/parquet/vparquet_column_reader.h @@ -486,7 +486,7 @@ class SkipReadingReader : public ParquetColumnReader { MutableColumnPtr data_column = doris_column->assert_mutable(); if (real_column_size > 0) { - if (doris_column->is_nullable()) { + if (is_column_nullable(*doris_column)) { auto* nullable_column = static_cast(data_column.get()); nullable_column->insert_many_defaults(real_column_size); } else { diff --git a/be/src/format/table/equality_delete.cpp b/be/src/format/table/equality_delete.cpp index dc94d8151f2048..9f58397238c144 100644 --- a/be/src/format/table/equality_delete.cpp +++ b/be/src/format/table/equality_delete.cpp @@ -17,6 +17,7 @@ #include "format/table/equality_delete.h" +#include "core/column/column_nullable.h" #include "exprs/create_predicate_function.h" namespace doris { @@ -64,12 +65,10 @@ Status SimpleEqualityDelete::filter_data_block( // reset the array capacity and fill all elements using the 0 _single_filter->assign(rows, UInt8(0)); } - if (column_and_type.column->is_nullable()) { - const NullMap& null_map = - reinterpret_cast(column_and_type.column.get()) - ->get_null_map_data(); - _hybrid_set->find_batch_nullable(*remove_nullable(column_and_type.column), rows, null_map, - *_single_filter); + auto column = column_and_type.column->convert_to_full_column_if_const(); + if (const auto* nullable = check_and_get_column(column.get())) { + const NullMap& null_map = nullable->get_null_map_data(); + _hybrid_set->find_batch_nullable(*remove_nullable(column), rows, null_map, *_single_filter); if (_hybrid_set->contain_null()) { auto* filter_data = _single_filter->data(); for (size_t i = 0; i < rows; ++i) { @@ -77,7 +76,7 @@ Status SimpleEqualityDelete::filter_data_block( } } } else { - _hybrid_set->find_batch(*column_and_type.column, rows, *_single_filter); + _hybrid_set->find_batch(*column, rows, *_single_filter); } // should reverse _filter auto* filter_data = filter.data(); diff --git a/be/src/format/transformer/vorc_transformer.cpp b/be/src/format/transformer/vorc_transformer.cpp index f31ac0e65548a8..eb2168285835fe 100644 --- a/be/src/format/transformer/vorc_transformer.cpp +++ b/be/src/format/transformer/vorc_transformer.cpp @@ -588,7 +588,7 @@ Status VOrcTransformer::_resize_row_batch(const DataTypePtr& type, const IColumn case TYPE_STRUCT: { auto* struct_batch = dynamic_cast(orc_col_batch); const auto& struct_col = - column.is_nullable() + is_column_nullable(column) ? assert_cast( assert_cast(column).get_nested_column()) : assert_cast(column); @@ -605,7 +605,7 @@ Status VOrcTransformer::_resize_row_batch(const DataTypePtr& type, const IColumn case TYPE_MAP: { auto* map_batch = dynamic_cast(orc_col_batch); const auto& map_column = - column.is_nullable() + is_column_nullable(column) ? assert_cast( assert_cast(column).get_nested_column()) : assert_cast(column); @@ -627,7 +627,7 @@ Status VOrcTransformer::_resize_row_batch(const DataTypePtr& type, const IColumn case TYPE_ARRAY: { auto* list_batch = dynamic_cast(orc_col_batch); const auto& array_col = - column.is_nullable() + is_column_nullable(column) ? assert_cast( assert_cast(column).get_nested_column()) : assert_cast(column); diff --git a/be/src/storage/iterator/olap_data_convertor.cpp b/be/src/storage/iterator/olap_data_convertor.cpp index 42b792ab1c7f18..47da182b68e05b 100644 --- a/be/src/storage/iterator/olap_data_convertor.cpp +++ b/be/src/storage/iterator/olap_data_convertor.cpp @@ -318,7 +318,7 @@ void OlapBlockDataConvertor::OlapColumnDataConvertorBase::set_source_column( _typed_column = typed_column; _row_pos = row_pos; _num_rows = num_rows; - if (_typed_column.column->is_nullable()) { + if (is_column_nullable(*_typed_column.column)) { const auto* nullable_column = assert_cast(_typed_column.column.get()); _nullmap = nullable_column->get_null_map_data().data(); @@ -1008,7 +1008,7 @@ void OlapBlockDataConvertor::OlapColumnDataConvertorVariant::set_source_column( const ColumnWithTypeAndName& typed_column, size_t row_pos, size_t num_rows) { // set const ColumnNullable* nullable_column = nullptr; - if (typed_column.column->is_nullable()) { + if (is_column_nullable(*typed_column.column)) { nullable_column = assert_cast(typed_column.column.get()); _nullmap = nullable_column->get_null_map_data().data(); } diff --git a/be/src/storage/iterator/vertical_block_reader.cpp b/be/src/storage/iterator/vertical_block_reader.cpp index 607ec28828894e..848a2a61542d70 100644 --- a/be/src/storage/iterator/vertical_block_reader.cpp +++ b/be/src/storage/iterator/vertical_block_reader.cpp @@ -633,7 +633,7 @@ void VerticalBlockReader::_prepare_sparse_columns(MutableColumns& columns, size_ for (size_t col_idx = 0; col_idx < column_count; ++col_idx) { auto& col = columns[col_idx]; - if (col->is_nullable()) { + if (is_column_nullable(*col)) { auto* nullable_col = assert_cast(col.get()); nullable_dst_cols[col_idx] = nullable_col; diff --git a/be/src/storage/predicate/bloom_filter_predicate.h b/be/src/storage/predicate/bloom_filter_predicate.h index 3b7479841fb64b..cc252394ce2b90 100644 --- a/be/src/storage/predicate/bloom_filter_predicate.h +++ b/be/src/storage/predicate/bloom_filter_predicate.h @@ -96,7 +96,7 @@ class BloomFilterColumnPredicate final : public ColumnPredicate { template uint16_t BloomFilterColumnPredicate::_evaluate_inner(const IColumn& column, uint16_t* sel, uint16_t size) const { - if (column.is_nullable()) { + if (is_column_nullable(column)) { const auto* nullable_col = assert_cast(&column); const auto& null_map_data = nullable_col->get_null_map_column().get_data(); return evaluate(nullable_col->get_nested_column(), null_map_data.data(), sel, size); diff --git a/be/src/storage/predicate/comparison_predicate.h b/be/src/storage/predicate/comparison_predicate.h index 192188cc7dc8e9..8bb6b6853fdec2 100644 --- a/be/src/storage/predicate/comparison_predicate.h +++ b/be/src/storage/predicate/comparison_predicate.h @@ -397,7 +397,7 @@ class ComparisonPredicateBase final : public ColumnPredicate { try_reset_judge_selectivity(); }); - if (column.is_nullable()) { + if (is_column_nullable(column)) { const auto* nullable_column_ptr = assert_cast(&column); const auto& nested_column = nullable_column_ptr->get_nested_column(); const auto& null_map = nullable_column_ptr->get_null_map_column().get_data(); @@ -490,7 +490,7 @@ class ComparisonPredicateBase final : public ColumnPredicate { private: uint16_t _evaluate_inner(const IColumn& column, uint16_t* sel, uint16_t size) const override { - if (column.is_nullable()) { + if (is_column_nullable(column)) { const auto* nullable_column_ptr = assert_cast(&column); const auto& nested_column = nullable_column_ptr->get_nested_column(); const auto& null_map = nullable_column_ptr->get_null_map_column().get_data(); @@ -543,7 +543,7 @@ class ComparisonPredicateBase final : public ColumnPredicate { template void _evaluate_bit(const IColumn& column, const uint16_t* sel, uint16_t size, bool* flags) const { - if (column.is_nullable()) { + if (is_column_nullable(column)) { const auto* nullable_column_ptr = assert_cast(&column); const auto& nested_column = nullable_column_ptr->get_nested_column(); const auto& null_map = nullable_column_ptr->get_null_map_column().get_data(); diff --git a/be/src/storage/predicate/in_list_predicate.h b/be/src/storage/predicate/in_list_predicate.h index 468760ac7ab0b6..53b1ebb1d64896 100644 --- a/be/src/storage/predicate/in_list_predicate.h +++ b/be/src/storage/predicate/in_list_predicate.h @@ -199,7 +199,7 @@ class InListPredicateBase final : public ColumnPredicate { template void _evaluate_bit(const IColumn& column, const uint16_t* sel, uint16_t size, bool* flags) const { - if (column.is_nullable()) { + if (is_column_nullable(column)) { const auto* nullable_col = assert_cast(&column); const auto& null_bitmap = nullable_col->get_null_map_column().get_data(); const auto& nested_col = nullable_col->get_nested_column(); @@ -458,7 +458,7 @@ class InListPredicateBase final : public ColumnPredicate { uint16_t _evaluate_inner(const IColumn& column, uint16_t* sel, uint16_t size) const override { int16_t new_size = 0; - if (column.is_nullable()) { + if (is_column_nullable(column)) { const auto* nullable_col = assert_cast(&column); const auto& null_map = nullable_col->get_null_map_column().get_data(); const auto& nested_col = nullable_col->get_nested_column(); diff --git a/be/src/storage/predicate/like_column_predicate.cpp b/be/src/storage/predicate/like_column_predicate.cpp index 17c63b2be97ca7..c1329bcc9ab16a 100644 --- a/be/src/storage/predicate/like_column_predicate.cpp +++ b/be/src/storage/predicate/like_column_predicate.cpp @@ -52,7 +52,7 @@ template uint16_t LikeColumnPredicate::_evaluate_inner(const IColumn& column, uint16_t* sel, uint16_t size) const { uint16_t new_size = 0; - if (column.is_nullable()) { + if (is_column_nullable(column)) { auto* nullable_col = assert_cast(&column); auto& null_map_data = nullable_col->get_null_map_column().get_data(); auto& nested_col = nullable_col->get_nested_column(); diff --git a/be/src/storage/predicate/like_column_predicate.h b/be/src/storage/predicate/like_column_predicate.h index 152518450517b9..69cfd6fbc196c9 100644 --- a/be/src/storage/predicate/like_column_predicate.h +++ b/be/src/storage/predicate/like_column_predicate.h @@ -96,7 +96,7 @@ class LikeColumnPredicate final : public ColumnPredicate { template void _evaluate_vec(const IColumn& column, uint16_t size, bool* flags) const { - if (column.is_nullable()) { + if (is_column_nullable(column)) { auto* nullable_col = assert_cast(&column); auto& null_map_data = nullable_col->get_null_map_column().get_data(); auto& nested_col = nullable_col->get_nested_column(); diff --git a/be/src/storage/schema_change/schema_change.cpp b/be/src/storage/schema_change/schema_change.cpp index be8d7b2d395d5f..13082f89401f29 100644 --- a/be/src/storage/schema_change/schema_change.cpp +++ b/be/src/storage/schema_change/schema_change.cpp @@ -34,6 +34,7 @@ #include "cloud/cloud_schema_change_job.h" #include "cloud/config.h" #include "common/cast_set.h" +#include "common/check.h" #include "common/consts.h" #include "common/logging.h" #include "common/signal_handler.h" @@ -43,6 +44,7 @@ #include "core/block/column_with_type_and_name.h" #include "core/column/column.h" #include "core/column/column_nullable.h" +#include "exec/common/util.hpp" #include "exec/common/variant_util.h" #include "exprs/aggregate/aggregate_function.h" #include "exprs/aggregate/aggregate_function_reader.h" @@ -91,6 +93,48 @@ using namespace ErrorCode; constexpr int ALTER_TABLE_BATCH_SIZE = 4064; +namespace { + +bool nullable_column_contains_null(const ColumnPtr& column) { + const auto* null_map = VectorizedUtils::get_null_map(column); + DORIS_CHECK(null_map != nullptr); + if (is_column_const(*column)) { + return (*null_map)[0]; + } + const auto* data = null_map->data(); + const size_t rows = column->size(); + for (size_t i = 0; i < rows; ++i) { + if (data[i]) { + return true; + } + } + return false; +} + +bool nullable_columns_have_different_null_maps(const ColumnPtr& lhs_column, + const ColumnPtr& rhs_column) { + const auto* lhs_null_map = VectorizedUtils::get_null_map(lhs_column); + const auto* rhs_null_map = VectorizedUtils::get_null_map(rhs_column); + DORIS_CHECK(lhs_null_map != nullptr); + DORIS_CHECK(rhs_null_map != nullptr); + + const bool lhs_is_single = is_column_const(*lhs_column); + const bool rhs_is_single = is_column_const(*rhs_column); + const auto* lhs_data = lhs_null_map->data(); + const auto* rhs_data = rhs_null_map->data(); + const size_t rows = lhs_column->size(); + for (size_t i = 0; i < rows; ++i) { + const auto lhs_null = lhs_is_single ? lhs_data[0] : lhs_data[i]; + const auto rhs_null = rhs_is_single ? rhs_data[0] : rhs_data[i]; + if (lhs_null != rhs_null) { + return true; + } + } + return false; +} + +} // namespace + class MultiBlockMerger { public: MultiBlockMerger(BaseTabletSPtr tablet) : _tablet(tablet), _cmp(*tablet) {} @@ -379,7 +423,7 @@ Status BlockChanger::change_block(Block* ref_block, Block* new_block) const { const auto& value = _schema_mapping[idx].default_value; auto column = new_block->get_by_position(idx).column->assert_mutable(); if (value.is_null()) { - DCHECK(column->is_nullable()); + DCHECK(is_column_nullable(*column)); column->insert_many_defaults(row_num); } else { column = column->convert_to_predicate_column_if_dictionary(); @@ -396,8 +440,8 @@ Status BlockChanger::change_block(Block* ref_block, Block* new_block) const { auto& ref_col = ref_block->get_by_position(it.first).column; auto& new_col = new_block->get_by_position(it.second).column; - bool ref_col_nullable = ref_col->is_nullable(); - bool new_col_nullable = new_col->is_nullable(); + bool ref_col_nullable = is_column_nullable(*ref_col); + bool new_col_nullable = is_column_nullable(*new_col); if (ref_col_nullable != new_col_nullable) { // not nullable to nullable @@ -438,45 +482,34 @@ Status BlockChanger::_check_cast_valid(ColumnPtr input_column, ColumnPtr output_ if (input_column->is_nullable() != output_column->is_nullable()) { if (input_column->is_nullable()) { - const auto* ref_null_map = assert_cast(*input_column) - .get_null_map_column() - .get_data() - .data(); - - bool is_changed = false; - for (size_t i = 0; i < input_column->size(); i++) { - is_changed |= ref_null_map[i]; - } - if (is_changed) { + if (nullable_column_contains_null(input_column)) { return Status::DataQualityError( "some null data is changed to not null, intput_column={}", input_column->get_name()); } } else { - const auto& output_nullable = assert_cast(*output_column); - const auto& null_map_column = output_nullable.get_null_map_column(); - const auto& nested_column = output_nullable.get_nested_column(); - const auto* new_null_map = null_map_column.get_data().data(); - - if (null_map_column.size() != output_column->size()) { - return Status::InternalError( - "null_map_column size mismatch output_column_size, " - "null_map_column_size={}, output_column_size={}; input_column={}", - null_map_column.size(), output_column->size(), input_column->get_name()); - } + if (const auto* output_nullable = + check_and_get_column(output_column.get())) { + const auto& null_map_column = output_nullable->get_null_map_column(); + const auto& nested_column = output_nullable->get_nested_column(); + + if (null_map_column.size() != output_column->size()) { + return Status::InternalError( + "null_map_column size mismatch output_column_size, " + "null_map_column_size={}, output_column_size={}; input_column={}", + null_map_column.size(), output_column->size(), + input_column->get_name()); + } - if (nested_column.size() != output_column->size()) { - return Status::InternalError( - "nested_column size is changed, nested_column_size={}, " - "ouput_column_size={}; input_column={}", - nested_column.size(), output_column->size(), input_column->get_name()); + if (nested_column.size() != output_column->size()) { + return Status::InternalError( + "nested_column size is changed, nested_column_size={}, " + "ouput_column_size={}; input_column={}", + nested_column.size(), output_column->size(), input_column->get_name()); + } } - bool is_changed = false; - for (size_t i = 0; i < input_column->size(); i++) { - is_changed |= new_null_map[i]; - } - if (is_changed) { + if (nullable_column_contains_null(output_column)) { return Status::DataQualityError( "some not null data is changed to null, intput_column={}", input_column->get_name()); @@ -485,20 +518,7 @@ Status BlockChanger::_check_cast_valid(ColumnPtr input_column, ColumnPtr output_ } if (input_column->is_nullable() && output_column->is_nullable()) { - const auto* ref_null_map = assert_cast(*input_column) - .get_null_map_column() - .get_data() - .data(); - const auto* new_null_map = assert_cast(*output_column) - .get_null_map_column() - .get_data() - .data(); - - bool is_changed = false; - for (size_t i = 0; i < input_column->size(); i++) { - is_changed |= (ref_null_map[i] != new_null_map[i]); - } - if (is_changed) { + if (nullable_columns_have_different_null_maps(input_column, output_column)) { return Status::DataQualityError( "null map is changed after calculation, input_column={}", input_column->get_name()); diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp index de526a432b89d1..1c6552b41753b8 100644 --- a/be/src/storage/segment/column_reader.cpp +++ b/be/src/storage/segment/column_reader.cpp @@ -318,7 +318,7 @@ void ColumnReader::check_data_by_zone_map_for_test(const MutableColumnPtr& dst) } auto* non_nullable_column = - dst->is_nullable() + is_column_nullable(*dst) ? assert_cast(dst.get())->get_nested_column_ptr().get() : dst.get(); @@ -972,7 +972,7 @@ Status MapFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* if (read_null_map_only()) { // NULL_MAP_ONLY mode: read null map, fill nested ColumnMap with empty defaults - DORIS_CHECK(dst->is_nullable()); + DORIS_CHECK(is_column_nullable(*dst)); auto& nullable_col = assert_cast(*dst); auto null_map_ptr = nullable_col.get_null_map_column_ptr(); size_t num_read = *n; @@ -995,7 +995,8 @@ Status MapFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* } auto& column_map = assert_cast( - dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); + is_column_nullable(*dst) ? static_cast(*dst).get_nested_column() + : *dst); auto column_offsets_ptr = IColumn::mutate(std::move(column_map.get_offsets_ptr())); Defer defer_offsets {[&] { auto typed_column_offsets_ptr = ColumnMap::COffsets::cast_to_column_mutptr( @@ -1035,7 +1036,7 @@ Status MapFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* } } - if (dst->is_nullable()) { + if (is_column_nullable(*dst)) { size_t num_read = *n; auto null_map_ptr = static_cast(*dst).get_null_map_column_ptr(); // in not-null to null linked-schemachange mode, @@ -1065,7 +1066,7 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t if (read_null_map_only()) { // NULL_MAP_ONLY mode: read null map by rowids, fill nested ColumnMap with empty defaults - DORIS_CHECK(dst->is_nullable()); + DORIS_CHECK(is_column_nullable(*dst)); auto& nullable_col = assert_cast(*dst); if (_null_iterator) { auto null_map_ptr = nullable_col.get_null_map_column_ptr(); @@ -1088,7 +1089,8 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t } // resolve ColumnMap and nullable wrapper auto& column_map = assert_cast( - dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); + is_column_nullable(*dst) ? static_cast(*dst).get_nested_column() + : *dst); auto offsets_ptr = IColumn::mutate(std::move(column_map.get_offsets_ptr())); Defer defer_offsets {[&] { auto typed_offsets_ptr = ColumnMap::COffsets::cast_to_column_mutptr( @@ -1103,7 +1105,7 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t std::vector null_mask; // 0: not null, 1: null if (_map_reader->is_nullable()) { // For nullable map columns, the destination column must also be nullable. - if (UNLIKELY(!dst->is_nullable())) { + if (UNLIKELY(!is_column_nullable(*dst))) { return Status::InternalError( "unexpected non-nullable destination column for nullable map reader"); } @@ -1117,7 +1119,7 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t for (size_t i = 0; i < count; ++i) { null_mask.push_back(null_map_col->get_element(null_before + i)); } - } else if (dst->is_nullable()) { + } else if (is_column_nullable(*dst)) { // in not-null to null linked-schemachange mode, // actually we do not change dat data include meta in footer, // so may dst from changed meta which is nullable but old data is not nullable, @@ -1401,7 +1403,7 @@ Status StructFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bo if (read_null_map_only()) { // NULL_MAP_ONLY mode: read null map, fill nested ColumnStruct with empty defaults - DORIS_CHECK(dst->is_nullable()); + DORIS_CHECK(is_column_nullable(*dst)); auto& nullable_col = assert_cast(*dst); auto null_map_ptr = nullable_col.get_null_map_column_ptr(); size_t num_read = *n; @@ -1424,7 +1426,8 @@ Status StructFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bo } auto& column_struct = assert_cast( - dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); + is_column_nullable(*dst) ? static_cast(*dst).get_nested_column() + : *dst); for (size_t i = 0; i < column_struct.tuple_size(); i++) { size_t num_read = *n; auto sub_column_ptr = IColumn::mutate(std::move(column_struct.get_column_ptr(i))); @@ -1436,7 +1439,7 @@ Status StructFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bo DCHECK(num_read == *n); } - if (dst->is_nullable()) { + if (is_column_nullable(*dst)) { size_t num_read = *n; auto null_map_ptr = static_cast(*dst).get_null_map_column_ptr(); // in not-null to null linked-schemachange mode, @@ -1766,7 +1769,7 @@ Status ArrayFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, boo if (read_null_map_only()) { // NULL_MAP_ONLY mode: read null map, fill nested ColumnArray with empty defaults - DORIS_CHECK(dst->is_nullable()); + DORIS_CHECK(is_column_nullable(*dst)); auto& nullable_col = assert_cast(*dst); auto null_map_ptr = nullable_col.get_null_map_column_ptr(); size_t num_read = *n; @@ -1789,7 +1792,8 @@ Status ArrayFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, boo } auto& column_array = assert_cast( - dst->is_nullable() ? static_cast(*dst).get_nested_column() : *dst); + is_column_nullable(*dst) ? static_cast(*dst).get_nested_column() + : *dst); bool offsets_has_null = false; auto column_offsets_ptr = std::move(*column_array.get_offsets_ptr()).mutate(); @@ -1824,7 +1828,7 @@ Status ArrayFileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, boo } } - if (dst->is_nullable()) { + if (is_column_nullable(*dst)) { auto null_map_ptr = static_cast(*dst).get_null_map_column_ptr(); size_t num_read = *n; // in not-null to null linked-schemachange mode, @@ -2143,7 +2147,7 @@ Status FileColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* ha if (read_null_map_only()) { DLOG(INFO) << "File column iterator column " << _column_name << " in NULL_MAP_ONLY mode, reading only null map."; - DORIS_CHECK(dst->is_nullable()); + DORIS_CHECK(is_column_nullable(*dst)); auto& nullable_col = assert_cast(*dst); auto& null_map_data = nullable_col.get_null_map_data(); @@ -2257,7 +2261,7 @@ Status FileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t co DLOG(INFO) << "File column iterator column " << _column_name << " in NULL_MAP_ONLY mode, reading only null map by rowids."; - DORIS_CHECK(dst->is_nullable()); + DORIS_CHECK(is_column_nullable(*dst)); auto& nullable_col = assert_cast(*dst); auto& null_map_data = nullable_col.get_null_map_data(); const size_t base_size = null_map_data.size(); diff --git a/be/src/storage/segment/historical_row_retriever.cpp b/be/src/storage/segment/historical_row_retriever.cpp index f61bde8d4c0b4c..b56daef47cb31a 100644 --- a/be/src/storage/segment/historical_row_retriever.cpp +++ b/be/src/storage/segment/historical_row_retriever.cpp @@ -46,9 +46,7 @@ #include "storage/tablet/tablet_meta.h" #include "storage/tablet/tablet_schema.h" -namespace doris { - -namespace segment_v2 { +namespace doris::segment_v2 { using namespace ErrorCode; @@ -56,7 +54,7 @@ namespace { void insert_value_to_nullable_column(IColumn* dst_column, const IColumn& src_column, size_t pos) { auto* nullable_column = assert_cast(dst_column); - if (src_column.is_nullable()) { + if (is_column_nullable(src_column)) { nullable_column->insert_from(src_column, pos); return; } @@ -284,6 +282,4 @@ void PrimaryKeyModelRowRetriever::_maybe_invalid_row_cache(const std::string& ke RowCache::instance()->erase({_context.tablet->tablet_id(), key}); } } - -} // namespace segment_v2 -} // namespace doris +} // namespace doris::segment_v2 diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index 0e8bedb7a4a379..1bf4dc261db777 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -2204,7 +2204,7 @@ bool SegmentIterator::_prune_column(ColumnId cid, MutableColumnPtr& column, bool if (!fill_defaults) { return true; } - if (column->is_nullable()) { + if (is_column_nullable(*column)) { auto nullable_col_ptr = reinterpret_cast(column.get()); nullable_col_ptr->get_null_map_column().insert_many_defaults(num_of_defaults); nullable_col_ptr->get_nested_column_ptr()->insert_many_defaults(num_of_defaults); @@ -2910,7 +2910,7 @@ Status SegmentIterator::copy_column_data_by_selector(IColumn* input_col_ptr, MutableColumnPtr& output_col, uint16_t* sel_rowid_idx, uint16_t select_size, size_t batch_size) { - if (output_col->is_nullable() != input_col_ptr->is_nullable()) { + if (is_column_nullable(*output_col) != is_column_nullable(*input_col_ptr)) { LOG(WARNING) << "nullable mismatch for output_column: " << output_col->dump_structure() << " input_column: " << input_col_ptr->dump_structure() << " select_size: " << select_size; @@ -3488,7 +3488,7 @@ bool SegmentIterator::_no_need_read_key_data(ColumnId cid, MutableColumnPtr& col return false; } - if (column->is_nullable()) { + if (is_column_nullable(*column)) { auto* nullable_col_ptr = reinterpret_cast(column.get()); nullable_col_ptr->get_null_map_column().insert_many_defaults(nrows_read); nullable_col_ptr->get_nested_column_ptr()->insert_many_defaults(nrows_read); diff --git a/be/src/storage/segment/variant/binary_column_extract_iterator.h b/be/src/storage/segment/variant/binary_column_extract_iterator.h index a9aef9ca5aa1af..81ca8d593f4c0d 100644 --- a/be/src/storage/segment/variant/binary_column_extract_iterator.h +++ b/be/src/storage/segment/variant/binary_column_extract_iterator.h @@ -137,7 +137,7 @@ class BinaryColumnExtractIterator : public BaseBinaryColumnProcessor { void _fill_path_column(MutableColumnPtr& dst) { ColumnNullable* nullable_column = nullptr; - if (dst->is_nullable()) { + if (is_column_nullable(*dst)) { nullable_column = assert_cast(dst.get()); } ColumnVariant& var = diff --git a/be/src/storage/segment/variant/hierarchical_data_iterator.cpp b/be/src/storage/segment/variant/hierarchical_data_iterator.cpp index 9e75f307280fe6..d1fbdb6f24aa49 100644 --- a/be/src/storage/segment/variant/hierarchical_data_iterator.cpp +++ b/be/src/storage/segment/variant/hierarchical_data_iterator.cpp @@ -565,9 +565,9 @@ Status HierarchicalDataIterator::_init_null_map_and_clear_columns(MutableColumnP container->clear(); _binary_column_reader->column->clear(); if (_root_reader) { - if (_root_reader->column->is_nullable()) { + if (is_column_nullable(*_root_reader->column)) { // fill nullmap - DCHECK(dst->is_nullable()); + DCHECK(is_column_nullable(*dst)); ColumnUInt8& dst_null_map = assert_cast(*dst).get_null_map_column(); ColumnUInt8& src_null_map = assert_cast(*_root_reader->column).get_null_map_column(); @@ -575,7 +575,7 @@ Status HierarchicalDataIterator::_init_null_map_and_clear_columns(MutableColumnP // clear nullmap and inner data src_null_map.clear(); } else { - if (dst->is_nullable()) { + if (is_column_nullable(*dst)) { // No nullable info exist in hirearchical data, fill nullmap with all none null ColumnUInt8& dst_null_map = assert_cast(*dst).get_null_map_column(); @@ -585,7 +585,7 @@ Status HierarchicalDataIterator::_init_null_map_and_clear_columns(MutableColumnP } _root_reader->column->clear(); } else { - if (dst->is_nullable()) { + if (is_column_nullable(*dst)) { // No nullable info exist in hirearchical data, fill nullmap with all none null ColumnUInt8& dst_null_map = assert_cast(*dst).get_null_map_column(); auto fake_nullable_column = ColumnUInt8::create(nrows, 0); @@ -595,7 +595,7 @@ Status HierarchicalDataIterator::_init_null_map_and_clear_columns(MutableColumnP // root column nullmap need to be reset, for example, the src_null_map is from the whole // variant column, but the root column rows should reset to null when empty ColumnVariant* variant = nullptr; - if (dst->is_nullable()) { + if (is_column_nullable(*dst)) { variant = &assert_cast( assert_cast(*dst).get_nested_column()); } else { diff --git a/be/src/storage/segment/variant/hierarchical_data_iterator.h b/be/src/storage/segment/variant/hierarchical_data_iterator.h index fb8f5cad819bec..b54e8a20ab4a6e 100644 --- a/be/src/storage/segment/variant/hierarchical_data_iterator.h +++ b/be/src/storage/segment/variant/hierarchical_data_iterator.h @@ -140,7 +140,7 @@ class HierarchicalDataIterator : public ColumnIterator { dst = IColumn::mutate(std::move(dst)); // // Read all sub columns, and merge with root column ColumnNullable* nullable_column = nullptr; - if (dst->is_nullable()) { + if (is_column_nullable(*dst)) { nullable_column = assert_cast(dst.get()); } auto& variant = nullable_column == nullptr diff --git a/be/src/storage/segment/variant/variant_column_reader.cpp b/be/src/storage/segment/variant/variant_column_reader.cpp index aa864519233b08..ded364b33c21bc 100644 --- a/be/src/storage/segment/variant/variant_column_reader.cpp +++ b/be/src/storage/segment/variant/variant_column_reader.cpp @@ -1551,12 +1551,13 @@ Status VariantColumnReader::infer_data_type_for_path(DataTypePtr* type, const Ta Status VariantRootColumnIterator::_process_root_column(MutableColumnPtr& dst, MutableColumnPtr& root_column, const DataTypePtr& most_common_type) { - auto& obj = dst->is_nullable() ? assert_cast( - assert_cast(*dst).get_nested_column()) - : assert_cast(*dst); + auto& obj = is_column_nullable(*dst) + ? assert_cast( + assert_cast(*dst).get_nested_column()) + : assert_cast(*dst); // fill nullmap - if (root_column->is_nullable() && dst->is_nullable()) { + if (is_column_nullable(*root_column) && is_column_nullable(*dst)) { ColumnUInt8& dst_null_map = assert_cast(*dst).get_null_map_column(); ColumnUInt8& src_null_map = assert_cast(*root_column).get_null_map_column(); @@ -1586,9 +1587,10 @@ Status VariantRootColumnIterator::_process_root_column(MutableColumnPtr& dst, Status VariantRootColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) { // read root column - auto& obj = dst->is_nullable() ? assert_cast( - assert_cast(*dst).get_nested_column()) - : assert_cast(*dst); + auto& obj = is_column_nullable(*dst) + ? assert_cast( + assert_cast(*dst).get_nested_column()) + : assert_cast(*dst); auto most_common_type = obj.get_most_common_type(); // NOLINT(readability-static-accessed-through-instance) @@ -1601,9 +1603,10 @@ Status VariantRootColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, b Status VariantRootColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count, MutableColumnPtr& dst) { // read root column - auto& obj = dst->is_nullable() ? assert_cast( - assert_cast(*dst).get_nested_column()) - : assert_cast(*dst); + auto& obj = is_column_nullable(*dst) + ? assert_cast( + assert_cast(*dst).get_nested_column()) + : assert_cast(*dst); auto most_common_type = obj.get_most_common_type(); // NOLINT(readability-static-accessed-through-instance) diff --git a/be/test/core/column/column_const_test.cpp b/be/test/core/column/column_const_test.cpp index e9f57df213bce3..da0c8e103eccd8 100644 --- a/be/test/core/column/column_const_test.cpp +++ b/be/test/core/column/column_const_test.cpp @@ -24,6 +24,7 @@ #include "common/exception.h" #include "core/column/column.h" #include "core/column/column_array.h" +#include "core/column/column_nullable.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_number.h" @@ -41,6 +42,38 @@ TEST(ColumnConstTest, TestCreate) { EXPECT_TRUE(!is_column_const(column_const2->get_data_column())); } +TEST(ColumnConstTest, ConstNullableIsSemanticallyNullable) { + auto nullable_data = ColumnHelper::create_nullable_column({7}, {0}); + auto const_nullable = ColumnConst::create(nullable_data, 3); + auto const_not_nullable = + ColumnConst::create(ColumnHelper::create_column({7}), 3); + + EXPECT_TRUE(nullable_data->is_nullable()); + EXPECT_TRUE(is_column_nullable(*nullable_data)); + EXPECT_TRUE(const_nullable->is_nullable()); + EXPECT_FALSE(is_column_nullable(*const_nullable)); + EXPECT_EQ(nullptr, check_and_get_column(const_nullable.get())); + + const auto* nested_nullable = check_and_get_column_with_const(*const_nullable); + ASSERT_NE(nullptr, nested_nullable); + EXPECT_FALSE(nested_nullable->is_null_at(0)); + + EXPECT_FALSE(const_not_nullable->is_nullable()); + EXPECT_FALSE(is_column_nullable(*const_not_nullable)); + EXPECT_EQ(nullptr, check_and_get_column_with_const(*const_not_nullable)); +} + +TEST(ColumnConstTest, ConstNullableNullValueKeepsNullSemantics) { + auto nullable_data = ColumnHelper::create_nullable_column({7}, {1}); + auto const_null = ColumnConst::create(nullable_data, 5); + + EXPECT_TRUE(const_null->is_nullable()); + EXPECT_FALSE(is_column_nullable(*const_null)); + EXPECT_TRUE(const_null->is_null_at(0)); + EXPECT_TRUE(const_null->is_null_at(4)); + EXPECT_TRUE(const_null->only_null()); +} + TEST(ColumnConstTest, clone_resized_clones_nested_data) { auto column_data = ColumnHelper::create_column({7}); auto column_const = ColumnConst::create(column_data, 3); @@ -322,4 +355,4 @@ TEST(ColumnConstTest, replace_float_special_values) { column_const->finalize(); } } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/test/exprs/function/function_jsonb_test.cpp b/be/test/exprs/function/function_jsonb_test.cpp index 8d9df92d40202f..852f878449f3d6 100644 --- a/be/test/exprs/function/function_jsonb_test.cpp +++ b/be/test/exprs/function/function_jsonb_test.cpp @@ -169,6 +169,52 @@ TEST(FunctionJsonbTEST, JsonbParseTest) { -1, -1, true)); } +TEST(FunctionJsonbTEST, JsonbParseConstNullableInputWithNonConstDefaultMultiRows) { + auto input_type = make_nullable(std::make_shared()); + auto default_type = std::make_shared(); + auto return_type = make_nullable(std::make_shared()); + constexpr size_t rows = 1024; + const std::string json = R"({"a":1})"; + + // Before const-aware null map merging this used row indexes on the one-row const null map. + auto str_col = ColumnString::create(); + str_col->insert_data(json.data(), json.size()); + auto nullable_col = ColumnNullable::create(std::move(str_col), ColumnUInt8::create(1, false)); + + Block block; + block.insert({ColumnConst::create(std::move(nullable_col), rows), input_type, "json"}); + block.insert({create_jsonb_column(rows), default_type, "default_value"}); + + FunctionBasePtr func = SimpleFunctionFactory::instance().get_function( + "json_parse_error_to_value", block.get_columns_with_type_and_name(), return_type); + DORIS_CHECK(func != nullptr); + + FunctionUtils fn_utils(return_type, {input_type, default_type}, false); + auto* fn_ctx = fn_utils.get_fn_ctx(); + ASSERT_TRUE(func->open(fn_ctx, FunctionContext::FRAGMENT_LOCAL).ok()); + ASSERT_TRUE(func->open(fn_ctx, FunctionContext::THREAD_LOCAL).ok()); + + block.insert({nullptr, return_type, "result"}); + auto result = block.columns() - 1; + auto st = func->execute(fn_ctx, block, {0, 1}, result, rows); + ASSERT_TRUE(st.ok()) << st.to_string(); + + auto expected_col = return_type->create_column(); + for (size_t i = 0; i < rows; ++i) { + ASSERT_TRUE(insert_cell(expected_col, return_type, STRING(R"({"a":1})"))); + } + + const auto result_col = block.get_by_position(result).column->convert_to_full_column_if_const(); + ASSERT_EQ(result_col->size(), rows); + for (size_t i = 0; i < rows; ++i) { + EXPECT_FALSE(result_col->is_null_at(i)); + EXPECT_EQ(0, result_col->compare_at(i, i, *expected_col, 1)); + } + + static_cast(func->close(fn_ctx, FunctionContext::THREAD_LOCAL)); + static_cast(func->close(fn_ctx, FunctionContext::FRAGMENT_LOCAL)); +} + TEST(FunctionJsonbTEST, JsonbParseErrorToNullTest) { std::string func_name = "json_parse_error_to_null"; InputTypeSet input_types = {Nullable {PrimitiveType::TYPE_VARCHAR}};