diff --git a/be/src/core/column/column_file.cpp b/be/src/core/column/column_file.cpp new file mode 100644 index 00000000000000..2b639781333e8c --- /dev/null +++ b/be/src/core/column/column_file.cpp @@ -0,0 +1,470 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/column/column_file.h" + +#include + +#include +#include + +#include "core/arena.h" +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_varbinary.h" +#include "core/column/column_vector.h" +#include "core/column/columns_common.h" +#include "core/custom_allocator.h" +#include "core/data_type/primitive_type.h" +#include "core/field.h" +#include "exec/common/sip_hash.h" +#include "util/hash_util.hpp" + +namespace doris { +namespace { + +MutableColumns make_file_children() { + MutableColumns children; + children.reserve(ColumnFile::NUM_CHILDREN); + children.push_back(ColumnNullable::create(ColumnString::create(), ColumnUInt8::create())); + children.push_back(ColumnNullable::create(ColumnInt64::create(), ColumnUInt8::create())); + children.push_back(ColumnNullable::create(ColumnInt64::create(), ColumnUInt8::create())); + children.push_back(ColumnNullable::create(ColumnString::create(), ColumnUInt8::create())); + children.push_back(ColumnNullable::create(ColumnString::create(), ColumnUInt8::create())); + children.push_back(ColumnNullable::create(ColumnVarbinary::create(), ColumnUInt8::create())); + return children; +} + +// VARBINARY has no column hash implementation. Hash the lossless row encoding, +// which includes every child null flag and length, without exposing SQL hashing. +template +void hash_file_rows(const ColumnFile& column, size_t start, size_t end, const uint8_t* null_map, + Update&& update) { + DorisVector buffer; + for (size_t row = start; row < end; ++row) { + if (null_map != nullptr && null_map[row] != 0) { + continue; + } + buffer.resize(column.serialize_size_at(row)); + column.serialize_impl(buffer.data(), row); + update(row, buffer.data(), buffer.size()); + } +} + +uint32_t file_crc(const char* data, size_t size, uint32_t hash) { + // A complete row can exceed the uint32 length accepted by zlib_crc_hash. + while (size != 0) { + const auto length = + static_cast(std::min(size, std::numeric_limits::max())); + hash = HashUtil::zlib_crc_hash(data, length, hash); + data += length; + size -= length; + } + return hash; +} + +} // namespace + +ColumnFile::ColumnFile() : ColumnFile(make_file_children()) {} + +ColumnFile::ColumnFile(MutableColumns&& columns) { + if (columns.size() != NUM_CHILDREN) { + throw Exception(ErrorCode::INTERNAL_ERROR, "ColumnFile requires six children, got {}", + columns.size()); + } + for (size_t i = 0; i < NUM_CHILDREN; ++i) { + _columns[i] = std::move(columns[i]); + } + sanity_check(); + check_const_only_in_top_level(); +} + +void ColumnFile::sanity_check() const { + for (size_t i = 0; i < NUM_CHILDREN; ++i) { + const auto* child = dynamic_cast(_columns[i].get()); + if (child == nullptr) { + throw Exception(ErrorCode::INTERNAL_ERROR, "ColumnFile child {} must be nullable", i); + } + const auto& nested = child->get_nested_column(); + bool expected_type; + if (i == 1 || i == 2) { + expected_type = dynamic_cast(&nested) != nullptr; + } else if (i == 5) { + expected_type = dynamic_cast(&nested) != nullptr; + } else { + // Both string offset widths represent the same physical child type. + expected_type = nested.is_column_string(); + } + if (!expected_type) { + throw Exception(ErrorCode::INTERNAL_ERROR, "Invalid ColumnFile child {} type {}", i, + nested.get_name()); + } + child->sanity_check(); + if (i != 0 && child->size() != size()) { + throw Exception(ErrorCode::INTERNAL_ERROR, + "ColumnFile child {} has {} rows, expected {}", i, child->size(), + size()); + } + } +} + +MutableColumnPtr ColumnFile::clone_resized(size_t size) const { + MutableColumns children; + children.reserve(NUM_CHILDREN); + for (const auto& column : _columns) { + children.push_back(column->clone_resized(size)); + } + return create(std::move(children)); +} + +Field ColumnFile::operator[](size_t n) const { + Field result; + get(n, result); + return result; +} + +void ColumnFile::get(size_t n, Field& res) const { + File value; + value.reserve(NUM_CHILDREN); + for (const auto& column : _columns) { + value.push_back((*column)[n]); + } + res = Field::create_field(std::move(value)); +} + +void ColumnFile::insert(const Field& value) { + DCHECK_EQ(value.get_type(), TYPE_FILE); + const auto& fields = value.get(); + if (fields.size() != NUM_CHILDREN) { + throw Exception(ErrorCode::INTERNAL_ERROR, "FILE Field requires six children, got {}", + fields.size()); + } + for (size_t i = 0; i < NUM_CHILDREN; ++i) { + _columns[i]->insert(fields[i]); + } +} + +void ColumnFile::insert_from(const IColumn& src, size_t n) { + const auto& file = assert_cast(src); + for (size_t i = 0; i < NUM_CHILDREN; ++i) { + _columns[i]->insert_from(file.get_column(i), n); + } +} + +void ColumnFile::insert_range_from(const IColumn& src, size_t start, size_t length) { + const auto& file = assert_cast(src); + for (size_t i = 0; i < NUM_CHILDREN; ++i) { + _columns[i]->insert_range_from(file.get_column(i), start, length); + } +} + +void ColumnFile::insert_range_from_ignore_overflow(const IColumn& src, size_t start, + size_t length) { + const auto& file = assert_cast(src); + for (size_t i = 0; i < NUM_CHILDREN; ++i) { + _columns[i]->insert_range_from_ignore_overflow(file.get_column(i), start, length); + } +} + +void ColumnFile::insert_indices_from(const IColumn& src, const uint32_t* begin, + const uint32_t* end) { + const auto& file = assert_cast(src); + for (size_t i = 0; i < NUM_CHILDREN; ++i) { + _columns[i]->insert_indices_from(file.get_column(i), begin, end); + } +} + +void ColumnFile::insert_many_from(const IColumn& src, size_t position, size_t length) { + const auto& file = assert_cast(src); + for (size_t i = 0; i < NUM_CHILDREN; ++i) { + _columns[i]->insert_many_from(file.get_column(i), position, length); + } +} + +void ColumnFile::replace_column_data(const IColumn&, size_t, size_t) { + // Like ColumnString/ColumnStruct, variable-length rows are replaced by rebuilding columns. + throw Exception(ErrorCode::INTERNAL_ERROR, + "Method replace_column_data is not supported for ColumnFile"); +} + +void ColumnFile::insert_default() { + for (auto& column : _columns) { + column->insert_default(); + } +} + +void ColumnFile::insert_many_defaults(size_t length) { + for (auto& column : _columns) { + column->insert_many_defaults(length); + } +} + +void ColumnFile::pop_back(size_t n) { + for (auto& column : _columns) { + column->pop_back(n); + } +} + +void ColumnFile::reserve(size_t n) { + for (auto& column : _columns) { + column->reserve(n); + } +} + +void ColumnFile::resize(size_t n) { + if (n > size()) { + // ColumnNullable::resize alone does not initialize new null-map entries. + insert_many_defaults(n - size()); + } else { + for (auto& column : _columns) { + column->resize(n); + } + } +} + +void ColumnFile::clear() { + for (auto& column : _columns) { + column->clear(); + } +} + +void ColumnFile::erase(size_t start, size_t length) { + DCHECK_LE(start, size()); + DCHECK_LE(length, size() - start); + // Filtering also works for the arena-backed VARBINARY child, which has no erase(). + Filter keep(size(), 1); + std::fill(keep.begin() + start, keep.begin() + start + length, 0); + filter(keep); +} + +ColumnPtr ColumnFile::filter(const Filter& filter, ssize_t result_size_hint) const { + MutableColumns children; + children.reserve(NUM_CHILDREN); + for (const auto& column : _columns) { + children.push_back(IColumn::mutate(column->filter(filter, result_size_hint))); + } + return create(std::move(children)); +} + +size_t ColumnFile::filter(const Filter& filter) { + column_match_filter_size(size(), filter.size()); + const size_t result_size = _columns[0]->filter(filter); + for (size_t i = 1; i < NUM_CHILDREN - 1; ++i) { + const size_t child_size = _columns[i]->filter(filter); + DCHECK_EQ(child_size, result_size); + } + auto& inline_column = assert_cast(*_columns[NUM_CHILDREN - 1]); + if (result_size == 0) { + inline_column.clear(); + return 0; + } + // The arena remains owned by this column. Compact views without copying large payloads. + auto& data = assert_cast(inline_column.get_nested_column()).get_data(); + size_t output_row = 0; + for (size_t row = 0; row < filter.size(); ++row) { + if (filter[row]) { + data[output_row++] = data[row]; + } + } + data.resize(output_row); + const size_t null_map_size = inline_column.get_null_map_column().filter(filter); + DCHECK_EQ(output_row, result_size); + DCHECK_EQ(null_map_size, result_size); + return result_size; +} + +Status ColumnFile::filter_by_selector(const uint16_t* selector, size_t count, + IColumn* destination) const { + auto& output = assert_cast(*destination); + DCHECK(output.empty()); + output.reserve(count); + // Row copies preserve selection order, duplicates and the binary child's owned bytes. + for (size_t i = 0; i < count; ++i) { + output.insert_from(*this, selector[i]); + } + return Status::OK(); +} + +MutableColumnPtr ColumnFile::permute(const Permutation& permutation, size_t limit) const { + MutableColumns children; + children.reserve(NUM_CHILDREN); + for (const auto& column : _columns) { + children.push_back(column->permute(permutation, limit)); + } + return create(std::move(children)); +} + +ColumnPtr ColumnFile::convert_column_if_overflow() { + for (auto& column : _columns) { + column = column->convert_column_if_overflow(); + } + return IColumn::convert_column_if_overflow(); +} + +size_t ColumnFile::byte_size() const { + size_t size = 0; + for (const auto& column : _columns) { + size += column->byte_size(); + } + return size; +} + +size_t ColumnFile::allocated_bytes() const { + size_t size = 0; + for (const auto& column : _columns) { + size += column->allocated_bytes(); + } + return size; +} + +bool ColumnFile::has_enough_capacity(const IColumn& src) const { + const auto& file = assert_cast(src); + for (size_t i = 0; i < NUM_CHILDREN; ++i) { + if (!_columns[i]->has_enough_capacity(file.get_column(i))) { + return false; + } + } + return true; +} + +void ColumnFile::for_each_subcolumn(ColumnCallback callback) { + for (auto& column : _columns) { + callback(column); + } +} + +StringRef ColumnFile::serialize_value_into_arena(size_t n, Arena& arena, const char*& begin) const { + char* pos = arena.alloc_continue(serialize_size_at(n), begin); + return {pos, serialize_impl(pos, n)}; +} + +const char* ColumnFile::deserialize_and_insert_from_arena(const char* pos) { + return pos + deserialize_impl(pos); +} + +size_t ColumnFile::serialize_size_at(size_t row) const { + size_t size = 0; + for (const auto& column : _columns) { + size += column->serialize_size_at(row); + } + return size; +} + +size_t ColumnFile::serialize_impl(char* pos, size_t row) const { + size_t size = 0; + for (const auto& column : _columns) { + size += column->serialize_impl(pos + size, row); + } + DCHECK_EQ(size, serialize_size_at(row)); + return size; +} + +size_t ColumnFile::deserialize_impl(const char* pos) { + size_t size = 0; + for (auto& column : _columns) { + size += column->deserialize_impl(pos + size); + } + return size; +} + +void ColumnFile::serialize_vec(StringRef* keys, size_t num_rows) const { + for (size_t i = 0; i < num_rows; ++i) { + // The caller owns these mutable row buffers, as in other column serializers. + keys[i].size += serialize_impl(const_cast(keys[i].data + keys[i].size), i); + } +} + +void ColumnFile::deserialize_vec(StringRef* keys, size_t num_rows) { + for (size_t i = 0; i < num_rows; ++i) { + const auto size = deserialize_impl(keys[i].data); + keys[i].data += size; + keys[i].size -= size; + } +} + +size_t ColumnFile::get_max_row_byte_size() const { + size_t size = 0; + for (const auto& column : _columns) { + size += column->get_max_row_byte_size(); + } + return size; +} + +void ColumnFile::update_hash_with_value(size_t n, SipHash& hash) const { + hash_file_rows(*this, n, n + 1, nullptr, + [&](size_t, const char* data, size_t size) { hash.update(data, size); }); +} + +void ColumnFile::update_hashes_with_value(uint64_t* __restrict hashes, + const uint8_t* __restrict null_data) const { + hash_file_rows(*this, 0, size(), null_data, [&](size_t row, const char* data, size_t size) { + hashes[row] = HashUtil::xxHash64WithSeed(data, size, hashes[row]); + }); +} + +void ColumnFile::update_xxHash_with_value(size_t start, size_t end, uint64_t& hash, + const uint8_t* __restrict null_data) const { + hash_file_rows(*this, start, end, null_data, [&](size_t, const char* data, size_t size) { + hash = HashUtil::xxHash64WithSeed(data, size, hash); + }); +} + +void ColumnFile::update_crcs_with_value(uint32_t* __restrict hashes, PrimitiveType /*type*/, + uint32_t rows, uint32_t /*offset*/, + const uint8_t* __restrict null_data) const { + DCHECK_EQ(rows, size()); + hash_file_rows(*this, 0, rows, null_data, [&](size_t row, const char* data, size_t size) { + hashes[row] = file_crc(data, size, hashes[row]); + }); +} + +void ColumnFile::update_crc_with_value(size_t start, size_t end, uint32_t& hash, + const uint8_t* __restrict null_data) const { + hash_file_rows(*this, start, end, null_data, [&](size_t, const char* data, size_t size) { + hash = file_crc(data, size, hash); + }); +} + +void ColumnFile::update_crc32c_batch(uint32_t* __restrict hashes, + const uint8_t* __restrict null_map) const { + hash_file_rows(*this, 0, size(), null_map, [&](size_t row, const char* data, size_t size) { + hashes[row] = crc32c_extend(hashes[row], reinterpret_cast(data), size); + }); +} + +void ColumnFile::update_crc32c_single(size_t start, size_t end, uint32_t& hash, + const uint8_t* __restrict null_map) const { + hash_file_rows(*this, start, end, null_map, [&](size_t, const char* data, size_t size) { + hash = crc32c_extend(hash, reinterpret_cast(data), size); + }); +} + +int ColumnFile::compare_at(size_t, size_t, const IColumn&, int) const { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "FILE comparison is not supported"); +} + +void ColumnFile::get_permutation(bool, size_t, int, HybridSorter&, Permutation&) const { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "FILE sorting is not supported"); +} + +void ColumnFile::sort_column(const ColumnSorter*, EqualFlags&, Permutation&, EqualRange&, + bool) const { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "FILE sorting is not supported"); +} + +} // namespace doris diff --git a/be/src/core/column/column_file.h b/be/src/core/column/column_file.h new file mode 100644 index 00000000000000..15b7e72ef50f06 --- /dev/null +++ b/be/src/core/column/column_file.h @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "core/column/column.h" +#include "core/cow.h" + +namespace doris { + +// Complete physical FILE values: uri, offset, size, content_type, checksum, inline. +// All six children are nullable; parent SQL NULL belongs to an outer ColumnNullable. +// Input-boundary validation is responsible for URI/range/metadata validity. +class ColumnFile final : public COWHelper { +private: + friend class COWHelper; + + ColumnFile(); + explicit ColumnFile(MutableColumns&& columns); + ColumnFile(const ColumnFile&) = default; + +public: + static constexpr size_t NUM_CHILDREN = 6; + using FileColumns = std::array; + + std::string get_name() const override { return "File"; } + size_t size() const override { return _columns[0]->size(); } + size_t tuple_size() const { return NUM_CHILDREN; } + bool is_variable_length() const override { return true; } + bool is_exclusive() const override { + return IColumn::is_exclusive() && + std::all_of(_columns.begin(), _columns.end(), + [](const auto& column) { return column->is_exclusive(); }); + } + void sanity_check() const override; + const IColumn& get_column(size_t i) const { return *_columns[i]; } + IColumn& get_column(size_t i) { return *_columns[i]; } + const FileColumns& get_columns() const { return _columns; } + Columns get_columns_copy() const { return {_columns.begin(), _columns.end()}; } + const ColumnPtr& get_column_ptr(size_t i) const { return _columns[i]; } + ColumnPtr& get_column_ptr(size_t i) { return _columns[i]; } + + MutableColumnPtr clone_resized(size_t size) const override; + Field operator[](size_t n) const override; + void get(size_t n, Field& res) const override; + void insert(const Field& value) override; + void insert_from(const IColumn& src, size_t n) override; + void insert_range_from(const IColumn& src, size_t start, size_t length) override; + void insert_range_from_ignore_overflow(const IColumn& src, size_t start, + size_t length) override; + void insert_indices_from(const IColumn& src, const uint32_t* begin, + const uint32_t* end) override; + void insert_many_from(const IColumn& src, size_t position, size_t length) override; + void replace_column_data(const IColumn& rhs, size_t row, size_t self_row = 0) override; + void insert_default() override; + void insert_many_defaults(size_t length) override; + void pop_back(size_t n) override; + void reserve(size_t n) override; + void resize(size_t n) override; + void clear() override; + void erase(size_t start, size_t length) override; + ColumnPtr filter(const Filter& filter, ssize_t result_size_hint) const override; + size_t filter(const Filter& filter) override; + Status filter_by_selector(const uint16_t* selector, size_t count, + IColumn* destination) const override; + MutableColumnPtr permute(const Permutation& permutation, size_t limit) const override; + ColumnPtr convert_column_if_overflow() override; + + size_t byte_size() const override; + size_t allocated_bytes() const override; + bool has_enough_capacity(const IColumn& src) const override; + void for_each_subcolumn(ColumnCallback callback) override; + + StringRef serialize_value_into_arena(size_t n, Arena& arena, const char*& begin) const override; + const char* deserialize_and_insert_from_arena(const char* pos) override; + size_t serialize_size_at(size_t row) const override; + size_t serialize_impl(char* pos, size_t row) const override; + size_t deserialize_impl(const char* pos) override; + void serialize_vec(StringRef* keys, size_t num_rows) const override; + void deserialize_vec(StringRef* keys, size_t num_rows) override; + size_t get_max_row_byte_size() const override; + + // These hashes cover serialized physical values for internal integrity only. + // They do not grant FILE SQL equality, hashing, or deduplication semantics. + void update_hash_with_value(size_t n, SipHash& hash) const override; + void update_hashes_with_value(uint64_t* __restrict hashes, + const uint8_t* __restrict null_data = nullptr) const override; + void update_xxHash_with_value(size_t start, size_t end, uint64_t& hash, + const uint8_t* __restrict null_data) const override; + void update_crcs_with_value(uint32_t* __restrict hashes, PrimitiveType type, uint32_t rows, + uint32_t offset = 0, + const uint8_t* __restrict null_data = nullptr) const override; + void update_crc_with_value(size_t start, size_t end, uint32_t& hash, + const uint8_t* __restrict null_data) const override; + void update_crc32c_batch(uint32_t* __restrict hashes, + const uint8_t* __restrict null_map) const override; + void update_crc32c_single(size_t start, size_t end, uint32_t& hash, + const uint8_t* __restrict null_map) const override; + + int compare_at(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint) const override; + void get_permutation(bool reverse, size_t limit, int nan_direction_hint, HybridSorter& sorter, + Permutation& res) const override; + void sort_column(const ColumnSorter* sorter, EqualFlags& flags, Permutation& permutation, + EqualRange& range, bool last_column) const override; + +private: + FileColumns _columns; +}; + +} // namespace doris diff --git a/be/src/core/data_type/data_type.cpp b/be/src/core/data_type/data_type.cpp index 358b3ac5f3fb54..63416c871558f6 100644 --- a/be/src/core/data_type/data_type.cpp +++ b/be/src/core/data_type/data_type.cpp @@ -143,6 +143,8 @@ PGenericType_TypeId IDataType::get_pdata_type(const IDataType* data_type) { return PGenericType::LIST; case PrimitiveType::TYPE_STRUCT: return PGenericType::STRUCT; + case PrimitiveType::TYPE_FILE: + return PGenericType::FILE; case PrimitiveType::TYPE_JSONB: return PGenericType::JSONB; case PrimitiveType::TYPE_MAP: diff --git a/be/src/core/data_type/data_type_factory.cpp b/be/src/core/data_type/data_type_factory.cpp index 4988b317c2df11..52fa74042ca38b 100644 --- a/be/src/core/data_type/data_type_factory.cpp +++ b/be/src/core/data_type/data_type_factory.cpp @@ -46,6 +46,7 @@ #include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_date_time.h" #include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_file.h" #include "core/data_type/data_type_fixed_length_object.h" #include "core/data_type/data_type_hll.h" #include "core/data_type/data_type_ipv4.h" @@ -327,6 +328,16 @@ DataTypePtr DataTypeFactory::create_data_type(const PColumnMeta& pcolumn) { nested = std::make_shared(create_data_type(pcolumn.children(0)), create_data_type(pcolumn.children(1))); break; + case PGenericType::FILE: { + DataTypes children; + Strings names; + for (const auto& child : pcolumn.children()) { + children.push_back(create_data_type(child)); + names.push_back(child.name()); + } + nested = std::make_shared(children, names); + break; + } case PGenericType::STRUCT: { int col_size = pcolumn.children_size(); DCHECK(col_size >= 1); @@ -512,6 +523,9 @@ DataTypePtr DataTypeFactory::create_data_type(const PrimitiveType primitive_type const_cast(reinterpret_cast(*nested)) .set_null_literal(true); break; + case TYPE_FILE: + nested = std::make_shared(); + break; case TYPE_VARBINARY: nested = std::make_shared(len, TYPE_VARBINARY); break; @@ -560,6 +574,8 @@ DataTypePtr DataTypeFactory::create_data_type(const std::vector& type doc_mode); } return is_nullable ? make_nullable(dt) : dt; + } else if (scalar_type.type == TPrimitiveType::FILE) { + throw Exception(ErrorCode::INVALID_ARGUMENT, "FILE requires a FILE type node"); } return create_data_type(thrift_to_type(scalar_type.type), is_nullable, scalar_type.__isset.precision ? scalar_type.precision : 0, @@ -577,7 +593,26 @@ DataTypePtr DataTypeFactory::create_data_type(const std::vector& type create_data_type(types, idx, node.contains_nulls[0])); break; } + case TTypeNodeType::FILE: case TTypeNodeType::STRUCT: { + if (node.type == TTypeNodeType::FILE && + (node.struct_fields.size() != DataTypeFile::FIELD_COUNT || + types.size() - static_cast(*idx) <= DataTypeFile::FIELD_COUNT)) { + throw Exception(ErrorCode::INVALID_ARGUMENT, "Invalid FILE type descriptor"); + } + if (node.type == TTypeNodeType::FILE) { + constexpr TPrimitiveType::type child_types[] = { + TPrimitiveType::VARCHAR, TPrimitiveType::BIGINT, TPrimitiveType::BIGINT, + TPrimitiveType::VARCHAR, TPrimitiveType::VARCHAR, TPrimitiveType::VARBINARY}; + for (size_t i = 0; i < DataTypeFile::FIELD_COUNT; ++i) { + const auto& child = types[*idx + i + 1]; + if (child.type != TTypeNodeType::SCALAR || !child.__isset.scalar_type || + child.scalar_type.type != child_types[i]) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "FILE requires its canonical scalar children"); + } + } + } DCHECK(!node.__isset.scalar_type); DCHECK_LT(*idx, types.size() - 1); DCHECK(!node.__isset.contains_nulls); @@ -592,7 +627,11 @@ DataTypePtr DataTypeFactory::create_data_type(const std::vector& type data_types.push_back(create_data_type(types, idx, node.struct_fields[i].contains_null)); names.push_back(node.struct_fields[i].name); } - nested = std::make_shared(data_types, names); + if (node.type == TTypeNodeType::FILE) { + nested = std::make_shared(data_types, names); + } else { + nested = std::make_shared(data_types, names); + } break; } case TTypeNodeType::MAP: { @@ -673,6 +712,8 @@ DataTypePtr DataTypeFactory::create_data_type( nested = std::make_shared(node.variant_max_subcolumns_count(), node.variant_enable_doc_mode()); } + } else if (primitive_type == TYPE_FILE) { + throw Exception(ErrorCode::INVALID_ARGUMENT, "FILE requires a FILE type node"); } else { return create_data_type(primitive_type, is_nullable, scalar_type.has_precision() ? scalar_type.precision() : 0, @@ -698,7 +739,26 @@ DataTypePtr DataTypeFactory::create_data_type( nested = std::make_shared(data_types[0], data_types[1]); break; } + case TTypeNodeType::FILE: case TTypeNodeType::STRUCT: { + if (node.type() == TTypeNodeType::FILE && + (node.struct_fields_size() != DataTypeFile::FIELD_COUNT || + types.size() - *idx <= DataTypeFile::FIELD_COUNT)) { + throw Exception(ErrorCode::INVALID_ARGUMENT, "Invalid FILE protobuf type descriptor"); + } + if (node.type() == TTypeNodeType::FILE) { + constexpr TPrimitiveType::type child_types[] = { + TPrimitiveType::VARCHAR, TPrimitiveType::BIGINT, TPrimitiveType::BIGINT, + TPrimitiveType::VARCHAR, TPrimitiveType::VARCHAR, TPrimitiveType::VARBINARY}; + for (int i = 0; i < DataTypeFile::FIELD_COUNT; ++i) { + const auto& child = types.Get(*idx + i + 1); + if (child.type() != TTypeNodeType::SCALAR || !child.has_scalar_type() || + child.scalar_type().type() != child_types[i]) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "FILE requires its canonical scalar children"); + } + } + } DataTypes data_types; Strings names; data_types.reserve(node.struct_fields_size()); @@ -706,10 +766,22 @@ DataTypePtr DataTypeFactory::create_data_type( for (size_t i = 0; i < node.struct_fields_size(); i++) { const auto& field = node.struct_fields(cast_set(i)); ++(*idx); - data_types.push_back(create_data_type(types, idx, field.contains_null())); + if (node.type() == TTypeNodeType::FILE) { + const auto& scalar_type = types.Get(*idx).scalar_type(); + data_types.push_back(create_data_type( + thrift_to_type(static_cast(scalar_type.type())), + field.contains_null(), 0, 0, + scalar_type.has_len() ? scalar_type.len() : -1)); + } else { + data_types.push_back(create_data_type(types, idx, field.contains_null())); + } names.push_back(field.name()); } - nested = std::make_shared(data_types, names); + if (node.type() == TTypeNodeType::FILE) { + nested = std::make_shared(data_types, names); + } else { + nested = std::make_shared(data_types, names); + } break; } case TTypeNodeType::VARIANT: { diff --git a/be/src/core/data_type/data_type_file.cpp b/be/src/core/data_type/data_type_file.cpp new file mode 100644 index 00000000000000..5b4a2bd28aec44 --- /dev/null +++ b/be/src/core/data_type/data_type_file.cpp @@ -0,0 +1,266 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type/data_type_file.h" + +#include +#include + +#include +#include +#include + +#include "core/column/column_const.h" +#include "core/column/column_file.h" +#include "core/column/column_nullable.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/data_type_serde/data_type_file_serde.h" +#include "core/value/file_value.h" +#include "util/string_util.h" + +namespace doris { +bool contains_file_type(const DataTypePtr& original) { + const auto type = remove_nullable(original); + switch (type->get_primitive_type()) { + case TYPE_FILE: + return true; + case TYPE_ARRAY: + return contains_file_type(assert_cast(*type).get_nested_type()); + case TYPE_MAP: { + const auto& map = assert_cast(*type); + return contains_file_type(map.get_key_type()) || contains_file_type(map.get_value_type()); + } + case TYPE_STRUCT: + for (const auto& child : assert_cast(*type).get_elements()) { + if (contains_file_type(child)) { + return true; + } + } + return false; + default: + return false; + } +} + +namespace { +std::optional file_text_at(const ColumnNullable& column, size_t row) { + if (column.is_null_at(row)) { + return std::nullopt; + } + const auto text = column.get_nested_column().get_data_at(row); + return std::string_view(text.data, text.size); +} + +Status validate_file_values(const ColumnFile& file, const NullMap* parent_nulls) { + std::array children; + for (size_t child = 0; child < children.size(); ++child) { + children[child] = &assert_cast(file.get_column(child)); + } + const auto& offsets = + assert_cast(children[1]->get_nested_column()).get_data(); + const auto& sizes = + assert_cast(children[2]->get_nested_column()).get_data(); + for (size_t row = 0; row < file.size(); ++row) { + if (parent_nulls && (*parent_nulls)[row]) { + continue; + } + const auto offset = + children[1]->is_null_at(row) ? std::nullopt : std::optional(offsets[row]); + const auto size = children[2]->is_null_at(row) ? std::nullopt : std::optional(sizes[row]); + RETURN_IF_ERROR(validate_file( + file_text_at(*children[0], row), offset, size, file_text_at(*children[3], row), + file_text_at(*children[4], row), children[5]->is_null_at(row))); + } + return Status::OK(); +} + +} // namespace + +Status validate_file_column(const IColumn& column, const DataTypePtr& type) { + DCHECK_EQ(remove_nullable(type)->get_primitive_type(), TYPE_FILE); + const IColumn* data = &column; + if (const auto* constant = check_and_get_column(column)) { + data = &constant->get_data_column(); + } + const NullMap* nulls = nullptr; + if (const auto* nullable = check_and_get_column(data)) { + // CAST merges ancestor NULL masks before validating the resulting FILE column. + nulls = &nullable->get_null_map_data(); + data = &nullable->get_nested_column(); + } + return validate_file_values(assert_cast(*data), nulls); +} + +DataTypeFile::DataTypeFile() + : _elements {make_nullable(std::make_shared(65533, TYPE_VARCHAR)), + make_nullable(std::make_shared()), + make_nullable(std::make_shared()), + make_nullable(std::make_shared(1024, TYPE_VARCHAR)), + make_nullable(std::make_shared(1024, TYPE_VARCHAR)), + make_nullable(std::make_shared( + std::numeric_limits::max()))}, + _names {"uri", "offset", "size", "content_type", "checksum", "inline"} {} + +DataTypeFile::DataTypeFile(const DataTypes& elements, const Strings& names) : DataTypeFile() { + if (names != _names || elements.size() != FIELD_COUNT) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "FILE requires its canonical six-child schema"); + } + for (size_t i = 0; i < FIELD_COUNT; ++i) { + if (!elements[i]->is_nullable()) { + throw Exception(ErrorCode::INVALID_ARGUMENT, "FILE child {} must be nullable", + _names[i]); + } + auto child = remove_nullable(elements[i]); + auto expected = remove_nullable(_elements[i]); + // Block metadata erases VARCHAR length/type to STRING. Restore the canonical type here. + bool compatible = child->equals(*expected); + if (i == 0 || i == 3 || i == 4) { + compatible = child->get_primitive_type() == TYPE_STRING || + (child->get_primitive_type() == TYPE_VARCHAR && + assert_cast(*child).len() == + assert_cast(*expected).len()); + } + if (!compatible) { + throw Exception(ErrorCode::INVALID_ARGUMENT, "Invalid FILE child type for {}", + _names[i]); + } + } +} + +MutableColumnPtr DataTypeFile::create_column() const { + MutableColumns children; + for (const auto& type : _elements) { + children.push_back(type->create_column()); + } + return ColumnFile::create(std::move(children)); +} + +Status DataTypeFile::check_column(const IColumn& column) const { + const auto* file = DORIS_TRY(check_column_nested_type(column)); + for (size_t i = 0; i < FIELD_COUNT; ++i) { + RETURN_IF_ERROR(_elements[i]->check_column(file->get_column(i))); + } + return Status::OK(); +} + +Field DataTypeFile::get_field(const TExprNode& node) const { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, + "FILE literals are materialized from child expressions"); +} + +bool DataTypeFile::equals(const IDataType& rhs) const { + return dynamic_cast(&rhs) != nullptr; +} + +std::optional DataTypeFile::try_get_position_by_name(const String& name) const { + for (size_t i = 0; i < FIELD_COUNT; ++i) { + if (iequal(_names[i], name)) { + return i; + } + } + return std::nullopt; +} + +int64_t DataTypeFile::get_uncompressed_serialized_bytes(const IColumn& column, int version) const { + const auto* data = &column; + if (is_column_const(column)) { + data = &assert_cast(column).get_data_column(); + } + const auto& file = assert_cast(*data); + int64_t size = sizeof(bool) + sizeof(size_t) + sizeof(size_t); + for (size_t i = 0; i < FIELD_COUNT; ++i) { + size += _elements[i]->get_uncompressed_serialized_bytes(file.get_column(i), version); + } + return size; +} + +char* DataTypeFile::serialize(const IColumn& column, char* buf, int version) const { + const auto* data = &column; + size_t rows = 0; + buf = serialize_const_flag_and_row_num(&data, buf, &rows); + const auto& file = assert_cast(*data); + for (size_t i = 0; i < FIELD_COUNT; ++i) { + buf = _elements[i]->serialize(file.get_column(i), buf, version); + } + return buf; +} + +const char* DataTypeFile::deserialize(const char* buf, MutableColumnPtr* column, + int version) const { + auto* original = column->get(); + size_t rows = 0; + buf = deserialize_const_flag_and_row_num(buf, column, &rows); + auto& file = assert_cast(*original); + for (size_t i = 0; i < FIELD_COUNT; ++i) { + auto child = std::move(*file.get_column_ptr(i)).mutate(); + buf = _elements[i]->deserialize(buf, &child, version); + file.get_column_ptr(i) = std::move(child); + } + file.sanity_check(); + return buf; +} + +void DataTypeFile::to_pb_column_meta(PColumnMeta* meta) const { + IDataType::to_pb_column_meta(meta); + for (size_t i = 0; i < FIELD_COUNT; ++i) { + auto* child = meta->add_children(); + child->set_name(_names[i]); + _elements[i]->to_pb_column_meta(child); + } +} + +void DataTypeFile::to_protobuf(PTypeDesc* type, PTypeNode* node, PScalarType*) const { + node->set_type(TTypeNodeType::FILE); + node->clear_scalar_type(); + for (const auto& name : _names) { + auto* child = node->add_struct_fields(); + child->set_name(name); + child->set_contains_null(true); + } + for (const auto& child : _elements) { + child->to_protobuf(type); + } +} + +DataTypeSerDeSPtr DataTypeFile::get_serde(int nesting_level) const { + return std::make_shared(nesting_level); +} + +#ifdef BE_TEST +void DataTypeFile::to_thrift(TTypeDesc& type, TTypeNode& node) const { + node.type = TTypeNodeType::FILE; + node.__set_struct_fields(std::vector()); + for (const auto& name : _names) { + TStructField child; + child.__set_name(name); + child.__set_contains_null(true); + node.struct_fields.push_back(std::move(child)); + } + for (const auto& child : _elements) { + child->to_thrift(type); + } +} +#endif + +} // namespace doris diff --git a/be/src/core/data_type/data_type_file.h b/be/src/core/data_type/data_type_file.h new file mode 100644 index 00000000000000..801c5ab6aec2bc --- /dev/null +++ b/be/src/core/data_type/data_type_file.h @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "core/data_type/data_type.h" + +namespace doris { + +bool contains_file_type(const DataTypePtr& type); +Status validate_file_column(const IColumn& column, const DataTypePtr& type); + +// A FILE always owns its canonical six-child schema. Scan selection is carried separately. +class DataTypeFile final : public IDataType { +public: + static constexpr PrimitiveType PType = TYPE_FILE; + static constexpr size_t FIELD_COUNT = 6; + static constexpr size_t INLINE_FIELD_INDEX = 5; + + DataTypeFile(); + DataTypeFile(const DataTypes& elements, const Strings& names); + + PrimitiveType get_primitive_type() const override { return TYPE_FILE; } + const std::string get_family_name() const override { return "File"; } + MutableColumnPtr create_column() const override; + Status check_column(const IColumn& column) const override; + Field get_field(const TExprNode& node) const override; + bool equals(const IDataType& rhs) const override; + + const DataTypes& get_elements() const { return _elements; } + const DataTypePtr& get_element(size_t i) const { return _elements.at(i); } + const Strings& get_element_names() const { return _names; } + std::optional try_get_position_by_name(const String& name) const; + + int64_t get_uncompressed_serialized_bytes(const IColumn& column, + int be_exec_version) const override; + char* serialize(const IColumn& column, char* buf, int be_exec_version) const override; + const char* deserialize(const char* buf, MutableColumnPtr* column, + int be_exec_version) const override; + void to_pb_column_meta(PColumnMeta* col_meta) const override; + void to_protobuf(PTypeDesc* ptype, PTypeNode* node, PScalarType* scalar_type) const override; + DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override; +#ifdef BE_TEST + using IDataType::to_thrift; + void to_thrift(TTypeDesc& thrift_type, TTypeNode& node) const override; +#endif + +private: + DataTypes _elements; + Strings _names; +}; + +} // namespace doris diff --git a/be/src/core/data_type/define_primitive_type.h b/be/src/core/data_type/define_primitive_type.h index 0dfe9c1dfb470e..4ad71f386c0626 100644 --- a/be/src/core/data_type/define_primitive_type.h +++ b/be/src/core/data_type/define_primitive_type.h @@ -75,7 +75,8 @@ enum PrimitiveType : PrimitiveNative { TYPE_VARBINARY, /* 41, varbinary */ TYPE_TIMESTAMPTZ, /* 42, timestamptz */ TYPE_TIMESTAMP_NS, /* 43, signed Int64 epoch nanoseconds */ - TYPE_UUID = 44 /* 44, uuid */ + TYPE_UUID = 44, /* 44, uuid */ + TYPE_FILE = 45 /* file value with a URI reference and/or inline content */ }; } // namespace doris diff --git a/be/src/core/data_type/primitive_type.cpp b/be/src/core/data_type/primitive_type.cpp index 4d729f95de2b9d..7b3375ba6fbcc2 100644 --- a/be/src/core/data_type/primitive_type.cpp +++ b/be/src/core/data_type/primitive_type.cpp @@ -131,6 +131,8 @@ PrimitiveType thrift_to_type(TPrimitiveType::type ttype) { case TPrimitiveType::STRUCT: return TYPE_STRUCT; + case TPrimitiveType::FILE: + return TYPE_FILE; case TPrimitiveType::LAMBDA_FUNCTION: // TYPE_LAMBDA_FUNCTION is deprecated. Lambda execution is driven by @@ -262,6 +264,8 @@ TPrimitiveType::type to_thrift(PrimitiveType ptype) { case TYPE_STRUCT: return TPrimitiveType::STRUCT; + case TYPE_FILE: + return TPrimitiveType::FILE; case TYPE_AGG_STATE: return TPrimitiveType::AGG_STATE; case TYPE_VARBINARY: @@ -382,6 +386,8 @@ std::string type_to_string(PrimitiveType t) { case TYPE_STRUCT: return "STRUCT"; + case TYPE_FILE: + return "FILE"; case TYPE_VARIANT: return "VARIANT"; diff --git a/be/src/core/data_type/primitive_type.h b/be/src/core/data_type/primitive_type.h index e7ebcef3f5e0cd..b1b5eb45b2fa12 100644 --- a/be/src/core/data_type/primitive_type.h +++ b/be/src/core/data_type/primitive_type.h @@ -44,12 +44,14 @@ class IColumnDummy; class ColumnMap; class ColumnVariant; class ColumnStruct; +class ColumnFile; class ColumnVarbinary; class VariantField; using ColumnString = ColumnStr; class JsonbField; struct Array; struct Struct; +struct File; struct Map; template struct Decimal; @@ -97,6 +99,7 @@ class DataTypeMap; class DataTypeNullable; class DataTypeVariant; class DataTypeStruct; +class DataTypeFile; class DataTypeBitMap; class DataTypeQuantileState; using DataTypeNullablePtr = std::shared_ptr; @@ -156,6 +159,7 @@ constexpr bool is_enumeration_type(PrimitiveType type) { case TYPE_BOOLEAN: case TYPE_ARRAY: case TYPE_STRUCT: + case TYPE_FILE: case TYPE_MAP: case TYPE_HLL: case TYPE_VARBINARY: @@ -222,7 +226,7 @@ constexpr bool is_var_len_object(PrimitiveType type) { } constexpr bool is_complex_type(PrimitiveType type) { - return type == TYPE_STRUCT || type == TYPE_ARRAY || type == TYPE_MAP; + return type == TYPE_STRUCT || type == TYPE_ARRAY || type == TYPE_MAP || type == TYPE_FILE; } constexpr bool is_variant_string_type(PrimitiveType type) { @@ -527,6 +531,13 @@ struct PrimitiveTypeTraits { using ColumnType = ColumnStruct; }; template <> +struct PrimitiveTypeTraits { + using CppType = File; + using StorageFieldType = CppType; + using DataType = DataTypeFile; + using ColumnType = ColumnFile; +}; +template <> struct PrimitiveTypeTraits { using CppType = VariantField; using StorageFieldType = CppType; diff --git a/be/src/core/data_type/storage_field_type.cpp b/be/src/core/data_type/storage_field_type.cpp index 7eb96d83029c87..8d958cdda92609 100644 --- a/be/src/core/data_type/storage_field_type.cpp +++ b/be/src/core/data_type/storage_field_type.cpp @@ -109,6 +109,7 @@ FieldType primitive_type_to_storage_field_type(PrimitiveType type) { case static_cast(33): // TYPE_LAMBDA_FUNCTION (deprecated) case PrimitiveType::TYPE_FIXED_LENGTH_OBJECT: case PrimitiveType::TYPE_VARBINARY: + case PrimitiveType::TYPE_FILE: break; } diff --git a/be/src/core/data_type_serde/data_type_file_serde.cpp b/be/src/core/data_type_serde/data_type_file_serde.cpp new file mode 100644 index 00000000000000..f286c723a34c70 --- /dev/null +++ b/be/src/core/data_type_serde/data_type_file_serde.cpp @@ -0,0 +1,457 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type_serde/data_type_file_serde.h" + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "core/column/column_const.h" +#include "core/column/column_file.h" +#include "core/column/column_nullable.h" +#include "core/custom_allocator.h" +#include "core/data_type/data_type_file.h" +#include "core/string_buffer.hpp" +#include "core/value/file_value.h" +#include "util/jsonb_writer.h" +#include "util/mysql_row_buffer.h" +#include "util/url_coding.h" + +namespace doris { +namespace { + +constexpr size_t FILE_FIELDS = DataTypeFile::FIELD_COUNT; +constexpr size_t INLINE_FIELD_INDEX = DataTypeFile::INLINE_FIELD_INDEX; + +int base64_digit(char ch) { + if (ch >= 'A' && ch <= 'Z') { + return ch - 'A'; + } + if (ch >= 'a' && ch <= 'z') { + return ch - 'a' + 26; + } + if (ch >= '0' && ch <= '9') { + return ch - '0' + 52; + } + if (ch == '+') { + return 62; + } + if (ch == '/') { + return 63; + } + return -1; +} + +Status check_inline_base64(StringRef encoded, size_t padding) { + const auto digits = encoded.size - padding; + for (size_t i = 0; i < digits; ++i) { + if (base64_digit(encoded.data[i]) < 0) { + return Status::InvalidArgument("FILE inline requires standard padded Base64"); + } + } + // RFC 4648's unused low bits must be zero, so each byte sequence has one encoding. + if (padding != 0 && (base64_digit(encoded.data[digits - 1]) & ((1 << (2 * padding)) - 1))) { + return Status::InvalidArgument("FILE inline Base64 has nonzero pad bits"); + } + return Status::OK(); +} + +// Multiples of three keep padding confined to the final chunk. Scratch space is bounded +// independently of the inline value's size. +void write_base64(StringRef bytes, BufferWritable& bw) { + constexpr size_t INPUT_CHUNK = 3 * 1024; + std::array output; + ColumnString::check_chars_length((bytes.size / 3 + (bytes.size % 3 != 0)) * 4 + 2, 1); + bw.write('"'); + for (size_t offset = 0; offset < bytes.size; offset += INPUT_CHUNK) { + const auto length = std::min(INPUT_CHUNK, bytes.size - offset); + const auto written = base64_encode( + reinterpret_cast(bytes.data + offset), length, output.data()); + bw.write(reinterpret_cast(output.data()), written); + } + bw.write('"'); +} + +Status append_file(IColumn& column, File value) { + RETURN_IF_ERROR(validate_file(value)); + column.insert(Field::create_field(std::move(value))); + return Status::OK(); +} + +Status check_pb_null_map(const PValues& value, int rows) { + if ((value.has_null() && value.null_map_size() != rows) || + (!value.has_null() && value.null_map_size() != 0)) { + return Status::InvalidArgument("FILE protobuf null map does not match row count {}", rows); + } + return Status::OK(); +} + +} // namespace + +DataTypeFileSerDe::DataTypeFileSerDe(int nesting_level) : DataTypeSerDe(nesting_level) { + const DataTypeFile type; + _names = type.get_element_names(); + for (const auto& element : type.get_elements()) { + _element_serdes.push_back(element->get_serde(nesting_level + 1)); + } +} + +void DataTypeFileSerDe::set_return_object_as_string(bool value) { + DataTypeSerDe::set_return_object_as_string(value); + for (auto& serde : _element_serdes) { + serde->set_return_object_as_string(value); + } +} + +void DataTypeFileSerDe::_write_json(const IColumn& column, int64_t row, BufferWritable& bw) const { + const auto [data, index] = check_column_const_set_readability(column, row); + const auto& file = assert_cast(*data); + bw.write('{'); + for (size_t i = 0; i < FILE_FIELDS; ++i) { + if (i != 0) { + bw.write(','); + } + bw.write_json_string(_names[i].data(), _names[i].size()); + bw.write(':'); + const auto& child = assert_cast(file.get_column(i)); + if (child.is_null_at(index)) { + bw.write("null", 4); + } else if (i == 1 || i == 2) { + bw.write_number(child.get_nested_column().get_int(index)); + } else { + const auto bytes = child.get_nested_column().get_data_at(index); + if (i == INLINE_FIELD_INDEX) { + write_base64(bytes, bw); + } else { + bw.write_json_string(bytes.data, bytes.size); + } + } + } + bw.write('}'); +} + +void DataTypeFileSerDe::to_string(const IColumn& column, size_t row_num, BufferWritable& bw, + const FormatOptions& options) const { + _write_json(column, row_num, bw); +} + +bool DataTypeFileSerDe::write_column_to_mysql_text(const IColumn& column, BufferWritable& bw, + int64_t row_idx, + const FormatOptions& options) const { + to_string(column, row_idx, bw, options); + return true; +} + +Status DataTypeFileSerDe::write_column_to_mysql_binary(const IColumn& column, + MysqlRowBinaryBuffer& row_buffer, + int64_t row_idx, bool col_const, + const FormatOptions& options) const { + auto output = ColumnString::create(); + BufferWritable bw(*output); + to_string(column, index_check_const(row_idx, col_const), bw, options); + bw.commit(); + const auto json = output->get_data_at(0); + if (row_buffer.push_string(json.data, json.size) != 0) { + return Status::InternalError("Packing FILE into MySQL binary buffer failed"); + } + return Status::OK(); +} + +Status DataTypeFileSerDe::serialize_one_cell_to_json(const IColumn& column, int64_t row_num, + BufferWritable& bw, + FormatOptions& options) const { + _write_json(column, row_num, bw); + return Status::OK(); +} + +Status DataTypeFileSerDe::serialize_column_to_json(const IColumn& column, int64_t start_idx, + int64_t end_idx, BufferWritable& bw, + FormatOptions& options) const { + SERIALIZE_COLUMN_TO_JSON(); +} + +Status DataTypeFileSerDe::from_string(StringRef& str, IColumn& column, + const FormatOptions& options) const { + return _read_json(str, column); +} + +Status DataTypeFileSerDe::_checked_inline_size(size_t encoded_size, size_t padding, + uint32_t* decoded_size) { + if (encoded_size % 4 != 0 || padding > 2 || (encoded_size == 0 && padding != 0)) { + return Status::InvalidArgument("FILE inline requires standard padded Base64"); + } + constexpr uint64_t MAX_INLINE_SIZE = std::numeric_limits::max(); + const auto groups = encoded_size / 4; + // Check before multiplication/allocation; the limit is VARBINARY's uint32_t length. + if (groups > (MAX_INLINE_SIZE + padding) / 3) { + return Status::InvalidArgument("FILE inline exceeds VARBINARY value length"); + } + *decoded_size = static_cast(groups * 3 - padding); + return Status::OK(); +} + +Status DataTypeFileSerDe::_decode_inline(StringRef encoded, DorisVector& scratch, + Field& value) { + size_t padding = 0; + if (encoded.size != 0 && encoded.data[encoded.size - 1] == '=') { + padding = encoded.size > 1 && encoded.data[encoded.size - 2] == '=' ? 2 : 1; + } + uint32_t decoded_size = 0; + RETURN_IF_ERROR(_checked_inline_size(encoded.size, padding, &decoded_size)); + RETURN_IF_ERROR(check_inline_base64(encoded, padding)); + // libbase64 needs room for 3/4 of the encoded input, including padding. + // The checked group count bounds this scratch allocation before it is made. + scratch.resize(static_cast(decoded_size) + padding); + if (decoded_size != 0 && + base64_decode(encoded.data, encoded.size, scratch.data()) != decoded_size) { + return Status::InvalidArgument("Invalid FILE inline Base64"); + } + value = Field::create_field(StringView(scratch.data(), decoded_size)); + return Status::OK(); +} + +Status DataTypeFileSerDe::_read_json(StringRef str, IColumn& column) const { + if (str.size == 0) { + return Status::InvalidArgument("FILE requires a JSON object"); + } + rapidjson::Document document; + rapidjson::MemoryStream stream(str.data, str.size); + document.ParseStream(stream); + if (document.HasParseError() || !document.IsObject() || stream.Tell() != str.size) { + return Status::InvalidArgument("FILE requires a JSON object"); + } + File value(FILE_FIELDS); + DorisVector inline_bytes; + std::array seen {}; + for (const auto& member : document.GetObject()) { + const std::string_view name(member.name.GetString(), member.name.GetStringLength()); + const auto it = std::ranges::find(_names, name); + if (it == _names.end()) { + return Status::InvalidArgument("Unknown FILE JSON field {}", name); + } + const auto index = cast_set(it - _names.begin()); + if (seen[index]) { + return Status::InvalidArgument("Duplicate FILE JSON field {}", name); + } + seen[index] = true; + const auto& field = member.value; + if (field.IsNull()) { + continue; + } + if (index == 1 || index == 2) { + if (!field.IsInt64()) { + return Status::InvalidArgument("FILE {} must be a BIGINT JSON integer", name); + } + value[index] = Field::create_field(field.GetInt64()); + } else { + if (!field.IsString()) { + return Status::InvalidArgument("FILE {} must be a JSON string", name); + } + if (index == INLINE_FIELD_INDEX) { + RETURN_IF_ERROR( + _decode_inline(StringRef(field.GetString(), field.GetStringLength()), + inline_bytes, value[index])); + } else { + value[index] = Field::create_field( + std::string(field.GetString(), field.GetStringLength())); + } + } + } + return append_file(column, std::move(value)); +} + +Status DataTypeFileSerDe::deserialize_one_cell_from_json(IColumn& column, Slice& slice, + const FormatOptions& options) const { + return _read_json(StringRef(slice.data, slice.size), column); +} + +Status DataTypeFileSerDe::deserialize_column_from_json_vector(IColumn& column, + std::vector& slices, + uint64_t* num_deserialized, + const FormatOptions& options) const { + DESERIALIZE_COLUMN_FROM_JSON_VECTOR(); + return Status::OK(); +} + +Status DataTypeFileSerDe::serialize_column_to_jsonb(const IColumn& column, int64_t row_num, + JsonbWriter& writer, + const FormatOptions& options) const { + const auto [data, index] = check_column_const_set_readability(column, row_num); + const auto& file = assert_cast(*data); + if (!writer.writeStartObject()) { + return Status::InternalError("Cannot start FILE JSON object"); + } + for (size_t i = 0; i < FILE_FIELDS; ++i) { + if (!writer.writeKey(_names[i].data(), cast_set(_names[i].size()))) { + return Status::InternalError("Cannot write FILE JSON key"); + } + if (i == INLINE_FIELD_INDEX) { + const auto& child = assert_cast(file.get_column(i)); + if (child.is_null_at(index)) { + if (!writer.writeNull()) { + return Status::InternalError("Cannot write FILE inline NULL"); + } + } else { + auto encoded = ColumnString::create(); + BufferWritable buffer(*encoded); + write_base64(child.get_nested_column().get_data_at(index), buffer); + buffer.commit(); + const auto text = encoded->get_data_at(0); + if (!writer.writeStartString() || + !writer.writeString(text.data + 1, text.size - 2) || !writer.writeEndString()) { + return Status::InternalError("Cannot write FILE inline Base64"); + } + } + } else { + RETURN_IF_ERROR(_element_serdes[i]->serialize_column_to_jsonb(file.get_column(i), index, + writer, options)); + } + } + if (!writer.writeEndObject()) { + return Status::InternalError("Cannot end FILE JSON object"); + } + return Status::OK(); +} + +Status DataTypeFileSerDe::deserialize_column_from_jsonb(IColumn& column, const JsonbValue* json, + CastParameters& cast_parameters) const { + if (!json->isObject()) { + return Status::InvalidArgument("FILE requires a JSON object"); + } + File value(FILE_FIELDS); + std::array seen {}; + DorisVector inline_bytes; + for (const auto& member : *json->unpack()) { + const std::string_view name(member.getKeyStr(), member.klen()); + const auto it = std::ranges::find(_names, name); + if (it == _names.end()) { + return Status::InvalidArgument("Unknown FILE JSON field {}", name); + } + const auto index = cast_set(it - _names.begin()); + if (seen[index]) { + return Status::InvalidArgument("Duplicate FILE JSON field {}", name); + } + seen[index] = true; + const auto* field = member.value(); + if (field->isNull()) { + continue; + } + if (index == 1 || index == 2) { + if (!field->isInt() || field->int_val() < std::numeric_limits::min() || + field->int_val() > std::numeric_limits::max()) { + return Status::InvalidArgument("FILE {} must be a BIGINT JSON integer", name); + } + value[index] = Field::create_field(static_cast(field->int_val())); + } else { + if (!field->isString()) { + return Status::InvalidArgument("FILE {} must be a string", name); + } + const auto* bytes = field->unpack(); + if (index == INLINE_FIELD_INDEX) { + RETURN_IF_ERROR(_decode_inline(StringRef(bytes->getBlob(), bytes->getBlobLen()), + inline_bytes, value[index])); + } else { + value[index] = Field::create_field( + std::string(bytes->getBlob(), bytes->getBlobLen())); + } + } + } + if (std::ranges::any_of(seen, [](bool present) { return !present; })) { + return Status::InvalidArgument("FILE CAST requires exactly six fields"); + } + return append_file(column, std::move(value)); +} + +Status DataTypeFileSerDe::write_column_to_pb(const IColumn& column, PValues& result, int64_t start, + int64_t end) const { + const auto& file = assert_cast(column); + result.mutable_type()->set_id(PGenericType::FILE); + for (size_t i = 0; i < FILE_FIELDS; ++i) { + RETURN_IF_ERROR(_element_serdes[i]->write_column_to_pb( + file.get_column(i), *result.add_child_element(), start, end)); + } + return Status::OK(); +} + +Status DataTypeFileSerDe::read_column_from_pb(IColumn& column, const PValues& arg) const { + if (arg.type().id() != PGenericType::FILE || + std::cmp_not_equal(arg.child_element_size(), FILE_FIELDS)) { + return Status::InvalidArgument("Expected FILE protobuf with six children"); + } + const int rows = arg.child_element(0).string_value_size(); + RETURN_IF_ERROR(check_pb_null_map(arg, rows)); + for (size_t i = 0; i < FILE_FIELDS; ++i) { + const auto& child = arg.child_element(cast_set(i)); + const bool integer = i == 1 || i == 2; + auto expected_type = integer ? PGenericType::INT64 : PGenericType::STRING; + auto count = integer ? child.int64_value_size() : child.string_value_size(); + if (i == INLINE_FIELD_INDEX) { + expected_type = PGenericType::VARBINARY; + count = child.bytes_value_size(); + } + if (child.type().id() != expected_type || count != rows) { + return Status::InvalidArgument("Invalid FILE protobuf child {} type or row count", + _names[i]); + } + RETURN_IF_ERROR(check_pb_null_map(child, rows)); + } + auto decoded = column.clone_empty(); + auto& file = assert_cast(*decoded); + for (size_t i = 0; i < FILE_FIELDS; ++i) { + RETURN_IF_ERROR(_element_serdes[i]->read_column_from_pb( + file.get_column(i), arg.child_element(cast_set(i)))); + } + // Internal protobuf transport restores values without repeating input validation. + column.insert_range_from(file, 0, rows); + return Status::OK(); +} + +void DataTypeFileSerDe::write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& writer, + Arena& arena, int32_t col_id, int64_t row_num, + const FormatOptions& options) const { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "FILE row-store JSONB is not supported"); +} + +void DataTypeFileSerDe::read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const { + throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "FILE row-store JSONB is not supported"); +} + +Status DataTypeFileSerDe::write_column_to_arrow(const IColumn&, const NullMap*, + arrow::ArrayBuilder*, int64_t, int64_t, + const cctz::time_zone&) const { + return Status::NotSupported("FILE Arrow output is not supported"); +} + +Status DataTypeFileSerDe::read_column_from_arrow(IColumn&, const arrow::Array*, int64_t, int64_t, + const cctz::time_zone&) const { + return Status::NotSupported("FILE Arrow input is not supported"); +} + +Status DataTypeFileSerDe::write_column_to_orc(const std::string&, const IColumn&, const NullMap*, + orc::ColumnVectorBatch*, int64_t, int64_t, Arena&, + const FormatOptions&) const { + return Status::NotSupported("FILE ORC output is not supported"); +} + +} // namespace doris diff --git a/be/src/core/data_type_serde/data_type_file_serde.h b/be/src/core/data_type_serde/data_type_file_serde.h new file mode 100644 index 00000000000000..f9d60f2d934bec --- /dev/null +++ b/be/src/core/data_type_serde/data_type_file_serde.h @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "core/custom_allocator.h" +#include "core/data_type_serde/data_type_serde.h" + +namespace doris { + +// FILE exposes six fields in query results. PB restores query values; public JSONB supports CAST. +// Row-store JSONB, Arrow and ORC are unsupported. +class DataTypeFileSerDe final : public DataTypeSerDe { +public: + explicit DataTypeFileSerDe(int nesting_level = 1); + std::string get_name() const override { return "File"; } + + Status from_string(StringRef& str, IColumn& column, + const FormatOptions& options) const override; + void to_string(const IColumn& column, size_t row_num, BufferWritable& bw, + const FormatOptions& options) const override; + Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw, + FormatOptions& options) const override; + Status serialize_column_to_json(const IColumn& column, int64_t start_idx, int64_t end_idx, + BufferWritable& bw, FormatOptions& options) const override; + Status deserialize_one_cell_from_json(IColumn& column, Slice& slice, + const FormatOptions& options) const override; + Status deserialize_column_from_json_vector(IColumn& column, std::vector& slices, + uint64_t* num_deserialized, + const FormatOptions& options) const override; + Status serialize_column_to_jsonb(const IColumn& column, int64_t row_num, JsonbWriter& writer, + const FormatOptions& options) const override; + Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* value, + CastParameters& cast_parameters) const override; + + bool write_column_to_mysql_text(const IColumn& column, BufferWritable& bw, int64_t row_idx, + const FormatOptions& options) const override; + Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, + int64_t row_idx, bool col_const, + const FormatOptions& options) const override; + Status write_column_to_pb(const IColumn& column, PValues& result, int64_t start, + int64_t end) const override; + Status read_column_from_pb(IColumn& column, const PValues& arg) const override; + void write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& writer, Arena& arena, + int32_t col_id, int64_t row_num, + const FormatOptions& options) const override; + void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override; + + Status write_column_to_arrow(const IColumn& column, const NullMap* null_map, + arrow::ArrayBuilder* builder, int64_t start, int64_t end, + const cctz::time_zone& ctz) const override; + Status read_column_from_arrow(IColumn& column, const arrow::Array* array, int64_t start, + int64_t end, const cctz::time_zone& ctz) const override; + Status write_column_to_orc(const std::string& timezone, const IColumn& column, + const NullMap* null_map, orc::ColumnVectorBatch* batch, + int64_t start, int64_t end, Arena& arena, + const FormatOptions& options) const override; + + DataTypeSerDeSPtrs get_nested_serdes() const override { return _element_serdes; } + void set_return_object_as_string(bool value) override; + +private: + Status _read_json(StringRef str, IColumn& column) const; + static Status _decode_inline(StringRef encoded, DorisVector& scratch, Field& value); + static Status _checked_inline_size(size_t encoded_size, size_t padding, uint32_t* decoded_size); + void _write_json(const IColumn& column, int64_t row, BufferWritable& bw) const; + + DataTypeSerDeSPtrs _element_serdes; + Strings _names; +}; + +} // namespace doris diff --git a/be/src/core/data_type_serde/data_type_varbinary_serde.cpp b/be/src/core/data_type_serde/data_type_varbinary_serde.cpp index b13bf5a7073933..8eb1469a875b0c 100644 --- a/be/src/core/data_type_serde/data_type_varbinary_serde.cpp +++ b/be/src/core/data_type_serde/data_type_varbinary_serde.cpp @@ -36,6 +36,26 @@ namespace doris { +Status DataTypeVarbinarySerDe::write_column_to_pb(const IColumn& column, PValues& result, + int64_t start, int64_t end) const { + result.mutable_type()->set_id(PGenericType::VARBINARY); + result.mutable_bytes_value()->Reserve(cast_set(end - start)); + for (int64_t row = start; row < end; ++row) { + const auto bytes = column.get_data_at(row); + result.add_bytes_value(bytes.data, bytes.size); + } + return Status::OK(); +} + +Status DataTypeVarbinarySerDe::read_column_from_pb(IColumn& column, const PValues& arg) const { + auto& output = assert_cast(column); + output.reserve(output.size() + arg.bytes_value_size()); + for (const auto& bytes : arg.bytes_value()) { + output.insert_data(bytes.data(), bytes.size()); + } + return Status::OK(); +} + namespace { class VarbinaryParquetConsumer final : public ParquetFixedValueConsumer, diff --git a/be/src/core/data_type_serde/data_type_varbinary_serde.h b/be/src/core/data_type_serde/data_type_varbinary_serde.h index 76e111e81b0fed..94da4396681f9c 100644 --- a/be/src/core/data_type_serde/data_type_varbinary_serde.h +++ b/be/src/core/data_type_serde/data_type_varbinary_serde.h @@ -71,13 +71,9 @@ class DataTypeVarbinarySerDe : public DataTypeSerDe { } Status write_column_to_pb(const IColumn& column, PValues& result, int64_t start, - int64_t end) const override { - return Status::NotSupported("write_column_to_pb with type " + column.get_name()); - } + int64_t end) const override; - Status read_column_from_pb(IColumn& column, const PValues& arg) const override { - return Status::NotSupported("read_column_from_pb with type " + column.get_name()); - } + Status read_column_from_pb(IColumn& column, const PValues& arg) const override; void write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena& mem_pool, int32_t col_id, int64_t row_num, diff --git a/be/src/core/field.cpp b/be/src/core/field.cpp index 00b389a49030a2..06554990023e41 100644 --- a/be/src/core/field.cpp +++ b/be/src/core/field.cpp @@ -249,6 +249,9 @@ void Field::create(Field&& field) { case PrimitiveType::TYPE_STRUCT: create_concrete(std::move(field.template get())); return; + case PrimitiveType::TYPE_FILE: + create_concrete(std::move(field.template get())); + return; case PrimitiveType::TYPE_MAP: create_concrete(std::move(field.template get())); return; @@ -386,6 +389,9 @@ void Field::create(const Field& field) { case PrimitiveType::TYPE_STRUCT: create_concrete(field.template get()); return; + case PrimitiveType::TYPE_FILE: + create_concrete(field.template get()); + return; case PrimitiveType::TYPE_MAP: create_concrete(field.template get()); return; @@ -450,6 +456,9 @@ void Field::destroy() { case PrimitiveType::TYPE_STRUCT: destroy(); break; + case PrimitiveType::TYPE_FILE: + destroy(); + break; case PrimitiveType::TYPE_MAP: destroy(); break; @@ -553,6 +562,9 @@ void Field::assign(Field&& field) { case PrimitiveType::TYPE_STRUCT: assign_concrete(std::move(field.template get())); return; + case PrimitiveType::TYPE_FILE: + assign_concrete(std::move(field.template get())); + return; case PrimitiveType::TYPE_MAP: assign_concrete(std::move(field.template get())); return; @@ -668,6 +680,9 @@ void Field::assign(const Field& field) { case PrimitiveType::TYPE_STRUCT: assign_concrete(field.template get()); return; + case PrimitiveType::TYPE_FILE: + assign_concrete(field.template get()); + return; case PrimitiveType::TYPE_MAP: assign_concrete(field.template get()); return; @@ -788,6 +803,7 @@ std::strong_ordering Field::operator<=>(const Field& rhs) const { } switch (type) { + case PrimitiveType::TYPE_FILE: case PrimitiveType::TYPE_BITMAP: case PrimitiveType::TYPE_HLL: case PrimitiveType::TYPE_QUANTILE_STATE: @@ -1149,6 +1165,10 @@ std::string Field::to_debug_string(int scale) const { typename PrimitiveTypeTraits::CppType && rhs); \ template void Field::FUNC_NAME( \ const typename PrimitiveTypeTraits::CppType& rhs); \ + template void Field::FUNC_NAME(typename PrimitiveTypeTraits::CppType && \ + rhs); \ + template void Field::FUNC_NAME( \ + const typename PrimitiveTypeTraits::CppType& rhs); \ template void Field::FUNC_NAME(typename PrimitiveTypeTraits::CppType && \ rhs); \ template void Field::FUNC_NAME( \ @@ -1213,6 +1233,7 @@ DECLARE_FUNCTION(TYPE_FLOAT) DECLARE_FUNCTION(TYPE_DOUBLE) DECLARE_FUNCTION(TYPE_JSONB) DECLARE_FUNCTION(TYPE_STRUCT) +DECLARE_FUNCTION(TYPE_FILE) DECLARE_FUNCTION(TYPE_MAP) DECLARE_FUNCTION(TYPE_BITMAP) DECLARE_FUNCTION(TYPE_TIMEV2) diff --git a/be/src/core/field.h b/be/src/core/field.h index 155278fa337919..a794e2a4ffda3c 100644 --- a/be/src/core/field.h +++ b/be/src/core/field.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -71,6 +72,10 @@ struct Struct : public FieldVector { using FieldVector::FieldVector; }; +struct File : public FieldVector { + using FieldVector::FieldVector; +}; + struct Map : public FieldVector { using FieldVector::FieldVector; }; @@ -238,7 +243,8 @@ class Field { bool is_complex_field() const { return type == PrimitiveType::TYPE_ARRAY || type == PrimitiveType::TYPE_MAP || - type == PrimitiveType::TYPE_STRUCT || type == PrimitiveType::TYPE_VARIANT; + type == PrimitiveType::TYPE_STRUCT || type == PrimitiveType::TYPE_VARIANT || + type == PrimitiveType::TYPE_FILE; } Field& operator=(Field&& rhs) { @@ -297,8 +303,8 @@ class Field { private: std::aligned_union_t storage; diff --git a/be/src/core/value/file_value.cpp b/be/src/core/value/file_value.cpp new file mode 100644 index 00000000000000..e4b598243087e2 --- /dev/null +++ b/be/src/core/value/file_value.cpp @@ -0,0 +1,326 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/value/file_value.h" + +#include +#include +#include +#include + +#include "core/field.h" + +namespace doris { +namespace { +// Keep helper names separate from other anonymous namespaces in Core unity builds. +namespace file_value_detail { + +constexpr std::array FIELD_NAMES = {"uri", "offset", "size", + "content_type", "checksum", "inline"}; +constexpr size_t MAX_METADATA_BYTES = 1024; + +bool ascii_digit(char ch) { + return ch >= '0' && ch <= '9'; +} + +bool mime_token_char(char ch) { + // RFC 2045 section 5.1: ASCII minus SPACE, CTLs and tspecials. + return ch > 0x20 && ch < 0x7f && + std::string_view("()<>@,;:\\\"/[]?=").find(ch) == std::string_view::npos; +} + +bool consume_mime_token(std::string_view& text) { + size_t length = 0; + while (length < text.size() && mime_token_char(text[length])) { + ++length; + } + text.remove_prefix(length); + return length != 0; +} + +bool consume_mime_fold(std::string_view& text) { + // RFC 822 section 3.1.1: a fold is CRLF followed by SP or HTAB. + if (text.size() < 3 || text[0] != '\r' || text[1] != '\n' || + (text[2] != ' ' && text[2] != '\t')) { + return false; + } + text.remove_prefix(3); + return true; +} + +bool consume_mime_delimited(std::string_view& text, bool comment) { + // RFC 822 section 3.3: qtext/ctext and quoted-pair use all ASCII, not + // HTTP qdtext. Count nested comments iteratively; never rewrite the value. + text.remove_prefix(1); // Opening '(' or DQUOTE. + size_t depth = 1; + while (!text.empty()) { + const auto ch = static_cast(text.front()); + if (ch >= 0x80) { + return false; + } + if (ch == '\r') { + if (!consume_mime_fold(text)) { + return false; + } + continue; + } + text.remove_prefix(1); + if (ch == '\\') { + if (text.empty() || static_cast(text.front()) >= 0x80) { + return false; + } + if (text.starts_with("\r\n")) { + // RFC 822 sections 3.4.3 and 3.4.5 require quoted CRLFs + // in comments and quoted strings to obey the folding rules. + if (!consume_mime_fold(text)) { + return false; + } + } else { + text.remove_prefix(1); // quoted-pair = backslash CHAR, including CTLs. + } + } else if (ch == (comment ? ')' : '"')) { + if (--depth == 0) { + return true; + } + } else if (comment && ch == '(') { + ++depth; + } + } + return false; +} + +bool consume_mime_cfws(std::string_view& text) { + // Comments and linear whitespace may surround each lexical token, but + // cannot join two separate token fragments (e.g. te(comment)xt/plain). + while (!text.empty()) { + const char ch = text.front(); + if (ch == ' ' || ch == '\t') { + text.remove_prefix(1); + } else if (ch == '\r') { + if (!consume_mime_fold(text)) { + return false; + } + } else if (ch == '(') { + if (!consume_mime_delimited(text, true)) { + return false; + } + } else { + break; + } + } + return true; +} + +bool consume_mime_separator(std::string_view& text, char separator) { + if (!consume_mime_cfws(text) || text.empty() || text.front() != separator) { + return false; + } + text.remove_prefix(1); + return consume_mime_cfws(text); +} + +bool mime(std::string_view text) { + // RFC 2045 section 5.1 with RFC 822 lexical rules. Validate syntax only: + // unknown types/parameters are allowed and all original bytes are preserved. + if (!consume_mime_cfws(text) || !consume_mime_token(text) || + !consume_mime_separator(text, '/') || !consume_mime_token(text) || + !consume_mime_cfws(text)) { + return false; + } + while (!text.empty()) { + if (!consume_mime_separator(text, ';') || !consume_mime_token(text) || + !consume_mime_separator(text, '=') || text.empty()) { + return false; + } + if (text.front() == '"') { + if (!consume_mime_delimited(text, false)) { + return false; + } + } else if (!consume_mime_token(text)) { + return false; + } + if (!consume_mime_cfws(text)) { + return false; + } + } + return true; +} + +bool equals_known_algorithm(std::string_view text, std::string_view canonical) { + return text.size() == canonical.size() && + std::equal(text.begin(), text.end(), canonical.begin(), [](char ch, char expected) { + return (ch >= 'a' && ch <= 'z' ? ch - 'a' + 'A' : ch) == expected; + }); +} + +bool checksum(std::string_view text) { + const auto colon = text.find(':'); + if (colon == std::string_view::npos || colon == 0 || colon + 1 == text.size()) { + return false; + } + const auto algorithm = text.substr(0, colon); + const auto digest = text.substr(colon + 1); + struct KnownAlgorithm { + std::string_view name; + size_t hex_length; + }; + constexpr std::array algorithms = {KnownAlgorithm {"ETAG", 0}, KnownAlgorithm {"MD5", 32}, + KnownAlgorithm {"CRC32", 8}, KnownAlgorithm {"CRC32C", 8}, + KnownAlgorithm {"SHA-256", 64}}; + for (const auto& known : algorithms) { + if (equals_known_algorithm(algorithm, known.name)) { + // Case-insensitive recognition prevents misspellings of known names + // from escaping validation as unknown algorithms. Never normalize. + if (algorithm != known.name) { + return false; + } + if (known.hex_length == 0) { + return true; // ETAG is opaque, including quotes and multipart suffixes. + } + return digest.size() == known.hex_length && + std::all_of(digest.begin(), digest.end(), + [](char ch) { return ascii_digit(ch) || (ch >= 'a' && ch <= 'f'); }); + } + } + return true; // Both parts of an unknown algorithm are nonempty opaque bytes. +} + +} // namespace file_value_detail +} // namespace + +bool is_valid_file_content_type(std::string_view content_type) { + return content_type.size() <= file_value_detail::MAX_METADATA_BYTES && + file_value_detail::mime(content_type); +} + +std::string_view infer_file_content_type_from_name(std::string_view name) { + // Shared by TO_FILE and list_files. Keep inference independent of URI rewriting. + static constexpr std::pair TYPES[] = { + {".csv", "text/csv"}, + {".tsv", "text/tab-separated-values"}, + {".json", "application/json"}, + {".jsonl", "application/x-ndjson"}, + {".parquet", "application/x-parquet"}, + {".orc", "application/x-orc"}, + {".avro", "application/avro"}, + {".txt", "text/plain"}, + {".log", "text/plain"}, + {".tbl", "text/plain"}, + {".xml", "application/xml"}, + {".html", "text/html"}, + {".htm", "text/html"}, + {".pdf", "application/pdf"}, + {".jpg", "image/jpeg"}, + {".jpeg", "image/jpeg"}, + {".png", "image/png"}, + {".gif", "image/gif"}, + {".bmp", "image/bmp"}, + {".svg", "image/svg+xml"}, + {".webp", "image/webp"}, + {".mp3", "audio/mpeg"}, + {".wav", "audio/wav"}, + {".mp4", "video/mp4"}, + {".avi", "video/x-msvideo"}, + {".gz", "application/gzip"}, + {".bz2", "application/x-bzip2"}, + {".zst", "application/zstd"}, + {".lz4", "application/x-lz4"}, + {".snappy", "application/x-snappy"}, + {".zip", "application/zip"}, + {".tar", "application/x-tar"}, + }; + const auto basename = name.substr(name.find_last_of('/') + 1); + const auto dot = basename.find_last_of('.'); + if (dot != std::string_view::npos) { + std::string extension(basename.substr(dot)); + for (auto& ch : extension) { + if (ch >= 'A' && ch <= 'Z') { + ch += 'a' - 'A'; + } + } + for (const auto& [suffix, type] : TYPES) { + if (extension == suffix) { + return type; + } + } + } + return "application/octet-stream"; +} + +std::optional file_checksum_from_etag(std::optional etag) { + if (!etag) { + return std::nullopt; + } + if (etag->size() >= 2 && etag->front() == '"' && etag->back() == '"') { + etag->remove_prefix(1); + etag->remove_suffix(1); + } + if (etag->empty()) { + return std::nullopt; + } + return "ETAG:" + std::string(*etag); +} + +Status validate_file(const File& value) { + // Callers establish the canonical field count and types when constructing or decoding FILE. + DCHECK_EQ(value.size(), file_value_detail::FIELD_NAMES.size()); + const auto text = [&value](size_t field) -> std::optional { + return value[field].is_null() ? std::nullopt : std::optional(value[field].as_string_view()); + }; + const auto number = [&value](size_t field) -> std::optional { + return value[field].is_null() ? std::nullopt + : std::optional(value[field].get()); + }; + return validate_file(text(0), number(1), number(2), text(3), text(4), value[5].is_null()); +} + +Status validate_file(std::optional uri_value, std::optional offset, + std::optional size, std::optional content_type, + std::optional checksum_value, bool inline_is_null) { + using namespace file_value_detail; + if (!uri_value && inline_is_null) { + return Status::InvalidArgument("FILE uri must not be NULL when inline is NULL"); + } + if (size && *size < 0) { + return Status::InvalidArgument("FILE size must be nonnegative"); + } + if (offset) { + if (*offset < 0) { + return Status::InvalidArgument("FILE offset must be nonnegative"); + } + if (!size) { + return Status::InvalidArgument("FILE offset requires size"); + } + if (*offset > std::numeric_limits::max() - *size) { + return Status::InvalidArgument("FILE offset plus size exceeds BIGINT"); + } + } + if (content_type && !is_valid_file_content_type(*content_type)) { + return Status::InvalidArgument( + "FILE content_type must be a nonempty valid media type within 1024 bytes"); + } + if (checksum_value && + (checksum_value->size() > MAX_METADATA_BYTES || !checksum(*checksum_value))) { + return Status::InvalidArgument( + "FILE checksum must be algorithm:digest within 1024 bytes; known algorithms " + "require canonical names and prescribed lowercase hex digests (ETAG is " + "opaque)"); + } + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/core/value/file_value.h b/be/src/core/value/file_value.h new file mode 100644 index 00000000000000..5ef5fc72b06cba --- /dev/null +++ b/be/src/core/value/file_value.h @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" + +namespace doris { + +struct File; + +// Validate the values of a non-NULL FILE without changing any bytes. +// The caller must supply six children with the canonical FILE field types. +// SQL NULL FILE values are handled by the caller, outside this function. +Status validate_file(const File& value); + +// Borrow metadata from FILE columns. Inline bytes are opaque to validation; +// only their SQL NULL state participates in the rules. +Status validate_file(std::optional uri, std::optional offset, + std::optional size, std::optional content_type, + std::optional checksum, bool inline_is_null); + +bool is_valid_file_content_type(std::string_view content_type); + +// Takes a filesystem name, not an encoded URI. The caller owns URI parsing. +std::string_view infer_file_content_type_from_name(std::string_view name); + +// S3 HEAD and LIST use the same FILE checksum: remove one enclosing quote pair, +// preserve the remaining opaque bytes, and return NULL for missing or empty ETags. +std::optional file_checksum_from_etag(std::optional etag); + +} // namespace doris diff --git a/be/src/exec/common/data_gen_functions/vlist_file_tvf.cpp b/be/src/exec/common/data_gen_functions/vlist_file_tvf.cpp new file mode 100644 index 00000000000000..aa7f935577dbe0 --- /dev/null +++ b/be/src/exec/common/data_gen_functions/vlist_file_tvf.cpp @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/common/data_gen_functions/vlist_file_tvf.h" + +#include +#include + +#include +#include + +#include "core/data_type/data_type_file.h" +#include "core/field.h" +#include "core/value/file_value.h" +#include "io/file_factory.h" +#include "io/fs/s3_file_system.h" +#include "runtime/runtime_state.h" +#include "util/s3_uri.h" + +namespace doris { +VListFileTVF::VListFileTVF(TupleId tuple_id, const TupleDescriptor* tuple_desc) + : VDataGenFunctionInf(tuple_id, tuple_desc) {} + +VListFileTVF::~VListFileTVF() = default; + +Status VListFileTVF::set_scan_ranges(const std::vector& scan_ranges) { + DCHECK(scan_ranges.size() == 1); + const auto& params = scan_ranges[0].scan_range.data_gen_scan_range.list_file_params; + S3URI uri(params.uri); + RETURN_IF_ERROR(uri.parse(true)); + _properties = params.resource.properties; + _path.bucket = uri.get_bucket(); + _path.prefix = uri.get_key(); + if (!_path.prefix.empty() && !_path.prefix.ends_with('/')) { + _path.prefix += '/'; + } + _path.delimiter = params.recursive ? "" : "/"; + static constexpr std::array NAMES {"path", "size", "file"}; + _column_indices.clear(); + for (const auto* slot : _tuple_desc->slots()) { + const auto* const field = std::ranges::find(NAMES, slot->col_name()); + DCHECK(field != NAMES.end()); + _column_indices.push_back(std::distance(NAMES.begin(), field)); + } + return Status::OK(); +} + +Status VListFileTVF::_fetch_next_page(RuntimeState* state) { + RETURN_IF_CANCELLED(state); + if (!_filesystem) { + io::FSPropertiesRef properties(TFileType::FILE_S3); + properties.properties = &_properties; + io::FileDescription description; + description.path = "s3://" + _path.bucket; + auto filesystem = DORIS_TRY(FileFactory::create_fs(properties, description)); + _filesystem = std::static_pointer_cast(filesystem); + RETURN_IF_CANCELLED(state); + } + + RETURN_IF_CANCELLED(state); + const auto client = _filesystem->client_holder()->get(); + // Release the previous page before the provider allocates its replacement. + _objects = std::vector(); + auto page = client->list_objects_page(_path, _continuation_token); + RETURN_IF_CANCELLED(state); + if (!page.resp.ok()) { + return {page.resp.status.code, std::move(page.resp.status.msg)}; + } + _objects = std::move(page.objects); + for (auto& prefix : page.common_prefixes) { + _objects.emplace_back(io::ObjectMeta {.key = std::move(prefix)}); + } + _next_index = 0; + _continuation_token = std::move(page.continuation_token); + _has_more = page.has_more; + return Status::OK(); +} + +Status VListFileTVF::get_next(RuntimeState* state, Block* block, bool* eos) { + DCHECK(block->rows() == 0); + DCHECK(!_tuple_desc->slots().empty()); + RETURN_IF_CANCELLED(state); + *eos = false; + + if (!block->mem_reuse()) { + for (const auto* slot : _tuple_desc->slots()) { + block->insert({slot->get_empty_mutable_column(), slot->get_data_type_ptr(), + slot->col_name()}); + } + } + // Restore borrowed column owners on success, cancellation and remote errors. + auto scoped_columns = block->mutate_columns_scoped(); + auto& columns = scoped_columns.mutable_columns(); + auto& output = columns[0]; + while (output->size() < static_cast(state->batch_size())) { + if (_next_index == _objects.size()) { + if (!output->empty()) { + break; + } + if (!_has_more) { + *eos = true; + break; + } + RETURN_IF_ERROR(_fetch_next_page(state)); + continue; + } + RETURN_IF_CANCELLED(state); + const auto& object = _objects[_next_index++]; + const bool is_directory = object.key.ends_with('/'); + if (is_directory && (_path.delimiter.empty() || object.key == _path.prefix)) { + continue; + } + std::array fields { + Field::create_field("s3://" + _path.bucket + "/" + object.key), + Field::create_field(is_directory ? 0 : object.size), Field()}; + if (!is_directory) { + File value(DataTypeFile::FIELD_COUNT); + value[0] = fields[0]; + value[2] = fields[1]; + value[3] = Field::create_field( + std::string(infer_file_content_type_from_name(object.key))); + if (auto checksum = file_checksum_from_etag(object.etag)) { + value[4] = Field::create_field(std::move(*checksum)); + } + RETURN_IF_ERROR(validate_file(value)); + fields[2] = Field::create_field(std::move(value)); + } + for (size_t i = 0; i < columns.size(); ++i) { + columns[i]->insert(fields[_column_indices[i]]); + } + } + *eos = _next_index == _objects.size() && !_has_more; + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/common/data_gen_functions/vlist_file_tvf.h b/be/src/exec/common/data_gen_functions/vlist_file_tvf.h new file mode 100644 index 00000000000000..65e4ad8eb76aec --- /dev/null +++ b/be/src/exec/common/data_gen_functions/vlist_file_tvf.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "cpp/obj-client/obj_storage_client.h" +#include "exec/common/data_gen_functions/vdata_gen_function_inf.h" + +namespace doris { +namespace io { +class S3FileSystem; +} + +// One provider page and the current output batch bound listing memory, independently of directory +// size. A populated page is returned before fetching the next one so the operator can apply LIMIT. +class VListFileTVF final : public VDataGenFunctionInf { +public: + VListFileTVF(TupleId tuple_id, const TupleDescriptor* tuple_desc); + ~VListFileTVF() override; + + Status set_scan_ranges(const std::vector& scan_ranges) override; + Status get_next(RuntimeState* state, Block* block, bool* eos) override; + +private: + Status _fetch_next_page(RuntimeState* state); + + std::vector _column_indices; + std::map _properties; + io::ObjStoragePath _path; + std::shared_ptr _filesystem; + std::vector _objects; + std::string _continuation_token; + size_t _next_index = 0; + bool _has_more = true; +}; + +} // namespace doris diff --git a/be/src/exec/operator/datagen_operator.cpp b/be/src/exec/operator/datagen_operator.cpp index c4d4e969fe8aee..357444d8c53f6e 100644 --- a/be/src/exec/operator/datagen_operator.cpp +++ b/be/src/exec/operator/datagen_operator.cpp @@ -20,6 +20,7 @@ #include #include "exec/common/data_gen_functions/vdata_gen_function_inf.h" +#include "exec/common/data_gen_functions/vlist_file_tvf.h" #include "exec/common/data_gen_functions/vnumbers_tvf.h" #include "exec/operator/operator.h" #include "exec/runtime_filter/runtime_filter_consumer.h" @@ -41,9 +42,13 @@ DataGenSourceOperatorX::DataGenSourceOperatorX(ObjectPool* pool, const TPlanNode Status DataGenSourceOperatorX::init(const TPlanNode& tnode, RuntimeState* state) { RETURN_IF_ERROR(OperatorX::init(tnode, state)); // set _table_func here - switch (tnode.data_gen_scan_node.func_name) { + _function_name = tnode.data_gen_scan_node.func_name; + switch (_function_name) { case TDataGenFunctionName::NUMBERS: break; + case TDataGenFunctionName::LIST_FILE: + _blockable = true; + break; default: return Status::InternalError("Unsupported function type"); } @@ -69,6 +74,17 @@ Status DataGenSourceOperatorX::get_block_impl(RuntimeState* state, Block* block, auto& local_state = get_local_state(state); SCOPED_TIMER(local_state.exec_time_counter()); SCOPED_PEAK_MEM(&local_state.estimate_memory_usage()); + if (_function_name == TDataGenFunctionName::LIST_FILE && _limit != -1 && + local_state.num_rows_returned() >= _limit) { + if (!block->mem_reuse()) { + for (const auto* slot : _tuple_desc->slots()) { + block->insert({slot->get_empty_mutable_column(), slot->get_data_type_ptr(), + slot->col_name()}); + } + } + *eos = true; + return Status::OK(); + } { SCOPED_TIMER(local_state._table_function_execution_timer); RETURN_IF_ERROR(local_state._table_func->get_next(state, block, eos)); @@ -89,7 +105,16 @@ Status DataGenLocalState::init(RuntimeState* state, LocalStateInfo& info) { _table_function_execution_timer = ADD_TIMER(custom_profile(), "TableFunctionExecutionTime"); _filter_timer = ADD_TIMER(custom_profile(), "FilterTime"); auto& p = _parent->cast(); - _table_func = std::make_shared(p._tuple_id, p._tuple_desc); + switch (p._function_name) { + case TDataGenFunctionName::NUMBERS: + _table_func = std::make_shared(p._tuple_id, p._tuple_desc); + break; + case TDataGenFunctionName::LIST_FILE: + _table_func = std::make_shared(p._tuple_id, p._tuple_desc); + break; + default: + return Status::InternalError("Unsupported function type"); + } _table_func->set_tuple_desc(p._tuple_desc); RETURN_IF_ERROR(_table_func->set_scan_ranges(info.scan_ranges)); diff --git a/be/src/exec/operator/datagen_operator.h b/be/src/exec/operator/datagen_operator.h index d8cc0fc4508d2d..ef5a7e046a4d5c 100644 --- a/be/src/exec/operator/datagen_operator.h +++ b/be/src/exec/operator/datagen_operator.h @@ -67,6 +67,7 @@ class DataGenSourceOperatorX final : public OperatorX { friend class DataGenLocalState; // Tuple id resolved in prepare() to set _tuple_desc; TupleId _tuple_id; + TDataGenFunctionName::type _function_name = TDataGenFunctionName::NUMBERS; // Descriptor of tuples generated const TupleDescriptor* _tuple_desc = nullptr; diff --git a/be/src/exprs/function/array/function_array_element.h b/be/src/exprs/function/array/function_array_element.h index 9d10034bb28549..f10c671fdbd9be 100644 --- a/be/src/exprs/function/array/function_array_element.h +++ b/be/src/exprs/function/array/function_array_element.h @@ -41,6 +41,7 @@ #include "core/column/column.h" #include "core/column/column_array.h" #include "core/column/column_decimal.h" +#include "core/column/column_file.h" #include "core/column/column_map.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" @@ -48,6 +49,7 @@ #include "core/column/column_vector.h" #include "core/data_type/data_type.h" #include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_file.h" #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" @@ -98,6 +100,8 @@ class FunctionArrayElement : public IFunction { throw doris::Exception(st); } return make_nullable(struct_type->get_elements()[index]); + } else if (arg_0->get_primitive_type() == TYPE_FILE) { + return get_file_element_return_type(arguments); } DataTypes data_types(arguments.size()); for (size_t i = 0; i < arguments.size(); ++i) { @@ -130,9 +134,18 @@ class FunctionArrayElement : public IFunction { Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, uint32_t result, size_t input_rows_count) const override { - if (remove_nullable(block.get_by_position(arguments[0]).type)->get_primitive_type() == - TYPE_STRUCT) { + if (block.get_by_position(arguments[0]).type->is_null_literal()) { + // Null-literal container: every row is NULL, matching the Nullable(Nothing) return type. + auto& res = block.get_by_position(result); + res.column = res.type->create_column_const_with_default_value(input_rows_count); + return Status::OK(); + } + const auto primitive_type = + remove_nullable(block.get_by_position(arguments[0]).type)->get_primitive_type(); + if (primitive_type == TYPE_STRUCT) { return _execute_struct(block, arguments, result, input_rows_count); + } else if (primitive_type == TYPE_FILE) { + return _execute_file(block, arguments, result, input_rows_count); } auto dst_null_column = ColumnUInt8::create(input_rows_count, 0); UInt8* dst_null_map = dst_null_column->get_data().data(); @@ -169,6 +182,82 @@ class FunctionArrayElement : public IFunction { } private: + static Status get_file_element_index(const DataTypeFile& type, + const ColumnWithTypeAndName& selector, size_t& index) { + if (!selector.column || + !is_string_type(remove_nullable(selector.type)->get_primitive_type()) || + selector.column->is_null_at(0)) { + return Status::InvalidArgument( + "FILE element_at requires a non-NULL constant field name"); + } + const auto field_name = selector.column->get_data_at(0).to_string(); + const auto position = type.try_get_position_by_name(field_name); + if (!position) { + return Status::InvalidArgument("Unknown FILE field {}", field_name); + } + index = *position; + return Status::OK(); + } + + DataTypePtr get_file_element_return_type(const ColumnsWithTypeAndName& arguments) const { + const auto& type = assert_cast(*remove_nullable(arguments[0].type)); + // Validate constness while binding. The default constant implementation unwraps + // the selector for single-row execution when both arguments are constant. + if (!arguments[1].column || !is_column_const(*arguments[1].column)) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "FILE element_at requires a non-NULL constant field name"); + } + size_t index = 0; + THROW_IF_ERROR(get_file_element_index(type, arguments[1], index)); + return type.get_element(index); + } + + Status _execute_file(Block& block, const ColumnNumbers& arguments, uint32_t result, + size_t input_rows_count) const { + const auto& argument = block.get_by_position(arguments[0]); + const auto& type = assert_cast(*remove_nullable(argument.type)); + size_t index = 0; + RETURN_IF_ERROR(get_file_element_index(type, block.get_by_position(arguments[1]), index)); + return execute_field(block, arguments[0], result, input_rows_count, index); + } + + template + static Status execute_field(Block& block, size_t argument, uint32_t result, + size_t input_rows_count, size_t index) { + auto [unpacked, is_const] = unpack_if_const(block.get_by_position(argument).column); + ColumnPtr source = unpacked; + const ColumnUInt8* outer_nulls = nullptr; + if (const auto* nullable = check_and_get_column(*source)) { + outer_nulls = &nullable->get_null_map_column(); + source = nullable->get_nested_column_ptr(); + } + const auto& fields = assert_cast(*source); + ColumnPtr selected = fields.get_column_ptr(index)->convert_to_full_column_if_const(); + const size_t source_rows = is_const ? 1 : input_rows_count; + auto nulls = ColumnUInt8::create(source_rows, 0); + if (const auto* nullable = check_and_get_column(*selected)) { + memcpy(nulls->get_data().data(), nullable->get_null_map_data().data(), source_rows); + selected = nullable->get_nested_column_ptr(); + } + if (outer_nulls) { + for (size_t row = 0; row < source_rows; ++row) { + nulls->get_data()[row] |= outer_nulls->get_data()[row]; + } + } + selected = ColumnNullable::create(selected, std::move(nulls)); + if (is_const) { + // Expand only the selected field, retaining shared storage for the other fields. + selected = ColumnConst::create(selected, input_rows_count)->convert_to_full_column(); + } + block.replace_by_position(result, std::move(selected)); + return Status::OK(); + } + + static bool valid_negative_array_index(Int64 index, size_t length) { + // Compare abs(index) - 1 so INT64_MIN stays representable and remains an out-of-range NULL. + return index < 0 && std::cmp_less(static_cast(-(index + 1)), length); + } + //=========================== struct element===========================// // Resolve the 0-based field offset selected by a constant int/string index. Mirrors the logic // of the former struct_element function, which element_at now subsumes. @@ -212,48 +301,13 @@ class FunctionArrayElement : public IFunction { Status _execute_struct(Block& block, const ColumnNumbers& arguments, uint32_t result, size_t input_rows_count) const { const auto& struct_arg = block.get_by_position(arguments[0]); - ColumnPtr struct_col_ptr = struct_arg.column->convert_to_full_column_if_const(); - // element_at manages nulls itself (use_default_implementation_for_nulls() == false), so a - // null struct row must be merged into the result null map manually. - const ColumnUInt8* outer_null_map = nullptr; - if (struct_col_ptr->is_nullable()) { - const auto* nullable = assert_cast(struct_col_ptr.get()); - outer_null_map = &nullable->get_null_map_column(); - struct_col_ptr = nullable->get_nested_column_ptr(); - } - const auto* struct_type = - check_and_get_data_type(remove_nullable(struct_arg.type).get()); - const auto* struct_col = check_and_get_column(struct_col_ptr.get()); - if (!struct_col || !struct_type) { - return Status::RuntimeError("unsupported types for function {}({}, {})", get_name(), - struct_arg.type->get_name(), - block.get_by_position(arguments[1]).type->get_name()); - } + const auto& struct_type = + assert_cast(*remove_nullable(struct_arg.type)); const auto& index_arg = block.get_by_position(arguments[1]); size_t index = 0; RETURN_IF_ERROR( - get_struct_element_index(*struct_type, index_arg.column, index_arg.type, &index)); - - ColumnPtr field_col = struct_col->get_column_ptr(index); - auto res_null_column = ColumnUInt8::create(input_rows_count, 0); - auto& res_null_map = res_null_column->get_data(); - ColumnPtr res_nested = field_col; - if (field_col->is_nullable()) { - const auto* field_nullable = assert_cast(field_col.get()); - const auto& field_null_map = field_nullable->get_null_map_column().get_data(); - memcpy(res_null_map.data(), field_null_map.data(), input_rows_count); - res_nested = field_nullable->get_nested_column_ptr(); - } - if (outer_null_map) { - const auto& outer = outer_null_map->get_data(); - for (size_t i = 0; i < input_rows_count; ++i) { - res_null_map[i] |= outer[i]; - } - } - block.replace_by_position( - result, ColumnNullable::create(res_nested->clone_resized(input_rows_count), - std::move(res_null_column))); - return Status::OK(); + get_struct_element_index(struct_type, index_arg.column, index_arg.type, &index)); + return execute_field(block, arguments[0], result, input_rows_count, index); } //=========================== map element===========================// diff --git a/be/src/exprs/function/cast/cast_to_array.h b/be/src/exprs/function/cast/cast_to_array.h index add20080e6636d..eb399a08d69ae0 100644 --- a/be/src/exprs/function/cast/cast_to_array.h +++ b/be/src/exprs/function/cast/cast_to_array.h @@ -92,4 +92,4 @@ WrapperType create_array_wrapper(FunctionContext* context, const DataTypePtr& fr }; } #include "common/compile_check_end.h" -} // namespace doris::CastWrapper \ No newline at end of file +} // namespace doris::CastWrapper diff --git a/be/src/exprs/function/cast/cast_to_variant.h b/be/src/exprs/function/cast/cast_to_variant.h index 8b95aca904fbfe..0b5731e1c1cb58 100644 --- a/be/src/exprs/function/cast/cast_to_variant.h +++ b/be/src/exprs/function/cast/cast_to_variant.h @@ -180,8 +180,8 @@ struct CastToVariant { }; // create corresponding variant value to wrap from_type -WrapperType create_cast_to_variant_wrapper(const DataTypePtr& from_type, - const DataTypeVariant& to_type) { +inline WrapperType create_cast_to_variant_wrapper(const DataTypePtr& from_type, + const DataTypeVariant& to_type) { if (from_type->get_primitive_type() == TYPE_VARIANT) { // variant_max_subcolumns_count is not equal return create_unsupport_wrapper(from_type->get_name(), to_type.get_name()); @@ -190,8 +190,8 @@ WrapperType create_cast_to_variant_wrapper(const DataTypePtr& from_type, } // create corresponding type convert from variant -WrapperType create_cast_from_variant_wrapper(const DataTypeVariant& from_type, - const DataTypePtr& to_type) { +inline WrapperType create_cast_from_variant_wrapper(const DataTypeVariant& from_type, + const DataTypePtr& to_type) { if (to_type->get_primitive_type() == TYPE_VARIANT) { // variant_max_subcolumns_count is not equal return create_unsupport_wrapper(from_type.get_name(), to_type->get_name()); diff --git a/be/src/exprs/function/cast/function_cast.cpp b/be/src/exprs/function/cast/function_cast.cpp index 3b647399805304..c4226e401213b1 100644 --- a/be/src/exprs/function/cast/function_cast.cpp +++ b/be/src/exprs/function/cast/function_cast.cpp @@ -20,6 +20,7 @@ #include "core/column/column_varbinary.h" #include "core/data_type/data_type_agg_state.h" #include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_file.h" #include "core/data_type/data_type_number.h" // IWYU pragma: keep #include "core/data_type/data_type_quantilestate.h" #include "core/data_type/data_type_timestamp_ns.h" // IWYU pragma: keep @@ -258,6 +259,10 @@ WrapperType prepare_impl(FunctionContext* context, const DataTypePtr& origin_fro return create_identity_wrapper(from_type); } + if (contains_file_type(from_type) || contains_file_type(to_type)) { + return create_unsupport_wrapper(from_type->get_name(), to_type->get_name()); + } + const auto* from_variant_v1 = dynamic_cast(remove_nullable(from_type).get()); const auto* to_variant_v1 = diff --git a/be/src/exprs/function/function_dict_get_many.cpp b/be/src/exprs/function/function_dict_get_many.cpp index 93e1f9fae5d9a6..0a9d402b47aae7 100644 --- a/be/src/exprs/function/function_dict_get_many.cpp +++ b/be/src/exprs/function/function_dict_get_many.cpp @@ -32,6 +32,7 @@ #include "exprs/function/dictionary_factory.h" #include "exprs/function/function.h" #include "exprs/function/simple_function_factory.h" +#include "runtime/exec_env.h" namespace doris { diff --git a/be/src/exprs/function/function_fake.cpp b/be/src/exprs/function/function_fake.cpp index 0966ed9af613c1..12bde7e7d858ec 100644 --- a/be/src/exprs/function/function_fake.cpp +++ b/be/src/exprs/function/function_fake.cpp @@ -26,6 +26,7 @@ #include #include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_file.h" #include "core/data_type/data_type_jsonb.h" #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" @@ -99,6 +100,18 @@ struct FunctionExplodeV2 { static std::string get_error_msg() { return "Fake function do not support execute"; } }; +struct FunctionExplodeFile { + static DataTypePtr get_return_type_impl(const DataTypes& arguments) { + DORIS_CHECK_EQ(arguments.size(), 1); + const auto& file = assert_cast(*arguments[0]); + const auto& elements = file.get_elements(); + const auto& names = file.get_element_names(); + return make_nullable(std::make_shared(elements, names)); + } + static DataTypes get_variadic_argument_types() { return {}; } + static std::string get_error_msg() { return "Fake function do not support execute"; } +}; + // explode map: make map k,v as struct field struct FunctionExplodeMap { static DataTypePtr get_return_type_impl(const DataTypes& arguments) { @@ -259,6 +272,7 @@ void register_function_fake(SimpleFunctionFactory& factory) { register_table_alternative_function_expand_outer(factory, "explode"); register_table_function_expand_outer(factory, "explode_map"); + register_table_function_expand_outer(factory, "explode_file"); register_table_function_expand_outer(factory, "explode_json_object"); register_table_function_expand_outer(factory, "json_each"); diff --git a/be/src/exprs/function/function_fl_get_binary.cpp b/be/src/exprs/function/function_fl_get_binary.cpp new file mode 100644 index 00000000000000..c31936ed998e03 --- /dev/null +++ b/be/src/exprs/function/function_fl_get_binary.cpp @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/column/column_file.h" +#include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_varbinary.h" +#include "exprs/function/function.h" +#include "exprs/function/simple_function_factory.h" + +namespace doris { + +class FunctionFlGetBinary : public IFunction { +public: + static constexpr auto name = "fl_get_binary"; + static FunctionPtr create() { return std::make_shared(); } + String get_name() const override { return name; } + size_t get_number_of_arguments() const override { return 1; } + bool use_default_implementation_for_constants() const override { return true; } + + DataTypePtr get_return_type_impl(const DataTypes&) const override { + return make_nullable(std::make_shared()); + } + + Status execute_impl(FunctionContext*, Block& block, const ColumnNumbers& arguments, + uint32_t result, size_t input_rows_count) const override { + const auto& file = + assert_cast(*block.get_by_position(arguments[0]).column); + const auto& inline_column = file.get_column_ptr(DataTypeFile::INLINE_FIELD_INDEX); + if (!inline_column->has_null()) { + block.replace_by_position(result, inline_column); + return Status::OK(); + } + + auto result_column = inline_column->clone_empty(); + result_column->reserve(input_rows_count); + for (size_t row = 0; row < input_rows_count; ++row) { + if (!inline_column->is_null_at(row)) { + result_column->insert_from(*inline_column, row); + } else { + result_column->insert_default(); + } + } + block.replace_by_position(result, std::move(result_column)); + return Status::OK(); + } +}; + +void register_function_fl_get_binary(SimpleFunctionFactory& factory) { + factory.register_function(); +} + +} // namespace doris diff --git a/be/src/exprs/function/function_to_file.cpp b/be/src/exprs/function/function_to_file.cpp new file mode 100644 index 00000000000000..33121cca19aeaf --- /dev/null +++ b/be/src/exprs/function/function_to_file.cpp @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exprs/function/function_to_file.h" + +#include +#include + +#include "core/column/column_const.h" +#include "core/column/column_file.h" +#include "core/column/column_nullable.h" +#include "core/data_type/data_type_file.h" +#include "core/value/file_value.h" +#include "exprs/function/simple_function_factory.h" +#include "io/file_factory.h" +#include "runtime/query_context.h" +#include "runtime/runtime_state.h" +#include "util/s3_uri.h" + +namespace doris { +namespace { + +struct ToFileState { + explicit ToFileState(std::shared_ptr resource_) + : resource(std::move(resource_)) {} + + const std::shared_ptr resource; + std::shared_ptr filesystem; + std::string bucket; + std::mutex mutex; + io::FileStatContext stat_context; +}; + +std::string to_file_content_type(std::string_view key, const io::FileStat& metadata) { + if (metadata.content_type && is_valid_file_content_type(*metadata.content_type)) { + return *metadata.content_type; + } + return std::string(infer_file_content_type_from_name(key)); +} + +} // namespace + +DataTypePtr FunctionToFile::get_return_type_impl(const ColumnsWithTypeAndName& arguments) const { + const auto& resource = arguments[0]; + const auto& uri = arguments[1]; + if (!is_string_type(remove_nullable(resource.type)->get_primitive_type()) || !resource.column || + !is_column_const(*resource.column) || resource.column->is_null_at(0)) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "TO_FILE resource must be a non-NULL constant string"); + } + if (!is_string_type(remove_nullable(uri.type)->get_primitive_type()) && + !uri.type->is_null_literal()) { + throw Exception(ErrorCode::INVALID_ARGUMENT, "TO_FILE URI must be string-like or NULL"); + } + DataTypePtr result = std::make_shared(); + return uri.type->is_nullable() || uri.type->is_null_literal() ? make_nullable(result) : result; +} + +Status FunctionToFile::create_filesystem(const TS3Resource& resource, const std::string& uri, + std::shared_ptr* filesystem) const { + io::FSPropertiesRef properties(TFileType::FILE_S3); + properties.properties = &resource.properties; + io::FileDescription description; + S3URI s3_uri(uri); + RETURN_IF_ERROR(s3_uri.parse(true)); + description.path = "s3://" + s3_uri.get_bucket(); + *filesystem = DORIS_TRY(FileFactory::create_fs(properties, description)); + return Status::OK(); +} + +Status FunctionToFile::open(FunctionContext* context, FunctionContext::FunctionStateScope scope) { + if (scope == FunctionContext::THREAD_LOCAL || context->get_function_state(scope)) { + return Status::OK(); + } + auto* runtime = context->state(); + if (!runtime || !runtime->get_query_ctx()) { + return Status::InvalidArgument("TO_FILE requires query resources"); + } + const auto& resources = runtime->get_query_ctx()->get_s3_resources(); + if (!resources) { + return Status::InvalidArgument("TO_FILE requires query resources"); + } + const auto* resource_column = context->get_constant_col(0); + if (!resource_column || resource_column->column_ptr->is_null_at(0)) { + return Status::InvalidArgument("TO_FILE resource must be a non-NULL constant string"); + } + const auto resource_name = resource_column->column_ptr->get_data_at(0).to_string(); + const auto resource = resources->find(resource_name); + if (resource == resources->end()) { + return Status::InvalidArgument("TO_FILE resource '{}' is missing from query resources", + resource_name); + } + // Defer client creation until a non-NULL URI needs it. + // Share the immutable query map's ownership without copying this resource per function. + auto state = std::make_shared( + std::shared_ptr(resources, &resource->second)); + // RuntimeState outlives the fragment contexts, including their clones. + state->stat_context.is_cancelled = [runtime] { return runtime->is_cancelled(); }; + context->set_function_state(scope, std::move(state)); + return Status::OK(); +} + +Status FunctionToFile::close(FunctionContext* context, FunctionContext::FunctionStateScope scope) { + if (scope == FunctionContext::FRAGMENT_LOCAL) { + context->set_function_state(scope, nullptr); + } + return Status::OK(); +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity): Keep per-row cancellation and client lifetime checks together. +Status FunctionToFile::execute_impl(FunctionContext* context, Block& block, + const ColumnNumbers& arguments, uint32_t result, + size_t input_rows_count) const { + auto* state = + static_cast(context->get_function_state(FunctionContext::FRAGMENT_LOCAL)); + if (!state) { + return Status::InternalError("TO_FILE has no fragment-local state"); + } + auto* runtime = context->state(); + const auto& source = block.get_by_position(arguments[1]).column; + auto output = ColumnFile::create(); + auto nulls = ColumnUInt8::create(input_rows_count, 0); + for (size_t row = 0; row < input_rows_count; ++row) { + RETURN_IF_CANCELLED(runtime); + if (source->is_null_at(row)) { + output->insert_default(); + nulls->get_data()[row] = 1; + continue; + } + const auto uri = source->get_data_at(row).to_string(); + io::FileStat metadata; + std::string bucket; + S3URI s3_uri(uri); + RETURN_IF_ERROR(s3_uri.parse(true)); + bucket = s3_uri.get_bucket(); + std::shared_ptr filesystem; + { + // FunctionContext clones share client initialization and bucket switching. + std::scoped_lock lock(state->mutex); + RETURN_IF_CANCELLED(runtime); + // S3FileSystem binds its bucket at construction. Keep only the current + // filesystem and switch when a row references a different bucket. + if (!state->filesystem || state->bucket != bucket) { + std::shared_ptr new_filesystem; + RETURN_IF_ERROR(create_filesystem(*state->resource, uri, &new_filesystem)); + DORIS_CHECK(new_filesystem != nullptr); + state->filesystem = std::move(new_filesystem); + state->bucket = std::move(bucket); + RETURN_IF_CANCELLED(runtime); + } + filesystem = state->filesystem; + } + // Pin this request's filesystem across concurrent bucket switches. Remote + // metadata calls must not serialize all scanners sharing the fragment state. + const auto stat_status = filesystem->stat(uri, &metadata, &state->stat_context); + RETURN_IF_CANCELLED(runtime); + RETURN_IF_ERROR(stat_status); + auto content_type = to_file_content_type(s3_uri.get_key(), metadata); + File value(DataTypeFile::FIELD_COUNT); + value[0] = Field::create_field(uri); + value[2] = Field::create_field(metadata.size); + value[3] = Field::create_field(std::move(content_type)); + if (metadata.checksum) { + value[4] = Field::create_field(*metadata.checksum); + } + RETURN_IF_ERROR(validate_file(value)); + output->insert(Field::create_field(std::move(value))); + } + if (block.get_by_position(result).type->is_nullable()) { + block.get_by_position(result).column = + ColumnNullable::create(std::move(output), std::move(nulls)); + } else { + block.get_by_position(result).column = std::move(output); + } + return Status::OK(); +} + +void register_function_to_file(SimpleFunctionFactory& factory) { + factory.register_function(); +} + +} // namespace doris diff --git a/be/src/exprs/function/function_to_file.h b/be/src/exprs/function/function_to_file.h new file mode 100644 index 00000000000000..fa8d46965ebcc3 --- /dev/null +++ b/be/src/exprs/function/function_to_file.h @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "exprs/function/function.h" + +namespace doris { +namespace io { +class FileSystem; +} + +class FunctionToFile : public IFunction { +public: + static constexpr auto name = "to_file"; + static FunctionPtr create() { return std::make_shared(); } + String get_name() const override { return name; } + size_t get_number_of_arguments() const override { return 2; } + ColumnNumbers get_arguments_that_are_always_constant() const override { return {0}; } + bool use_default_implementation_for_constants() const override { return false; } + bool use_default_implementation_for_nulls() const override { return false; } + bool is_blockable() const override { return true; } + + DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override; + Status open(FunctionContext* context, FunctionContext::FunctionStateScope scope) override; + Status close(FunctionContext* context, FunctionContext::FunctionStateScope scope) override; + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + uint32_t result, size_t input_rows_count) const override; + +protected: + virtual Status create_filesystem(const TS3Resource& resource, const std::string& uri, + std::shared_ptr* filesystem) const; +}; + +} // namespace doris diff --git a/be/src/exprs/function/function_variant_type.cpp b/be/src/exprs/function/function_variant_type.cpp index 5122e3851d1f12..0dce5e95864ace 100644 --- a/be/src/exprs/function/function_variant_type.cpp +++ b/be/src/exprs/function/function_variant_type.cpp @@ -25,6 +25,7 @@ #include "core/column/column_variant.h" #include "core/column/column_vector.h" #include "core/column/variant_v2/column_variant_v2.h" +#include "core/data_type/data_type_string.h" #include "exprs/function/simple_function_factory.h" #include "util/string_util.h" diff --git a/be/src/exprs/function/simple_function_factory.h b/be/src/exprs/function/simple_function_factory.h index 83140018cc0d58..951dd183946bc2 100644 --- a/be/src/exprs/function/simple_function_factory.h +++ b/be/src/exprs/function/simple_function_factory.h @@ -115,6 +115,8 @@ void register_function_assert_true(SimpleFunctionFactory& factory); void register_function_compress(SimpleFunctionFactory& factory); void register_function_bit_test(SimpleFunctionFactory& factory); void register_function_dict_get(SimpleFunctionFactory& factory); +void register_function_to_file(SimpleFunctionFactory& factory); +void register_function_fl_get_binary(SimpleFunctionFactory& factory); void register_function_dict_get_many(SimpleFunctionFactory& factory); void register_function_ai(SimpleFunctionFactory& factory); void register_function_score(SimpleFunctionFactory& factory); @@ -350,6 +352,8 @@ class SimpleFunctionFactory { register_function_format(instance); register_function_compress(instance); register_function_dict_get(instance); + register_function_to_file(instance); + register_function_fl_get_binary(instance); register_function_dict_get_many(instance); register_function_ai(instance); register_function_score(instance); diff --git a/be/src/exprs/table_function/table_function_factory.cpp b/be/src/exprs/table_function/table_function_factory.cpp index 4e64fdab1e7adb..9c865f13c63d47 100644 --- a/be/src/exprs/table_function/table_function_factory.cpp +++ b/be/src/exprs/table_function/table_function_factory.cpp @@ -30,6 +30,7 @@ #include "exprs/table_function/udf_table_function.h" #include "exprs/table_function/vexplode.h" #include "exprs/table_function/vexplode_bitmap.h" +#include "exprs/table_function/vexplode_file.h" #include "exprs/table_function/vexplode_json_object.h" #include "exprs/table_function/vexplode_map.h" #include "exprs/table_function/vexplode_numbers.h" @@ -50,6 +51,7 @@ const std::unordered_map()}, {"explode_bitmap", TableFunctionCreator()}, {"explode_map", TableFunctionCreator {}}, + {"explode_file", TableFunctionCreator {}}, {"explode_json_object", TableFunctionCreator {}}, {"json_each", TableFunctionCreator {}}, {"json_each_text", TableFunctionCreator {}}, diff --git a/be/src/exprs/table_function/vexplode_file.cpp b/be/src/exprs/table_function/vexplode_file.cpp new file mode 100644 index 00000000000000..ebae3d92f1a29f --- /dev/null +++ b/be/src/exprs/table_function/vexplode_file.cpp @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exprs/table_function/vexplode_file.h" + +#include "core/column/column_file.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/data_type/data_type_file.h" +#include "exprs/vexpr.h" + +namespace doris { + +VExplodeFileTableFunction::VExplodeFileTableFunction() { + _fn_name = "vexplode_file"; +} + +Status VExplodeFileTableFunction::process_init(Block* block, RuntimeState* state) { + const auto& children = _expr_context->root()->children(); + DORIS_CHECK(children.size() == 1); + ColumnPtr input; + RETURN_IF_ERROR( + children[0]->execute_column(_expr_context.get(), block, nullptr, block->rows(), input)); + _input = input->convert_to_full_column_if_const(); + const IColumn* nested = _input.get(); + _nulls = nullptr; + if (const auto* nullable = check_and_get_column(*nested)) { + _nulls = nullable->get_null_map_data().data(); + nested = &nullable->get_nested_column(); + } + _file = check_and_get_column(nested); + if (!_file) return Status::InvalidArgument("explode_file requires FILE input"); + return Status::OK(); +} + +void VExplodeFileTableFunction::process_row(size_t row_idx) { + DCHECK_LT(row_idx, _input->size()); + TableFunction::process_row(row_idx); + _row = row_idx; + _cur_size = (!_nulls || !_nulls[row_idx]) ? 1 : 0; +} + +void VExplodeFileTableFunction::process_close() { + _input = nullptr; + _file = nullptr; + _nulls = nullptr; + _row = 0; +} + +void VExplodeFileTableFunction::get_same_many_values(MutableColumnPtr& column, int length) { + if (current_empty()) { + if (is_outer()) column->insert_many_defaults(length); + return; + } + IColumn* nested = column.get(); + if (_is_nullable) { + auto& nullable = assert_cast(*nested); + nullable.get_null_map_column().insert_many_defaults(length); + nested = &nullable.get_nested_column(); + } + auto& structure = assert_cast(*nested); + DCHECK_EQ(structure.tuple_size(), DataTypeFile::FIELD_COUNT); + for (size_t i = 0; i < DataTypeFile::FIELD_COUNT; ++i) { + structure.get_column(i).insert_many_from(_file->get_column(i), _row, length); + } +} + +int VExplodeFileTableFunction::get_value(MutableColumnPtr& column, int max_step) { + if (max_step <= 0 || eos()) return 0; + if (current_empty() && !is_outer()) { + forward(); + return 0; + } + get_same_many_values(column, 1); + forward(); + return 1; +} + +} // namespace doris diff --git a/be/src/exprs/table_function/vexplode_file.h b/be/src/exprs/table_function/vexplode_file.h new file mode 100644 index 00000000000000..66f08ef952ca10 --- /dev/null +++ b/be/src/exprs/table_function/vexplode_file.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "exprs/table_function/table_function.h" + +namespace doris { + +class ColumnFile; + +class VExplodeFileTableFunction final : public TableFunction { + ENABLE_FACTORY_CREATOR(VExplodeFileTableFunction); + +public: + VExplodeFileTableFunction(); + Status process_init(Block* block, RuntimeState* state) override; + void process_row(size_t row_idx) override; + void process_close() override; + void get_same_many_values(MutableColumnPtr& column, int length) override; + int get_value(MutableColumnPtr& column, int max_step) override; + +private: + ColumnPtr _input; + const ColumnFile* _file = nullptr; + const UInt8* _nulls = nullptr; + size_t _row = 0; +}; + +} // namespace doris diff --git a/be/src/exprs/vcase_expr.h b/be/src/exprs/vcase_expr.h index 806ada9e3798f4..d2c6288020aebe 100644 --- a/be/src/exprs/vcase_expr.h +++ b/be/src/exprs/vcase_expr.h @@ -25,6 +25,7 @@ #include "core/column/column_complex.h" #include "core/column/column_const.h" #include "core/column/column_decimal.h" +#include "core/column/column_file.h" #include "core/column/column_map.h" #include "core/column/column_nullable.h" #include "core/column/column_struct.h" @@ -87,6 +88,7 @@ class VCaseExpr final : public VExpr { std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || @@ -152,6 +154,7 @@ class VCaseExpr final : public VExpr { CASE_TYPE(TYPE_ARRAY, ColumnArray) CASE_TYPE(TYPE_MAP, ColumnMap) CASE_TYPE(TYPE_STRUCT, ColumnStruct) + CASE_TYPE(TYPE_FILE, ColumnFile) CASE_TYPE(TYPE_BITMAP, ColumnBitmap) CASE_TYPE(TYPE_HLL, ColumnHLL) CASE_TYPE(TYPE_QUANTILE_STATE, ColumnQuantileState) diff --git a/be/src/exprs/vcondition_expr.cpp b/be/src/exprs/vcondition_expr.cpp index 057cadf0b2f54d..e6caf7aaf42e77 100644 --- a/be/src/exprs/vcondition_expr.cpp +++ b/be/src/exprs/vcondition_expr.cpp @@ -678,6 +678,7 @@ Status VectorizedCoalesceExpr::execute_column_impl(VExprContext* context, const bool cannot_random_write = result_column->is_column_string() || result_type->get_primitive_type() == PrimitiveType::TYPE_MAP || result_type->get_primitive_type() == PrimitiveType::TYPE_STRUCT || + result_type->get_primitive_type() == PrimitiveType::TYPE_FILE || result_type->get_primitive_type() == PrimitiveType::TYPE_ARRAY || result_type->get_primitive_type() == PrimitiveType::TYPE_VARIANT || result_type->get_primitive_type() == PrimitiveType::TYPE_JSONB; diff --git a/be/src/exprs/vectorized_fn_call.cpp b/be/src/exprs/vectorized_fn_call.cpp index 137fe34e97d018..d8058b96b70e99 100644 --- a/be/src/exprs/vectorized_fn_call.cpp +++ b/be/src/exprs/vectorized_fn_call.cpp @@ -668,7 +668,8 @@ bool VectorizedFnCall::can_push_down_to_index() const { bool VectorizedFnCall::is_deterministic() const { // array_shuffle gives the rows of a block different orders, even with a seed. static const std::set NON_DETERMINISTIC_FUNCTIONS = { - "random", "rand", "random_bytes", "uuid", "uuid_numeric", "array_shuffle", "shuffle"}; + "random", "rand", "random_bytes", "uuid", + "uuid_numeric", "to_file", "array_shuffle", "shuffle"}; return !NON_DETERMINISTIC_FUNCTIONS.contains(_function_name) && VExpr::is_deterministic(); } diff --git a/be/src/format/transformer/vcsv_transformer.cpp b/be/src/format/transformer/vcsv_transformer.cpp index 97ccf54f4046b9..90bded1a8f713e 100644 --- a/be/src/format/transformer/vcsv_transformer.cpp +++ b/be/src/format/transformer/vcsv_transformer.cpp @@ -26,6 +26,7 @@ #include "common/status.h" #include "core/block/column_with_type_and_name.h" #include "core/column/column_string.h" +#include "core/data_type/data_type_file.h" #include "core/data_type/primitive_type.h" #include "core/data_type_serde/data_type_serde.h" #include "core/string_buffer.hpp" @@ -84,6 +85,12 @@ VCSVTransformer::VCSVTransformer(RuntimeState* state, doris::io::FileWriter* fil } Status VCSVTransformer::open() { + for (const auto& context : _output_vexpr_ctxs) { + if (contains_file_type(context->root()->data_type())) { + return Status::NotSupported("CSV output does not support FILE"); + } + } + RETURN_IF_ERROR(get_block_compression_codec(_compress_type, &_compress_codec)); if (_with_bom) { Slice bom_slice(reinterpret_cast(bom), sizeof(bom)); diff --git a/be/src/format/transformer/vparquet_writer.cpp b/be/src/format/transformer/vparquet_writer.cpp index 7ec34b7e3b9507..0ce809e2c5a97f 100644 --- a/be/src/format/transformer/vparquet_writer.cpp +++ b/be/src/format/transformer/vparquet_writer.cpp @@ -36,6 +36,7 @@ #include "common/config.h" #include "common/status.h" +#include "core/data_type/data_type_file.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" #include "format/arrow/arrow_row_batch.h" @@ -299,6 +300,14 @@ arrow::Status VParquetWriter::_open_file_writer() { } Status VParquetWriter::open() { + for (const auto& context : _output_vexpr_ctxs) { + const auto& type = context->root()->data_type(); + if (contains_file_type(type)) { + return Status::NotSupported("Parquet output does not support type {}", + type->get_name()); + } + } + _timezone = _state->timezone(); _timezone_obj = _state->timezone_obj(); if (_parquet_options.enable_int96_timestamps && _parquet_options.int96_timezone.has_value()) { diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index abd1a277dd9339..6cd851643de65d 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -2896,6 +2896,11 @@ std::map BlockFileCache::get_stats_unsafe() { return stats; } +template FileBlockCell* BlockFileCache::get_cell(const UInt128Wrapper& hash, size_t offset, + std::lock_guard& cache_lock); +template FileBlockCell* BlockFileCache::get_cell(const UInt128Wrapper& hash, size_t offset, + std::unique_lock& cache_lock); + template void BlockFileCache::remove(FileBlockSPtr file_block, std::lock_guard& cache_lock, std::lock_guard& block_lock, bool sync); diff --git a/be/src/io/fs/file_system.cpp b/be/src/io/fs/file_system.cpp index e6b5ef7df1a8f5..c5d81220997bcf 100644 --- a/be/src/io/fs/file_system.cpp +++ b/be/src/io/fs/file_system.cpp @@ -77,6 +77,20 @@ Status FileSystem::file_size(const Path& file, int64_t* file_size) const { FILESYSTEM_M(file_size_impl(path, file_size)); } +Status FileSystem::stat(const Path& path, FileStat* metadata, FileStatContext* context) const { + if (context && context->is_cancelled && context->is_cancelled()) { + return Status::Cancelled("File stat cancelled"); + } + Path fs_path; + RETURN_IF_ERROR(absolute_path(path, fs_path)); + FileStatContext local_context; + FILESYSTEM_M(stat_impl(fs_path, metadata, context ? context : &local_context)); +} + +Status FileSystem::stat_impl(const Path& path, FileStat* metadata, FileStatContext* context) const { + return Status::NotSupported("FileSystem::stat metadata is not supported by this backend"); +} + Status FileSystem::list(const Path& dir, bool only_file, std::vector* files, bool* exists) { Path path; diff --git a/be/src/io/fs/file_system.h b/be/src/io/fs/file_system.h index 2cf63b2ff008db..6455ac40b65617 100644 --- a/be/src/io/fs/file_system.h +++ b/be/src/io/fs/file_system.h @@ -21,7 +21,9 @@ #include #include +#include #include +#include #include #include @@ -80,6 +82,18 @@ struct FileInfo { bool is_file; }; +struct FileStat { + int64_t size = 0; + std::optional content_type {}; + // Provider ETag as an opaque ETAG:digest, when available. + std::optional checksum {}; +}; + +// Query-owned cancellation, checked before and after the stock metadata operation. +struct FileStatContext { + std::function is_cancelled {}; +}; + // `FileSystem` providing an interface to access FS metadata. Each `RemoteFileSystem` instance // (except temporary Remote FS) corresponds to a `StorageResource` configured in FE. class FileSystem { @@ -100,6 +114,8 @@ class FileSystem { Status batch_delete(const std::vector& files); Status exists(const Path& path, bool* res) const; Status file_size(const Path& file, int64_t* file_size) const; + // On error, metadata is unchanged. + Status stat(const Path& path, FileStat* metadata, FileStatContext* context = nullptr) const; Status list(const Path& dir, bool only_file, std::vector* files, bool* exists); Status rename(const Path& orig_name, const Path& new_name); @@ -147,6 +163,8 @@ class FileSystem { /// return ERR otherwise virtual Status file_size_impl(const Path& file, int64_t* file_size) const = 0; + virtual Status stat_impl(const Path& path, FileStat* metadata, FileStatContext* context) const; + /// return OK and list all objects in "dir", save in "files" /// return ERR otherwise /// will not traverse dir recursively. diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp index 4797b573961a77..2eba694501a481 100644 --- a/be/src/io/fs/s3_file_system.cpp +++ b/be/src/io/fs/s3_file_system.cpp @@ -35,6 +35,7 @@ #include "common/config.h" #include "common/logging.h" #include "common/status.h" +#include "core/value/file_value.h" #include "cpp/obj-client/obj_storage_client.h" #include "cpp/obj-client/s3_common.h" #include "cpp/sync_point.h" @@ -292,6 +293,32 @@ Status S3FileSystem::file_size_impl(const Path& file, int64_t* file_size) const return Status::OK(); } +Status S3FileSystem::stat_impl(const Path& path, FileStat* metadata, + FileStatContext* context) const { + S3URI uri(path.native()); + RETURN_IF_ERROR(uri.parse(true)); + const auto& key = uri.get_key(); + const auto client = _client->get(); + CHECK_S3_CLIENT(client); + const ObjStoragePath object {.bucket = _bucket, .key = key}; + if (context->is_cancelled && context->is_cancelled()) { + return Status::Cancelled("File stat cancelled"); + } + auto result = client->head_object(object); + if (context->is_cancelled && context->is_cancelled()) { + return Status::Cancelled("File stat cancelled"); + } + if (!result.resp.ok()) { + return std::move( + Status(result.resp.status.code, std::move(result.resp.status.msg)) + .append(fmt::format("failed to head s3 file {}", full_s3_path(key)))); + } + FileStat value {.size = result.file_size, .content_type = std::move(result.content_type)}; + value.checksum = file_checksum_from_etag(result.etag); + *metadata = std::move(value); + return Status::OK(); +} + Status S3FileSystem::list_impl(const Path& dir, bool only_file, std::vector* files, bool* exists) { // For object storage, this path is always not exist. diff --git a/be/src/io/fs/s3_file_system.h b/be/src/io/fs/s3_file_system.h index d1b969eedeca95..5cad8cdf02df89 100644 --- a/be/src/io/fs/s3_file_system.h +++ b/be/src/io/fs/s3_file_system.h @@ -103,6 +103,7 @@ class S3FileSystem final : public RemoteFileSystem { Status batch_delete_impl(const std::vector& files) override; Status exists_impl(const Path& path, bool* res) const override; Status file_size_impl(const Path& file, int64_t* file_size) const override; + Status stat_impl(const Path& path, FileStat* metadata, FileStatContext* context) const override; Status list_impl(const Path& dir, bool only_file, std::vector* files, bool* exists) override; Status rename_impl(const Path& orig_name, const Path& new_name) override; diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp index 5437711b894d12..8046e3814ba890 100644 --- a/be/src/runtime/fragment_mgr.cpp +++ b/be/src/runtime/fragment_mgr.cpp @@ -533,6 +533,9 @@ Status FragmentMgr::_get_or_create_query_ctx(const TPipelineFragmentParams& para if (params.__isset.ai_resources) { query_ctx->set_ai_resources(params.ai_resources); } + if (params.__isset.s3_resources) { + query_ctx->set_s3_resources(params.s3_resources); + } RETURN_IF_ERROR(query_ctx->set_workload_group(workload_group_ptr)); diff --git a/be/src/runtime/query_context.h b/be/src/runtime/query_context.h index f735c2f7a4a1a6..10472550ad8809 100644 --- a/be/src/runtime/query_context.h +++ b/be/src/runtime/query_context.h @@ -290,6 +290,15 @@ class QueryContext : public std::enable_shared_from_this { return _ai_resources; } + void set_s3_resources(std::map s3_resources) { + _s3_resources = + std::make_shared>(std::move(s3_resources)); + } + + const std::shared_ptr>& get_s3_resources() const { + return _s3_resources; + } + std::unordered_map> get_using_brpc_stubs() { std::lock_guard lock(_brpc_stubs_mutex); @@ -402,6 +411,7 @@ class QueryContext : public std::enable_shared_from_this { std::shared_ptr> _ai_resources; mutable std::mutex _ai_statistics_mutex; std::unordered_map> _ai_statistics; + std::shared_ptr> _s3_resources; void _report_query_profile(); diff --git a/be/src/util/s3_uri.cpp b/be/src/util/s3_uri.cpp index 16c3bf3ffda47f..a6c2860ca7f096 100644 --- a/be/src/util/s3_uri.cpp +++ b/be/src/util/s3_uri.cpp @@ -37,11 +37,16 @@ const std::string S3URI::_FRAGMENT_DELIM = "#"; /// _schema: s3 /// _bucket: bucket1 /// _key: path/to/file.txt -Status S3URI::parse() { +Status S3URI::parse(bool preserve_key) { if (_location.empty()) { return Status::InvalidArgument("location is empty"); } - std::vector scheme_split = absl::StrSplit(_location, _SCHEME_DELIM); + std::vector scheme_split; + if (preserve_key) { + scheme_split = absl::StrSplit(_location, absl::MaxSplits(_SCHEME_DELIM, 1)); + } else { + scheme_split = absl::StrSplit(_location, _SCHEME_DELIM); + } std::string rest; if (scheme_split.size() == 2) { if (scheme_split[0] == _SCHEME_S3) { @@ -49,23 +54,28 @@ Status S3URI::parse() { rest = scheme_split[1]; std::vector authority_split = absl::StrSplit(rest, absl::MaxSplits(_PATH_DELIM, 1)); - if (authority_split.size() < 1) { + if (authority_split.empty()) { return Status::InvalidArgument("Invalid S3 URI: {}", _location); } _bucket = authority_split[0]; // support s3://bucket1 - _key = authority_split.size() == 1 ? "/" : authority_split[1]; + if (authority_split.size() == 1) { + _key = preserve_key ? "" : "/"; + } else { + _key = authority_split[1]; + } } else if (scheme_split[0] == _SCHEME_HTTP || scheme_split[0] == _SCHEME_HTTPS) { // has scheme, eg: http(s)://host/bucket1/path/to/file.txt rest = scheme_split[1]; std::vector authority_split = absl::StrSplit(rest, absl::MaxSplits(_PATH_DELIM, 2)); - if (authority_split.size() != 3) { + if (authority_split.size() != 3 && + !(preserve_key && authority_split.size() == 2 && !authority_split[1].empty())) { return Status::InvalidArgument("Invalid S3 HTTP URI: {}", _location); } - // authority_split[1] is host + // authority_split[0] is host _bucket = authority_split[1]; - _key = authority_split[2]; + _key = authority_split.size() == 3 ? authority_split[2] : ""; } else { return Status::InvalidArgument("Invalid S3 URI: {}", _location); } @@ -76,10 +86,15 @@ Status S3URI::parse() { } else { return Status::InvalidArgument("Invalid S3 URI: {}", _location); } - absl::StripAsciiWhitespace(&_key); - if (_key.empty()) { + if (!preserve_key) { + absl::StripAsciiWhitespace(&_key); + } + if (_key.empty() && !(preserve_key && !_bucket.empty())) { return Status::InvalidArgument("Invalid S3 key: {}", _location); } + if (preserve_key) { + return Status::OK(); + } // Strip query and fragment if they exist std::vector _query_split = absl::StrSplit(_key, _QUERY_DELIM); std::vector _fragment_split = absl::StrSplit(_query_split[0], _FRAGMENT_DELIM); diff --git a/be/src/util/s3_uri.h b/be/src/util/s3_uri.h index 769053450bac36..961a3bf6841c2e 100644 --- a/be/src/util/s3_uri.h +++ b/be/src/util/s3_uri.h @@ -34,7 +34,8 @@ namespace doris { class S3URI { public: S3URI(const std::string& location) : _location(location) {} - Status parse(); + // Preserve object keys verbatim when the path denotes an object rather than a URL. + Status parse(bool preserve_key = false); const std::string& get_bucket() const { return _bucket; } const std::string& get_key() const { return _key; } const std::string& get_location() const { return _location; } diff --git a/be/test/core/column/column_file_test.cpp b/be/test/core/column/column_file_test.cpp new file mode 100644 index 00000000000000..ec4a51780ee5f5 --- /dev/null +++ b/be/test/core/column/column_file_test.cpp @@ -0,0 +1,612 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/column/column_file.h" + +#include + +#include +#include +#include +#include + +#include "common/exception.h" +#include "core/arena.h" +#include "core/column/column_array.h" +#include "core/column/column_const.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" +#include "core/column/column_varbinary.h" +#include "core/column/column_vector.h" +#include "core/data_type/primitive_type.h" +#include "exec/common/sip_hash.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/thread_context.h" +#include "vec/core/hybrid_sorter.h" + +namespace doris { + +class ColumnFileTest : public ::testing::Test { +protected: + // Exercise both StringView's small value and arena-backed representations. + const std::string small_bytes = std::string("\0\xff\x80z", 4); + const std::string large_bytes = std::string(4096, '\xff') + std::string("\0end", 4); + + static MutableColumns children() { + MutableColumns result; + result.push_back(ColumnNullable::create(ColumnString::create(), ColumnUInt8::create())); + result.push_back(ColumnNullable::create(ColumnInt64::create(), ColumnUInt8::create())); + result.push_back(ColumnNullable::create(ColumnInt64::create(), ColumnUInt8::create())); + result.push_back(ColumnNullable::create(ColumnString::create(), ColumnUInt8::create())); + result.push_back(ColumnNullable::create(ColumnString::create(), ColumnUInt8::create())); + result.push_back(ColumnNullable::create(ColumnVarbinary::create(), ColumnUInt8::create())); + return result; + } + + static Field file(const std::string* bytes, bool metadata = true) { + File value(6); + value[0] = Field::create_field("s3://bucket/a%2Fb?versionId=AbC"); + if (metadata) { + value[1] = Field::create_field(7); + value[2] = Field::create_field(1024); + value[3] = Field::create_field("application/octet-stream"); + value[4] = Field::create_field("ETAG:opaque-2"); + } + if (bytes != nullptr) { + value[5] = Field::create_field( + StringView(bytes->data(), cast_set(bytes->size()))); + } + return Field::create_field(std::move(value)); + } + + static void expect_file(const IColumn& column, size_t row, const Field& expected) { + const auto actual = column[row]; + ASSERT_EQ(actual.get_type(), TYPE_FILE); + const auto& fields = actual.get(); + const auto& expected_fields = expected.get(); + ASSERT_EQ(fields.size(), 6); + for (size_t i = 0; i < fields.size(); ++i) { + SCOPED_TRACE(i); + ASSERT_EQ(fields[i].is_null(), expected_fields[i].is_null()); + if (expected_fields[i].is_null()) { + continue; + } + ASSERT_EQ(fields[i].get_type(), expected_fields[i].get_type()); + if (i == 5) { + EXPECT_EQ(fields[i].get().to_string_ref().to_string(), + expected_fields[i].get().to_string_ref().to_string()); + } else if (i == 1 || i == 2) { + EXPECT_EQ(fields[i].get(), expected_fields[i].get()); + } else { + EXPECT_EQ(fields[i].get(), expected_fields[i].get()); + } + } + } + + static uint64_t sip_hash(const IColumn& column, size_t row) { + SipHash hash; + column.update_hash_with_value(row, hash); + return hash.get64(); + } + + template + static void expect_not_implemented(F&& operation) { + try { + operation(); + FAIL() << "FILE ordering must fail"; + } catch (const Exception& e) { + EXPECT_EQ(e.code(), ErrorCode::NOT_IMPLEMENTED_ERROR); + } + } +}; + +TEST_F(ColumnFileTest, PredicateSelectorPreservesRowsAndOwnedInline) { + auto source = ColumnFile::create(); + source->insert(file(&large_bytes)); + source->insert(file(nullptr, false)); + source->insert(file(&small_bytes)); + const std::array selector {2, 0, 2, 1}; + auto selected = ColumnFile::create(); + ASSERT_TRUE(source->filter_by_selector(selector.data(), selector.size(), selected.get()).ok()); + source->clear(); + ASSERT_EQ(selected->size(), 4); + expect_file(*selected, 0, file(&small_bytes)); + expect_file(*selected, 1, file(&large_bytes)); + expect_file(*selected, 2, file(&small_bytes)); + expect_file(*selected, 3, file(nullptr, false)); + auto empty = ColumnFile::create(); + ASSERT_TRUE(selected->filter_by_selector(nullptr, 0, empty.get()).ok()); + EXPECT_TRUE(empty->empty()); +} + +TEST_F(ColumnFileTest, PredicateSelectorPreservesParentNullsAndEmptyInline) { + auto source = ColumnNullable::create(ColumnFile::create(), ColumnUInt8::create()); + const std::string empty; + source->insert(file(&large_bytes)); + source->insert_default(); + source->insert(file(&empty)); + source->insert(file(nullptr, false)); + const std::array selector {3, 1, 2, 0, 1}; + auto selected = source->clone_empty(); + ASSERT_TRUE(source->filter_by_selector(selector.data(), selector.size(), selected.get()).ok()); + source->clear(); + ASSERT_EQ(selected->size(), 5); + expect_file(*selected, 0, file(nullptr, false)); + EXPECT_TRUE(selected->is_null_at(1)); + expect_file(*selected, 2, file(&empty)); + expect_file(*selected, 3, file(&large_bytes)); + EXPECT_TRUE(selected->is_null_at(4)); +} + +TEST_F(ColumnFileTest, OwnedInlineRemainsTrackedUntilLastFieldCopyIsReleased) { + auto tracker = MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER, + "FILE-inline-ownership-test"); + SCOPED_ATTACH_TASK(tracker); + const auto consumption = [&] { + thread_context()->thread_mem_tracker_mgr->flush_untracked_mem(); + return tracker->consumption(); + }; + const auto baseline = consumption(); + Field retained; + { + auto source = ColumnFile::create(); + source->insert(file(&large_bytes)); + retained = (*source)[0]; + } + // The source's tracked arena is gone, but the extracted FILE owns its payload. + const auto owned = consumption(); + EXPECT_GE(owned - baseline, large_bytes.size()); + Field copy = retained; + // VARBINARY Field copies own and track their bytes independently. + EXPECT_GE(consumption() - owned, large_bytes.size()); + retained = Field(); + EXPECT_EQ(consumption(), owned); + EXPECT_EQ(copy.get()[5].get().to_string_ref().to_string(), + large_bytes); + copy = Field(); + EXPECT_EQ(consumption(), baseline); +} + +TEST_F(ColumnFileTest, FixedNullableShapeAndIndependentIdentity) { + static_assert(!std::is_base_of_v); + auto column = ColumnFile::create(children()); + EXPECT_EQ(column->tuple_size(), 6); + EXPECT_EQ(column->get_columns().size(), 6); + EXPECT_EQ(column->size(), 0); + auto wide_children = children(); + wide_children[0] = ColumnNullable::create(ColumnString64::create(), ColumnUInt8::create()); + auto wide = ColumnFile::create(std::move(wide_children)); + EXPECT_NE(dynamic_cast( + &assert_cast(wide->get_column(0)).get_nested_column()), + nullptr); + column->insert_default(); + for (size_t i = 0; i < 6; ++i) { + const auto& child = assert_cast(column->get_column(i)); + EXPECT_TRUE(child.is_null_at(0)); + if (i == 5) { + EXPECT_NE(dynamic_cast(&child.get_nested_column()), nullptr); + } else if (i == 1 || i == 2) { + EXPECT_NE(dynamic_cast(&child.get_nested_column()), nullptr); + } else { + EXPECT_NE(dynamic_cast(&child.get_nested_column()), nullptr); + } + } + column->sanity_check(); +} + +TEST_F(ColumnFileTest, RejectMalformedPhysicalChildren) { + auto wrong_count = children(); + wrong_count.pop_back(); + EXPECT_THROW(ColumnFile::create(std::move(wrong_count)), Exception); + auto non_nullable = children(); + non_nullable[0] = ColumnString::create(); + EXPECT_THROW(ColumnFile::create(std::move(non_nullable)), Exception); + auto wrong_binary = children(); + wrong_binary[5] = ColumnNullable::create(ColumnString::create(), ColumnUInt8::create()); + EXPECT_THROW(ColumnFile::create(std::move(wrong_binary)), Exception); + auto wrong_integer = children(); + wrong_integer[1] = ColumnNullable::create(ColumnInt32::create(), ColumnUInt8::create()); + EXPECT_THROW(ColumnFile::create(std::move(wrong_integer)), Exception); + auto constant = children(); + constant[0]->insert_default(); + constant[0] = ColumnConst::create(std::move(constant[0]), 0); + EXPECT_THROW(ColumnFile::create(std::move(constant)), Exception); + auto uneven = children(); + uneven[2]->insert_default(); + EXPECT_THROW(ColumnFile::create(std::move(uneven)), Exception); + + auto column = ColumnFile::create(); + EXPECT_THROW(column->insert(Field::create_field(File(5))), Exception); + EXPECT_EQ(column->size(), 0); +} + +TEST_F(ColumnFileTest, FieldRoundTripPreservesNullEmptyAndBinary) { + const std::string empty; + auto column = ColumnFile::create(); + std::vector values {file(nullptr, false), file(&empty), file(&small_bytes), + file(&large_bytes)}; + for (const auto& value : values) { + column->insert(value); + } + for (size_t i = 0; i < values.size(); ++i) { + expect_file(*column, i, values[i]); + } + Field result; + column->get(3, result); + auto copy = ColumnFile::create(); + copy->insert(result); + column->clear(); + expect_file(*copy, 0, values[3]); +} + +TEST_F(ColumnFileTest, FileFieldsOwnInlineBytesAcrossColumnReuseAndFieldCopies) { + std::string source = large_bytes; + auto value = file(&source); + source.assign(source.size(), 'x'); + const auto& bytes = value.get()[5].get(); + EXPECT_EQ(bytes.str(), large_bytes); + + auto column = ColumnFile::create(); + column->insert(value); + Field extracted; + column->get(0, extracted); + Field copied(extracted); + Field assigned = file(nullptr); + assigned = extracted; + Field nested = Field::create_field(Array {extracted}); + column->clear(); + column->insert(file(&source)); + extracted = Field(); + value = Field(); + + Field moved(std::move(copied)); + Field move_assigned = file(nullptr); + move_assigned = std::move(assigned); + auto restored = ColumnFile::create(); + restored->insert(moved); + restored->insert(move_assigned); + restored->insert(nested.get()[0]); + const auto expected = file(&large_bytes); + for (size_t row = 0; row < restored->size(); ++row) { + expect_file(*restored, row, expected); + } +} + +TEST_F(ColumnFileTest, TransportAndReplicationKeepEveryChildAligned) { + auto source = ColumnFile::create(); + const auto a = file(&small_bytes); + const auto b = file(&large_bytes, false); + source->insert(a); + source->insert(b); + auto target = ColumnFile::create(); + target->insert_from(*source, 1); + target->insert_range_from(*source, 0, 2); + target->insert_range_from_ignore_overflow(*source, 1, 1); + target->insert_many_from(*source, 0, 2); + const uint32_t indices[] = {1, 0, 1}; + target->insert_indices_from(*source, indices, indices + 3); + target->insert_duplicate_fields(b, 2); + const std::vector expected {b, a, b, b, a, a, b, a, b, b, b}; + source->clear(); + ASSERT_EQ(target->size(), expected.size()); + for (size_t i = 0; i < expected.size(); ++i) { + expect_file(*target, i, expected[i]); + } + target->sanity_check(); +} + +TEST_F(ColumnFileTest, FilterPermuteEraseAndResize) { + auto source = ColumnFile::create(); + const auto a = file(nullptr, false); + const auto b = file(&small_bytes); + const auto c = file(&large_bytes); + source->insert(a); + source->insert(b); + source->insert(c); + const IColumn::Filter filter {1, 0, 1}; + auto filtered = source->filter(filter, -1); + ASSERT_EQ(filtered->size(), 2); + expect_file(*filtered, 0, a); + expect_file(*filtered, 1, c); + IColumn::Permutation permutation {2, 0, 1}; + auto permuted = source->permute(permutation, 2); + ASSERT_EQ(permuted->size(), 2); + expect_file(*permuted, 0, c); + expect_file(*permuted, 1, a); + EXPECT_EQ(source->filter(filter), 2); + expect_file(*source, 1, c); + source->erase(0, 1); + expect_file(*source, 0, c); + source->reserve(8); + source->resize(3); + auto resized = source->clone_resized(5); + const auto placeholder = Field::create_field(File(6)); + for (size_t i = 1; i < 5; ++i) { + expect_file(*resized, i, placeholder); + } + source->resize(1); + expect_file(*source, 0, c); + source->pop_back(1); + EXPECT_EQ(source->size(), 0); + source->sanity_check(); + auto empty = source->clone_empty(); + EXPECT_TRUE(empty->empty()); + EXPECT_NE(dynamic_cast(empty.get()), nullptr); +} + +TEST_F(ColumnFileTest, InPlaceFilterReusesInlineArenaAcrossRepeatedFilters) { + const std::string payload(1024 * 1024, '\xff'); + const std::string empty; + auto source = ColumnFile::create(); + source->insert(file(&payload)); + source->insert(file(nullptr)); + source->insert(file(&empty)); + source->insert(file(&small_bytes)); + source->insert(file(&payload)); + auto& inline_column = assert_cast(source->get_column(5)); + auto& binary = assert_cast(inline_column.get_nested_column()); + const auto* first = binary.get_data()[0].data(); + const auto* last = binary.get_data()[4].data(); + const auto allocated = binary.allocated_bytes(); + const auto bytes = binary.byte_size(); + EXPECT_EQ(source->filter(IColumn::Filter {1, 1, 1, 0, 1}), 4); + EXPECT_EQ(binary.get_data()[0].data(), first); + EXPECT_EQ(binary.get_data()[3].data(), last); + EXPECT_EQ(binary.allocated_bytes(), allocated); + EXPECT_LE(binary.byte_size(), bytes); + expect_file(*source, 0, file(&payload)); + expect_file(*source, 1, file(nullptr)); + expect_file(*source, 2, file(&empty)); + expect_file(*source, 3, file(&payload)); + EXPECT_EQ(source->filter(IColumn::Filter {0, 1, 1, 1}), 3); + EXPECT_EQ(binary.get_data()[2].data(), last); + EXPECT_EQ(binary.allocated_bytes(), allocated); + EXPECT_TRUE(inline_column.has_null()); + EXPECT_EQ(source->filter(IColumn::Filter {0, 1, 1}), 2); + EXPECT_FALSE(inline_column.has_null()); + EXPECT_EQ(binary.get_data()[1].data(), last); + EXPECT_EQ(binary.allocated_bytes(), allocated); + source->sanity_check(); + source->erase(0, 1); + expect_file(*source, 0, file(&payload)); + EXPECT_EQ(binary.get_data()[0].data(), last); + EXPECT_EQ(binary.allocated_bytes(), allocated); + EXPECT_EQ(source->filter(IColumn::Filter {1}), 1); + EXPECT_EQ(binary.get_data()[0].data(), last); + EXPECT_EQ(binary.allocated_bytes(), allocated); + EXPECT_EQ(source->filter(IColumn::Filter {0}), 0); + EXPECT_TRUE(binary.empty()); + EXPECT_EQ(source->filter(IColumn::Filter {}), 0); + source->sanity_check(); +} + +TEST_F(ColumnFileTest, InPlaceFilterPreservesCopyOnWriteAndIndependentOutput) { + auto source = ColumnFile::create(); + source->insert(file(&large_bytes)); + source->insert(file(&small_bytes)); + source->insert(file(nullptr)); + ColumnPtr original = std::move(source); + auto detached = IColumn::mutate(original); + auto& binary = assert_cast( + assert_cast(assert_cast(*detached).get_column(5)) + .get_nested_column()); + const auto* payload = binary.get_data()[0].data(); + const auto allocated = binary.allocated_bytes(); + auto independent = detached->filter(IColumn::Filter {1, 0, 0}, -1); + ASSERT_EQ(detached->filter(IColumn::Filter {1, 0, 1}), 2); + EXPECT_EQ(binary.get_data()[0].data(), payload); + EXPECT_EQ(binary.allocated_bytes(), allocated); + expect_file(*original, 0, file(&large_bytes)); + expect_file(*original, 1, file(&small_bytes)); + expect_file(*original, 2, file(nullptr)); + detached->clear(); + expect_file(*independent, 0, file(&large_bytes)); + expect_file(*original, 0, file(&large_bytes)); +} + +TEST_F(ColumnFileTest, CopyOnWriteDetachesParentAndSharedChild) { + auto original = ColumnFile::create(); + const auto value = file(&large_bytes); + original->insert(value); + ColumnPtr snapshot = std::move(original); + ColumnPtr shared_child = assert_cast(*snapshot).get_column_ptr(5); + auto changed = IColumn::mutate(snapshot); + changed->clear(); + changed->insert(file(&small_bytes)); + expect_file(*snapshot, 0, value); + expect_file(*changed, 0, file(&small_bytes)); + EXPECT_EQ((*shared_child)[0].get().to_string_ref().to_string(), large_bytes); + + auto unique = ColumnFile::create(); + unique->insert(value); + ColumnPtr child_only = std::as_const(*unique).get_column_ptr(5); + ColumnPtr owner = std::move(unique); + auto detached = IColumn::mutate(std::move(owner)); + EXPECT_TRUE(detached->is_exclusive()); + detached->clear(); + EXPECT_EQ((*child_only)[0].get().to_string_ref().to_string(), large_bytes); +} + +TEST_F(ColumnFileTest, ArenaAndBatchRowSerialization) { + const std::string empty; + const std::vector values {file(nullptr), file(&empty), file(&small_bytes), + file(&large_bytes, false)}; + auto source = ColumnFile::create(); + for (const auto& value : values) { + source->insert(value); + } + Arena arena; + auto arena_copy = ColumnFile::create(); + std::vector> buffers(values.size()); + std::vector keys(values.size()); + for (size_t i = 0; i < values.size(); ++i) { + const char* begin = nullptr; + const auto serialized = source->serialize_value_into_arena(i, arena, begin); + EXPECT_EQ(serialized.size, source->serialize_size_at(i)); + EXPECT_LE(serialized.size, source->get_max_row_byte_size()); + EXPECT_EQ(arena_copy->deserialize_and_insert_from_arena(serialized.data), + serialized.data + serialized.size); + // A preceding serialized column already occupies one byte in each row. + buffers[i].resize(serialized.size + 1); + buffers[i][0] = 'p'; + keys[i] = StringRef(buffers[i].data(), 1); + } + source->serialize_vec(keys.data(), keys.size()); + for (size_t i = 0; i < keys.size(); ++i) { + EXPECT_EQ(buffers[i][0], 'p'); + EXPECT_EQ(keys[i].size, buffers[i].size()); + ++keys[i].data; + --keys[i].size; + } + auto batch_copy = ColumnFile::create(); + batch_copy->deserialize_vec(keys.data(), keys.size()); + source->clear(); + for (size_t i = 0; i < values.size(); ++i) { + EXPECT_EQ(keys[i].size, 0); + expect_file(*arena_copy, i, values[i]); + expect_file(*batch_copy, i, values[i]); + } +} + +TEST_F(ColumnFileTest, OuterNullAndNestedArraySerialization) { + auto nullable = ColumnNullable::create(ColumnFile::create(), ColumnUInt8::create()); + const auto value = file(&large_bytes); + nullable->insert_default(); + nullable->insert(value); + auto array = ColumnArray::create(std::move(nullable), ColumnArray::ColumnOffsets::create()); + array->get_offsets().push_back(2); + array->get_offsets().push_back(2); + Arena arena; + auto copy = array->clone_empty(); + for (size_t i = 0; i < 2; ++i) { + const char* begin = nullptr; + auto serialized = array->serialize_value_into_arena(i, arena, begin); + EXPECT_EQ(copy->deserialize_and_insert_from_arena(serialized.data), + serialized.data + serialized.size); + } + const auto& copied_array = assert_cast(*copy); + EXPECT_EQ(copied_array.get_offsets()[0], 2); + EXPECT_EQ(copied_array.get_offsets()[1], 2); + const auto& data = assert_cast(copied_array.get_data()); + EXPECT_TRUE(data.is_null_at(0)); + EXPECT_FALSE(data.is_null_at(1)); + expect_file(data.get_nested_column(), 1, value); +} + +TEST_F(ColumnFileTest, IntegrityHashIncludesEveryChildAndInlineNullness) { + auto column = ColumnFile::create(); + const auto original = file(&large_bytes); + column->insert(original); + for (size_t i = 0; i < 6; ++i) { + auto modified = original; + modified.get()[i] = Field(); + column->insert(modified); + EXPECT_NE(sip_hash(*column, 0), sip_hash(*column, i + 1)); + } + const std::string empty; + column->insert(file(&empty)); + column->insert(file(&small_bytes)); + EXPECT_NE(sip_hash(*column, 6), sip_hash(*column, 7)); + EXPECT_NE(sip_hash(*column, 0), sip_hash(*column, 8)); + auto copy = column->clone_resized(column->size()); + for (size_t i = 0; i < column->size(); ++i) { + EXPECT_EQ(sip_hash(*column, i), sip_hash(*copy, i)); + } +} + +TEST_F(ColumnFileTest, BatchAndRangeHashesSkipOuterNullPayload) { + auto column = ColumnFile::create(); + column->insert(file(&small_bytes)); + column->insert(file(&large_bytes)); + const uint8_t null_map[] = {0, 1}; + std::array xx {17, 17}; + std::array crc {17, 17}; + std::array crc32c {17, 17}; + column->update_hashes_with_value(xx.data(), null_map); + column->update_crcs_with_value(crc.data(), TYPE_FILE, 2, 0, null_map); + column->update_crc32c_batch(crc32c.data(), null_map); + uint64_t expected_xx = 17; + uint32_t expected_crc = 17; + uint32_t expected_crc32c = 17; + column->update_xxHash_with_value(0, 1, expected_xx, nullptr); + column->update_crc_with_value(0, 1, expected_crc, nullptr); + column->update_crc32c_single(0, 1, expected_crc32c, nullptr); + EXPECT_EQ(xx[0], expected_xx); + EXPECT_EQ(crc[0], expected_crc); + EXPECT_EQ(crc32c[0], expected_crc32c); + EXPECT_NE(xx[0], 17); + EXPECT_NE(crc[0], 17); + EXPECT_NE(crc32c[0], 17); + EXPECT_EQ(xx[1], 17); + EXPECT_EQ(crc[1], 17); + EXPECT_EQ(crc32c[1], 17); + uint64_t masked_xx = 17; + uint32_t masked_crc = 17; + uint32_t masked_crc32c = 17; + column->update_xxHash_with_value(0, 2, masked_xx, null_map); + column->update_crc_with_value(0, 2, masked_crc, null_map); + column->update_crc32c_single(0, 2, masked_crc32c, null_map); + EXPECT_EQ(masked_xx, expected_xx); + EXPECT_EQ(masked_crc, expected_crc); + EXPECT_EQ(masked_crc32c, expected_crc32c); +} + +TEST_F(ColumnFileTest, RejectComparisonAndSortingEvenWhenEmpty) { + auto column = ColumnFile::create(); + HybridSorter sorter; + IColumn::Permutation permutation; + EqualFlags flags; + EqualRange range {0, 0}; + for (size_t rows = 0; rows < 2; ++rows) { + expect_not_implemented([&] { column->compare_at(0, 0, *column, 1); }); + expect_not_implemented([&] { column->get_permutation(false, 0, 1, sorter, permutation); }); + expect_not_implemented( + [&] { column->sort_column(nullptr, flags, permutation, range, true); }); + column->insert(file(&small_bytes)); + } +} + +TEST_F(ColumnFileTest, OuterNullHashesIgnoreHiddenPayload) { + auto first = ColumnNullable::create(ColumnFile::create(), ColumnUInt8::create()); + auto second = ColumnNullable::create(ColumnFile::create(), ColumnUInt8::create()); + first->insert(file(&small_bytes)); + second->insert(file(&large_bytes)); + first->get_null_map_data()[0] = 1; + second->get_null_map_data()[0] = 1; + EXPECT_EQ(sip_hash(*first, 0), sip_hash(*second, 0)); + uint64_t first_xx = 17; + uint64_t second_xx = 17; + uint32_t first_crc = 17; + uint32_t second_crc = 17; + uint32_t first_crc32c = 17; + uint32_t second_crc32c = 17; + first->update_hashes_with_value(&first_xx, nullptr); + second->update_hashes_with_value(&second_xx, nullptr); + first->update_crcs_with_value(&first_crc, TYPE_FILE, 1, 0, nullptr); + second->update_crcs_with_value(&second_crc, TYPE_FILE, 1, 0, nullptr); + first->update_crc32c_batch(&first_crc32c, nullptr); + second->update_crc32c_batch(&second_crc32c, nullptr); + EXPECT_EQ(first_xx, second_xx); + EXPECT_EQ(first_crc, second_crc); + EXPECT_EQ(first_crc32c, second_crc32c); + EXPECT_NE(first_xx, 17); + EXPECT_NE(first_crc, 17); + EXPECT_NE(first_crc32c, 17); +} + +} // namespace doris diff --git a/be/test/core/data_type/data_type_file_test.cpp b/be/test/core/data_type/data_type_file_test.cpp new file mode 100644 index 00000000000000..243e75028f31f8 --- /dev/null +++ b/be/test/core/data_type/data_type_file_test.cpp @@ -0,0 +1,314 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type/data_type_file.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "agent/be_exec_version_manager.h" +#include "core/block/block.h" +#include "core/column/column_const.h" +#include "core/column/column_file.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/column/column_varbinary.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" + +namespace doris { + +TEST(DataTypeFileTest, CanonicalIdentityAndMetadata) { + DataTypeFile type; + EXPECT_EQ(type.get_primitive_type(), TYPE_FILE); + EXPECT_EQ(type.get_element_names(), + (Strings {"uri", "offset", "size", "content_type", "checksum", "inline"})); + ASSERT_EQ(type.get_elements().size(), 6); + for (const auto& child : type.get_elements()) { + EXPECT_TRUE(child->is_nullable()); + } + EXPECT_EQ(remove_nullable(type.get_element(5))->get_primitive_type(), TYPE_VARBINARY); + EXPECT_FALSE(type.equals(DataTypeStruct(type.get_elements(), type.get_element_names()))); + EXPECT_EQ(type.try_get_position_by_name("URI"), 0); + + PColumnMeta meta; + type.to_pb_column_meta(&meta); + EXPECT_EQ(meta.type(), PGenericType::FILE); + EXPECT_EQ(meta.children_size(), 6); + EXPECT_TRUE(type.equals(*DataTypeFactory::instance().create_data_type(meta))); + TTypeDesc thrift; + type.to_thrift(thrift); + ASSERT_EQ(thrift.types.size(), 7); + EXPECT_EQ(thrift.types[0].type, TTypeNodeType::FILE); + EXPECT_TRUE(type.equals(*DataTypeFactory::instance().create_data_type(thrift, false))); + PTypeDesc proto; + static_cast(type).to_protobuf(&proto); + EXPECT_TRUE(type.equals(*DataTypeFactory::instance().create_data_type(proto, false))); + // FILE metadata must agree with FE's canonical inline VARBINARY length. + EXPECT_EQ(proto.types(6).scalar_type().len(), std::numeric_limits::max()); +} + +TEST(DataTypeFileTest, RejectMalformedTypeDescriptors) { + DataTypeFile type; + TTypeDesc original; + type.to_thrift(original); + auto malformed = original; + malformed.types.pop_back(); + EXPECT_THROW(DataTypeFactory::instance().create_data_type(malformed, false), Exception); + malformed = original; + malformed.types[1].type = TTypeNodeType::ARRAY; + EXPECT_THROW(DataTypeFactory::instance().create_data_type(malformed, false), Exception); + malformed = original; + malformed.types[1].scalar_type.__set_len(1024); + EXPECT_THROW(DataTypeFactory::instance().create_data_type(malformed, false), Exception); + malformed = original; + malformed.types[0].struct_fields[0].__set_contains_null(false); + EXPECT_THROW(DataTypeFactory::instance().create_data_type(malformed, false), Exception); + malformed = original; + malformed.types[0].struct_fields[5].__set_name("payload"); + EXPECT_THROW(DataTypeFactory::instance().create_data_type(malformed, false), Exception); + + PTypeDesc proto; + static_cast(type).to_protobuf(&proto); + proto.mutable_types(6)->mutable_scalar_type()->set_type(TPrimitiveType::VARCHAR); + EXPECT_THROW(DataTypeFactory::instance().create_data_type(proto, false), Exception); +} + +TEST(DataTypeFileTest, ValidationRespectsFileNulls) { + const auto type = make_nullable(std::make_shared()); + auto column = type->create_column(); + column->insert_default(); + EXPECT_TRUE(validate_file_column(*column, type).ok()); + auto& nullable = assert_cast(*column); + nullable.get_null_map_data()[0] = 0; + EXPECT_FALSE(validate_file_column(*column, type).ok()); + nullable.get_null_map_data()[0] = 1; + auto constant = ColumnConst::create(std::move(column), 8); + EXPECT_TRUE(validate_file_column(*constant, type).ok()); +} + +namespace { +void validate_protected_inline() { + const auto page_size = static_cast(sysconf(_SC_PAGESIZE)); + auto* bytes = static_cast( + mmap(nullptr, page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); + if (bytes == MAP_FAILED) { + std::_Exit(1); + } + const auto type = std::make_shared(); + auto column = type->create_column(); + column->insert_default(); + column->insert( + Field::create_field(File {Field::create_field("urn:valid"), + Field(), Field(), Field(), Field(), Field()})); + column->insert_default(); + auto& inline_column = + assert_cast(assert_cast(*column).get_column(5)); + auto& payloads = assert_cast(inline_column.get_nested_column()).get_data(); + payloads[0] = StringView(bytes, static_cast(page_size)); + payloads[1] = StringView(bytes, static_cast(page_size)); + inline_column.get_null_map_data().assign(size_t {3}, UInt8 {0}); + if (mprotect(bytes, page_size, PROT_NONE) != 0 || !validate_file_column(*column, type).ok()) { + std::_Exit(2); + } + // Empty inline is still non-NULL; NULL uri + NULL inline is invalid. + inline_column.get_null_map_data()[2] = 1; + if (validate_file_column(*column, type).ok()) { + std::_Exit(3); + } + column->resize(1); + auto constant = ColumnConst::create(std::move(column), 8); + if (!validate_file_column(*constant, type).ok()) { + std::_Exit(4); + } + if (munmap(bytes, page_size) != 0) { + std::_Exit(5); + } + std::_Exit(0); +} +} // namespace + +TEST(DataTypeFileDeathTest, BoundaryValidationNeverReadsInlineBytes) { + // A large VARBINARY view points to a protected page; reading or copying + // its bytes during validation makes the child process fail. + ASSERT_EXIT(validate_protected_inline(), ::testing::ExitedWithCode(0), ""); +} + +TEST(DataTypeFileTest, BoundaryValidationChecksMetadataWithInline) { + const auto type = std::make_shared(); + const std::string bytes(4096, '\xff'); + File valid {Field(), Field(), Field(), + Field(), Field(), Field::create_field(StringView(bytes))}; + for (const auto& [index, bad_value] : + std::vector> {{1, Field::create_field(Int64 {-1})}, + {1, Field::create_field(Int64 {0})}, + {2, Field::create_field(Int64 {-1})}, + {3, Field::create_field("text")}, + {4, Field::create_field("MD5:abc")}}) { + auto fields = valid; + fields[index] = bad_value; + auto column = type->create_column(); + column->insert(Field::create_field(std::move(fields))); + EXPECT_FALSE(validate_file_column(*column, type).ok()) << index; + } + valid[1] = Field::create_field(std::numeric_limits::max()); + valid[2] = Field::create_field(Int64 {1}); + auto column = type->create_column(); + column->insert(Field::create_field(valid)); + EXPECT_FALSE(validate_file_column(*column, type).ok()); + valid[1] = Field::create_field(Int64 {0}); + column->clear(); + column->insert(Field::create_field(std::move(valid))); + EXPECT_TRUE(validate_file_column(*column, type).ok()); +} + +TEST(DataTypeFileTest, SerializationRetainsBinaryInlineAndNulls) { + DataTypeFile type; + auto column = type.create_column(); + const std::string bytes("\0\xff\0", 3); + column->insert(Field::create_field( + File {Field::create_field(String {"s3://bucket/object?versionId=v1"}), + Field(), Field::create_field(Int64 {0}), Field(), Field(), + Field::create_field(StringView(bytes))})); + column->insert(Field::create_field( + File {Field::create_field(String {"hdfs://host/a"}), Field(), Field(), + Field(), Field(), Field()})); + column->insert(Field::create_field( + File {Field(), Field(), Field(), Field(), Field(), + Field::create_field(StringView(bytes))})); + const auto version = BeExecVersionManager::get_newest_version(); + const auto size = type.get_uncompressed_serialized_bytes(*column, version); + std::string buffer(size, '\0'); + const char* end = type.serialize(*column, buffer.data(), version); + auto restored = type.create_column(); + EXPECT_EQ(type.deserialize(buffer.data(), &restored, version), end); + ASSERT_EQ(restored->size(), 3); + auto value = (*restored)[0]; + const auto& inline_value = value.get()[5].get(); + EXPECT_EQ(std::string(inline_value.data(), inline_value.size()), bytes); + EXPECT_TRUE((*restored)[1].get()[5].is_null()); + EXPECT_TRUE((*restored)[2].get()[0].is_null()); + EXPECT_EQ((*restored)[2].get()[5].as_string_view(), bytes); + EXPECT_TRUE(validate_file_column(*restored, std::make_shared()).ok()); +} + +TEST(DataTypeFileTest, BlockRoundTripPreservesNestedAndConstantFileValues) { + const auto file = make_nullable(std::make_shared()); + const auto array = std::make_shared(file); + const std::string bytes(4096, '\xff'); + const auto value = [&bytes](bool empty_inline) { + File fields(6); + fields[0] = Field::create_field("s3://bucket/object?versionId=v1"); + fields[1] = Field::create_field(Int64 {7}); + fields[2] = Field::create_field(Int64 {11}); + fields[3] = Field::create_field("application/octet-stream"); + fields[4] = Field::create_field("ETAG:opaque"); + fields[5] = Field::create_field(empty_inline ? StringView("", 0) + : StringView(bytes)); + return Field::create_field(std::move(fields)); + }; + + auto files = file->create_column(); + files->insert_default(); + files->insert(value(true)); + files->insert(value(false)); + auto arrays = array->create_column(); + arrays->insert(Field::create_field(Array {value(false), Field()})); + arrays->insert_default(); + arrays->insert(Field::create_field(Array {value(true), value(false)})); + auto constant_data = file->create_column(); + constant_data->insert(value(false)); + auto constant = ColumnConst::create(std::move(constant_data), 3); + Block source {{std::move(files), file, "file"}, + {std::move(arrays), array, "nested"}, + {std::move(constant), file, "constant"}}; + + for (const auto compression : {segment_v2::NO_COMPRESSION, segment_v2::SNAPPY}) { + PBlock wire; + size_t uncompressed_bytes = 0; + size_t compressed_bytes = 0; + int64_t codec_time = 0; + ASSERT_TRUE(source.serialize(BeExecVersionManager::get_newest_version(), &wire, + &uncompressed_bytes, &compressed_bytes, &codec_time, + compression) + .ok()); + Block restored; + ASSERT_TRUE(restored.deserialize(wire, &uncompressed_bytes, &codec_time).ok()); + ASSERT_EQ(restored.rows(), 3); + ASSERT_EQ(restored.columns(), 3); + EXPECT_TRUE(restored.get_by_position(0).type->equals(*file)); + EXPECT_TRUE(restored.get_by_position(1).type->equals(*array)); + EXPECT_TRUE(is_column_const(*restored.get_by_position(2).column)); + + const auto& restored_files = *restored.get_by_position(0).column; + EXPECT_TRUE(restored_files.is_null_at(0)); + const auto empty = restored_files[1]; + EXPECT_FALSE(empty.get()[5].is_null()); + EXPECT_EQ(empty.get()[5].get().size(), 0); + const auto populated = restored_files[2]; + const auto& inline_value = populated.get()[5].get(); + EXPECT_EQ(std::string(inline_value.data(), inline_value.size()), bytes); + + PBlock second_wire; + ASSERT_TRUE(restored.serialize(BeExecVersionManager::get_newest_version(), &second_wire, + &uncompressed_bytes, &compressed_bytes, &codec_time, + compression) + .ok()); + EXPECT_EQ(wire.SerializeAsString(), second_wire.SerializeAsString()); + } +} + +TEST(DataTypeFileTest, BlockDeserializePreservesValuesWithoutRevalidating) { + const auto file = std::make_shared(); + const auto parent = + make_nullable(std::make_shared(DataTypes {file}, Strings {"file"})); + auto column = parent->create_column(); + column->insert( + Field::create_field(Struct {Field::create_field(File(6))})); + auto& null_map = assert_cast(*column).get_null_map_data(); + for (const bool parent_is_null : {true, false}) { + null_map[0] = parent_is_null; + Block source {{column->get_ptr(), parent, "parent"}}; + PBlock wire; + size_t uncompressed_bytes = 0; + size_t compressed_bytes = 0; + int64_t codec_time = 0; + ASSERT_TRUE(source.serialize(BeExecVersionManager::get_newest_version(), &wire, + &uncompressed_bytes, &compressed_bytes, &codec_time, + segment_v2::SNAPPY) + .ok()); + Block restored; + const auto status = restored.deserialize(wire, &uncompressed_bytes, &codec_time); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(restored.get_by_position(0).column->is_null_at(0), parent_is_null); + } +} + +} // namespace doris diff --git a/be/test/core/data_type/storage_field_type_test.cpp b/be/test/core/data_type/storage_field_type_test.cpp index f2f40f2a265429..144105d50f70c3 100644 --- a/be/test/core/data_type/storage_field_type_test.cpp +++ b/be/test/core/data_type/storage_field_type_test.cpp @@ -91,6 +91,7 @@ TEST(StorageFieldTypeTest, UnsupportedPrimitiveTypesThrow) { static_cast(33), // TYPE_LAMBDA_FUNCTION (deprecated) PrimitiveType::TYPE_FIXED_LENGTH_OBJECT, PrimitiveType::TYPE_VARBINARY, + PrimitiveType::TYPE_FILE, static_cast(255), }; diff --git a/be/test/core/data_type_serde/data_type_file_serde_test.cpp b/be/test/core/data_type_serde/data_type_file_serde_test.cpp new file mode 100644 index 00000000000000..aa584afbf77078 --- /dev/null +++ b/be/test/core/data_type_serde/data_type_file_serde_test.cpp @@ -0,0 +1,632 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type_serde/data_type_file_serde.h" + +#include +#include +#include + +#include + +#include "core/arena.h" +#include "core/column/column_const.h" +#include "core/column/column_file.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type_serde/data_type_nullable_serde.h" +#include "core/string_buffer.hpp" +#include "core/value/jsonb_value.h" +#include "exprs/function/cast/cast_parameters.h" +#include "util/jsonb_utils.h" +#include "util/jsonb_writer.h" +#include "util/mysql_row_buffer.h" + +namespace doris { +class DataTypeFileSerDeTest : public testing::Test { +protected: + DataTypeFile type; + DataTypeFileSerDe serde; + DataTypeSerDe::FormatOptions options; + + static Field value(const std::string* bytes = nullptr) { + File fields(6); + fields[0] = Field::create_field("s3://bucket/a?versionId=AbC"); + fields[2] = Field::create_field(1024); + fields[3] = Field::create_field("image/png"); + fields[4] = Field::create_field("ETAG:opaque-2"); + if (bytes) { + fields[5] = Field::create_field( + StringView(bytes->data(), cast_set(bytes->size()))); + } + return Field::create_field(std::move(fields)); + } + + static std::string public_json(const std::string& inline_json = "null") { + return R"({"uri":"s3://bucket/a?versionId=AbC","offset":null,"size":1024,"content_type":"image/png","checksum":"ETAG:opaque-2","inline":)" + + inline_json + "}"; + } + + std::string text(const IColumn& column, size_t row, bool format) { + auto output = ColumnString::create(); + BufferWritable buffer(*output); + if (format) { + EXPECT_TRUE(serde.serialize_one_cell_to_json(column, row, buffer, options).ok()); + } else { + serde.to_string(column, row, buffer, options); + } + buffer.commit(); + return output->get_data_at(0).to_string(); + } + + Status parse(IColumn& column, const std::string& json) { + StringRef input(json); + return serde.from_string(input, column, options); + } + + Status parse_format(IColumn& column, std::string json, bool csv = false) { + Slice input(json); + return csv ? serde.deserialize_one_cell_from_csv(column, input, options) + : serde.deserialize_one_cell_from_json(column, input, options); + } + + static void expect_bytes(const IColumn& column, size_t row, const std::string* expected) { + auto field = column[row]; + ASSERT_EQ(field.get_type(), TYPE_FILE); + const auto& fields = field.get(); + ASSERT_EQ(fields.size(), 6); + ASSERT_EQ(fields[5].is_null(), expected == nullptr); + if (expected) { + EXPECT_EQ(fields[5].get().to_string_ref().to_string(), *expected); + } + } +}; + +TEST_F(DataTypeFileSerDeTest, TextRoundTripPreservesOpaqueMetadataBytes) { + for (const size_t field : {0, 4}) { + SCOPED_TRACE(field); + auto input = value(); + auto& fields = input.get(); + const std::string bytes = (field == 0 ? std::string("urn:") : std::string("ETAG:")) + + std::string("\xff\0x", 3); + fields[field] = Field::create_field(bytes); + auto source = type.create_column(); + source->insert(input); + for (const bool format : {false, true}) { + const auto encoded = text(*source, 0, format); + auto restored = type.create_column(); + ASSERT_TRUE( + (format ? parse_format(*restored, encoded) : parse(*restored, encoded)).ok()); + ASSERT_EQ(restored->size(), 1); + const auto output = (*restored)[0]; + const auto& output_fields = output.get(); + EXPECT_EQ(output_fields[field].get(), bytes); + EXPECT_EQ(output_fields[0].get(), fields[0].get()); + EXPECT_EQ(output_fields[4].get(), fields[4].get()); + expect_bytes(*restored, 0, nullptr); + } + } +} + +TEST_F(DataTypeFileSerDeTest, InlineOnlyJsonbRoundTrip) { + auto source = type.create_column(); + ASSERT_TRUE(parse(*source, R"({"inline":""})").ok()); + ASSERT_TRUE(parse(*source, R"({"uri":null,"inline":"AP8="})").ok()); + auto jsonb_restored = type.create_column(); + CastParameters parameters; + for (size_t row = 0; row < source->size(); ++row) { + const auto value = (*source)[row]; + for (size_t child = 0; child < 5; ++child) { + EXPECT_TRUE(value.get()[child].is_null()); + } + JsonbWriter writer; + ASSERT_TRUE(serde.serialize_column_to_jsonb(*source, row, writer, {}).ok()); + const JsonbDocument* document = nullptr; + ASSERT_TRUE(JsonbDocument::checkAndCreateDocument(writer.getOutput()->getBuffer(), + writer.getOutput()->getSize(), &document) + .ok()); + ASSERT_TRUE(serde.deserialize_column_from_jsonb(*jsonb_restored, document->getValue(), + parameters) + .ok()); + EXPECT_EQ(text(*jsonb_restored, row, false), text(*source, row, false)); + } +} + +TEST_F(DataTypeFileSerDeTest, InlineOnlyProtobufRoundTripAndRequiresLocator) { + auto source = type.create_column(); + ASSERT_TRUE(parse(*source, R"({"inline":""})").ok()); + ASSERT_TRUE(parse(*source, R"({"uri":null,"inline":"AP8="})").ok()); + PValues pb; + ASSERT_TRUE(serde.write_column_to_pb(*source, pb, 0, source->size()).ok()); + auto pb_restored = type.create_column(); + ASSERT_TRUE(serde.read_column_from_pb(*pb_restored, pb).ok()); + ASSERT_TRUE(validate_file_column(*pb_restored, std::make_shared()).ok()); + for (size_t row = 0; row < source->size(); ++row) { + EXPECT_EQ(text(*pb_restored, row, false), text(*source, row, false)); + } + for (const auto* json : {R"({"inline":null})", R"({"uri":null,"inline":null})"}) { + EXPECT_FALSE(parse(*source, json).ok()); + EXPECT_EQ(source->size(), 2); + } +} + +TEST_F(DataTypeFileSerDeTest, JsonAcceptsUriWithoutFormatChecks) { + auto column = type.create_column(); + for (const auto* json : {R"({"uri":""})", R"({"uri":"relative/path"})", + R"({"uri":"s3://bucket/a b+中文%ZZ#fragment"})"}) { + ASSERT_TRUE(parse(*column, json).ok()) << json; + } + EXPECT_EQ((*column)[0].get()[0].get(), ""); + EXPECT_EQ((*column)[1].get()[0].get(), "relative/path"); + EXPECT_EQ((*column)[2].get()[0].get(), + "s3://bucket/a b+中文%ZZ#fragment"); +} + +TEST_F(DataTypeFileSerDeTest, PublicJsonbRoundTripsInlineNullEmptyAndBinary) { + const std::string fixtures[] {"", "f", std::string("\0\xff\x80", 3)}; + auto source = type.create_column(); + source->insert(value()); + for (const auto& bytes : fixtures) { + source->insert(value(&bytes)); + } + auto restored = type.create_column(); + CastParameters parameters; + for (size_t row = 0; row < source->size(); ++row) { + JsonbWriter writer; + ASSERT_TRUE(serde.serialize_column_to_jsonb(*source, row, writer, {}).ok()); + const JsonbDocument* document = nullptr; + ASSERT_TRUE(JsonbDocument::checkAndCreateDocument(writer.getOutput()->getBuffer(), + writer.getOutput()->getSize(), &document) + .ok()); + const auto* object = document->getValue()->unpack(); + ASSERT_EQ(object->numElem(), DataTypeFile::FIELD_COUNT); + ASSERT_NE(object->find("inline"), nullptr); + ASSERT_TRUE(serde.deserialize_column_from_jsonb(*restored, document->getValue(), parameters) + .ok()); + expect_bytes(*restored, row, row == 0 ? nullptr : &fixtures[row - 1]); + EXPECT_EQ(text(*restored, row, false), text(*source, row, true)); + } +} + +TEST_F(DataTypeFileSerDeTest, PublicOutputIncludesInlineAndBinaryUsesLengthEncodedJson) { + const std::string bytes("\0\xff", 2); + auto column = type.create_column(); + column->insert(value(&bytes)); + EXPECT_EQ(text(*column, 0, false), public_json("\"AP8=\"")); + + auto mysql_text = ColumnString::create(); + BufferWritable text_buffer(*mysql_text); + ASSERT_TRUE(serde.write_column_to_mysql_text(*column, text_buffer, 0, options)); + text_buffer.commit(); + EXPECT_EQ(mysql_text->get_data_at(0).to_string(), public_json("\"AP8=\"")); + + JsonbWriter writer; + ASSERT_TRUE(serde.serialize_column_to_jsonb(*column, 0, writer, {}).ok()); + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(writer.getOutput()->getBuffer(), + writer.getOutput()->getSize()), + public_json("\"AP8=\"")); + + MysqlRowBinaryBuffer binary; + binary.start_binary_row(1); + ASSERT_TRUE(serde.write_column_to_mysql_binary(*column, binary, 0, false, options).ok()); + const auto expected = public_json("\"AP8=\""); + ASSERT_EQ(binary.length(), expected.size() + 3); + EXPECT_EQ(static_cast(binary.buf()[2]), expected.size()); + EXPECT_EQ(std::string(binary.buf() + 3, expected.size()), expected); +} + +TEST_F(DataTypeFileSerDeTest, FormatOutputPreservesNullEmptyAndPaddedBase64) { + auto column = type.create_column(); + column->insert(value()); + std::vector> fixtures = { + {"", ""}, + {"f", "Zg=="}, + {"fo", "Zm8="}, + {"foo", "Zm9v"}, + {std::string("\0\xff\x80", 3), "AP+A"}}; + const auto prefix = public_json().substr(0, public_json().size() - 5); + EXPECT_EQ(text(*column, 0, true), prefix + "null}"); + for (const auto& [bytes, encoded] : fixtures) { + column->insert(value(&bytes)); + EXPECT_EQ(text(*column, column->size() - 1, true), prefix + "\"" + encoded + "\"}"); + } + // Crossing the encoder's chunks must not insert padding in the middle. + const std::string large(3 * 4096 + 1, 'f'); + column->insert(value(&large)); + std::string encoded; + for (size_t i = 0; i < 4096; ++i) encoded += "ZmZm"; + encoded += "Zg=="; + EXPECT_EQ(text(*column, column->size() - 1, true), prefix + "\"" + encoded + "\"}"); + + auto constant = ColumnConst::create(column->clone_resized(1), 3); + EXPECT_EQ(text(*constant, 2, true), prefix + "null}"); +} + +TEST_F(DataTypeFileSerDeTest, PublicInputMatchesNamesAndRejectsInvalidWholeValue) { + auto column = type.create_column(); + ASSERT_TRUE(parse(*column, R"({"size":0,"uri":"s3://bucket/a"})").ok()); + expect_bytes(*column, 0, nullptr); + const std::vector invalid = { + "[]", + "null", + R"("s3://bucket/a")", + "{}", + R"({"uri":null})", + R"({"uri":"s3://bucket/a","extra":null})", + R"({"uri":"s3://bucket/a","uri":"s3://bucket/b"})", + R"({"URI":"s3://bucket/a"})", + R"({"uri":"s3://bucket/a","size":"1"})", + R"({"uri":"s3://bucket/a","size":1.0})", + R"({"uri":"s3://bucket/a","offset":0})", + R"({"uri":"s3://bucket/a","size":9223372036854775808})", + R"({"uri":"s3://bucket/a","checksum":"md5:d41d8cd98f00b204e9800998ecf8427e"})", + R"({"uri":"s3://bucket/a","checksum":"MD5:D41D8CD98F00B204E9800998ECF8427E"})"}; + for (const auto& json : invalid) { + SCOPED_TRACE(json); + EXPECT_FALSE(parse(*column, json).ok()); + EXPECT_EQ(column->size(), 1); + const auto& file = assert_cast(*column); + for (size_t i = 0; i < 6; ++i) EXPECT_EQ(file.get_column(i).size(), 1); + } +} + +TEST_F(DataTypeFileSerDeTest, PublicTextInputPreservesNullEmptyAndBinaryInline) { + auto column = type.create_column(); + ASSERT_TRUE(parse(*column, R"({"uri":"s3://bucket/a","inline":null})").ok()); + ASSERT_TRUE(parse(*column, R"({"uri":"s3://bucket/a","inline":""})").ok()); + ASSERT_TRUE(parse(*column, R"({"uri":"s3://bucket/a","inline":"AP+A"})").ok()); + const std::string empty; + const std::string bytes("\0\xff\x80", 3); + expect_bytes(*column, 0, nullptr); + expect_bytes(*column, 1, &empty); + expect_bytes(*column, 2, &bytes); + for (size_t row = 0; row < column->size(); ++row) { + EXPECT_EQ(text(*column, row, false), text(*column, row, true)); + } +} + +TEST_F(DataTypeFileSerDeTest, FormatInputRoundTripsAllSixFieldsAndInlineBytes) { + auto source = type.create_column(); + source->insert(value()); + const std::vector fixtures = { + "", "f", "fo", "foo", std::string("\0\xff\x80", 3), std::string(3 * 4096 + 1, '\xff')}; + for (const auto& bytes : fixtures) source->insert(value(&bytes)); + for (const bool csv : {false, true}) { + auto result = type.create_column(); + for (size_t row = 0; row < source->size(); ++row) { + const auto json = text(*source, row, true); + ASSERT_TRUE(parse_format(*result, json, csv).ok()); + EXPECT_EQ(text(*result, row, true), json); + expect_bytes(*result, row, row == 0 ? nullptr : &fixtures[row - 1]); + } + EXPECT_TRUE(type.check_column(*result).ok()); + } +} + +TEST_F(DataTypeFileSerDeTest, FormatInputMatchesNamesAndFillsMissingOptionalFields) { + auto column = type.create_column(); + ASSERT_TRUE(parse_format(*column, + " \n" + R"({"uri":"s3://bucket/a"})" + "\r\n") + .ok()); + auto fields = (*column)[0].get(); + for (size_t i = 1; i < 6; ++i) EXPECT_TRUE(fields[i].is_null()); + ASSERT_TRUE(parse_format(*column, R"({"inline":null,"size":null,"uri":"s3://bucket/a"})").ok()); + expect_bytes(*column, 1, nullptr); + ASSERT_TRUE(parse_format(*column, R"({"inline":"","uri":"s3://bucket/a","offset":0,"size":0})") + .ok()); + const std::string empty; + expect_bytes(*column, 2, &empty); + fields = (*column)[2].get(); + EXPECT_EQ(fields[1].get(), 0); + EXPECT_EQ(fields[2].get(), 0); +} + +TEST_F(DataTypeFileSerDeTest, FormatInputRejectsInvalidWholeValuesWithoutAppending) { + auto column = type.create_column(); + column->insert(value()); + const auto original = text(*column, 0, true); + const std::vector invalid = { + "", + "[]", + "null", + R"("s3://bucket/a")", + "{}", + R"({"uri":null})", + R"({"uri":"s3://bucket/a","extra":null})", + R"({"URI":"s3://bucket/a"})", + R"({"uri":"s3://bucket/a","Inline":""})", + R"({"uri":"s3://bucket/a","\u0075ri":"s3://bucket/b"})", + R"({"uri":"s3://bucket/a","inline":"","inline":null})", + R"({"inline":"Zm9vYmFyZm9vYmFyZm9v","uri":"s3://bucket/a","unknown":0})", + R"({"uri":"s3://bucket/a","inline":0})", + R"({"uri":"s3://bucket/a","inline":true})", + R"({"uri":"s3://bucket/a","inline":[]})", + R"({"uri":"s3://bucket/a","size":"1"})", + R"({"uri":"s3://bucket/a","size":1.0})", + R"({"uri":"s3://bucket/a","size":1e0})", + R"({"uri":"s3://bucket/a","size":9223372036854775808})", + R"({"uri":"s3://bucket/a","size":-1})", + R"({"uri":"s3://bucket/a","offset":0})", + R"({"uri":"s3://bucket/a","size":1,"offset":9223372036854775807})", + R"({"uri":"s3://bucket/a","checksum":"md5:d41d8cd98f00b204e9800998ecf8427e"})", + R"({"uri":"s3://bucket/a","checksum":"MD5:D41D8CD98F00B204E9800998ECF8427E"})", + R"({"uri":"s3://bucket/a"} trailing)", + std::string(R"({"uri":"s3://bucket/a"})") + '\0' + "{}"}; + for (const auto& json : invalid) { + SCOPED_TRACE(json); + EXPECT_FALSE(parse_format(*column, json).ok()); + ASSERT_EQ(column->size(), 1); + const auto& file = assert_cast(*column); + for (size_t i = 0; i < 6; ++i) EXPECT_EQ(file.get_column(i).size(), 1); + EXPECT_EQ(text(*column, 0, true), original); + } +} + +TEST_F(DataTypeFileSerDeTest, FormatInputRequiresCanonicalPaddedBase64) { + auto column = type.create_column(); + const std::vector invalid = { + "Zg", "Zg=", "Zg===", "=m9v", "Zm=v", "Zm9v=", + "Zg==AAAA", "====", "Zh==", "Zm9=", "AP-A", "AP_A", + " Zg==", "Zg== ", "Zg==\\n", "Zg==\\r", "Zg==\\t", "\\u0000AAA"}; + for (const auto& encoded : invalid) { + SCOPED_TRACE(encoded); + const std::string json = R"({"uri":"s3://bucket/a","inline":")" + encoded + R"("})"; + EXPECT_FALSE(parse_format(*column, json).ok()); + EXPECT_FALSE(parse(*column, json).ok()); + EXPECT_EQ(column->size(), 0); + } +} + +TEST_F(DataTypeFileSerDeTest, InlineBase64LengthChecksDoNotNeedPayloadOrAllocation) { + uint32_t length = 99; + ASSERT_TRUE(DataTypeFileSerDe::_checked_inline_size(0, 0, &length).ok()); + EXPECT_EQ(length, 0); + for (size_t padding = 0; padding <= 2; ++padding) { + ASSERT_TRUE(DataTypeFileSerDe::_checked_inline_size(4, padding, &length).ok()); + EXPECT_EQ(length, 3 - padding); + } + const uint64_t max_length = std::numeric_limits::max(); + const auto max_encoded = static_cast(max_length / 3 * 4); + ASSERT_TRUE(DataTypeFileSerDe::_checked_inline_size(max_encoded, 0, &length).ok()); + EXPECT_EQ(length, max_length); + length = 99; + EXPECT_FALSE(DataTypeFileSerDe::_checked_inline_size(max_encoded + 4, 2, &length).ok()); + EXPECT_FALSE(DataTypeFileSerDe::_checked_inline_size(std::numeric_limits::max() - 3, 2, + &length) + .ok()); + EXPECT_FALSE(DataTypeFileSerDe::_checked_inline_size(3, 0, &length).ok()); + EXPECT_FALSE(DataTypeFileSerDe::_checked_inline_size(0, 1, &length).ok()); + EXPECT_FALSE(DataTypeFileSerDe::_checked_inline_size(4, 3, &length).ok()); + EXPECT_EQ(length, 99); +} + +TEST_F(DataTypeFileSerDeTest, FormatInputPreservesNestedFileValuesAndNulls) { + const auto file = make_nullable(std::make_shared()); + const auto array = make_nullable(std::make_shared(file)); + const auto map = make_nullable( + std::make_shared(make_nullable(std::make_shared()), file)); + const DataTypeStruct schema({file, array, map}, {"asset", "files", "lookup"}); + auto column = schema.create_column(); + std::string json = + R"({"asset":{"uri":"s3://bucket/a","inline":"AP+A"},"files":[null,{"uri":"s3://bucket/b","inline":""}],"lookup":{"raw":{"uri":"s3://bucket/c","inline":"Zg=="},"none":null}})"; + Slice input(json); + ASSERT_TRUE(schema.get_serde()->deserialize_one_cell_from_json(*column, input, options).ok()); + const auto fields = (*column)[0].get(); + EXPECT_EQ(fields[0].get()[5].get().to_string_ref().to_string(), + std::string("\0\xff\x80", 3)); + const auto& items = fields[1].get(); + ASSERT_EQ(items.size(), 2); + EXPECT_TRUE(items[0].is_null()); + const auto& empty_inline = items[1].get()[5]; + EXPECT_FALSE(empty_inline.is_null()); + EXPECT_EQ(empty_inline.get().size(), 0); + const auto& entries = fields[2].get(); + ASSERT_EQ(entries.size(), 2); + const auto& values = entries[1].get(); + ASSERT_EQ(values.size(), 2); + EXPECT_EQ(values[0].get()[5].get().to_string_ref().to_string(), "f"); + EXPECT_TRUE(values[1].is_null()); +} + +TEST_F(DataTypeFileSerDeTest, ProtobufPreservesFileIdentityInlineAndOuterNullMask) { + const std::string bytes("\0\xffinline", 8); + auto source = ColumnNullable::create(type.create_column(), ColumnUInt8::create()); + source->insert_default(); // The nested URI is invalid, but the FILE is NULL. + source->insert(value(&bytes)); + source->insert(value()); + DataTypeNullableSerDe nullable(std::make_shared(), 1); + PValues pb; + ASSERT_TRUE(nullable.write_column_to_pb(*source, pb, 0, 3).ok()); + EXPECT_EQ(pb.type().id(), PGenericType::FILE); + ASSERT_EQ(pb.child_element_size(), 6); + EXPECT_EQ(pb.child_element(5).type().id(), PGenericType::VARBINARY); + auto restored = ColumnNullable::create(type.create_column(), ColumnUInt8::create()); + ASSERT_TRUE(nullable.read_column_from_pb(*restored, pb).ok()); + ASSERT_EQ(restored->size(), 3); + EXPECT_TRUE(restored->is_null_at(0)); + EXPECT_FALSE(restored->is_null_at(1)); + expect_bytes(restored->get_nested_column(), 1, &bytes); + expect_bytes(restored->get_nested_column(), 2, nullptr); + + pb.mutable_type()->set_id(PGenericType::STRUCT); + EXPECT_FALSE(nullable.read_column_from_pb(*restored, pb).ok()); + EXPECT_EQ(restored->size(), 3); +} + +TEST_F(DataTypeFileSerDeTest, JsonbCastRequiresAllSixFieldsWhileJsonParsingKeepsDefaults) { + auto column = type.create_column(); + JsonBinaryValue encoded; + ASSERT_TRUE(encoded.from_json_string(R"({"uri":"s3://bucket/a"})").ok()); + const JsonbDocument* document = nullptr; + ASSERT_TRUE( + JsonbDocument::checkAndCreateDocument(encoded.value(), encoded.size(), &document).ok()); + CastParameters parameters; + EXPECT_FALSE( + serde.deserialize_column_from_jsonb(*column, document->getValue(), parameters).ok()); + EXPECT_EQ(column->size(), 0); + ASSERT_TRUE(parse_format(*column, R"({"uri":"s3://bucket/a","inline":"AA=="})").ok()); + const std::string bytes(1, '\0'); + expect_bytes(*column, 0, &bytes); +} + +TEST_F(DataTypeFileSerDeTest, JsonbCastRejectsMissingInlineAndInvalidBase64Atomically) { + const std::string prefix = + R"({"uri":"s3://bucket/a","offset":null,"size":null,"content_type":null,"checksum":null)"; + const std::vector invalid {prefix + "}", + prefix + R"(,"inline":"AA"})", + prefix + R"(,"inline":"AB=="})", + prefix + R"(,"inline":"_w=="})", + prefix + R"(,"inline":1})", + prefix + R"(,"inline":"","inline":null})", + prefix + R"(,"inline":null,"extra":null})"}; + auto column = type.create_column(); + column->insert(value()); + CastParameters parameters; + for (const auto& json : invalid) { + SCOPED_TRACE(json); + JsonBinaryValue encoded; + ASSERT_TRUE(encoded.from_json_string(json).ok()); + const JsonbDocument* document = nullptr; + ASSERT_TRUE( + JsonbDocument::checkAndCreateDocument(encoded.value(), encoded.size(), &document) + .ok()); + EXPECT_FALSE(serde.deserialize_column_from_jsonb(*column, document->getValue(), parameters) + .ok()); + EXPECT_EQ(column->size(), 1); + } +} + +TEST_F(DataTypeFileSerDeTest, JsonbCastRejectsIncompleteDuplicateAndFloatingPointWithoutAppending) { + auto column = type.create_column(); + CastParameters parameters; + for (int invalid_case = 0; invalid_case < 3; ++invalid_case) { + JsonbWriter writer; + ASSERT_TRUE(writer.writeStartObject()); + ASSERT_TRUE(writer.writeKey("uri", 3)); + ASSERT_TRUE(writer.writeStartString()); + ASSERT_TRUE(writer.writeString("s3://bucket/a")); + ASSERT_TRUE(writer.writeEndString()); + if (invalid_case == 0) { + ASSERT_TRUE(writer.writeKey("inline", 6)); + ASSERT_TRUE(writer.writeNull()); + } else if (invalid_case == 1) { + ASSERT_TRUE(writer.writeKey("uri", 3)); + ASSERT_TRUE(writer.writeNull()); + } else { + ASSERT_TRUE(writer.writeKey("size", 4)); + ASSERT_TRUE(writer.writeDouble(1.0)); + } + ASSERT_TRUE(writer.writeEndObject()); + const JsonbDocument* document = nullptr; + ASSERT_TRUE(JsonbDocument::checkAndCreateDocument(writer.getOutput()->getBuffer(), + writer.getOutput()->getSize(), &document) + .ok()); + EXPECT_FALSE(serde.deserialize_column_from_jsonb(*column, document->getValue(), parameters) + .ok()); + EXPECT_EQ(column->size(), 0); + } + auto source = type.create_column(); + source->insert(value()); + JsonbWriter writer; + ASSERT_TRUE(serde.serialize_column_to_jsonb(*source, 0, writer, {}).ok()); + const JsonbDocument* document = nullptr; + ASSERT_TRUE(JsonbDocument::checkAndCreateDocument(writer.getOutput()->getBuffer(), + writer.getOutput()->getSize(), &document) + .ok()); + ASSERT_TRUE( + serde.deserialize_column_from_jsonb(*column, document->getValue(), parameters).ok()); + expect_bytes(*column, 0, nullptr); + EXPECT_EQ(text(*column, 0, false), public_json()); +} + +TEST_F(DataTypeFileSerDeTest, ProtobufShapeValidationIsAtomicForAllSixChildren) { + auto source = type.create_column(); + source->insert(value()); + PValues pb; + ASSERT_TRUE(serde.write_column_to_pb(*source, pb, 0, 1).ok()); + auto restored = type.create_column(); + restored->insert(value()); + pb.mutable_child_element(2)->clear_int64_value(); + EXPECT_FALSE(serde.read_column_from_pb(*restored, pb).ok()); + EXPECT_EQ(restored->size(), 1); + pb.mutable_child_element(2)->add_int64_value(1024); + pb.mutable_child_element(5)->clear_bytes_value(); + EXPECT_FALSE(serde.read_column_from_pb(*restored, pb).ok()); + EXPECT_EQ(restored->size(), 1); +} + +TEST_F(DataTypeFileSerDeTest, ProtobufPreservesNullableParentWithNonNullableFileChild) { + const auto outer_type = make_nullable(std::make_shared( + DataTypes {std::make_shared()}, Strings {"payload"})); + const auto outer_serde = outer_type->get_serde(); + auto source = outer_type->create_column(); + source->insert_default(); + source->insert(Field::create_field(Struct {value()})); + PValues pb; + ASSERT_TRUE(outer_serde->write_column_to_pb(*source, pb, 0, 2).ok()); + // FILE is nonnullable within a nullable STRUCT. Its invalid default URI is hidden. + ASSERT_FALSE(pb.child_element(0).has_null()); + auto restored = outer_type->create_column(); + ASSERT_TRUE(outer_serde->read_column_from_pb(*restored, pb).ok()); + ASSERT_EQ(restored->size(), 2); + EXPECT_TRUE(restored->is_null_at(0)); + EXPECT_FALSE(restored->is_null_at(1)); + const auto& parent = assert_cast(*restored); + const auto& structure = assert_cast(parent.get_nested_column()); + EXPECT_TRUE(structure.get_column(0)[0].get()[0].is_null()); + EXPECT_EQ(structure.get_column(0)[1], value()); +} + +TEST_F(DataTypeFileSerDeTest, ArrowConversionIsUnsupported) { + auto column = type.create_column(); + const auto timezone = cctz::utc_time_zone(); + EXPECT_FALSE(serde.write_column_to_arrow(*column, nullptr, nullptr, 0, 0, timezone).ok()); + EXPECT_FALSE(serde.read_column_from_arrow(*column, nullptr, 0, 0, timezone).ok()); +} + +TEST_F(DataTypeFileSerDeTest, OrcConversionIsUnsupported) { + auto column = type.create_column(); + Arena arena; + EXPECT_FALSE( + serde.write_column_to_orc("UTC", *column, nullptr, nullptr, 0, 0, arena, options).ok()); + EXPECT_FALSE(serde.read_column_from_orc(*column, OrcDecodedColumnView {}).ok()); +} + +TEST_F(DataTypeFileSerDeTest, MysqlNullableWrapperUsesProtocolNull) { + auto column = ColumnNullable::create(type.create_column(), ColumnUInt8::create()); + column->insert_default(); + DataTypeNullableSerDe nullable(std::make_shared()); + auto output = ColumnString::create(); + BufferWritable bw(*output); + EXPECT_FALSE(nullable.write_column_to_mysql_text(*column, bw, 0, options)); + bw.commit(); + EXPECT_EQ(output->get_data_at(0).size, 0); + MysqlRowBinaryBuffer binary; + binary.start_binary_row(1); + ASSERT_TRUE(nullable.write_column_to_mysql_binary(*column, binary, 0, false, options).ok()); + ASSERT_EQ(binary.length(), 2); + EXPECT_EQ(static_cast(binary.buf()[1]), 4); +} +} // namespace doris diff --git a/be/test/core/data_type_serde/data_type_serde_varbinary_test.cpp b/be/test/core/data_type_serde/data_type_serde_varbinary_test.cpp index 700dda99a15e18..5f0e25230c0ee2 100644 --- a/be/test/core/data_type_serde/data_type_serde_varbinary_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_varbinary_test.cpp @@ -34,8 +34,11 @@ #include "core/assert_cast.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" +#include "core/column/column_struct.h" #include "core/column/column_varbinary.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" #include "core/data_type/data_type_varbinary.h" #include "core/data_type_serde/data_type_serde.h" #include "core/data_type_serde/data_type_varbinary_serde.h" @@ -219,19 +222,47 @@ TEST_F(DataTypeVarbinarySerDeTest, JsonTextBehavior) { } } -TEST_F(DataTypeVarbinarySerDeTest, ProtobufNotSupported) { +TEST_F(DataTypeVarbinarySerDeTest, ProtobufPreservesRawBytesAndSelectedRange) { DataTypeVarbinarySerDe serde; - auto col = ColumnVarbinary::create(); - auto* vb = col.get(); - std::string v = make_bytes(4); - vb->insert_data(v.data(), v.size()); - - PValues pv; - auto st1 = serde.write_column_to_pb(*col, pv, 0, 1); - EXPECT_FALSE(st1.ok()); + auto source = ColumnVarbinary::create(); + const std::vector values {"skip", "", make_bytes(8), make_bytes(4096)}; + for (const auto& bytes : values) { + source->insert_data(bytes.data(), bytes.size()); + } + PValues pb; + ASSERT_TRUE(serde.write_column_to_pb(*source, pb, 1, source->size()).ok()); + EXPECT_EQ(pb.type().id(), PGenericType::VARBINARY); + ASSERT_EQ(pb.bytes_value_size(), 3); + auto restored = ColumnVarbinary::create(); + ASSERT_TRUE(serde.read_column_from_pb(*restored, pb).ok()); + source->clear(); + ASSERT_EQ(restored->size(), 3); + for (size_t row = 0; row < restored->size(); ++row) { + EXPECT_EQ(restored->get_data_at(row).to_string(), values[row + 1]); + } +} - auto st2 = serde.read_column_from_pb(*vb, pv); - EXPECT_FALSE(st2.ok()); +TEST_F(DataTypeVarbinarySerDeTest, ProtobufPreservesBinaryStructFieldAndNulls) { + const auto binary = make_nullable(std::make_shared()); + const auto type = std::make_shared(DataTypes {binary}, Strings {"payload"}); + auto source = type->create_column(); + const std::string bytes = make_bytes(4096); + source->insert(Field::create_field( + Struct {Field::create_field(StringView(bytes))})); + source->insert(Field::create_field(Struct {Field()})); + source->insert(Field::create_field( + Struct {Field::create_field(StringView())})); + PValues pb; + const auto serde = type->get_serde(); + ASSERT_TRUE(serde->write_column_to_pb(*source, pb, 0, source->size()).ok()); + auto restored = type->create_column(); + ASSERT_TRUE(serde->read_column_from_pb(*restored, pb).ok()); + ASSERT_EQ(restored->size(), 3); + const auto& field = assert_cast(*restored).get_column(0); + EXPECT_EQ(field.get_data_at(0).to_string(), bytes); + EXPECT_TRUE(field.is_null_at(1)); + EXPECT_FALSE(field.is_null_at(2)); + EXPECT_EQ(field.get_data_at(2).size, 0); } TEST_F(DataTypeVarbinarySerDeTest, JsonbThrows) { diff --git a/be/test/core/field_test.cpp b/be/test/core/field_test.cpp index efe94cef96892d..7c34dfb13adb6a 100644 --- a/be/test/core/field_test.cpp +++ b/be/test/core/field_test.cpp @@ -33,6 +33,28 @@ #include "gtest/gtest_pred_impl.h" // IWYU pragma: keep namespace doris { +TEST(VFieldTest, file_identity_and_copy) { + File value {Field::create_field(String {"s3://bucket/object"}), + Field(), + Field::create_field(Int64 {0}), + Field(), + Field(), + Field()}; + Field original = Field::create_field(value); + Field copied(original); + Field moved(std::move(copied)); + Field assigned; + assigned = moved; + EXPECT_EQ(assigned.get_type(), TYPE_FILE); + ASSERT_EQ(assigned.get().size(), 6); + EXPECT_EQ(assigned.get()[0].get(), "s3://bucket/object"); + EXPECT_EQ(assigned.get()[2].get(), 0); + EXPECT_TRUE(assigned.get()[5].is_null()); + EXPECT_EQ(original <=> assigned, std::strong_ordering::equal); + assigned.get()[0] = Field::create_field(String {"urn:other"}); + EXPECT_EQ(original <=> assigned, std::strong_ordering::equal); +} + TEST(VFieldTest, detects_floating_point_nan) { EXPECT_TRUE(Field::create_field(std::numeric_limits::quiet_NaN()).is_nan()); EXPECT_TRUE( diff --git a/be/test/core/value/file_value_test.cpp b/be/test/core/value/file_value_test.cpp new file mode 100644 index 00000000000000..792683e2adb19f --- /dev/null +++ b/be/test/core/value/file_value_test.cpp @@ -0,0 +1,400 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/value/file_value.h" + +#include + +#include +#include +#include +#include +#include + +#include "core/field.h" + +namespace doris { + +class FileValueTest : public ::testing::Test { +protected: + static File file(std::string uri = "s3://bucket/object") { + File value(6); + value[0] = Field::create_field(std::move(uri)); + return value; + } + + static Status with_text(size_t index, std::string text) { + auto value = file(); + value[index] = Field::create_field(std::move(text)); + return validate_file(value); + } +}; + +TEST_F(FileValueTest, ConvertsProviderEtagsWithoutRewritingOpaqueBytes) { + EXPECT_EQ(file_checksum_from_etag(std::nullopt), std::nullopt); + EXPECT_EQ(file_checksum_from_etag(""), std::nullopt); + EXPECT_EQ(file_checksum_from_etag("\"\""), std::nullopt); + for (const auto* etag : {"opaque-2", "\"opaque-2\""}) { + EXPECT_EQ(file_checksum_from_etag(etag), "ETAG:opaque-2"); + } + for (const std::string etag : {"MiXeD:opaque-2", "\"", " leading trailing "}) { + EXPECT_EQ(file_checksum_from_etag(etag), "ETAG:" + etag); + } + EXPECT_EQ(file_checksum_from_etag("\"\"opaque-2\"\""), "ETAG:\"opaque-2\""); + EXPECT_EQ(file_checksum_from_etag(std::string_view("\"a\0b\"", 5)), + std::string("ETAG:a\0b", 8)); +} + +TEST_F(FileValueTest, RequiresUriOrInline) { + EXPECT_TRUE(validate_file(file()).ok()); + EXPECT_FALSE(validate_file(File(6)).ok()); + EXPECT_TRUE(validate_file(file("")).ok()); +} + +TEST_F(FileValueTest, InlineAllowsAllOtherChildrenToBeNull) { + for (const auto& bytes : {std::string(), std::string("\0\xff", 2)}) { + File value(6); + value[5] = Field::create_field(StringView(bytes)); + EXPECT_TRUE(validate_file(value).ok()); + value[0] = Field::create_field("relative/path"); + EXPECT_TRUE(validate_file(value).ok()); + value[0] = Field(); + value[2] = Field::create_field(Int64 {-1}); + EXPECT_FALSE(validate_file(value).ok()); + value[2] = Field(); + value[1] = Field::create_field(Int64 {0}); + EXPECT_FALSE(validate_file(value).ok()); + value[1] = Field(); + value[5] = Field(); + EXPECT_FALSE(validate_file(value).ok()); + } +} + +TEST_F(FileValueTest, PreservesUriBytesWithoutFormatChecks) { + for (const auto& uri : {std::string(), std::string("relative/path"), + std::string("s3://bucket/a b+中文%ZZ#fragment"), std::string("a\0b", 3), + std::string(65534, 'a')}) { + const auto value = file(uri); + EXPECT_TRUE(validate_file(value).ok()); + EXPECT_EQ(value[0].get(), uri); + } +} + +TEST_F(FileValueTest, RangeBoundaries) { + constexpr Int64 max = std::numeric_limits::max(); + struct RangeCase { + bool has_offset; + Int64 offset; + bool has_size; + Int64 size; + bool valid; + }; + for (const auto& range : std::array { + RangeCase {false, 0, false, 0, true}, + RangeCase {false, 0, true, 0, true}, + RangeCase {false, 0, true, max, true}, + RangeCase {false, 0, true, -1, false}, + RangeCase {true, 0, false, 0, false}, + RangeCase {true, 1, false, 0, false}, + RangeCase {true, -1, true, 0, false}, + RangeCase {true, 0, true, -1, false}, + RangeCase {true, 0, true, 0, true}, + RangeCase {true, 0, true, max, true}, + RangeCase {true, max, true, 0, true}, + RangeCase {true, max - 1, true, 1, true}, + RangeCase {true, max, true, 1, false}, + RangeCase {true, 1, true, max, false}, + RangeCase {true, max, true, max, false}, + RangeCase {true, std::numeric_limits::min(), true, 0, false}, + }) { + auto value = file(); + if (range.has_offset) { + value[1] = Field::create_field(range.offset); + } + if (range.has_size) { + value[2] = Field::create_field(range.size); + } + EXPECT_EQ(validate_file(value).ok(), range.valid) + << range.has_offset << ":" << range.offset << ", " << range.has_size << ":" + << range.size; + } +} + +TEST_F(FileValueTest, Rfc2045MimeCorpusAndRawPreservation) { + using namespace std::string_view_literals; + for (const auto mime : { + "image/png"sv, + "Application/X.Unknown+Thing"sv, + "unknown/unknown"sv, + "x-{type}/x-{subtype}"sv, + "x/!#$%&'*+-.^_`{|}~"sv, + "text/plain; charset=UTF-8"sv, + "text/plain\t;\tcharset=utf-8;format=flowed"sv, + "multipart/mixed;boundary=\"a;b=c/\\\"quote\\\\end\""sv, + "text/plain;empty=\"\""sv, + "text/plain;v=\"a\tb\""sv, + "text/plain;a=one;a=two"sv, + " text/plain"sv, + "text/plain "sv, + "text /plain"sv, + "text/ plain"sv, + "text/plain(comment)"sv, + "text/plain; charset =utf-8"sv, + "text/plain; charset= utf-8"sv, + "(before)text(type)/(slash)plain(subtype);(semi)charset(attr)=(eq)utf-8(value)"sv, + "(one)(two) text/plain (three)"sv, + "text/plain (outer(inner(deep))tail)"sv, + "text/plain (escaped \\( \\) \\\\ delimiters)"sv, + "text/plain (a \"quote\" is just comment text)"sv, + "text/plain; title=\"(not a comment); x=y\""sv, + "text/plain; name=\"a\\qb\""sv, + "text\r\n /\r\n\tplain\r\n ;\r\n x\r\n =\r\n \"a\r\n b\"\r\n "sv, + "text/plain (a\r\n\tb)"sv, + "text/plain; p=\"a\r\n \r\n\tb\""sv, + "text/plain; p=\"a\\\r\n b\""sv, + "text/plain (a\\\r\n b)"sv, + "text/plain; p=\"\000\001\177\""sv, + "text/plain (\000\001\177)"sv, + "text/plain; p=\"a\nb\""sv, + "text/plain (a\nb)"sv, + "text/plain; p=\"a\\\rb\""sv, + "text/plain (a\\\rb)"sv, + }) { + auto value = file(); + value[3] = Field::create_field(std::string(mime)); + EXPECT_TRUE(is_valid_file_content_type(mime)) << mime; + EXPECT_TRUE(validate_file(value).ok()) << mime; + EXPECT_EQ(value[3].as_string_view(), mime); + } + for (const auto mime : { + ""sv, + " "sv, + "(only a comment)"sv, + "text"sv, + "/plain"sv, + "text/"sv, + "text/plain/extra"sv, + "text/plain, image/png"sv, + "text/plain;"sv, + "text/plain; ;charset=utf-8;\t"sv, + "text/plain;(comment)"sv, + "text/plain; charset"sv, + "text/plain; =utf-8"sv, + "text/plain; charset="sv, + "text/plain; charset=(empty)"sv, + "text/plain; charset=utf 8"sv, + "text/plain; a=b=c"sv, + "text/plain; a=b/c"sv, + "text/plain; a=\"unfinished"sv, + "text/plain; a=\"dangling\\"sv, + "text/plain; a=\"x\"suffix"sv, + "text/plain; a=\"x\"(comment)suffix"sv, + "te(comment)xt/plain"sv, + "text/pl(comment)ain"sv, + "text/plain; char(comment)set=utf-8"sv, + "text/plain; charset=utf(comment)-8"sv, + "\"text\"/plain"sv, + "text/\"plain\""sv, + "text/plain; \"charset\"=utf-8"sv, + "text/plain (unfinished"sv, + "text/plain (a(b)"sv, + "text/plain (dangling\\"sv, + "text/plain )"sv, + "text/plain (x))"sv, + "text/plain; a=b\\c"sv, + "text/plain\r\n; charset=utf-8"sv, + "text/plain\r\n"sv, + "text/plain\r"sv, + "text/plain\n"sv, + "text/plain (a\rb)"sv, + "text/plain; p=\"a\rb\""sv, + "text/plain (a\r\nb)"sv, + "text/plain; p=\"a\r\nb\""sv, + "text/plain\r\n\r\n x=y"sv, + "text/pl\303\244in"sv, + "text/plain; p=\"\303\251\""sv, + "text/plain (\303\251)"sv, + "text/plain; p=\"\\\303\251\""sv, + "text/plain (\\\303\251)"sv, + "text/plain; p=[a]"sv, + "text/plain; p={a?}"sv, + "text/plain; a=\000"sv, + }) { + EXPECT_FALSE(is_valid_file_content_type(mime)) << mime; + EXPECT_FALSE(with_text(3, std::string(mime)).ok()) << mime; + } +} + +TEST_F(FileValueTest, Rfc2045QuotedCrlfRequiresFoldingWhitespace) { + // RFC 822 sections 3.4.3 and 3.4.5 explicitly cover quoted CRLFs. + for (const auto* mime : {"text/plain; p=\"a\\\r\n b\"", "text/plain; p=\"a\\\r\n\tb\"", + "text/plain (a\\\r\n b)", "text/plain (a\\\r\n\tb)"}) { + EXPECT_TRUE(with_text(3, mime).ok()); + } + for (const auto* mime : {"text/plain; p=\"a\\\r\nb\"", "text/plain; p=\"a\\\r\n\"", + "text/plain (a\\\r\nb)", "text/plain (a\\\r\n)"}) { + EXPECT_FALSE(with_text(3, mime).ok()); + } +} + +TEST_F(FileValueTest, Rfc2045MimeAsciiLexicalClasses) { + for (int code = 0; code < 256; ++code) { + SCOPED_TRACE(code); + const char ch = static_cast(code); + const bool token = code > 32 && code < 127 && + std::string_view("()<>@,;:\\\"/[]?=").find(ch) == std::string_view::npos; + EXPECT_EQ(is_valid_file_content_type(std::string("x/") + ch), token); + EXPECT_EQ(is_valid_file_content_type(std::string("x/y; p=\"") + ch + "\""), + code < 128 && ch != '\\' && ch != '"' && ch != '\r'); + EXPECT_EQ(is_valid_file_content_type(std::string("x/y; p=\"\\") + ch + "\""), code < 128); + EXPECT_EQ(is_valid_file_content_type(std::string("x/y (") + ch + ")"), + code < 128 && ch != '(' && ch != ')' && ch != '\\' && ch != '\r'); + EXPECT_EQ(is_valid_file_content_type(std::string("x/y (\\") + ch + ")"), code < 128); + } +} + +TEST_F(FileValueTest, Rfc2045MimeCommentDepthAndFoldedLengthLimit) { + const std::string nested = "x/y" + std::string(510, '(') + std::string(510, ')'); + EXPECT_TRUE(is_valid_file_content_type(nested)); + EXPECT_FALSE(is_valid_file_content_type(nested.substr(0, nested.size() - 1))); + EXPECT_FALSE(is_valid_file_content_type("x/y" + std::string(511, '(') + std::string(511, ')'))); + std::string folded = "x/y"; + for (int i = 0; i < 340; ++i) { + folded += "\r\n "; + } + folded += ' '; + EXPECT_EQ(folded.size(), 1024); + EXPECT_TRUE(is_valid_file_content_type(folded)); + EXPECT_FALSE(is_valid_file_content_type(folded + ' ')); +} + +TEST_F(FileValueTest, RequiresCanonicalKnownChecksums) { + struct Algorithm { + std::string_view name; + std::string_view lowercase_name; + size_t length; + }; + for (const auto& algorithm : std::array { + Algorithm {"MD5", "md5", 32}, + Algorithm {"CRC32", "crc32", 8}, + Algorithm {"CRC32C", "crc32c", 8}, + Algorithm {"SHA-256", "sha-256", 64}, + }) { + const std::string prefix = std::string(algorithm.name) + ":"; + EXPECT_TRUE(with_text(4, prefix + std::string(algorithm.length, 'a')).ok()); + EXPECT_TRUE(with_text(4, prefix + std::string(algorithm.length, '0')).ok()); + EXPECT_FALSE(with_text(4, prefix + std::string(algorithm.length - 1, 'a')).ok()); + EXPECT_FALSE(with_text(4, prefix + std::string(algorithm.length + 1, 'a')).ok()); + EXPECT_FALSE(with_text(4, prefix + std::string(algorithm.length, 'A')).ok()); + EXPECT_FALSE(with_text(4, prefix + std::string(algorithm.length, 'g')).ok()); + EXPECT_FALSE(with_text(4, std::string(algorithm.lowercase_name) + ":" + + std::string(algorithm.length, 'a')) + .ok()); + } + EXPECT_FALSE(with_text(4, "md5:0123456789ABCDEF0123456789ABCDEF").ok()); + EXPECT_FALSE(with_text(4, "Md5:0123456789abcdef0123456789abcdef").ok()); + EXPECT_FALSE(with_text(4, "etag:opaque").ok()); + EXPECT_FALSE(with_text(4, "ETag:opaque").ok()); + for (const auto* checksum : {"", ":", ":digest", "unknown", "unknown:", "ETAG:"}) { + EXPECT_FALSE(with_text(4, checksum).ok()) << checksum; + } +} + +TEST_F(FileValueTest, PreservesOpaqueChecksums) { + for (const std::string& checksum : { + std::string("ETAG:\"AbC-123\""), + std::string("ETAG:opaque:token-2"), + std::string("Future algorithm:Uppercase / opaque : digest"), + std::string("UNKNOWN:\0\xff", 10), + std::string("ETAG: "), + }) { + auto value = file(); + value[4] = Field::create_field(checksum); + EXPECT_TRUE(validate_file(value).ok()); + EXPECT_EQ(value[4].get(), checksum); + } +} + +TEST_F(FileValueTest, EnforcesByteLengthLimits) { + const std::string mime_prefix = "application/x-example;p="; + EXPECT_TRUE(with_text(3, mime_prefix + std::string(1024 - mime_prefix.size(), 'a')).ok()); + EXPECT_FALSE(with_text(3, mime_prefix + std::string(1025 - mime_prefix.size(), 'a')).ok()); + EXPECT_TRUE(with_text(4, "ETAG:" + std::string(1019, 'A')).ok()); + EXPECT_FALSE(with_text(4, "ETAG:" + std::string(1020, 'A')).ok()); + EXPECT_TRUE(with_text(4, "X:" + std::string(1022, 'A')).ok()); + EXPECT_FALSE(with_text(4, "X:" + std::string(1023, 'A')).ok()); +} + +TEST_F(FileValueTest, PreservesUriMimeAndInlineWithoutReadingOrComparingContent) { + const std::string uri = "Custom://User:Secret@Host:00080/a/../b%2fc?signature=Token%2F"; + const std::string mime = "Application/X.Unknown; Name=\"Mixed Case\""; + const std::string bytes = std::string(4096, '\xff') + std::string("\0end", 4); + auto value = file(uri); + value[2] = Field::create_field(0); + value[3] = Field::create_field(mime); + value[4] = Field::create_field("MD5:00000000000000000000000000000000"); + value[5] = Field::create_field(StringView(bytes)); + ASSERT_TRUE(validate_file(value).ok()); + EXPECT_EQ(value[0].get(), uri); + EXPECT_EQ(value[3].get(), mime); + EXPECT_EQ(std::string_view(value[5].get()), bytes); + + value[5] = Field::create_field(StringView("")); + EXPECT_TRUE(validate_file(value).ok()); + value[0] = Field(); + EXPECT_TRUE(validate_file(value).ok()); + value[5] = Field(); + EXPECT_FALSE(validate_file(value).ok()); +} + +TEST_F(FileValueTest, ErrorsDoNotIncludeSensitiveValues) { + const std::string uri = "s3://user:PrivatePassword@bucket/path?sig=PrivateSignature#bad"; + auto value = file(uri); + value[2] = Field::create_field(Int64 {-1}); + const auto uri_status = validate_file(value); + ASSERT_FALSE(uri_status.ok()); + EXPECT_EQ(uri_status.to_string().find(uri), std::string::npos); + EXPECT_EQ(uri_status.to_string().find("PrivatePassword"), std::string::npos); + EXPECT_EQ(uri_status.to_string().find("PrivateSignature"), std::string::npos); + EXPECT_EQ(value[0].get(), uri); + + const std::string checksum = "MD5:PrivateChecksum"; + value = file(); + value[4] = Field::create_field(checksum); + const auto checksum_status = validate_file(value); + ASSERT_FALSE(checksum_status.ok()); + EXPECT_EQ(checksum_status.to_string().find(checksum), std::string::npos); + EXPECT_EQ(checksum_status.to_string().find("PrivateChecksum"), std::string::npos); + EXPECT_EQ(value[4].get(), checksum); +} + +TEST_F(FileValueTest, InfersContentTypeFromFilesystemNames) { + EXPECT_EQ(infer_file_content_type_from_name("prefix/a.PNG"), "image/png"); + EXPECT_EQ(infer_file_content_type_from_name("a.JpEg"), "image/jpeg"); + EXPECT_EQ(infer_file_content_type_from_name("document.pdf"), "application/pdf"); + EXPECT_EQ(infer_file_content_type_from_name("part.parquet"), "application/x-parquet"); + EXPECT_EQ(infer_file_content_type_from_name("part.jsonl"), "application/x-ndjson"); + EXPECT_EQ(infer_file_content_type_from_name("directory.png/no_extension"), + "application/octet-stream"); + EXPECT_EQ(infer_file_content_type_from_name("a.unrecognized"), "application/octet-stream"); + EXPECT_EQ(infer_file_content_type_from_name(""), "application/octet-stream"); + EXPECT_TRUE(is_valid_file_content_type("image/png")); + EXPECT_FALSE(is_valid_file_content_type("")); + EXPECT_FALSE(is_valid_file_content_type("not a mime type")); +} + +} // namespace doris diff --git a/be/test/exec/common/data_gen_functions/vlist_file_tvf_test.cpp b/be/test/exec/common/data_gen_functions/vlist_file_tvf_test.cpp new file mode 100644 index 00000000000000..1caad1f39fea2b --- /dev/null +++ b/be/test/exec/common/data_gen_functions/vlist_file_tvf_test.cpp @@ -0,0 +1,660 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/common/data_gen_functions/vlist_file_tvf.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "core/column/column_file.h" +#include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_string.h" +#include "core/field.h" +#include "cpp/obj-client/s3_obj_storage_client.h" +#include "exec/operator/datagen_operator.h" +#include "exec/operator/operator_helper.h" +#include "exprs/function/function_to_file.h" +#include "io/fs/s3_file_system.h" +#include "testutil/mock/mock_descriptors.h" +#include "testutil/mock/mock_query_context.h" +#include "testutil/mock/mock_runtime_state.h" + +namespace doris { +using namespace Aws::S3::Model; + +class ListFileS3Client : public Aws::S3::S3Client { +public: + MOCK_METHOD(HeadObjectOutcome, HeadObject, (const HeadObjectRequest&), (const, override)); + MOCK_METHOD(ListObjectsV2Outcome, ListObjectsV2, (const ListObjectsV2Request&), + (const, override)); +}; + +namespace { +ListObjectsV2Result page(std::initializer_list> objects, + const std::string& token = "") { + ListObjectsV2Result result; + result.SetIsTruncated(!token.empty()); + result.SetNextContinuationToken(token); + for (const auto& [key, size] : objects) { + Object object; + object.SetKey(key); + object.SetSize(size); + result.AddContents(object); + } + return result; +} + +std::vector ranges(std::string uri = "s3://bucket/dir", bool recursive = false) { + TTVFListFileScanRange params; + params.resource.properties = {{"AWS_BUCKET", "bucket"}}; + params.uri = std::move(uri); + params.recursive = recursive; + TScanRangeParams range; + range.scan_range.data_gen_scan_range.__set_list_file_params(params); + return {range}; +} + +class ListFileTVFTest : public testing::Test { +protected: + static void SetUpTestSuite() { S3ClientFactory::instance(); } + + void SetUp() override { + descriptor = std::make_unique( + std::vector {std::make_shared(), + make_nullable(std::make_shared())}, + &pool); + const std::array names {"path", "file"}; + for (size_t i = 0; i < names.size(); ++i) { + descriptor->tuple_desc_map[0]->slots()[i]->_col_name = names[i]; + } + function = std::make_unique(0, descriptor->tuple_desc_map[0]); + // Replace only remote I/O; parsing, paging, FILE construction and batching stay real. + s3 = std::make_shared>(); + auto filesystem = std::shared_ptr( + new io::S3FileSystem(S3Conf {}, "list-file-test")); + filesystem->_client = std::make_shared(S3ClientConf {}); + filesystem->_client->_client = std::make_shared(s3); + filesystem->_bucket = "bucket"; + function->_filesystem = filesystem; + } + + File row(const Block& block, size_t index) { + return (*block.get_by_position(1).column)[index].get(); + } + + void check_projection(const std::vector& positions) { + std::vector types; + for (const auto position : positions) { + types.push_back(descriptor->tuple_desc_map[0]->slots()[position]->get_data_type_ptr()); + } + MockRowDescriptor projected(types, &pool); + for (size_t i = 0; i < positions.size(); ++i) { + projected.tuple_desc_map[0]->slots()[i]->_col_name = + descriptor->tuple_desc_map[0]->slots()[positions[i]]->col_name(); + } + function->set_tuple_desc(projected.tuple_desc_map[0]); + ASSERT_TRUE(function->set_scan_ranges(ranges()).ok()); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([](const auto&) { + return ListObjectsV2Outcome(page({{"dir/a.txt", 7}})); + }); + Block block; + bool eos = false; + ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); + ASSERT_EQ(block.rows(), 1); + ASSERT_EQ(block.columns(), positions.size()); + EXPECT_TRUE(eos); + for (size_t i = 0; i < positions.size(); ++i) { + const auto value = (*block.get_by_position(i).column)[0]; + switch (positions[i]) { + case 0: + EXPECT_EQ(value.get(), "s3://bucket/dir/a.txt"); + break; + case 1: + EXPECT_EQ(value.get()[0].get(), "s3://bucket/dir/a.txt"); + EXPECT_EQ(value.get()[2].get(), 7); + break; + } + } + } + + void expect_etag_listing(const std::array, 5>& etags) { + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([etags](const auto& request) { + EXPECT_EQ(request.GetBucket(), "bucket"); + EXPECT_EQ(request.GetPrefix(), "dir/"); + ListObjectsV2Result result; + for (size_t i = 0; i < etags.size(); ++i) { + Object object; + object.SetKey("dir/" + std::to_string(i)); + object.SetSize(7); + if (etags[i]) { + object.SetETag(*etags[i]); + } + result.AddContents(std::move(object)); + } + return ListObjectsV2Outcome(result); + }); + } + + void expect_etag_head(size_t index, const std::optional& etag) { + EXPECT_CALL(*s3, HeadObject(testing::_)).WillOnce([index, etag](const auto& request) { + EXPECT_EQ(request.GetBucket(), "bucket"); + EXPECT_EQ(request.GetKey(), "dir/" + std::to_string(index)); + EXPECT_FALSE(request.ChecksumModeHasBeenSet()); + HeadObjectResult result; + result.SetContentLength(7); + if (etag) { + result.SetETag(*etag); + } + return HeadObjectOutcome(result); + }); + } + + void check_etag_consistency(const Block& block, size_t index) { + const auto value = row(block, index); + io::FileStat metadata; + ASSERT_TRUE(function->_filesystem->stat(value[0].get(), &metadata).ok()); + if (index < 3) { + EXPECT_TRUE(value[4].is_null()); + EXPECT_FALSE(metadata.checksum.has_value()); + return; + } + ASSERT_FALSE(value[4].is_null()); + EXPECT_EQ(value[4].get(), "ETAG:MiXeD:opaque-2"); + EXPECT_EQ(metadata.checksum, value[4].get()); + } + + ObjectPool pool; + MockRuntimeState state; + std::unique_ptr descriptor; + std::unique_ptr function; + std::shared_ptr> s3; +}; + +TEST_F(ListFileTVFTest, EmitsCompleteFilesAndSkipsDirectoryMarkers) { + ASSERT_TRUE(function->set_scan_ranges(ranges()).ok()); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([](const auto& request) { + EXPECT_EQ(request.GetBucket(), "bucket"); + EXPECT_EQ(request.GetPrefix(), "dir/"); + EXPECT_EQ(request.GetDelimiter(), "/"); + EXPECT_FALSE(request.ContinuationTokenHasBeenSet()); + return ListObjectsV2Outcome( + page({{"dir/", 0}, {"dir/empty.txt", 0}, {"dir/image.PNG", 27}, {"dir/raw", 4}})); + }); + Block block; + bool eos = false; + ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); + ASSERT_EQ(block.rows(), 3); + EXPECT_TRUE(eos); + EXPECT_TRUE(block.get_by_position(1).type->is_nullable()); + EXPECT_EQ((*block.get_by_position(0).column)[0].get(), + "s3://bucket/dir/empty.txt"); + const auto empty = row(block, 0); + EXPECT_EQ(empty[0].get(), "s3://bucket/dir/empty.txt"); + EXPECT_TRUE(empty[1].is_null()); + EXPECT_EQ(empty[2].get(), 0); + EXPECT_EQ(empty[3].get(), "text/plain"); + EXPECT_TRUE(empty[4].is_null()); + EXPECT_TRUE(empty[5].is_null()); + EXPECT_EQ(row(block, 1)[3].get(), "image/png"); + EXPECT_EQ(row(block, 2)[3].get(), "application/octet-stream"); +} + +TEST_F(ListFileTVFTest, EmitsListingEtagsWithoutHeadRequests) { + ASSERT_TRUE(function->set_scan_ranges(ranges()).ok()); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([](const auto&) { + ListObjectsV2Result result; + for (const auto& etag : + {std::optional("\"opaque-2\""), std::optional(""), + std::optional(), std::optional("\"\""), + std::optional("opaque-2")}) { + Object object; + object.SetKey("dir/" + std::to_string(result.GetContents().size())); + object.SetSize(7); + if (etag) { + object.SetETag(*etag); + } + result.AddContents(std::move(object)); + } + return ListObjectsV2Outcome(result); + }); + Block block; + bool eos = false; + ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); + ASSERT_EQ(block.rows(), 5); + EXPECT_TRUE(eos); + ASSERT_FALSE(row(block, 0)[4].is_null()); + EXPECT_EQ(row(block, 0)[4].get(), "ETAG:opaque-2"); + EXPECT_TRUE(row(block, 1)[4].is_null()); + EXPECT_TRUE(row(block, 2)[4].is_null()); + EXPECT_TRUE(row(block, 3)[4].is_null()); + EXPECT_EQ(row(block, 4)[4].get(), "ETAG:opaque-2"); +} + +TEST_F(ListFileTVFTest, ListingAndHeadProduceTheSameFileChecksum) { + const std::array, 5> etags {std::nullopt, "", "\"\"", + "MiXeD:opaque-2", "\"MiXeD:opaque-2\""}; + ASSERT_TRUE(function->set_scan_ranges(ranges()).ok()); + expect_etag_listing(etags); + Block block; + bool eos = false; + ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); + ASSERT_EQ(block.rows(), etags.size()); + EXPECT_TRUE(eos); + for (size_t i = 0; i < etags.size(); ++i) { + expect_etag_head(i, etags[i]); + check_etag_consistency(block, i); + } +} + +TEST_F(ListFileTVFTest, SplitsPagesIntoBatchesWithoutPrefetching) { + state._batch_size = 2; + ASSERT_TRUE(function->set_scan_ranges(ranges("s3://bucket/dir/", true)).ok()); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([](const auto& request) { + EXPECT_EQ(request.GetPrefix(), "dir/"); + EXPECT_FALSE(request.DelimiterHasBeenSet()); + return ListObjectsV2Outcome( + page({{"dir/a.txt", 1}, {"dir/b.txt", 2}, {"dir/deep/c.txt", 3}}, "next")); + }); + Block block; + bool eos = false; + ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); + ASSERT_EQ(block.rows(), 2); + EXPECT_FALSE(eos); + auto previous = block.get_by_position(1).column; + block.clear_column_data(); + ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); + ASSERT_EQ(block.rows(), 1); + EXPECT_FALSE(eos); + EXPECT_EQ(row(block, 0)[0].get(), "s3://bucket/dir/deep/c.txt"); + EXPECT_EQ(previous->size(), 2); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([](const auto& request) { + EXPECT_EQ(request.GetContinuationToken(), "next"); + return ListObjectsV2Outcome(page({{"dir/d.txt", 4}})); + }); + block.clear_column_data(); + ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); + EXPECT_EQ(block.rows(), 1); + EXPECT_TRUE(eos); +} + +TEST_F(ListFileTVFTest, ContinuesPastEmptyAndMarkerOnlyIntermediatePages) { + ASSERT_TRUE(function->set_scan_ranges(ranges("s3://bucket/dir/", true)).ok()); + testing::InSequence sequence; + EXPECT_CALL(*s3, ListObjectsV2(testing::_)) + .WillOnce(testing::Return(ListObjectsV2Outcome(page({}, "empty")))); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([](const auto& request) { + EXPECT_EQ(request.GetContinuationToken(), "empty"); + return ListObjectsV2Outcome(page({{"dir/subdir/", 0}}, "marker")); + }); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([](const auto& request) { + EXPECT_EQ(request.GetContinuationToken(), "marker"); + return ListObjectsV2Outcome(page({{"dir/file.csv", 8}})); + }); + Block block; + bool eos = false; + ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); + EXPECT_EQ(block.rows(), 1); + EXPECT_TRUE(eos); +} + +TEST_F(ListFileTVFTest, ReusesOutputColumnsAcrossPages) { + ASSERT_TRUE(function->set_scan_ranges(ranges()).ok()); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)) + .WillOnce(testing::Return(ListObjectsV2Outcome(page({{"dir/a.txt", 1}}, "next")))) + .WillOnce(testing::Return(ListObjectsV2Outcome(page({{"dir/b.txt", 2}})))); + Block block; + bool eos = false; + ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); + ASSERT_FALSE(eos); + const auto* path_column = block.get_by_position(0).column.get(); + const auto* file_column = block.get_by_position(1).column.get(); + block.clear_column_data(); + ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); + EXPECT_EQ(block.get_by_position(0).column.get(), path_column); + EXPECT_EQ(block.get_by_position(1).column.get(), file_column); + ASSERT_EQ(block.rows(), 1); + EXPECT_EQ(row(block, 0)[0].get(), "s3://bucket/dir/b.txt"); + EXPECT_TRUE(eos); +} + +TEST_F(ListFileTVFTest, RestoresReusedColumnsAfterListingError) { + ASSERT_TRUE(function->set_scan_ranges(ranges()).ok()); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)) + .WillOnce(testing::Return(ListObjectsV2Outcome(page({{"dir/a.txt", 1}}, "next")))) + .WillOnce([](const auto&) { + Aws::S3::S3Error error; + error.SetMessage("listing failed"); + return ListObjectsV2Outcome(error); + }); + Block block; + bool eos = false; + ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); + const auto* path_column = block.get_by_position(0).column.get(); + const auto* file_column = block.get_by_position(1).column.get(); + block.clear_column_data(); + EXPECT_FALSE(function->get_next(&state, &block, &eos).ok()); + ASSERT_NE(block.get_by_position(0).column.get(), nullptr); + ASSERT_NE(block.get_by_position(1).column.get(), nullptr); + EXPECT_EQ(block.get_by_position(0).column.get(), path_column); + EXPECT_EQ(block.get_by_position(1).column.get(), file_column); + EXPECT_EQ(block.rows(), 0); +} + +TEST_F(ListFileTVFTest, UsesUriBucketWithoutResourceBucketConstraint) { + for (const auto& properties : {std::map {}, + std::map {{"AWS_BUCKET", "other"}}}) { + auto parameters = ranges(); + parameters[0].scan_range.data_gen_scan_range.list_file_params.resource.properties = + properties; + ASSERT_TRUE(function->set_scan_ranges(parameters).ok()); + EXPECT_EQ(function->_path.bucket, "bucket"); + } +} + +TEST_F(ListFileTVFTest, DirectoryPathCanContainAnotherSchemeDelimiter) { + ASSERT_TRUE(function->set_scan_ranges(ranges("s3://bucket/http://dir/")).ok()); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([](const auto& request) { + EXPECT_EQ(request.GetPrefix(), "http://dir/"); + return ListObjectsV2Outcome(page({{"http://dir/a.txt", 1}})); + }); + Block block; + bool eos = false; + ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); + EXPECT_EQ(row(block, 0)[0].get(), "s3://bucket/http://dir/a.txt"); + EXPECT_TRUE(eos); +} + +TEST_F(ListFileTVFTest, HttpUrisUseParsedBucketAndCompleteDirectoryKey) { + for (const auto* scheme : {"http://", "https://"}) { + auto listing = std::make_unique(0, descriptor->tuple_desc_map[0]); + listing->_filesystem = function->_filesystem; + ASSERT_TRUE(listing->set_scan_ranges(ranges(std::string(scheme) + + "host/bucket/dir?part#raw://nested")) + .ok()); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([](const auto& request) { + EXPECT_EQ(request.GetBucket(), "bucket"); + EXPECT_EQ(request.GetPrefix(), "dir?part#raw://nested/"); + EXPECT_EQ(request.GetDelimiter(), "/"); + return ListObjectsV2Outcome(page({{"dir?part#raw://nested/a.jpg", 7}})); + }); + Block block; + bool eos = false; + ASSERT_TRUE(listing->get_next(&state, &block, &eos).ok()); + ASSERT_EQ(block.rows(), 1); + EXPECT_EQ((*block.get_by_position(0).column)[0].get(), + "s3://bucket/dir?part#raw://nested/a.jpg"); + EXPECT_EQ(row(block, 0)[3].get(), "image/jpeg"); + EXPECT_TRUE(eos); + } +} + +TEST_F(ListFileTVFTest, ObjectKeysAndDirectoryPrefixRemainRaw) { + ASSERT_TRUE(function->set_scan_ranges(ranges("s3://bucket/dir%20%25%3F%23%E4%B8%AD+")).ok()); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([](const auto& request) { + EXPECT_EQ(request.GetPrefix(), "dir%20%25%3F%23%E4%B8%AD+/"); + return ListObjectsV2Outcome(page({{"dir%20%25%3F%23%E4%B8%AD+/file %?#中+.txt", 3}})); + }); + Block block; + bool eos = false; + ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); + const std::string expected = "s3://bucket/dir%20%25%3F%23%E4%B8%AD+/file %?#中+.txt"; + EXPECT_EQ((*block.get_by_position(0).column)[0].get(), expected); + EXPECT_EQ(row(block, 0)[0].get(), expected); + EXPECT_EQ(row(block, 0)[3].get(), "text/plain"); +} + +TEST_F(ListFileTVFTest, AcceptsCanonicalS3BucketProperty) { + auto parameters = ranges(); + auto& properties = + parameters[0].scan_range.data_gen_scan_range.list_file_params.resource.properties; + properties = {{"s3.bucket", "bucket"}}; + ASSERT_TRUE(function->set_scan_ranges(parameters).ok()); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([](const auto& request) { + EXPECT_EQ(request.GetBucket(), "bucket"); + return ListObjectsV2Outcome(page({{"dir/a.csv", 1}})); + }); + Block block; + bool eos = false; + ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); + EXPECT_EQ(block.rows(), 1); + EXPECT_TRUE(eos); +} + +TEST_F(ListFileTVFTest, BucketRootFormsListWithoutPrefix) { + for (const auto* uri : {"s3://bucket", "s3://bucket/", "http://host/bucket", + "http://host/bucket/", "https://host/bucket", "https://host/bucket/"}) { + auto listing = std::make_unique(0, descriptor->tuple_desc_map[0]); + listing->_filesystem = function->_filesystem; + ASSERT_TRUE(listing->set_scan_ranges(ranges(uri)).ok()) << uri; + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([](const auto& request) { + EXPECT_TRUE(request.GetPrefix().empty()); + EXPECT_EQ(request.GetDelimiter(), "/"); + return ListObjectsV2Outcome(page({{"root.csv", 2}})); + }); + Block block; + bool eos = false; + ASSERT_TRUE(listing->get_next(&state, &block, &eos).ok()); + EXPECT_EQ(row(block, 0)[0].get(), "s3://bucket/root.csv"); + EXPECT_TRUE(eos); + } +} + +TEST_F(ListFileTVFTest, ExtraLeadingSlashIsPartOfDirectoryKey) { + ASSERT_TRUE(function->set_scan_ranges(ranges("s3://bucket//")).ok()); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([](const auto& request) { + EXPECT_EQ(request.GetPrefix(), "/"); + return ListObjectsV2Outcome(page({{"/a.csv", 1}})); + }); + Block block; + bool eos = false; + ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); + EXPECT_EQ(row(block, 0)[0].get(), "s3://bucket//a.csv"); + EXPECT_TRUE(eos); +} + +TEST_F(ListFileTVFTest, EmptyListingReturnsEmptyFileColumn) { + ASSERT_TRUE(function->set_scan_ranges(ranges()).ok()); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)) + .WillOnce(testing::Return(ListObjectsV2Outcome(page({})))); + Block block; + bool eos = false; + ASSERT_TRUE(function->get_next(&state, &block, &eos).ok()); + EXPECT_EQ(block.rows(), 0); + ASSERT_EQ(block.columns(), 2); + EXPECT_EQ(block.get_by_position(0).type->get_name(), "String"); + EXPECT_EQ(block.get_by_position(1).type->get_name(), "Nullable(File)"); + EXPECT_TRUE(eos); +} + +class ToFileWithListedFilesystem : public FunctionToFile { +public: + explicit ToFileWithListedFilesystem(std::shared_ptr filesystem) + : _filesystem(std::move(filesystem)) {} + Status create_filesystem(const TS3Resource&, const std::string&, + std::shared_ptr* result) const override { + *result = _filesystem; + return Status::OK(); + } + std::shared_ptr _filesystem; +}; + +TEST_F(ListFileTVFTest, RawPathsRoundTripThroughToFileAndS3Head) { + ToFileWithListedFilesystem to_file(function->_filesystem); + const std::array keys { + "dir/a://b.txt", "dir/a?b.txt", "dir/a#b.txt", + "dir/ a.txt ", "dir/中文+space .txt", "dir/a%20b.txt", + "dir/a?b#c://d.txt", "dir/photo?part.jpg", "dir/photo#part.PNG"}; + ASSERT_TRUE(function->set_scan_ranges(ranges()).ok()); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([&](const auto&) { + ListObjectsV2Result result; + for (const auto& key : keys) { + Aws::S3::Model::Object object; + object.SetKey(key); + object.SetSize(7); + result.AddContents(std::move(object)); + } + return ListObjectsV2Outcome(result); + }); + Block listed; + bool eos = false; + ASSERT_TRUE(function->get_next(&state, &listed, &eos).ok()); + ASSERT_EQ(listed.rows(), keys.size()); + auto text = std::make_shared(); + auto query = MockQueryContext::create(); + query->set_s3_resources({{"resource", TS3Resource {}}}); + state._query_ctx = query.get(); + auto resource = + text->create_column_const(listed.rows(), Field::create_field("resource")); + auto context = + FunctionContext::create_context(&state, std::make_shared(), {text, text}); + context->set_constant_cols({std::make_shared(resource), nullptr}); + ASSERT_TRUE(to_file.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + size_t head_index = 0; + EXPECT_CALL(*s3, HeadObject(testing::_)) + .Times(keys.size()) + .WillRepeatedly([&](const auto& request) { + EXPECT_EQ(request.GetBucket(), "bucket"); + EXPECT_EQ(request.GetKey(), keys[head_index++]); + HeadObjectResult result; + result.SetContentLength(7); + return HeadObjectOutcome(result); + }); + Block output {{resource, text, "resource"}, + listed.get_by_position(0), + {nullptr, std::make_shared(), "result"}}; + ASSERT_TRUE(to_file.execute(context.get(), output, {0, 1}, 2, listed.rows()).ok()); + for (size_t i = 0; i < keys.size(); ++i) { + const auto value = (*output.get_by_position(2).column)[i].get(); + EXPECT_EQ(value[0].get(), "s3://bucket/" + keys[i]); + EXPECT_EQ(value[3], row(listed, i)[3]) << keys[i]; + } + EXPECT_EQ(row(listed, 7)[3].get(), "image/jpeg"); + EXPECT_EQ(row(listed, 8)[3].get(), "image/png"); + ASSERT_TRUE(to_file.close(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + state._query_ctx = nullptr; +} + +TEST_F(ListFileTVFTest, CancellationBeforeIoDoesNotFetchPage) { + ASSERT_TRUE(function->set_scan_ranges(ranges()).ok()); + state.cancel(Status::Cancelled("cancel before list")); + Block block; + bool eos = false; + EXPECT_FALSE(function->get_next(&state, &block, &eos).ok()); + EXPECT_EQ(block.rows(), 0); +} + +TEST_F(ListFileTVFTest, CancellationAfterEmptyPageStopsBeforeNextRequest) { + ASSERT_TRUE(function->set_scan_ranges(ranges()).ok()); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([&](const auto&) { + state.cancel(Status::Cancelled("cancel during list")); + return ListObjectsV2Outcome(page({}, "next")); + }); + Block block; + bool eos = false; + const auto status = function->get_next(&state, &block, &eos); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(block.rows(), 0); +} + +TEST_F(ListFileTVFTest, PropagatesListingErrors) { + ASSERT_TRUE(function->set_scan_ranges(ranges()).ok()); + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([](const auto&) { + Aws::S3::S3Error error; + error.SetResponseCode(Aws::Http::HttpResponseCode::FORBIDDEN); + error.SetMessage("listing denied"); + return ListObjectsV2Outcome(error); + }); + Block block; + bool eos = false; + const auto status = function->get_next(&state, &block, &eos); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("listing denied"), std::string::npos); +} + +TEST_F(ListFileTVFTest, ListingUsesBlockingSchedulerAndNumbersKeepsDefault) { + DataGenSourceOperatorX listing; + TPlanNode node; + node.node_type = TPlanNodeType::DATA_GEN_SCAN_NODE; + node.data_gen_scan_node.func_name = TDataGenFunctionName::LIST_FILE; + ASSERT_TRUE(listing.init(node, &state).ok()); + EXPECT_TRUE(listing._blockable); + DataGenSourceOperatorX numbers; + node.data_gen_scan_node.func_name = TDataGenFunctionName::NUMBERS; + ASSERT_TRUE(numbers.init(node, &state).ok()); + EXPECT_FALSE(numbers._blockable); +} + +TEST_F(ListFileTVFTest, ZeroLimitDoesNotFetchAnyPage) { + OperatorContext context; + DataGenSourceOperatorX op; + op._tuple_id = 0; + op._tuple_desc = descriptor->tuple_desc_map[0]; + op._function_name = TDataGenFunctionName::LIST_FILE; + op._limit = 0; + OperatorHelper::init_local_state(context, op, ranges()); + auto& local = op.get_local_state(&context.state); + static_cast(local._table_func.get())->_filesystem = function->_filesystem; + Block block; + bool eos = false; + ASSERT_TRUE(op.get_block(&context.state, &block, &eos).ok()); + EXPECT_EQ(block.rows(), 0); + EXPECT_EQ(block.columns(), 2); + EXPECT_TRUE(eos); +} + +TEST_F(ListFileTVFTest, LimitStopsBeforeFetchingFollowingPage) { + OperatorContext context; + DataGenSourceOperatorX op; + op._tuple_id = 0; + op._tuple_desc = descriptor->tuple_desc_map[0]; + op._function_name = TDataGenFunctionName::LIST_FILE; + op._limit = 1; + OperatorHelper::init_local_state(context, op, ranges()); + auto& local = op.get_local_state(&context.state); + static_cast(local._table_func.get())->_filesystem = function->_filesystem; + EXPECT_CALL(*s3, ListObjectsV2(testing::_)).WillOnce([](const auto&) { + return ListObjectsV2Outcome(page({{"dir/a.txt", 1}, {"dir/b.txt", 2}}, "next")); + }); + Block block; + bool eos = false; + ASSERT_TRUE(op.get_block(&context.state, &block, &eos).ok()); + EXPECT_EQ(block.rows(), 1); + EXPECT_TRUE(eos); +} + +TEST_F(ListFileTVFTest, PrunedPathProjection) { + check_projection({0}); +} + +TEST_F(ListFileTVFTest, PrunedFileProjection) { + check_projection({1}); +} + +TEST_F(ListFileTVFTest, ReorderedPrunedProjection) { + check_projection({1, 0}); +} + +} // namespace +} // namespace doris diff --git a/be/test/exec/common/data_gen_functions/vlist_files_rows_test.cpp b/be/test/exec/common/data_gen_functions/vlist_files_rows_test.cpp new file mode 100644 index 00000000000000..d8feb1c50002bb --- /dev/null +++ b/be/test/exec/common/data_gen_functions/vlist_files_rows_test.cpp @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include + +#include + +#include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "exec/common/data_gen_functions/vlist_file_tvf.h" +#include "runtime/descriptors.h" +#include "runtime/runtime_state.h" + +namespace doris { + +TEST(ListFilesRowsTest, DirectoryNullsSizesAndProjectionsAcrossBatches) { + const std::array names {"path", "size", "file"}; + const std::array types {std::make_shared(), + std::make_shared(), + std::make_shared()}; + const std::vector> projections {{0, 1, 2}, {0}, {1}, {2}, {0, 1}, + {0, 2}, {1, 2}, {2, 0}, {2, 1, 0}}; + for (bool recursive : {false, true}) { + for (const auto& projection : projections) { + TTupleDescriptor tuple_spec; + tuple_spec.__set_id(0); + TupleDescriptor tuple(tuple_spec, true); + for (const auto index : projection) { + TSlotDescriptor slot; + slot.__set_id(static_cast(index)); + slot.__set_parent(0); + slot.__set_colName(names[index]); + slot.__set_slotType(types[index]->to_thrift()); + slot.__set_nullIndicatorBit(index == 2 ? 0 : -1); + tuple.add_slot(new SlotDescriptor(slot)); + } + VListFileTVF function(0, &tuple); + TTVFListFileScanRange params; + params.__set_uri("s3://bucket/a/"); + params.__set_recursive(recursive); + TScanRangeParams range; + range.scan_range.data_gen_scan_range.__set_list_file_params(params); + ASSERT_TRUE(function.set_scan_ranges({range}).ok()); + function._objects = {{.key = "a/"}, + {.key = "a/empty.txt"}, + {.key = "a/b/"}, + {.key = "a/c.png", .size = 3}, + {.key = "a/marker/", .size = 7}}; + function._has_more = false; + RuntimeState state; + state._query_options.__set_batch_size(1); + Block block; + bool eos = false; + size_t rows = 0; + while (!eos) { + ASSERT_TRUE(function.get_next(&state, &block, &eos).ok()); + for (size_t row = 0; row < block.rows(); ++row) { + const bool directory = !recursive && (rows == 1 || rows == 3); + const int64_t size = rows == (recursive ? 1U : 2U) ? 3 : 0; + for (size_t column = 0; column < projection.size(); ++column) { + const auto field = (*block.get_by_position(column).column)[row]; + switch (projection[column]) { + case 0: + EXPECT_EQ(field.get().ends_with('/'), directory); + break; + case 1: + EXPECT_EQ(field.get(), size); + break; + case 2: + EXPECT_EQ(field.is_null(), directory); + if (!directory) { + EXPECT_EQ(field.get()[2].get(), size); + } + break; + } + } + ++rows; + } + block.clear_column_data(); + } + EXPECT_EQ(rows, recursive ? 2 : 4); + function._objects.clear(); + function._next_index = 0; + eos = false; + Block empty; + ASSERT_TRUE(function.get_next(&state, &empty, &eos).ok()); + EXPECT_EQ(empty.rows(), 0); + EXPECT_EQ(empty.columns(), projection.size()); + for (size_t column = 0; column < projection.size(); ++column) { + EXPECT_EQ(empty.get_by_position(column).name, names[projection[column]]); + EXPECT_EQ(empty.get_by_position(column).type->is_nullable(), + projection[column] == 2); + } + EXPECT_TRUE(eos); + } + } +} + +} // namespace doris diff --git a/be/test/exec/operator/query_cache_operator_test.cpp b/be/test/exec/operator/query_cache_operator_test.cpp index 91c73b99077247..49e029d4835346 100644 --- a/be/test/exec/operator/query_cache_operator_test.cpp +++ b/be/test/exec/operator/query_cache_operator_test.cpp @@ -21,6 +21,10 @@ #include #include "core/block/block.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" #include "exec/operator/cache_sink_operator.h" #include "exec/operator/cache_source_operator.h" #include "exec/operator/repeat_operator.h" @@ -173,6 +177,168 @@ TEST_F(QueryCacheOperatorTest, test_no_hit_cache1) { EXPECT_TRUE(source_local_state->_need_insert_cache); } +TEST_F(QueryCacheOperatorTest, FilePayloadMissCommitAndRepeatedHit) { + const auto file_type = make_nullable(std::make_shared()); + const auto array_type = make_nullable(std::make_shared(file_type)); + const auto payload_type = make_nullable( + std::make_shared(DataTypes {array_type}, Strings {"files"})); + const auto id_type = std::make_shared(); + const std::vector inline_bytes { + std::string("\0\xff\x80", 3) + std::string(4096, 'a'), + std::string("\xff\0z", 3) + std::string(4096, 'b')}; + + auto make_block = [&](size_t batch) { + File file {Field::create_field("urn:query-cache:nested"), + Field::create_field(7), + Field::create_field(4099), + Field::create_field("application/octet-stream"), + Field::create_field("ETAG:nested-cache"), + Field::create_field(StringView(inline_bytes[batch]))}; + Array files {Field::create_field(file), Field()}; + file[5] = Field::create_field(StringView()); + files.push_back(Field::create_field(file)); + file[5] = Field(); + files.push_back(Field::create_field(file)); + auto payload = payload_type->create_column(); + payload->insert_default(); // NULL struct ancestor. + payload->insert(Field::create_field(Struct {Field()})); // NULL array ancestor. + payload->insert(Field::create_field( + Struct {Field::create_field(Array {})})); + payload->insert(Field::create_field( + Struct {Field::create_field(std::move(files))})); + auto ids = id_type->create_column(); + for (int64_t row = 0; row < 4; ++row) { + ids->insert(Field::create_field(static_cast(batch) * 10 + row)); + } + Block block; + block.insert({std::move(ids), id_type, "id"}); + block.insert({std::move(payload), payload_type, "payload"}); + return block; + }; + auto expect_block = [&](const Block& block, size_t batch, bool reordered) { + ASSERT_EQ(block.rows(), 4); + ASSERT_EQ(block.columns(), 2); + const auto& ids = block.get_by_position(reordered ? 1 : 0); + const auto& payload = block.get_by_position(reordered ? 0 : 1); + ASSERT_TRUE(ids.type->equals(*id_type)); + ASSERT_TRUE(payload.type->equals(*payload_type)); + for (int64_t row = 0; row < 4; ++row) { + EXPECT_EQ((*ids.column)[row].get(), + static_cast(batch) * 10 + row); + } + ASSERT_TRUE(payload.column->is_null_at(0)); + for (size_t row = 1; row < 4; ++row) { + ASSERT_FALSE(payload.column->is_null_at(row)); + } + const auto null_array = (*payload.column)[1]; + EXPECT_TRUE(null_array.get()[0].is_null()); + const auto empty_array = (*payload.column)[2]; + EXPECT_TRUE(empty_array.get()[0].get().empty()); + const auto populated = (*payload.column)[3]; + const auto& files = populated.get()[0].get(); + ASSERT_EQ(files.size(), 4); + EXPECT_TRUE(files[1].is_null()); + for (size_t item : {0, 2, 3}) { + ASSERT_EQ(files[item].get_type(), TYPE_FILE); + const auto& fields = files[item].get(); + ASSERT_EQ(fields.size(), 6); + EXPECT_EQ(fields[0].get(), "urn:query-cache:nested"); + EXPECT_EQ(fields[1].get(), 7); + EXPECT_EQ(fields[2].get(), 4099); + EXPECT_EQ(fields[3].get(), "application/octet-stream"); + EXPECT_EQ(fields[4].get(), "ETAG:nested-cache"); + if (item == 3) { + EXPECT_TRUE(fields[5].is_null()); + } else { + ASSERT_FALSE(fields[5].is_null()); + EXPECT_EQ(fields[5].get().to_string_ref().to_string(), + item == 0 ? inline_bytes[batch] : std::string()); + } + } + }; + + auto start_query = [&](bool reordered) { + // Release the old query's handles/local state; keep only the global cache. + state.reset(); + source.reset(); + sink.reset(); + state = std::make_shared(); + state->_batch_size = 10; + sink = std::make_unique(); + auto row_desc = std::make_unique( + reordered ? DataTypes {payload_type, id_type} : DataTypes {id_type, payload_type}, + &pool); + auto& slots = static_cast(row_desc->tuple_desc_map.front())->Slots; + slots[0]->_id = 100; + slots[1]->_id = 101; + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "file-payload-query"; + cache_param.output_slot_mapping[100] = reordered ? 11 : 10; + cache_param.output_slot_mapping[101] = reordered ? 10 : 11; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + source = std::make_unique(&pool, 0, 0, cache_param); + // Keep the output schema on subsequent pulls, as the pipeline driver does. + source->_row_descriptor = *row_desc; + child_op->_mock_row_desc = std::move(row_desc); + ASSERT_TRUE(source->set_child(child_op)); + create_local_state(); + }; + + ASSERT_NO_FATAL_FAILURE(start_query(false)); + ASSERT_EQ(source_local_state->_hit_cache_results, nullptr); + ASSERT_TRUE(source_local_state->_need_insert_cache); + for (size_t batch = 0; batch < inline_bytes.size(); ++batch) { + auto input = make_block(batch); + const auto st = sink->sink(state.get(), &input, batch == 1); + ASSERT_TRUE(st.ok()) << st; + } + Block output; + for (size_t batch = 0; batch < inline_bytes.size(); ++batch) { + bool eos = false; + const auto st = source->get_block(state.get(), &output, &eos); + ASSERT_TRUE(st.ok()) << st; + EXPECT_EQ(eos, batch == 1); + ASSERT_NO_FATAL_FAILURE(expect_block(output, batch, false)); + output.clear_column_data(2); + } + ASSERT_EQ(query_cache->get_element_count(), 1); + EXPECT_TRUE(source_local_state->_need_insert_cache); + + for (int query = 0; query < 2; ++query) { + SCOPED_TRACE(query); + ASSERT_NO_FATAL_FAILURE(start_query(true)); + ASSERT_NE(source_local_state->_hit_cache_results, nullptr); + const auto* hit = source_local_state->custom_profile()->get_info_string("HitCache"); + ASSERT_NE(hit, nullptr); + EXPECT_EQ(*hit, "1"); + EXPECT_EQ(source_local_state->_hit_cache_column_orders, (std::vector {1, 0})); + // A hit skips the scan: only eos arrives from the sink, never replacement rows. + Block empty; + ASSERT_TRUE(sink->sink(state.get(), &empty, true).ok()); + output.clear(); + for (size_t batch = 0; batch < inline_bytes.size(); ++batch) { + bool eos = false; + const auto st = source->get_block(state.get(), &output, &eos); + ASSERT_TRUE(st.ok()) << st; + EXPECT_FALSE(eos); + ASSERT_NO_FATAL_FAILURE(expect_block(output, batch, true)); + // Clearing a returned block must not mutate the next cache hit. The 4.1 + // source clones the cached schema before applying the requested slot order. + output.clear_column_data(2); + output.clear(); + } + bool eos = false; + ASSERT_TRUE(source->get_block(state.get(), &output, &eos).ok()); + EXPECT_TRUE(eos); + EXPECT_EQ(output.rows(), 0); + EXPECT_EQ(query_cache->get_element_count(), 1); + } +} + TEST_F(QueryCacheOperatorTest, test_no_hit_cache2) { sink = std::make_unique(); source = std::make_unique(); diff --git a/be/test/exec/pipeline/query_cache_test.cpp b/be/test/exec/pipeline/query_cache_test.cpp index ec0f8c6c697686..82e7499606f4d7 100644 --- a/be/test/exec/pipeline/query_cache_test.cpp +++ b/be/test/exec/pipeline/query_cache_test.cpp @@ -22,6 +22,13 @@ #include #include +#include "cloud/config.h" +#include "common/config.h" +#include "common/metrics/doris_metrics.h" +#include "core/column/column_file.h" +#include "core/column/column_nullable.h" +#include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "testutil/column_helper.h" @@ -285,6 +292,108 @@ TEST_F(QueryCacheTest, insert_and_lookup) { } } +TEST_F(QueryCacheTest, FilePayloadInsertLookupOwnsAllChildren) { + // Keep both versioned payloads within one shard's budget so LRU-K admission + // does not mask the value-ownership and replacement checks below. + std::unique_ptr cache(QueryCache::create_global_cache(1024 * 1024, 1)); + const auto type = make_nullable(std::make_shared()); + const std::string bytes = std::string("\0\xff\x80", 3) + std::string(4096, 'q'); + auto column = type->create_column(); + File value {Field::create_field("s3://bucket/a%2Fb?versionId=cache"), + Field::create_field(7), + Field::create_field(4099), + Field::create_field("application/octet-stream"), + Field::create_field("ETAG:cache"), + Field::create_field(StringView(bytes))}; + column->insert(Field::create_field(value)); + value[5] = Field::create_field(StringView()); + column->insert(Field::create_field(value)); + for (size_t child = 1; child < value.size(); ++child) { + value[child] = Field(); + } + column->insert(Field::create_field(value)); + column->insert_default(); + + auto expect_payload = [&](const Block& block) { + ASSERT_EQ(block.rows(), 4); + ASSERT_EQ(block.columns(), 1); + ASSERT_TRUE(block.get_by_position(0).type->equals(*type)); + const auto& nullable = assert_cast(*block.get_by_position(0).column); + const auto& file = assert_cast(nullable.get_nested_column()); + ASSERT_EQ(file.get_columns().size(), 6); + EXPECT_TRUE(nullable.is_null_at(3)); + for (size_t row = 0; row < 3; ++row) { + ASSERT_FALSE(nullable.is_null_at(row)); + const auto actual = file[row]; + ASSERT_EQ(actual.get_type(), TYPE_FILE); + const auto& fields = actual.get(); + ASSERT_EQ(fields.size(), 6); + EXPECT_EQ(fields[0].get(), "s3://bucket/a%2Fb?versionId=cache"); + if (row == 2) { + for (size_t child = 1; child < fields.size(); ++child) { + EXPECT_TRUE(fields[child].is_null()); + } + } else { + EXPECT_EQ(fields[1].get(), 7); + EXPECT_EQ(fields[2].get(), 4099); + EXPECT_EQ(fields[3].get(), "application/octet-stream"); + EXPECT_EQ(fields[4].get(), "ETAG:cache"); + ASSERT_FALSE(fields[5].is_null()); + EXPECT_EQ(fields[5].get().to_string_ref().to_string(), + row == 0 ? bytes : std::string()); + } + } + }; + + CacheResult input; + input.emplace_back(std::make_unique()); + input.back()->insert({std::move(column), type, "f"}); + cache->insert("file-payload", 42, input, {17}, input.back()->allocated_bytes()); + QueryCacheHandle handle; + ASSERT_TRUE(cache->lookup("file-payload", 42, &handle)); + ASSERT_EQ(handle.get_cache_result()->size(), 1); + EXPECT_EQ(*handle.get_cache_slot_orders(), (std::vector {17})); + EXPECT_EQ(handle.get_cache_result()->front()->rows(), 4); + const auto& original = assert_cast( + assert_cast(*input.front()->get_by_position(0).column) + .get_nested_column()); + const auto& cached = assert_cast( + assert_cast( + *handle.get_cache_result()->front()->get_by_position(0).column) + .get_nested_column()); + for (size_t child = 0; child < 6; ++child) { + EXPECT_NE(&original.get_column(child), &cached.get_column(child)); + } + EXPECT_NE(original.get_column(5).get_data_at(0).data, cached.get_column(5).get_data_at(0).data); + // Reuse the producer's block and release its storage before another lookup. + input.front()->clear_column_data(1); + input.front()->get_by_position(0).column->assert_mutable()->insert_default(); + input.clear(); + expect_payload(*handle.get_cache_result()->front()); + { + QueryCacheHandle again; + ASSERT_TRUE(cache->lookup("file-payload", 42, &again)); + expect_payload(*again.get_cache_result()->front()); + } + QueryCacheHandle wrong_version; + EXPECT_FALSE(cache->lookup("file-payload", 43, &wrong_version)); + + CacheResult replacement; + replacement.emplace_back(std::make_unique()); + auto null_column = type->create_column(); + null_column->insert_default(); + replacement.back()->insert({std::move(null_column), type, "f"}); + cache->insert("file-payload", 43, replacement, {17}, replacement.back()->allocated_bytes()); + replacement.clear(); + // Replacement must not invalidate a pinned old entry's FILE children/inline arena. + QueryCacheHandle pinned(std::move(handle)); + expect_payload(*pinned.get_cache_result()->front()); + QueryCacheHandle latest; + ASSERT_TRUE(cache->lookup("file-payload", 43, &latest)); + EXPECT_EQ(latest.get_cache_result()->front()->rows(), 1); + EXPECT_TRUE(latest.get_cache_result()->front()->get_by_position(0).column->is_null_at(0)); +} + // ./run-be-ut.sh --run --filter=DataQueueTest.* } // namespace doris diff --git a/be/test/exprs/aggregate/aggregate_function_file_test.cpp b/be/test/exprs/aggregate/aggregate_function_file_test.cpp new file mode 100644 index 00000000000000..027297b895773e --- /dev/null +++ b/be/test/exprs/aggregate/aggregate_function_file_test.cpp @@ -0,0 +1,250 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include + +#include "agent/be_exec_version_manager.h" +#include "core/arena.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/field.h" +#include "core/string_buffer.hpp" +#include "core/value/file_value.h" +#include "exprs/aggregate/aggregate_function.h" +#include "exprs/aggregate/aggregate_function_simple_factory.h" + +namespace doris { +namespace { + +Field file_aggregate_value(int id) { + static const std::string bytes = std::string(4096, '\xff') + std::string("\0end", 4); + return Field::create_field( + File {Field::create_field("urn:aggregate:" + std::to_string(id)), Field(), + Field::create_field(Int64 {id}), Field(), + Field::create_field("ETAG:opaque-" + std::to_string(id)), + id == 1 ? Field::create_field(StringView(bytes)) + : Field::create_field(StringView(""))}); +} + +void expect_file_payload(const Field& value, int id) { + ASSERT_EQ(value.get_type(), TYPE_FILE); + const auto& file = value.get(); + ASSERT_EQ(file.size(), 6); + EXPECT_EQ(file[0].get(), "urn:aggregate:" + std::to_string(id)); + EXPECT_TRUE(file[1].is_null()); + EXPECT_EQ(file[2].get(), id); + EXPECT_TRUE(file[3].is_null()); + EXPECT_EQ(file[4].get(), "ETAG:opaque-" + std::to_string(id)); + ASSERT_EQ(file[5].get_type(), TYPE_VARBINARY); + const auto bytes = file[5].get(); + EXPECT_EQ(std::string(bytes.data(), bytes.size()), + id == 1 ? std::string(4096, '\xff') + std::string("\0end", 4) : std::string()); +} + +void expect_opaque_file_bytes(const Field& value, const Field& expected) { + ASSERT_EQ(value.get_type(), expected.get_type()); + if (value.is_null()) { + return; + } + switch (value.get_type()) { + case TYPE_FILE: { + const auto& file = value.get(); + ASSERT_EQ(file.size(), 6); + EXPECT_EQ(file[0].get(), std::string("urn:\xff\0x", 7)); + EXPECT_TRUE(file[1].is_null()); + EXPECT_EQ(file[2].get(), 7); + EXPECT_TRUE(file[3].is_null()); + EXPECT_EQ(file[4].get(), std::string("ETAG:\xff\0x", 8)); + const auto bytes = file[5].get(); + EXPECT_EQ(std::string(bytes.data(), bytes.size()), std::string("\xff\0x", 3)); + break; + } + case TYPE_ARRAY: { + const auto& children = value.get(); + const auto& expected_children = expected.get(); + ASSERT_EQ(children.size(), expected_children.size()); + for (size_t i = 0; i < children.size(); ++i) { + expect_opaque_file_bytes(children[i], expected_children[i]); + } + break; + } + case TYPE_STRUCT: { + const auto& children = value.get(); + const auto& expected_children = expected.get(); + ASSERT_EQ(children.size(), expected_children.size()); + for (size_t i = 0; i < children.size(); ++i) { + expect_opaque_file_bytes(children[i], expected_children[i]); + } + break; + } + case TYPE_MAP: { + const auto& children = value.get(); + const auto& expected_children = expected.get(); + ASSERT_EQ(children.size(), expected_children.size()); + for (size_t i = 0; i < children.size(); ++i) { + expect_opaque_file_bytes(children[i], expected_children[i]); + } + break; + } + default: + ASSERT_EQ(value.get_type(), TYPE_STRING); + EXPECT_EQ(value.get(), "key"); + } +} + +} // namespace + +TEST(FileAggregateTest, CollectListPartialStatesPreserveOpaqueBytes) { + const auto file_type = std::make_shared(); + const auto nullable_file = make_nullable(file_type); + const auto file = Field::create_field( + File {Field::create_field(std::string("urn:\xff\0x", 7)), Field(), + Field::create_field(Int64 {7}), Field(), + Field::create_field(std::string("ETAG:\xff\0x", 8)), + Field::create_field(StringView("\xff\0x", 3))}); + ASSERT_TRUE(validate_file(file.get()).ok()); + const std::vector> cases = { + {file_type, file}, + {std::make_shared(nullable_file), + Field::create_field(Array {file, Field()})}, + {std::make_shared(DataTypes {nullable_file}, Strings {"f"}), + Field::create_field(Struct {file})}, + {std::make_shared(make_nullable(std::make_shared()), + nullable_file), + Field::create_field(Map {Field::create_field(Array { + Field::create_field("key")}), + Field::create_field(Array {file})})}}; + for (const auto& [type, value] : cases) { + SCOPED_TRACE(type->get_name()); + for (const auto [nullable, has_limit] : {std::pair {false, false}, std::pair {true, false}, + std::pair {false, true}, std::pair {true, true}}) { + SCOPED_TRACE(nullable); + SCOPED_TRACE(has_limit); + auto source_type = nullable ? make_nullable(type) : type; + DataTypes argument_types {source_type}; + if (has_limit) { + argument_types.push_back(std::make_shared()); + } + auto function = AggregateFunctionSimpleFactory::instance().get( + "collect_list", argument_types, nullptr, false, + BeExecVersionManager::get_newest_version()); + ASSERT_NE(function, nullptr); + AggregateFunctionGuard partial(function.get()); + AggregateFunctionGuard restored(function.get()); + AggregateFunctionGuard global(function.get()); + Arena arena; + + ColumnString empty_buffer; + VectorBufferWriter empty_writer(empty_buffer); + function->serialize(partial.data(), empty_writer); + empty_writer.commit(); + VectorBufferReader empty_reader(empty_buffer.get_data_at(0)); + ASSERT_NO_THROW(function->deserialize(restored.data(), empty_reader, arena)); + EXPECT_FALSE(empty_reader.has_remaining()); + auto empty_result = function->get_return_type()->create_column(); + function->insert_result_into(restored.data(), *empty_result); + EXPECT_TRUE((*empty_result)[0].get().empty()); + + auto source = source_type->create_column(); + source->insert(value); + if (nullable) { + source->insert_default(); + } + auto limits = ColumnInt32::create(); + limits->insert_value(1); + if (nullable) { + limits->insert_value(1); + } + const IColumn* columns[] = {source.get(), limits.get()}; + function->add_batch_single_place(source->size(), partial.data(), columns, arena); + + ColumnString buffer; + VectorBufferWriter writer(buffer); + function->serialize(partial.data(), writer); + writer.commit(); + source->clear(); + function->reset(partial.data()); + VectorBufferReader reader(buffer.get_data_at(0)); + ASSERT_NO_THROW(function->deserialize(restored.data(), reader, arena)); + EXPECT_FALSE(reader.has_remaining()); + function->merge(global.data(), restored.data(), arena); + function->merge(global.data(), restored.data(), arena); + + auto result = function->get_return_type()->create_column(); + function->insert_result_into(global.data(), *result); + const auto result_value = (*result)[0]; + const auto& collected = result_value.get(); + ASSERT_EQ(collected.size(), has_limit ? 1 : 2); + for (const auto& item : collected) { + ASSERT_FALSE(item.is_null()); + expect_opaque_file_bytes(item, value); + } + } + } +} + +TEST(FileAggregateTest, ReaderReplaceRetainsInlineAfterSourceColumnReuse) { + const auto file = std::make_shared(); + for (const auto& name : {"replace_reader", "replace_if_not_null_reader"}) { + auto function = AggregateFunctionSimpleFactory::instance().get( + name, {file}, nullptr, false, BeExecVersionManager::get_newest_version()); + ASSERT_NE(function.get(), nullptr); + AggregateFunctionGuard state(function.get()); + Arena arena; + auto source = file->create_column(); + source->insert(file_aggregate_value(1)); + const IColumn* columns[] = {source.get()}; + function->add(state.data(), columns, 0, arena); + source->clear(); + source->insert(file_aggregate_value(2)); + function->add(state.data(), columns, 0, arena); + auto result = function->get_return_type()->create_column(); + function->insert_result_into(state.data(), *result); + expect_file_payload((*result)[0], 1); + } +} + +TEST(FileAggregateTest, CountReadsOnlyTheOuterNullMask) { + const auto type = make_nullable(std::make_shared()); + auto source = type->create_column(); + source->insert(file_aggregate_value(1)); + source->insert_default(); + source->insert(file_aggregate_value(2)); + auto function = AggregateFunctionSimpleFactory::instance().get( + "count", {type}, nullptr, false, BeExecVersionManager::get_newest_version()); + ASSERT_NE(function, nullptr); + AggregateFunctionGuard state(function.get()); + Arena arena; + const IColumn* columns[] = {source.get()}; + function->add_batch_single_place(source->size(), state.data(), columns, arena); + auto result = function->get_return_type()->create_column(); + function->insert_result_into(state.data(), *result); + ASSERT_EQ(result->size(), 1); + EXPECT_EQ((*result)[0].get(), 2); +} + +} // namespace doris diff --git a/be/test/exprs/file_expr_test.cpp b/be/test/exprs/file_expr_test.cpp new file mode 100644 index 00000000000000..9a52f9b773ab1c --- /dev/null +++ b/be/test/exprs/file_expr_test.cpp @@ -0,0 +1,844 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include + +#include "core/column/column_array.h" +#include "core/column/column_const.h" +#include "core/column/column_file.h" +#include "core/column/column_map.h" +#include "core/column/column_struct.h" +#include "core/column/column_varbinary.h" +#include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/data_type/data_type_variant.h" +#include "core/data_type/data_type_variant_v2.h" +#include "core/value/file_value.h" +#include "core/value/jsonb_value.h" +#include "exprs/function/array/function_array_element.h" +#include "exprs/function/cast/cast_base.h" +#include "exprs/function/simple_function_factory.h" +#include "exprs/table_function/table_function_factory.h" +#include "exprs/table_function/vexplode_file.h" +#include "exprs/vectorized_fn_call.h" +#include "exprs/vexpr_context.h" +#include "exprs/vslot_ref.h" +#include "runtime/runtime_state.h" +#include "util/jsonb_document.h" +#include "util/jsonb_utils.h" + +namespace doris { + +class FileExprTest : public testing::Test { +protected: + const DataTypePtr text = std::make_shared(); + const DataTypePtr binary = std::make_shared(); + const DataTypePtr bigint = std::make_shared(); + const DataTypePtr file_type = std::make_shared(); + + static Field string(const std::string& value) { + return Field::create_field(value); + } + static Field integer(Int64 value) { return Field::create_field(value); } + static Field structure(std::initializer_list fields) { + return Field::create_field(Struct(fields)); + } + static Field public_value(const std::string& uri, Field size = Field()) { + return structure({string(uri), Field(), std::move(size), Field(), Field(), Field()}); + } + static Field file_value(const std::string& uri = "s3://bucket/a%2Fb?versionId=AbC") { + File value(6); + value[0] = string(uri); + return Field::create_field(std::move(value)); + } + + static ColumnWithTypeAndName column(const DataTypePtr& type, + std::initializer_list values) { + auto result = type->create_column(); + for (const auto& value : values) result->insert(value); + return {std::move(result), type, "input"}; + } + + static Status cast(const ColumnWithTypeAndName& input, const DataTypePtr& target, bool strict, + ColumnPtr& output) { + FunctionContext context; + context.set_enable_strict_mode(strict); + Block block {input, {nullptr, target, "output"}}; + auto wrapper = get_cast_wrapper(&context, input.type, target); + RETURN_IF_ERROR(wrapper(&context, block, {0}, 1, input.column->size(), nullptr)); + output = block.get_by_position(1).column; + return Status::OK(); + } + + static ColumnPtr selector(const std::string& name, size_t rows = 1) { + auto nested = ColumnString::create(); + nested->insert_data(name.data(), name.size()); + return ColumnConst::create(std::move(nested), rows); + } + + static Field json(const std::string& input) { + JsonBinaryValue value; + auto status = value.from_json_string(input); + EXPECT_TRUE(status.ok()) << status; + return Field::create_field(JsonbField(value.value(), value.size())); + } + + DataTypePtr public_struct() const { + const auto& file = assert_cast(*file_type); + return std::make_shared(file.get_elements(), file.get_element_names()); + } + + static void check_constant_inline_row(const IColumn& output, size_t row, const Field& payload, + bool parent_null) { + EXPECT_EQ(output.is_null_at(row), parent_null || payload.is_null()); + if (!parent_null && !payload.is_null()) { + EXPECT_EQ(output[row].get().str(), payload.get().str()); + } + } + + void check_constant_metadata_projection(const ColumnPtr& file, size_t rows, + const std::string& expected_uri) const { + const auto nullable_file = make_nullable(file_type); + ColumnsWithTypeAndName args {{file, nullable_file, "file"}, + {selector("uri", rows), text, "field"}}; + auto getter = SimpleFunctionFactory::instance().get_function("element_at", args, + make_nullable(text)); + ASSERT_NE(getter, nullptr); + Block projection {args[0], args[1], {nullptr, make_nullable(text), "uri"}}; + ASSERT_TRUE(getter->execute(nullptr, projection, {0, 1}, 2, rows).ok()); + const auto& uri = projection.get_by_position(2).column; + ASSERT_TRUE(is_column_const(*uri)); + EXPECT_EQ((*uri)[rows - 1].get(), expected_uri); + } + + void check_constant_inline_getter(const Field& payload, bool parent_null) const { + auto value = file_value(); + value.get()[5] = payload; + auto input = + make_nullable(file_type)->create_column_const(4, parent_null ? Field() : value); + Block block {{input, make_nullable(file_type), "file"}, + {nullptr, make_nullable(binary), "output"}}; + auto function = SimpleFunctionFactory::instance().get_function( + "fl_get_binary", {block.get_by_position(0)}, make_nullable(binary)); + ASSERT_NE(function, nullptr); + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 4).ok()); + const auto& output = block.get_by_position(1).column; + ASSERT_EQ(output->size(), 4); + ASSERT_TRUE(is_column_const(*output)); + for (size_t row = 0; row < 4; ++row) { + check_constant_inline_row(*output, row, payload, parent_null); + } + } +}; + +TEST_F(FileExprTest, InlineGetterPreservesBinary) { + const std::string bytes("\0\xff\x80", 3); + auto value = file_value("urn:binary"); + value.get()[5] = Field::create_field(StringView(bytes)); + auto empty = file_value("urn:empty"); + empty.get()[5] = Field::create_field(StringView()); + const auto input = + column(make_nullable(file_type), {value, empty, file_value("urn:missing"), Field()}); + auto function = SimpleFunctionFactory::instance().get_function("fl_get_binary", {input}, + make_nullable(binary)); + ASSERT_NE(function, nullptr); + EXPECT_EQ(remove_nullable(function->get_return_type())->get_primitive_type(), TYPE_VARBINARY); + Block block {input, {nullptr, make_nullable(binary), "output"}}; + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 4).ok()); + const auto& output = block.get_by_position(1).column; + EXPECT_EQ((*output)[0].get().str(), bytes); + EXPECT_FALSE(output->is_null_at(1)); + EXPECT_EQ((*output)[1].get().size(), 0); + EXPECT_TRUE(output->is_null_at(2)); + EXPECT_TRUE(output->is_null_at(3)); +} + +TEST_F(FileExprTest, BinaryGetterPreservesParentNullsAndSharesPayload) { + std::string bytes(1 << 20, '\xff'); + bytes[0] = '\0'; + auto value = file_value(); + value.get()[5] = Field::create_field(StringView(bytes)); + auto values = make_nullable(file_type)->create_column(); + values->insert(value); + values->insert(value); + values->insert(value); + auto& source = assert_cast(*values); + source.get_null_map_data()[0] = 1; + ColumnWithTypeAndName input {std::move(values), make_nullable(file_type), "input"}; + const auto& inline_column = assert_cast( + assert_cast(source.get_nested_column()).get_column(5)); + auto function = SimpleFunctionFactory::instance().get_function("fl_get_binary", {input}, + make_nullable(binary)); + ASSERT_NE(function, nullptr); + Block block {input, {nullptr, make_nullable(binary), "output"}}; + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 3).ok()); + const auto& output = assert_cast(*block.get_by_position(1).column); + EXPECT_TRUE(output.is_null_at(0)); + EXPECT_FALSE(output.is_null_at(1)); + EXPECT_FALSE(output.is_null_at(2)); + EXPECT_EQ(output[1].get().str(), bytes); + EXPECT_EQ(output.get_nested_column_ptr(), inline_column.get_nested_column_ptr()); + EXPECT_FALSE(inline_column.is_null_at(0)); + EXPECT_FALSE(source.is_null_at(1)); +} + +TEST_F(FileExprTest, BinaryGetterInsertsNullForMissingInline) { + const std::string bytes("\0\xff\x80", 3); + auto value = file_value(); + value.get()[5] = Field::create_field(StringView(bytes)); + auto empty = file_value(); + empty.get()[5] = Field::create_field(StringView()); + auto values = file_type->create_column(); + values->insert(value); + values->insert(value); + values->insert(empty); + auto& inline_column = assert_cast( + assert_cast(*values).get_column(DataTypeFile::INLINE_FIELD_INDEX)); + inline_column.get_null_map_data()[1] = 1; + ColumnWithTypeAndName input {std::move(values), file_type, "input"}; + auto function = SimpleFunctionFactory::instance().get_function("fl_get_binary", {input}, + make_nullable(binary)); + ASSERT_NE(function, nullptr); + Block block {input, {nullptr, make_nullable(binary), "output"}}; + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 3).ok()); + const auto& output = assert_cast(*block.get_by_position(1).column); + EXPECT_EQ(output[0].get().str(), bytes); + EXPECT_TRUE(output.is_null_at(1)); + EXPECT_TRUE(output.get_nested_column()[1].get().empty()); + EXPECT_FALSE(output.is_null_at(2)); + EXPECT_TRUE(output[2].get().empty()); + EXPECT_EQ(inline_column.get_nested_column()[1].get().str(), bytes); +} + +TEST_F(FileExprTest, BinaryGetterHandlesEmptyColumns) { + const auto input = column(file_type, {}); + auto function = SimpleFunctionFactory::instance().get_function("fl_get_binary", {input}, + make_nullable(binary)); + ASSERT_NE(function, nullptr); + Block block {input, {nullptr, make_nullable(binary), "output"}}; + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 0).ok()); + EXPECT_EQ(block.get_by_position(1).column->size(), 0); +} + +TEST_F(FileExprTest, CastRejectsScalarInputsAndFileToString) { + for (const auto& input : {column(text, {string("s3://b/key")}), column(bigint, {integer(1)})}) { + for (bool strict : {false, true}) { + ColumnPtr output; + EXPECT_FALSE(cast(input, make_nullable(file_type), strict, output).ok()); + } + } + for (bool strict : {false, true}) { + ColumnPtr output; + EXPECT_FALSE( + cast(column(file_type, {file_value()}), make_nullable(text), strict, output).ok()); + const auto array_type = std::make_shared(make_nullable(file_type)); + EXPECT_FALSE( + cast(column(array_type, {Field::create_field(Array {file_value()})}), + make_nullable(text), strict, output) + .ok()); + } +} + +TEST_F(FileExprTest, VariantCastsAreUnsupported) { + for (const auto& variant_type : + DataTypes {make_nullable(std::make_shared()), + make_nullable(std::make_shared())}) { + const auto json_type = make_nullable(std::make_shared()); + const std::string bytes("a\0\xff", 3); + auto value = file_value("s3://b/variant"); + value.get()[1] = integer(0); + value.get()[2] = integer(3); + value.get()[3] = string("application/octet-stream"); + value.get()[4] = string("ETAG:opaque"); + value.get()[5] = Field::create_field(StringView(bytes)); + const auto input = column(make_nullable(file_type), {value, Field()}); + ColumnPtr variant; + const auto json_input = column( + json_type, + {json(R"({"uri":"s3://b/variant","offset":0,"size":3,"content_type":null,"checksum":null,"inline":null})")}); + ASSERT_TRUE(cast(json_input, variant_type, true, variant).ok()); + for (bool strict : {false, true}) { + ColumnPtr output; + EXPECT_FALSE(cast(input, variant_type, strict, output).ok()); + EXPECT_FALSE(cast({variant, variant_type, "variant"}, make_nullable(file_type), strict, + output) + .ok()); + } + EXPECT_EQ((*input.column)[0].get()[5].get().str(), bytes); + } +} + +TEST_F(FileExprTest, VariantCastsRejectAllInputValues) { + const auto json_type = make_nullable(std::make_shared()); + const auto variant_type = make_nullable(std::make_shared()); + const auto input = column( + json_type, + {json(R"({"checksum":null,"content_type":null,"size":4,"offset":0,"uri":"s3://b/variant","inline":null})"), + json("null"), Field(), json("[]"), json("1"), json(R"("s3://b/variant")"), + json(R"({"offset":null,"size":null,"content_type":null,"checksum":null,"inline":null})"), + json(R"({"uri":"s3://b/variant","size":null,"content_type":null,"checksum":null,"inline":null})"), + json(R"({"uri":"s3://b/variant","offset":null,"content_type":null,"checksum":null,"inline":null})"), + json(R"({"uri":"s3://b/variant","offset":null,"size":null,"checksum":null,"inline":null})"), + json(R"({"uri":"s3://b/variant","offset":null,"size":null,"content_type":null})"), + json(R"({"uri":"s3://b/variant","offset":null,"size":null,"content_type":null,"checksum":null,"inline":"AA"})"), + json(R"({"uri":"s3://b/variant","offset":null,"size":"4","content_type":null,"checksum":null,"inline":null})"), + json(R"({"uri":"s3://b/variant","offset":null,"size":null,"content_type":false,"checksum":null,"inline":null})"), + json(R"({"uri":null,"offset":null,"size":null,"content_type":null,"checksum":null,"inline":null})")}); + ColumnPtr variant; + ASSERT_TRUE(cast(input, variant_type, true, variant).ok()); + ColumnWithTypeAndName source {variant, variant_type, "variant"}; + ColumnPtr output; + EXPECT_FALSE(cast(source, make_nullable(file_type), false, output).ok()); + for (bool strict : {false, true}) { + for (size_t row = 0; row < variant->size(); ++row) { + EXPECT_FALSE(cast({variant->cut(row, 1), variant_type, "variant"}, + make_nullable(file_type), strict, output) + .ok()); + } + } +} + +TEST_F(FileExprTest, JsonCastsAreUnsupported) { + const auto json_type = std::make_shared(); + auto input = column( + json_type, + {json(R"({"uri":"s3://b/key","offset":null,"size":0,"content_type":null,"checksum":null,"inline":null})"), + json("null"), + json(R"({"uri":"s3://b/key","offset":null,"size":null,"content_type":null,"checksum":null,"inline":"AA"})"), + json(R"({"uri":"s3://b/key","offset":1,"size":null,"content_type":null,"checksum":null,"inline":null})"), + json(R"({"uri":"s3://b/key","offset":null,"size":1.5,"content_type":null,"checksum":null,"inline":null})"), + json(R"({"uri":"s3://b/key","offset":null,"size":null,"content_type":null,"checksum":"MD5:ABCDEF0123456789ABCDEF0123456789","inline":null})")}); + ColumnPtr output; + for (bool strict : {false, true}) { + for (size_t row = 0; row < input.column->size(); ++row) { + EXPECT_FALSE(cast({input.column->cut(row, 1), json_type, "input"}, + make_nullable(file_type), strict, output) + .ok()); + } + EXPECT_FALSE(cast(column(file_type, {file_value()}), json_type, strict, output).ok()); + } +} + +TEST_F(FileExprTest, ToJsonExposesExactlySixFields) { + const std::string bytes("\0\xff", 2); + auto value = file_value("s3://b/key"); + value.get()[5] = Field::create_field( + StringView(bytes.data(), cast_set(bytes.size()))); + const auto input = column(make_nullable(file_type), {value, Field()}); + ColumnPtr output; + + const auto expected = + R"({"uri":"s3://b/key","offset":null,"size":null,"content_type":null,"checksum":null,"inline":"AP8="})"; + const auto json_type = make_nullable(std::make_shared()); + RuntimeState runtime; + auto context = FunctionContext::create_context(&runtime, json_type, {input.type}); + auto function = SimpleFunctionFactory::instance().get_function("to_json", {input}, json_type); + ASSERT_NE(function, nullptr); + Block block {input, {nullptr, json_type, "json"}}; + ASSERT_TRUE(function->execute(context.get(), block, {0}, 1, input.column->size()).ok()); + output = block.get_by_position(1).column; + const auto& json_column = assert_cast(*output).get_nested_column(); + const auto encoded = json_column.get_data_at(0); + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(encoded.data, encoded.size), expected); + EXPECT_TRUE(output->is_null_at(1)); + // SQL output must preserve the source bytes. + EXPECT_EQ((*input.column)[0].get()[5].get().size(), bytes.size()); +} + +TEST_F(FileExprTest, FileToStructCastIsUnsupported) { + const auto& file = assert_cast(*file_type); + ColumnPtr output; + auto source_value = file_value("s3://b/key"); + source_value.get()[1] = integer(3); + source_value.get()[2] = integer(12); + const std::string bytes("\0\xff", 2); + source_value.get()[5] = Field::create_field(StringView(bytes)); + const auto input = column(file_type, {source_value}); + auto six = std::make_shared(file.get_elements(), file.get_element_names()); + EXPECT_FALSE(cast(input, six, true, output).ok()); + EXPECT_FALSE(cast(input, six, false, output).ok()); + auto reordered = std::make_shared( + DataTypes {make_nullable(bigint), make_nullable(text), make_nullable(bigint), + make_nullable(text), make_nullable(text), make_nullable(binary)}, + Strings {"SIZE", "UrI", "offset", "content_type", "checksum", "inline"}); + EXPECT_FALSE(cast(input, reordered, true, output).ok()); + auto nonnullable = std::make_shared( + DataTypes {text, make_nullable(bigint), make_nullable(bigint), make_nullable(text), + make_nullable(text), make_nullable(binary)}, + Strings {"uri", "offset", "size", "content_type", "checksum", "inline"}); + EXPECT_FALSE(cast(input, nonnullable, true, output).ok()); +} + +TEST_F(FileExprTest, CastRejectsStructInputsAndNestedFileTypes) { + auto source_file = make_nullable(public_struct()); + auto target_file = make_nullable(file_type); + auto good = public_value("s3://b/key"); + ColumnPtr output; + EXPECT_FALSE(cast(column(source_file, {good}), target_file, true, output).ok()); + auto array_source = std::make_shared(source_file); + auto array_target = std::make_shared(target_file); + const auto valid_array = Field::create_field(Array {good, Field()}); + EXPECT_FALSE(cast(column(array_source, {valid_array}), array_target, false, output).ok()); + + auto struct_source = std::make_shared(DataTypes {source_file}, Strings {"f"}); + auto struct_target = std::make_shared(DataTypes {target_file}, Strings {"f"}); + EXPECT_FALSE( + cast(column(struct_source, {structure({good})}), struct_target, true, output).ok()); + + auto map_source = std::make_shared(make_nullable(bigint), source_file); + auto map_target = std::make_shared(make_nullable(bigint), target_file); + auto map_value = + Field::create_field(Map {Field::create_field(Array {integer(1)}), + Field::create_field(Array {good})}); + EXPECT_FALSE(cast(column(map_source, {map_value}), map_target, true, output).ok()); +} + +TEST_F(FileExprTest, GetterUsesConstantPublicNamesAndMergesNullsWithoutMutation) { + auto value = file_value(); + value.get()[2] = integer(7); + auto input = column(make_nullable(file_type), {value, Field(), file_value()}); + ColumnsWithTypeAndName args {input, {selector("SiZe", 3), text, "selector"}}; + auto function = SimpleFunctionFactory::instance().get_function("element_at", args, + make_nullable(bigint)); + ASSERT_NE(function, nullptr); + EXPECT_TRUE(function->get_return_type()->is_nullable()); + Block block {args[0], args[1], {nullptr, make_nullable(bigint), "output"}}; + ASSERT_TRUE(function->execute(nullptr, block, {0, 1}, 2, 3).ok()); + auto output = block.get_by_position(2).column; + ASSERT_EQ(output->size(), 3); + EXPECT_EQ((*output)[0].get(), 7); + EXPECT_TRUE(output->is_null_at(1)); + EXPECT_TRUE(output->is_null_at(2)); + auto changed = IColumn::mutate(output); + changed->clear(); + EXPECT_EQ((*input.column)[0].get()[2].get(), 7); +} + +TEST_F(FileExprTest, GetterRejectsDynamicNumericNullAndUnknownSelectors) { + FunctionArrayElement function; + auto bad_selectors = std::vector { + {selector("unknown"), text, "selector"}, + column(text, {string("uri")}), + column(bigint, {integer(1)}), + {make_nullable(text)->create_column_const_with_default_value(1), make_nullable(text), + "null selector"}}; + for (const auto& index : bad_selectors) { + ColumnsWithTypeAndName args {{nullptr, file_type, "file"}, index}; + EXPECT_THROW(function.get_return_type_impl(args), Exception); + } +} + +TEST_F(FileExprTest, JsonCastRejectsMissingDuplicateUnknownWrongTypeAndOutOfRangeFields) { + const auto type = std::make_shared(); + const auto input = column( + type, + {json(R"({"uri":"s3://b/a"})"), + json(R"({"uri":"s3://b/a","offset":null,"size":null,"content_type":null})"), + json(R"({"uri":"s3://b/a","uri":"s3://b/b","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null})"), + json(R"({"URI":"s3://b/a","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null})"), + json(R"({"uri":"s3://b/a","offset":null,"size":9223372036854775808,"content_type":null,"checksum":null,"inline":null})"), + json(R"({"uri":"s3://b/a","offset":9223372036854775807,"size":1,"content_type":null,"checksum":null,"inline":null})"), + json(R"({"uri":"s3://b/a","offset":null,"size":null,"content_type":null,"checksum":"md5:0123456789abcdef0123456789abcdef","inline":null})"), + json(R"({"uri":"s3://b/a","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null,"other":null})"), + json(R"({"uri":1,"offset":null,"size":null,"content_type":null,"checksum":null,"inline":null})"), + json(R"({"uri":"s3://b/a","offset":null,"size":"12","content_type":null,"checksum":null,"inline":null})"), + json(R"({"uri":"s3://b/a","offset":null,"size":null,"content_type":false,"checksum":null,"inline":null})"), + json("[]"), json("1")}); + ColumnPtr output; + EXPECT_FALSE(cast(input, make_nullable(file_type), false, output).ok()); + for (size_t row = 0; row < input.column->size(); ++row) { + EXPECT_FALSE(cast({input.column->cut(row, 1), type, "input"}, make_nullable(file_type), + true, output) + .ok()) + << row; + } +} + +TEST_F(FileExprTest, GetterSupportsConstantFileAndConstantNullableSelector) { + auto input = file_type->create_column_const(4, file_value("s3://b/key")); + auto name = make_nullable(text)->create_column_const(4, string("URI")); + ColumnsWithTypeAndName args {{input, file_type, "file"}, {name, make_nullable(text), "field"}}; + auto function = + SimpleFunctionFactory::instance().get_function("element_at", args, make_nullable(text)); + ASSERT_NE(function, nullptr); + Block block {args[0], args[1], {nullptr, make_nullable(text), "output"}}; + ASSERT_TRUE(function->execute(nullptr, block, {0, 1}, 2, 4).ok()); + const auto output = block.get_by_position(2).column; + ASSERT_EQ(output->size(), 4); + for (size_t row = 0; row < 4; ++row) EXPECT_EQ((*output)[row].get(), "s3://b/key"); +} + +namespace { +void project_constant_fields_with_protected_payload() { + constexpr size_t payload_size = 1 << 20; + constexpr size_t rows = 4096; + auto* bytes = static_cast(mmap(nullptr, payload_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); + if (bytes == MAP_FAILED) { + std::_Exit(1); + } + std::memset(bytes, '\xff', payload_size); + const auto type = std::make_shared(); + auto column = type->create_column(); + column->insert(Field::create_field( + File {Field::create_field("urn:constant"), Field(), + Field::create_field(7), Field(), Field(), Field()})); + auto& inline_column = + assert_cast(assert_cast(*column).get_column(5)); + auto& payloads = assert_cast(inline_column.get_nested_column()).get_data(); + payloads[0] = StringView(bytes, static_cast(payload_size)); + inline_column.get_null_map_data()[0] = 0; + if (mprotect(bytes, payload_size, PROT_NONE) != 0) { + std::_Exit(2); + } + const auto& schema = *type; + const auto text = std::make_shared(); + const ColumnPtr source = std::move(column); + FunctionArrayElement function; + Columns struct_fields; + for (size_t child = 0; child < DataTypeFile::FIELD_COUNT; ++child) { + struct_fields.push_back(assert_cast(*source).get_column_ptr(child)); + } + const auto struct_type = + std::make_shared(schema.get_elements(), schema.get_element_names()); + const ColumnPtr struct_source = ColumnStruct::create(struct_fields); + for (bool is_struct : {false, true}) { + const ColumnPtr field_source = is_struct ? struct_source : source; + const DataTypePtr field_type = is_struct ? DataTypePtr(struct_type) : DataTypePtr(type); + for (int parent = 0; parent < 3; ++parent) { + ColumnPtr input = field_source; + DataTypePtr input_type = field_type; + if (parent != 0) { + input = ColumnNullable::create(input, ColumnUInt8::create(1, parent == 2)); + input_type = make_nullable(field_type); + } + input = ColumnConst::create(input, rows); + for (size_t index : {size_t {0}, size_t {2}, size_t {4}}) { + const auto& name = schema.get_element_names()[index]; + auto field_name = ColumnString::create(); + field_name->insert_data(name.data(), name.size()); + Block block {{input, input_type, "file"}, + {ColumnConst::create(std::move(field_name), rows), text, "field"}, + {nullptr, schema.get_element(index), "output"}}; + if (!function.execute_impl(nullptr, block, {0, 1}, 2, rows).ok()) { + std::_Exit(3); + } + const auto& output = block.get_by_position(2).column; + if (output->size() != rows) { + std::_Exit(4); + } + for (size_t row = 0; row < rows; ++row) { + const bool expected_null = parent == 2 || index == 4; + if (output->is_null_at(row) != expected_null) { + std::_Exit(5); + } + if (!expected_null && + ((index == 0 && output->get_data_at(row).to_string() != "urn:constant") || + (index == 2 && (*output)[row].get() != 7))) { + std::_Exit(6); + } + } + } + } + } + if (munmap(bytes, payload_size) != 0) { + std::_Exit(7); + } + std::_Exit(0); +} +} // namespace + +TEST(FileExprDeathTest, ConstantFieldAccessNeverReadsUnselectedBinary) { + // Both FILE and STRUCT retain a protected 1 MiB binary field while extracting metadata. + // Any read or copy of the unselected binary payload faults immediately. + ASSERT_EXIT(project_constant_fields_with_protected_payload(), ::testing::ExitedWithCode(0), ""); +} + +TEST_F(FileExprTest, ConstantInlineGetterPreservesBinaryAndNulls) { + const std::string bytes("\0\xff\x80", 3); + for (const auto& payload : {Field(), Field::create_field(StringView()), + Field::create_field(StringView(bytes))}) { + for (bool parent_null : {false, true}) { + check_constant_inline_getter(payload, parent_null); + } + } +} + +TEST_F(FileExprTest, BinaryGetterSharesStoredPayloadAndDetachesBeforeMutation) { + const std::string bytes(128 * 1024, '\xff'); + auto value = file_value("urn:binary"); + value.get()[5] = Field::create_field(StringView(bytes)); + auto input = column(file_type, {value, value}); + const auto& source = assert_cast(*input.column); + auto function = SimpleFunctionFactory::instance().get_function("fl_get_binary", {input}, + make_nullable(binary)); + ASSERT_NE(function, nullptr); + Block block {input, {nullptr, make_nullable(binary), "binary"}}; + ASSERT_TRUE(function->execute(nullptr, block, {0}, 1, 2).ok()); + const auto output = block.get_by_position(1).column; + EXPECT_EQ(output.get(), source.get_column_ptr(DataTypeFile::INLINE_FIELD_INDEX).get()); + auto changed = IColumn::mutate(output); + changed->clear(); + EXPECT_EQ(source.size(), 2); + EXPECT_EQ(source.get_column(5)[1].get().str(), bytes); + block.clear(); + input.column = nullptr; + EXPECT_EQ((*output)[0].get().str(), bytes); +} + +TEST_F(FileExprTest, ToJsonPreservesConstantFileAndNull) { + const auto type = make_nullable(file_type); + const auto json_type = make_nullable(std::make_shared()); + auto value = file_value("urn:constant"); + value.get()[5] = Field::create_field(StringView()); + for (bool is_null : {false, true}) { + const ColumnWithTypeAndName input {type->create_column_const(3, is_null ? Field() : value), + type, "input"}; + RuntimeState runtime; + auto context = FunctionContext::create_context(&runtime, json_type, {type}); + auto function = + SimpleFunctionFactory::instance().get_function("to_json", {input}, json_type); + ASSERT_NE(function, nullptr); + Block block {input, {nullptr, json_type, "json"}}; + ASSERT_TRUE(function->execute(context.get(), block, {0}, 1, 3).ok()); + const auto& output = block.get_by_position(1).column; + ASSERT_EQ(output->size(), 3); + for (size_t row = 0; row < 3; ++row) { + EXPECT_EQ(output->is_null_at(row), is_null); + if (!is_null) { + const auto encoded = output->get_data_at(row); + EXPECT_EQ( + JsonbToJson::jsonb_to_json_string(encoded.data, encoded.size), + R"({"uri":"urn:constant","offset":null,"size":null,"content_type":null,"checksum":null,"inline":""})"); + } + } + } +} + +TEST_F(FileExprTest, ConstantContainersKeepSingleFileResultAndMetadataProjection) { + constexpr size_t rows = 4096; + const std::string bytes(1 << 20, '\xff'); + auto value = file_value("urn:constant-container"); + value.get()[5] = Field::create_field(StringView(bytes)); + const auto nullable_file = make_nullable(file_type); + const auto array_type = std::make_shared(nullable_file); + const auto map_type = std::make_shared(make_nullable(bigint), nullable_file); + const auto struct_type = + std::make_shared(DataTypes {nullable_file}, Strings {"asset"}); + const auto array = Field::create_field(Array {value}); + const auto map = + Field::create_field(Map {Field::create_field(Array {integer(1)}), + Field::create_field(Array {value})}); + const std::vector inputs { + {array_type->create_column_const(rows, array), array_type, "array"}, + {map_type->create_column_const(rows, map), map_type, "map"}, + {struct_type->create_column_const(rows, structure({value})), struct_type, "struct"}}; + for (const auto& input : inputs) { + SCOPED_TRACE(input.name); + auto key = input.type == struct_type ? selector("asset", rows) + : bigint->create_column_const(rows, integer(1)); + const DataTypePtr key_type = input.type == struct_type ? text : bigint; + ColumnsWithTypeAndName args {input, {key, key_type, "selector"}}; + auto function = + SimpleFunctionFactory::instance().get_function("element_at", args, nullable_file); + ASSERT_NE(function, nullptr); + Block block {args[0], args[1], {nullptr, nullable_file, "file"}}; + ASSERT_TRUE(function->execute(nullptr, block, {0, 1}, 2, rows).ok()); + const auto& file = block.get_by_position(2).column; + ASSERT_TRUE(is_column_const(*file)); + EXPECT_LT(file->allocated_bytes(), 4 * bytes.size()); + EXPECT_EQ(file->size(), rows); + EXPECT_EQ((*file)[rows - 1].get()[5].get().str(), bytes); + check_constant_metadata_projection(file, rows, "urn:constant-container"); + } +} + +TEST_F(FileExprTest, ConstantArrayAndMapStillAcceptDynamicSelectors) { + constexpr size_t rows = 4; + const auto nullable_file = make_nullable(file_type); + const auto first = file_value("urn:first"); + const auto second = file_value("urn:second"); + const auto array_type = std::make_shared(nullable_file); + const auto map_type = std::make_shared(make_nullable(bigint), nullable_file); + const auto array = Field::create_field(Array {first, second}); + const auto map = Field::create_field( + Map {Field::create_field(Array {integer(1), integer(2)}), + Field::create_field(Array {first, second})}); + const std::vector inputs { + {array_type->create_column_const(rows, array), array_type, "array"}, + {map_type->create_column_const(rows, map), map_type, "map"}}; + for (const auto& input : inputs) { + SCOPED_TRACE(input.name); + ColumnsWithTypeAndName args {input, column(make_nullable(bigint), + {integer(2), integer(1), integer(3), Field()})}; + auto function = + SimpleFunctionFactory::instance().get_function("element_at", args, nullable_file); + ASSERT_NE(function, nullptr); + Block block {args[0], args[1], {nullptr, nullable_file, "file"}}; + ASSERT_TRUE(function->execute(nullptr, block, {0, 1}, 2, rows).ok()); + const auto& output = block.get_by_position(2).column; + EXPECT_FALSE(is_column_const(*output)); + EXPECT_EQ((*output)[0].get()[0].get(), "urn:second"); + EXPECT_EQ((*output)[1].get()[0].get(), "urn:first"); + EXPECT_TRUE(output->is_null_at(2)); + EXPECT_TRUE(output->is_null_at(3)); + } +} + +TEST_F(FileExprTest, NestedFileToStructCastIsUnsupportedAndIdentityPreservesInline) { + auto value = file_value("s3://b/key"); + const std::string bytes("a\0b", 3); + value.get()[5] = Field::create_field(StringView(bytes)); + const auto input_type = std::make_shared(make_nullable(file_type)); + const auto input = + column(input_type, {Field::create_field(Array {value, Field()})}); + ColumnPtr output; + ASSERT_TRUE(cast(input, input_type, true, output).ok()); + EXPECT_EQ((*output)[0].get()[0].get()[5].get().size(), + 3); + EXPECT_FALSE(cast(input, std::make_shared(make_nullable(public_struct())), true, + output) + .ok()); +} + +TEST_F(FileExprTest, ExplodeFilePublicFieldsNullAndOuterRows) { + auto value = file_value("s3://b/key"); + value.get()[2] = integer(17); + const std::string bytes("\0\xff\x80", 3); + value.get()[5] = Field::create_field(StringView(bytes)); + Block block {column(make_nullable(file_type), {value, Field(), file_value("s3://b/last")})}; + auto root = std::make_shared(); + auto child = std::make_shared(0, 0, -1, make_nullable(file_type), "file"); + root->add_child(child); + auto context = std::make_shared(root); + for (bool outer : {false, true}) { + TFunction definition; + definition.name.function_name = outer ? "explode_file_outer" : "explode_file"; + definition.binary_type = TFunctionBinaryType::BUILTIN; + ObjectPool pool; + TableFunction* function = nullptr; + ASSERT_TRUE(TableFunctionFactory::get_fn(definition, &pool, &function, 0).ok()); + ASSERT_NE(dynamic_cast(function), nullptr); + EXPECT_EQ(function->is_outer(), outer); + function->set_expr_context(context); + function->set_nullable(); + ASSERT_TRUE(function->process_init(&block, nullptr).ok()); + auto output = make_nullable(public_struct())->create_column(); + function->process_row(0); + EXPECT_FALSE(function->current_empty()); + EXPECT_EQ(function->get_value(output, 8), 1); + EXPECT_TRUE(function->eos()); + EXPECT_EQ((*output)[0].get().size(), 6); + EXPECT_EQ((*output)[0].get()[2].get(), 17); + EXPECT_EQ((*output)[0].get()[5].get().str(), bytes); + function->process_row(1); + EXPECT_TRUE(function->current_empty()); + EXPECT_EQ(function->get_value(output, 8), outer ? 1 : 0); + if (outer) EXPECT_TRUE(output->is_null_at(1)); + function->process_row(2); + EXPECT_EQ(function->get_value(output, 1), 1); + EXPECT_EQ(output->size(), outer ? 3 : 2); + EXPECT_EQ((*output)[output->size() - 1].get()[0].get(), + "s3://b/last"); + function->process_row(0); + auto repeated = make_nullable(public_struct())->create_column(); + function->get_same_many_values(repeated, 3); + ASSERT_EQ(repeated->size(), 3); + EXPECT_EQ((*repeated)[2].get()[2].get(), 17); + EXPECT_EQ((*repeated)[2].get()[5].get().str(), bytes); + function->process_close(); + } +} + +TEST_F(FileExprTest, JsonCastsRejectNestedFileTypes) { + const auto source = std::make_shared(); + const auto arrays = column( + source, + {json(R"([{"uri":"s3://b/good","offset":null,"size":4,"content_type":null,"checksum":null,"inline":null}, + {"uri":"s3://b/bad","offset":null,"size":-1,"content_type":null,"checksum":null,"inline":null},null])"), + json("[]"), json("null")}); + const auto array_target = + make_nullable(std::make_shared(make_nullable(file_type))); + const auto structs = column( + source, + {json(R"({"file":{"uri":"s3://b/bad","offset":null,"size":-1,"content_type":null,"checksum":null,"inline":null},"label":"kept"})"), + json(R"({"file":{"uri":"s3://b/good","offset":null,"size":4,"content_type":null,"checksum":null,"inline":null},"label":"also"})")}); + const auto struct_target = make_nullable(std::make_shared( + DataTypes {make_nullable(file_type), make_nullable(text)}, Strings {"file", "label"})); + for (bool strict : {false, true}) { + SCOPED_TRACE(strict ? "strict" : "non-strict"); + ColumnPtr output; + EXPECT_FALSE(cast(arrays, array_target, strict, output).ok()); + EXPECT_FALSE(cast(structs, struct_target, strict, output).ok()); + const auto files = + column(array_target, {Field::create_field(Array {file_value()})}); + EXPECT_FALSE(cast(files, make_nullable(source), strict, output).ok()); + } +} + +TEST_F(FileExprTest, JsonCastsKeepExistingNonFileBehavior) { + const auto source = std::make_shared(); + const auto target = make_nullable(std::make_shared( + DataTypes {make_nullable(text), make_nullable(bigint)}, Strings {"label", "number"})); + const auto input = column(source, {json(R"({"label":"first","number":7})"), + json(R"({"label":"second","number":8})")}); + ColumnPtr output; + ASSERT_TRUE(cast(input, target, true, output).ok()); + ASSERT_EQ(output->size(), 2); + EXPECT_EQ((*output)[0].get()[0].get(), "first"); + EXPECT_EQ((*output)[0].get()[1].get(), 7); + EXPECT_EQ((*output)[1].get()[0].get(), "second"); + EXPECT_EQ((*output)[1].get()[1].get(), 8); +} + +TEST_F(FileExprTest, ElementAtStillSupportsDynamicArraySelectors) { + const auto array_type = std::make_shared(make_nullable(bigint)); + auto array = + column(array_type, {Field::create_field(Array {integer(10), integer(20)}), + Field::create_field(Array {integer(30), integer(40)})}); + auto indexes = column(bigint, {integer(2), integer(1)}); + auto function = SimpleFunctionFactory::instance().get_function("element_at", {array, indexes}, + make_nullable(bigint)); + ASSERT_NE(function, nullptr); + Block block {array, indexes, {nullptr, make_nullable(bigint), "result"}}; + ASSERT_TRUE(function->execute(nullptr, block, {0, 1}, 2, 2).ok()); + EXPECT_EQ((*block.get_by_position(2).column)[0].get(), 20); + EXPECT_EQ((*block.get_by_position(2).column)[1].get(), 30); +} + +TEST_F(FileExprTest, ReverseStructRejectsIntegerWideningAndNarrowing) { + const auto& schema = assert_cast(*file_type); + auto value = file_value(); + value.get()[2] = integer(17); + for (const auto& integer_type : + DataTypes {std::make_shared(), std::make_shared()}) { + auto types = schema.get_elements(); + types[2] = make_nullable(integer_type); + auto target = std::make_shared( + types, Strings {"uri", "offset", "size", "content_type", "checksum", "inline"}); + for (bool strict : {false, true}) { + ColumnPtr output; + EXPECT_FALSE(cast(column(file_type, {value}), target, strict, output).ok()); + } + } +} + +} // namespace doris diff --git a/be/test/exprs/function/function_to_file_test.cpp b/be/test/exprs/function/function_to_file_test.cpp new file mode 100644 index 00000000000000..77789234af6edd --- /dev/null +++ b/be/test/exprs/function/function_to_file_test.cpp @@ -0,0 +1,788 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exprs/function/function_to_file.h" + +#include +#include + +#include + +#include "core/column/column_const.h" +#include "core/column/column_file.h" +#include "core/column/column_nullable.h" +#include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "exprs/function/simple_function_factory.h" +#include "exprs/vectorized_fn_call.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "io/fs/file_system.h" +#include "io/fs/s3_file_system.h" +#include "runtime/descriptors.h" +#include "runtime/runtime_state.h" +#include "runtime/thread_context.h" +#include "testutil/mock/mock_query_context.h" + +namespace doris { + +using testing::_; +using testing::Invoke; +using testing::Return; + +class ToFileMockFS : public io::FileSystem { +public: + ToFileMockFS() : FileSystem("test-resource", io::FileSystemType::S3) {} + MOCK_METHOD(Status, stat_impl, (const io::Path&, io::FileStat*, io::FileStatContext*), + (const, override)); + MOCK_METHOD(Status, create_file_impl, + (const io::Path&, io::FileWriterPtr*, const io::FileWriterOptions*), (override)); + MOCK_METHOD(Status, open_file_impl, + (const io::Path&, io::FileReaderSPtr*, const io::FileReaderOptions*), (override)); + MOCK_METHOD(Status, create_directory_impl, (const io::Path&, bool), (override)); + MOCK_METHOD(Status, delete_file_impl, (const io::Path&), (override)); + MOCK_METHOD(Status, batch_delete_impl, (const std::vector&), (override)); + MOCK_METHOD(Status, delete_directory_impl, (const io::Path&), (override)); + MOCK_METHOD(Status, exists_impl, (const io::Path&, bool*), (const, override)); + MOCK_METHOD(Status, file_size_impl, (const io::Path&, int64_t*), (const, override)); + MOCK_METHOD(Status, list_impl, (const io::Path&, bool, std::vector*, bool*), + (override)); + MOCK_METHOD(Status, rename_impl, (const io::Path&, const io::Path&), (override)); + Status absolute_path(const io::Path& path, io::Path& result) const override { + result = path; + return Status::OK(); + } +}; + +class ToFileWithMockFS : public FunctionToFile { +public: + MOCK_METHOD(Status, create_filesystem, + (const TS3Resource&, const std::string&, std::shared_ptr*), + (const, override)); +}; + +class ToFileWithRealFS : public FunctionToFile { +public: + using FunctionToFile::create_filesystem; +}; + +class FunctionToFileTest : public testing::Test { +protected: + DataTypePtr text = std::make_shared(); + DataTypePtr file = std::make_shared(); + std::shared_ptr filesystem = std::make_shared(); + ToFileWithMockFS function; + std::shared_ptr query = MockQueryContext::create(); + RuntimeState query_runtime; + + TS3Resource resource_config() const { return query->get_s3_resources()->at("test-resource"); } + + void set_resource_config(const TS3Resource& resource) { + query->set_s3_resources({{"test-resource", resource}}); + } + + static Field string(const std::string& value) { + return Field::create_field(value); + } + + ColumnPtr resource_column(size_t rows = 1) { + return text->create_column_const(rows, string("test-resource")); + } + + std::unique_ptr make_context(RuntimeState* runtime = nullptr, + ColumnPtr constant_uri = nullptr) { + if (!runtime) runtime = &query_runtime; + runtime->_query_ctx = query.get(); + auto context = FunctionContext::create_context(runtime, make_nullable(file), + {text, make_nullable(text)}); + context->set_constant_cols( + {std::make_shared(resource_column()), + constant_uri ? std::make_shared(constant_uri) : nullptr}); + TS3Resource resource; + + resource.properties = {{"AWS_BUCKET", "bucket"}, {"AWS_ENDPOINT", "endpoint"}}; + set_resource_config(resource); + return context; + } + + ColumnWithTypeAndName uris(std::initializer_list values) { + auto type = make_nullable(text); + auto column = type->create_column(); + for (const auto& value : values) column->insert(value); + return {std::move(column), type, "uri"}; + } + + Status execute(FunctionContext* context, const ColumnWithTypeAndName& input, + ColumnPtr& output) { + const auto type = input.type->is_nullable() ? make_nullable(file) : file; + Block block {{context->get_constant_col(0)->column_ptr->clone_resized(input.column->size()), + text, "resource"}, + input, + {output, type, "result"}}; + auto status = function.execute(context, block, {0, 1}, 2, input.column->size()); + output = block.get_by_position(2).column; + return status; + } + + void expect_client_creation() { + EXPECT_CALL(function, create_filesystem(_, _, _)) + .WillOnce(Invoke([this](const TS3Resource& resource, const std::string&, + std::shared_ptr* result) { + EXPECT_EQ(resource.properties.at("AWS_BUCKET"), "bucket"); + *result = filesystem; + return Status::OK(); + })); + } +}; + +TEST_F(FunctionToFileTest, AllNullBatchesAndQueriesNeverCreateClient) { + EXPECT_CALL(function, create_filesystem(_, _, _)).Times(0); + EXPECT_CALL(*filesystem, stat_impl(_, _, _)).Times(0); + auto context = make_context(); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + auto clone = context->clone(); + ASSERT_TRUE(function.open(clone.get(), FunctionContext::THREAD_LOCAL).ok()); + const auto null_input = uris({Field(), Field()}); + ASSERT_FALSE(is_column_const(*null_input.column)); + for (auto* active : {context.get(), clone.get()}) { + ColumnPtr output; + ASSERT_TRUE(execute(active, null_input, output).ok()); + ASSERT_EQ(output->size(), 2); + EXPECT_TRUE(output->is_null_at(0)); + EXPECT_TRUE(output->is_null_at(1)); + ASSERT_TRUE(execute(active, uris({}), output).ok()); + EXPECT_EQ(output->size(), 0); + } + ASSERT_TRUE(function.close(clone.get(), FunctionContext::THREAD_LOCAL).ok()); + ASSERT_TRUE(function.close(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + + auto null_uri = make_nullable(text)->create_column_const(4, Field()); + auto null_context = make_context(nullptr, null_uri); + ASSERT_TRUE(function.open(null_context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + ColumnPtr output; + ASSERT_TRUE(execute(null_context.get(), {null_uri, make_nullable(text), "uri"}, output).ok()); + ASSERT_EQ(output->size(), 4); + for (size_t row = 0; row < 4; ++row) EXPECT_TRUE(output->is_null_at(row)); +} + +TEST_F(FunctionToFileTest, CreatesClientOnFirstNonNullUsingConfigurationCapturedAtOpen) { + auto context = make_context(); + auto resource = resource_config(); + + resource.properties = {{"AWS_ENDPOINT", "http://original"}}; + set_resource_config(resource); + EXPECT_CALL(function, create_filesystem(_, _, _)).Times(0); + EXPECT_CALL(*filesystem, stat_impl(_, _, _)).Times(0); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + auto clone = context->clone(); + ASSERT_TRUE(function.open(clone.get(), FunctionContext::THREAD_LOCAL).ok()); + ColumnPtr output; + ASSERT_TRUE(execute(context.get(), uris({Field(), Field()}), output).ok()); + ASSERT_TRUE(execute(clone.get(), uris({Field()}), output).ok()); + ASSERT_TRUE(testing::Mock::VerifyAndClearExpectations(&function)); + ASSERT_TRUE(testing::Mock::VerifyAndClearExpectations(filesystem.get())); + + // Lazy creation must retain the configuration pinned by fragment-local open. + resource.properties["AWS_ENDPOINT"] = "http://changed"; + set_resource_config(resource); + EXPECT_CALL(function, create_filesystem(_, _, _)) + .WillOnce(Invoke([&](const TS3Resource& resource, const std::string&, + std::shared_ptr* result) { + EXPECT_EQ(resource.properties.at("AWS_ENDPOINT"), "http://original"); + *result = filesystem; + return Status::OK(); + })); + EXPECT_CALL(*filesystem, stat_impl(_, _, _)) + .Times(2) + .WillRepeatedly( + Invoke([](const io::Path&, io::FileStat* metadata, io::FileStatContext*) { + *metadata = io::FileStat {.size = 0}; + return Status::OK(); + })); + ASSERT_TRUE(execute(clone.get(), uris({Field(), string("s3://bucket/a.txt")}), output).ok()); + EXPECT_TRUE(output->is_null_at(0)); + EXPECT_FALSE(output->is_null_at(1)); + ASSERT_TRUE(execute(context.get(), uris({string("s3://bucket/b.txt"), Field()}), output).ok()); + EXPECT_FALSE(output->is_null_at(0)); + EXPECT_TRUE(output->is_null_at(1)); + ASSERT_TRUE(execute(context.get(), uris({Field(), Field()}), output).ok()); + EXPECT_TRUE(output->is_null_at(0)); + EXPECT_TRUE(output->is_null_at(1)); +} + +TEST_F(FunctionToFileTest, LazyClientCreationFailureDoesNotStatOrPublishOutput) { + auto context = make_context(); + EXPECT_CALL(function, create_filesystem(_, _, _)).Times(0); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + ASSERT_TRUE(testing::Mock::VerifyAndClearExpectations(&function)); + EXPECT_CALL(function, create_filesystem(_, _, _)) + .WillOnce(Return(Status::IOError("client connection failed"))); + EXPECT_CALL(*filesystem, stat_impl(_, _, _)).Times(0); + ColumnPtr output = make_nullable(file)->create_column_const(2, Field()); + const auto original = output; + EXPECT_FALSE(execute(context.get(), uris({Field(), string("s3://bucket/a.txt")}), output).ok()); + EXPECT_EQ(output.get(), original.get()); + ASSERT_TRUE(execute(context.get(), uris({Field(), Field()}), output).ok()); + EXPECT_TRUE(output->is_null_at(0)); + EXPECT_TRUE(output->is_null_at(1)); +} + +TEST_F(FunctionToFileTest, CancellationBeforeLazyCreationDoesNotCreateClient) { + RuntimeState runtime; + auto context = make_context(&runtime); + EXPECT_CALL(function, create_filesystem(_, _, _)).Times(0); + EXPECT_CALL(*filesystem, stat_impl(_, _, _)).Times(0); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + runtime.cancel(Status::Cancelled("cancel before client creation")); + ColumnPtr output; + EXPECT_FALSE(execute(context.get(), uris({string("s3://bucket/a.txt")}), output).ok()); + EXPECT_EQ(output.get(), nullptr); +} + +TEST_F(FunctionToFileTest, CancellationDuringLazyCreationDoesNotStat) { + RuntimeState runtime; + auto context = make_context(&runtime); + EXPECT_CALL(function, create_filesystem(_, _, _)).Times(0); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + ASSERT_TRUE(testing::Mock::VerifyAndClearExpectations(&function)); + EXPECT_CALL(function, create_filesystem(_, _, _)) + .WillOnce(Invoke([&](const TS3Resource&, const std::string&, + std::shared_ptr* result) { + *result = filesystem; + runtime.cancel(Status::Cancelled("cancel during client creation")); + return Status::OK(); + })); + EXPECT_CALL(*filesystem, stat_impl(_, _, _)).Times(0); + ColumnPtr output; + EXPECT_FALSE(execute(context.get(), uris({string("s3://bucket/a.txt")}), output).ok()); + EXPECT_EQ(output.get(), nullptr); +} + +TEST_F(FunctionToFileTest, RealS3FactoryUsesUriWithoutAwsBucketProperty) { + ToFileWithRealFS real_function; + TS3Resource resource; + + resource.properties = {{"AWS_ENDPOINT", "http://127.0.0.1:1"}, + {"AWS_REGION", "us-east-1"}, + {"AWS_ACCESS_KEY", "test-ak"}, + {"AWS_SECRET_KEY", "test-sk"}, + {"use_path_style", "true"}}; + std::shared_ptr result; + ASSERT_TRUE(real_function.create_filesystem(resource, "s3://uri-bucket/a.txt", &result).ok()); + ASSERT_NE(result, nullptr); + EXPECT_EQ(assert_cast(*result).bucket(), "uri-bucket"); + for (const auto* key : {"dir/a://b.txt", "dir/a?b.txt", "dir/a#b.txt", "dir/ a.txt "}) { + ASSERT_TRUE( + real_function + .create_filesystem(resource, std::string("s3://uri-bucket/") + key, &result) + .ok()); + EXPECT_EQ(assert_cast(*result).bucket(), "uri-bucket"); + } + resource.properties["AWS_BUCKET"] = "different-resource-bucket"; + ASSERT_TRUE(real_function.create_filesystem(resource, "s3://other-bucket/a.txt", &result).ok()); + EXPECT_EQ(assert_cast(*result).bucket(), "other-bucket"); +} + +// A filesystem is bound to its S3 bucket. Reusing it for another bucket must +// not send metadata requests to the first bucket. +TEST_F(FunctionToFileTest, SwitchesFilesystemWhenUriBucketChanges) { + auto context = make_context(); + std::vector created_uris; + EXPECT_CALL(function, create_filesystem(_, _, _)) + .Times(3) + .WillRepeatedly(Invoke([&](const TS3Resource&, const std::string& uri, + std::shared_ptr* result) { + created_uris.push_back(uri); + *result = filesystem; + return Status::OK(); + })); + EXPECT_CALL(*filesystem, stat_impl(_, _, _)) + .Times(4) + .WillRepeatedly( + Invoke([](const io::Path&, io::FileStat* metadata, io::FileStatContext*) { + *metadata = io::FileStat {.size = 1}; + return Status::OK(); + })); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + ColumnPtr output; + ASSERT_TRUE(execute(context.get(), + uris({string("s3://bucket/a.txt"), string("s3://bucket/b.txt"), + string("s3://other/a.txt"), string("s3://bucket/c.txt")}), + output) + .ok()); + EXPECT_EQ(created_uris, (std::vector {"s3://bucket/a.txt", "s3://other/a.txt", + "s3://bucket/c.txt"})); +} + +TEST_F(FunctionToFileTest, ReusesClientAcrossBlocksAndClones) { + auto context = make_context(); + expect_client_creation(); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + auto clone = context->clone(); + EXPECT_EQ(clone->get_function_state(FunctionContext::FRAGMENT_LOCAL), + context->get_function_state(FunctionContext::FRAGMENT_LOCAL)); + ASSERT_TRUE(function.open(clone.get(), FunctionContext::THREAD_LOCAL).ok()); + io::FileStatContext* first_context = nullptr; + size_t calls = 0; + EXPECT_CALL(*filesystem, stat_impl(_, _, _)) + .Times(3) + .WillRepeatedly(Invoke([&](const io::Path&, io::FileStat* metadata, + io::FileStatContext* stat_context) { + if (calls++ == 0) { + first_context = stat_context; + EXPECT_TRUE(stat_context->is_cancelled); + EXPECT_FALSE(stat_context->is_cancelled()); + } else { + EXPECT_EQ(stat_context, first_context); + } + *metadata = io::FileStat {.size = 1}; + return Status::OK(); + })); + ColumnPtr output; + ASSERT_TRUE(execute(context.get(), uris({string("s3://bucket/a.txt"), Field()}), output).ok()); + EXPECT_TRUE(output->is_null_at(1)); + ASSERT_TRUE(execute(clone.get(), uris({string("s3://bucket/b.txt")}), output).ok()); + ASSERT_TRUE(function.close(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + ASSERT_TRUE(execute(clone.get(), uris({string("s3://bucket/c.txt")}), output).ok()); +} + +TEST_F(FunctionToFileTest, ClonesStatConcurrentlyDuringSameBucketReuseAndBucketSwitch) { + for (const bool switch_bucket : {false, true}) { + SCOPED_TRACE(switch_bucket); + auto context = make_context(); + auto other_filesystem = std::make_shared(); + EXPECT_CALL(function, create_filesystem(_, _, _)) + .Times(switch_bucket ? 2 : 1) + .WillRepeatedly(Invoke([&](const TS3Resource&, const std::string& uri, + std::shared_ptr* result) { + *result = uri.starts_with("s3://other/") ? other_filesystem : filesystem; + return Status::OK(); + })); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + auto clone = context->clone(); + ASSERT_TRUE(function.open(clone.get(), FunctionContext::THREAD_LOCAL).ok()); + std::promise first_entered; + std::promise second_entered; + std::promise release_first; + auto first_ready = first_entered.get_future(); + auto second_ready = second_entered.get_future(); + auto release = release_first.get_future().share(); + EXPECT_CALL(*filesystem, stat_impl(io::Path("s3://bucket/first.txt"), _, _)) + .WillOnce( + Invoke([&](const io::Path&, io::FileStat* metadata, io::FileStatContext*) { + first_entered.set_value(); + release.wait(); + *metadata = io::FileStat {.size = 1}; + return Status::OK(); + })); + const std::string second_uri = + switch_bucket ? "s3://other/second.txt" : "s3://bucket/second.txt"; + EXPECT_CALL(*(switch_bucket ? other_filesystem : filesystem), + stat_impl(io::Path(second_uri), _, _)) + .WillOnce( + Invoke([&](const io::Path&, io::FileStat* metadata, io::FileStatContext*) { + second_entered.set_value(); + *metadata = io::FileStat {.size = 2}; + return Status::OK(); + })); + ColumnPtr first_output; + ColumnPtr second_output; + auto first = std::async(std::launch::async, [&] { + SCOPED_INIT_THREAD_CONTEXT(); + return execute(context.get(), uris({string("s3://bucket/first.txt")}), first_output); + }); + const auto first_status = first_ready.wait_for(std::chrono::seconds(5)); + const std::weak_ptr first_filesystem = filesystem; + if (switch_bucket && first_status == std::future_status::ready) { + // Leave ownership with function state and the pending request only. + filesystem.reset(); + } + auto second = std::async(std::launch::async, [&] { + SCOPED_INIT_THREAD_CONTEXT(); + return execute(clone.get(), uris({string(second_uri)}), second_output); + }); + // The second request must enter stat while the first is still blocked. + // The deadline only prevents a failed implementation from hanging the test. + const auto second_status = second_ready.wait_for(std::chrono::seconds(5)); + EXPECT_FALSE(first_filesystem.expired()); + release_first.set_value(); + EXPECT_TRUE(first.get().ok()); + EXPECT_TRUE(second.get().ok()); + EXPECT_EQ(first_status, std::future_status::ready); + EXPECT_EQ(second_status, std::future_status::ready); + EXPECT_EQ((*first_output)[0].get()[2].get(), 1); + EXPECT_EQ((*second_output)[0].get()[2].get(), 2); + ASSERT_TRUE(testing::Mock::VerifyAndClearExpectations(&function)); + if (switch_bucket && !filesystem) { + EXPECT_TRUE(first_filesystem.expired()); + } else { + ASSERT_TRUE(testing::Mock::VerifyAndClearExpectations(filesystem.get())); + } + } +} + +TEST_F(FunctionToFileTest, PreservesRawUriServerMetadataAndZeroSize) { + const std::string uri = "s3://bucket/a%2Fb.PNG?versionId=Ab%2FC"; + auto context = make_context(); + expect_client_creation(); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + EXPECT_CALL(*filesystem, stat_impl(_, _, _)) + .WillOnce( + Invoke([&](const io::Path& path, io::FileStat* metadata, io::FileStatContext*) { + EXPECT_EQ(path.native(), uri); + *metadata = io::FileStat {.size = 0, + .content_type = "Image/PNG; name=Original", + .checksum = "ETAG:Opaque-2"}; + return Status::OK(); + })); + ColumnPtr output; + ASSERT_TRUE(execute(context.get(), uris({string(uri)}), output).ok()); + ASSERT_FALSE(output->is_null_at(0)); + const auto value = (*output)[0]; + ASSERT_EQ(value.get_type(), TYPE_FILE); + const auto& fields = value.get(); + ASSERT_EQ(fields.size(), 6); + EXPECT_EQ(fields[0].get(), uri); + EXPECT_TRUE(fields[1].is_null()); + EXPECT_EQ(fields[2].get(), 0); + EXPECT_EQ(fields[3].get(), "Image/PNG; name=Original"); + EXPECT_EQ(fields[4].get(), "ETAG:Opaque-2"); + EXPECT_TRUE(fields[5].is_null()); +} + +TEST_F(FunctionToFileTest, InfersRawKeyExtensionsAndDefaultsUnknownNames) { + auto context = make_context(); + expect_client_creation(); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + size_t calls = 0; + EXPECT_CALL(*filesystem, stat_impl(_, _, _)) + .Times(3) + .WillRepeatedly( + Invoke([&](const io::Path&, io::FileStat* metadata, io::FileStatContext*) { + *metadata = io::FileStat {.size = 5}; + if (calls++ == 1) metadata->content_type = "not a MIME type"; + return Status::OK(); + })); + ColumnPtr output; + ASSERT_TRUE(execute(context.get(), + uris({string("s3://bucket/a.PNG?name=wrong.txt"), + string("s3://bucket/a.csv"), string("s3://bucket/unknown")}), + output) + .ok()); + EXPECT_EQ((*output)[0].get()[3].get(), "text/plain"); + EXPECT_EQ((*output)[1].get()[3].get(), "text/csv"); + EXPECT_EQ((*output)[2].get()[3].get(), "application/octet-stream"); + EXPECT_TRUE((*output)[0].get()[4].is_null()); +} + +TEST_F(FunctionToFileTest, MetadataErrorsPropagateWithoutPublishingPartialOutput) { + auto context = make_context(); + expect_client_creation(); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + { + testing::InSequence sequence; + EXPECT_CALL(*filesystem, stat_impl(_, _, _)) + .WillOnce(Invoke([](const io::Path&, io::FileStat* metadata, io::FileStatContext*) { + *metadata = io::FileStat {.size = 1}; + return Status::OK(); + })); + EXPECT_CALL(*filesystem, stat_impl(_, _, _)) + .WillOnce(Return(Status::IOError("stat failed"))); + } + ColumnPtr output = make_nullable(file)->create_column_const(2, Field()); + const auto original = output; + EXPECT_FALSE(execute(context.get(), + uris({string("s3://bucket/a.txt"), string("s3://bucket/b.txt")}), output) + .ok()); + EXPECT_EQ(output.get(), original.get()); +} + +TEST_F(FunctionToFileTest, InfersExtensionFromCompleteRawKey) { + auto context = make_context(); + expect_client_creation(); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + const std::vector> cases { + {"s3://bucket/a%2E%50%4eg?filename=a.txt", "text/plain"}, + {"s3://bucket/a%252Epng", "application/octet-stream"}, + {"s3://bucket/directory.txt/filename", "application/octet-stream"}, + {"s3://bucket/directory%2Etxt/filename", "application/octet-stream"}, + {"s3://bucket/a%2Fb%2Ecsv?filename=a.pdf", "application/pdf"}, + {"s3://bucket/a%3Fpart%2Etxt", "application/octet-stream"}, + {"s3://bucket/a.txt%2F", "application/octet-stream"}, + {"s3://bucket/a%2Epng%00", "application/octet-stream"}, + {"s3://bucket/a.png?filename=a.txt", "text/plain"}, + {"s3://bucket/a%2Fb.csv?filename=a.pdf", "application/pdf"}, + {"s3://bucket/dir/photo?part.jpg", "image/jpeg"}, + {"s3://bucket/dir/photo#part.PNG", "image/png"}, + {"https://host/bucket/dir/photo?part.jpg", "image/jpeg"}, + {"http://host/bucket/dir/photo#part.PNG", "image/png"}}; + size_t calls = 0; + EXPECT_CALL(*filesystem, stat_impl(_, _, _)) + .Times(static_cast(cases.size())) + .WillRepeatedly( + Invoke([&](const io::Path& path, io::FileStat* metadata, io::FileStatContext*) { + EXPECT_EQ(path.native(), cases[calls++].first); + *metadata = io::FileStat {.size = 1}; + return Status::OK(); + })); + for (const auto& [uri, content_type] : cases) { + ColumnPtr output; + ASSERT_TRUE(execute(context.get(), uris({string(uri)}), output).ok()); + EXPECT_EQ((*output)[0].get()[0].get(), uri); + EXPECT_EQ((*output)[0].get()[3].get(), content_type); + } +} + +TEST_F(FunctionToFileTest, ConstantUriStillExecutesEachRow) { + auto uri = text->create_column_const(2, string("s3://bucket/a.txt")); + auto context = make_context(nullptr, uri); + expect_client_creation(); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + int64_t size = 0; + EXPECT_CALL(*filesystem, stat_impl(_, _, _)) + .Times(2) + .WillRepeatedly( + Invoke([&](const io::Path&, io::FileStat* metadata, io::FileStatContext*) { + *metadata = io::FileStat {.size = ++size}; + return Status::OK(); + })); + ColumnPtr output; + ASSERT_TRUE(execute(context.get(), {uri, text, "uri"}, output).ok()); + EXPECT_FALSE(is_column_const(*output)); + EXPECT_EQ((*output)[0].get()[2].get(), 1); + EXPECT_EQ((*output)[1].get()[2].get(), 2); +} + +TEST_F(FunctionToFileTest, + ExpressionOpensUsingQueryResourcesWithoutFunctionDescriptorConfiguration) { + auto source = make_context(); + TFunction definition; + definition.name.function_name = "to_file"; + definition.binary_type = TFunctionBinaryType::BUILTIN; + TExprNode node; + node.__set_node_type(TExprNodeType::FUNCTION_CALL); + node.__set_type(file->to_thrift()); + node.__set_is_nullable(true); + node.__set_fn(definition); + auto expression = VectorizedFnCall::create_shared(node); + expression->add_child(VLiteral::create_shared(text, string("test-resource"))); + expression->add_child(VLiteral::create_shared(make_nullable(text), Field())); + auto context = VExprContext::create_shared(expression); + ASSERT_TRUE(context->prepare(source->state(), RowDescriptor()).ok()); + ASSERT_TRUE(context->open(source->state()).ok()); + Block input {{resource_column(), text, "resource"}}; + ColumnPtr result; + ASSERT_TRUE(context->execute(&input, result).ok()); + ASSERT_EQ(result->size(), 1); + EXPECT_TRUE(result->is_null_at(0)); + context->close(); +} + +TEST_F(FunctionToFileTest, PreservesUriAndValidatesMetadataAfterSuccessfulStat) { + auto context = make_context(); + expect_client_creation(); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + ColumnPtr output; + for (const std::string uri : {"s3://bucket/a%ZZ.txt", "s3://bucket/a%"}) { + EXPECT_CALL(*filesystem, stat_impl(_, _, _)) + .WillOnce(Invoke( + [&](const io::Path& path, io::FileStat* metadata, io::FileStatContext*) { + EXPECT_EQ(path.native(), uri); + *metadata = io::FileStat {.size = 1, .content_type = "text/plain"}; + return Status::OK(); + })); + ASSERT_TRUE(execute(context.get(), uris({string(uri)}), output).ok()); + EXPECT_EQ((*output)[0].get()[0].get(), uri); + } + EXPECT_CALL(*filesystem, stat_impl(_, _, _)) + .WillOnce(Invoke([](const io::Path&, io::FileStat* metadata, io::FileStatContext*) { + *metadata = io::FileStat {.size = -1}; + return Status::OK(); + })); + EXPECT_FALSE(execute(context.get(), uris({string("s3://bucket/a.txt")}), output).ok()); + EXPECT_CALL(*filesystem, stat_impl(_, _, _)) + .WillOnce(Invoke([](const io::Path&, io::FileStat* metadata, io::FileStatContext*) { + *metadata = io::FileStat {.size = 1, + .checksum = "MD5:ABCDEF0123456789ABCDEF0123456789"}; + return Status::OK(); + })); + EXPECT_FALSE(execute(context.get(), uris({string("s3://bucket/a.txt")}), output).ok()); +} + +TEST_F(FunctionToFileTest, CancellationStopsFurtherMetadataCalls) { + RuntimeState runtime; + auto context = make_context(&runtime); + expect_client_creation(); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + EXPECT_CALL(*filesystem, stat_impl(_, _, _)) + .WillOnce(Invoke([&](const io::Path&, io::FileStat* metadata, + io::FileStatContext* stat_context) { + EXPECT_TRUE(stat_context->is_cancelled); + EXPECT_FALSE(stat_context->is_cancelled()); + *metadata = io::FileStat {.size = 1}; + runtime.cancel(Status::Cancelled("cancel during stat")); + EXPECT_TRUE(stat_context->is_cancelled()); + return Status::OK(); + })); + ColumnPtr output; + EXPECT_FALSE(execute(context.get(), + uris({string("s3://bucket/a.txt"), string("s3://bucket/b.txt")}), output) + .ok()); + EXPECT_EQ(output.get(), nullptr); + EXPECT_FALSE(execute(context.get(), uris({string("s3://bucket/c.txt")}), output).ok()); +} + +TEST_F(FunctionToFileTest, CancellationCallbackSurvivesOriginalContextCloseAndDestruction) { + RuntimeState runtime; + auto context = make_context(&runtime); + expect_client_creation(); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + auto clone = context->clone(); + ASSERT_TRUE(function.open(clone.get(), FunctionContext::THREAD_LOCAL).ok()); + ASSERT_TRUE(function.close(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + context.reset(); + const auto cancellation = Status::Cancelled("cancel cloned metadata request"); + EXPECT_CALL(*filesystem, stat_impl(_, _, _)) + .WillOnce( + Invoke([&](const io::Path&, io::FileStat*, io::FileStatContext* stat_context) { + EXPECT_TRUE(stat_context->is_cancelled); + EXPECT_FALSE(stat_context->is_cancelled()); + runtime.cancel(cancellation); + EXPECT_TRUE(stat_context->is_cancelled()); + return Status::IOError("metadata request aborted"); + })); + ColumnPtr output; + const auto status = execute( + clone.get(), uris({string("s3://bucket/a.txt"), string("s3://bucket/b.txt")}), output); + EXPECT_EQ(status.to_string(), cancellation.to_string()); + EXPECT_EQ(output.get(), nullptr); + ASSERT_TRUE(function.close(clone.get(), FunctionContext::THREAD_LOCAL).ok()); +} + +TEST_F(FunctionToFileTest, QueryCancellationReasonTakesPrecedenceOverAbortedStatError) { + RuntimeState runtime; + auto context = make_context(&runtime); + expect_client_creation(); + ASSERT_TRUE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + const auto cancellation = Status::Cancelled("query cancelled during metadata request"); + EXPECT_CALL(*filesystem, stat_impl(_, _, _)) + .WillOnce(Invoke([&](const io::Path&, io::FileStat*, io::FileStatContext*) { + runtime.cancel(cancellation); + return Status::IOError("metadata request aborted"); + })); + ColumnPtr output; + const auto status = + execute(context.get(), uris({string("s3://bucket/a.txt"), string("s3://bucket/b.txt")}), + output); + EXPECT_EQ(status.to_string(), cancellation.to_string()); + EXPECT_EQ(output.get(), nullptr); +} + +TEST_F(FunctionToFileTest, QueryConfigurationIsCopiedAndMissingResourceFailsOpen) { + auto context = make_context(); + auto resource = resource_config(); + resource.properties["AWS_BUCKET"] = "original"; + set_resource_config(resource); + resource.properties["AWS_BUCKET"] = "changed"; + EXPECT_EQ(resource_config().properties.at("AWS_BUCKET"), "original"); + EXPECT_CALL(function, create_filesystem(_, _, _)).Times(0); + query->set_s3_resources({{"other-resource", resource}}); + EXPECT_FALSE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + query->set_s3_resources({}); + EXPECT_FALSE(function.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + auto missing = FunctionContext::create_context(nullptr, file, {text, text}); + EXPECT_FALSE(function.open(missing.get(), FunctionContext::FRAGMENT_LOCAL).ok()); +} + +TEST_F(FunctionToFileTest, SelectsResourcesByNameWithinOneQuery) { + auto first = make_context(); + TS3Resource other; + + other.properties = {{"AWS_ENDPOINT", "other-endpoint"}}; + query->set_s3_resources({{"test-resource", resource_config()}, {"other-resource", other}}); + auto second = FunctionContext::create_context(&query_runtime, make_nullable(file), + {text, make_nullable(text)}); + second->set_constant_cols({std::make_shared( + text->create_column_const(1, string("other-resource"))), + nullptr}); + ASSERT_TRUE(function.open(first.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + ASSERT_TRUE(function.open(second.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + expect_client_creation(); + EXPECT_CALL(function, create_filesystem(_, "s3://other-bucket/a.txt", _)) + .WillOnce(Invoke([&](const TS3Resource& resource, const std::string&, + std::shared_ptr* result) { + EXPECT_EQ(resource.properties, other.properties); + *result = filesystem; + return Status::OK(); + })); + EXPECT_CALL(*filesystem, stat_impl(_, _, _)) + .Times(2) + .WillRepeatedly( + Invoke([](const io::Path&, io::FileStat* metadata, io::FileStatContext*) { + *metadata = io::FileStat {.size = 1}; + return Status::OK(); + })); + ColumnPtr output; + ASSERT_TRUE(execute(first.get(), uris({string("s3://bucket/a.txt")}), output).ok()); + EXPECT_EQ((*output)[0].get()[0].get(), "s3://bucket/a.txt"); + ASSERT_TRUE(execute(second.get(), uris({string("s3://other-bucket/a.txt")}), output).ok()); + EXPECT_EQ((*output)[0].get()[0].get(), "s3://other-bucket/a.txt"); +} + +TEST_F(FunctionToFileTest, RegistersBlockableNonConstantFunctionWithNullableUriResult) { + ColumnsWithTypeAndName arguments {{resource_column(), text, "resource"}, + {nullptr, make_nullable(text), "uri"}}; + auto registered = SimpleFunctionFactory::instance().get_function("to_file", arguments, + make_nullable(file)); + ASSERT_NE(registered, nullptr); + EXPECT_TRUE(registered->is_blockable()); + EXPECT_FALSE(registered->is_use_default_implementation_for_constants()); + EXPECT_TRUE(registered->get_return_type()->is_nullable()); + arguments[1].type = text; + auto nonnullable = SimpleFunctionFactory::instance().get_function("to_file", arguments, file); + ASSERT_NE(nonnullable, nullptr); + EXPECT_FALSE(nonnullable->get_return_type()->is_nullable()); + TExprNode node; + node.__set_node_type(TExprNodeType::FUNCTION_CALL); + node.__set_type(file->to_thrift()); + node.__set_is_nullable(false); + TFunction definition; + definition.name.function_name = "to_file"; + definition.binary_type = TFunctionBinaryType::BUILTIN; + node.__set_fn(definition); + auto expression = VectorizedFnCall::create_shared(node); + expression->_function = nonnullable; + EXPECT_FALSE(expression->is_constant()); + EXPECT_FALSE(expression->is_deterministic()); +} + +TEST_F(FunctionToFileTest, RejectsDynamicResourceAndNonStringUriTypes) { + auto dynamic_resource = text->create_column(); + dynamic_resource->insert(string("test-resource")); + EXPECT_THROW(function.get_return_type_impl(ColumnsWithTypeAndName { + {std::move(dynamic_resource), text, "resource"}, {nullptr, text, "uri"}}), + Exception); + EXPECT_THROW(function.get_return_type_impl(ColumnsWithTypeAndName { + {resource_column(), text, "resource"}, + {nullptr, std::make_shared(), "uri"}}), + Exception); +} + +} // namespace doris diff --git a/be/test/exprs/function/table_function_test.cpp b/be/test/exprs/function/table_function_test.cpp index f5b70595b779dc..7defadae7f4190 100644 --- a/be/test/exprs/function/table_function_test.cpp +++ b/be/test/exprs/function/table_function_test.cpp @@ -24,6 +24,10 @@ #include #include +#include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_varbinary.h" #include "core/types.h" #include "exprs/function/function_test_util.h" #include "exprs/mock_vexpr.h" @@ -32,7 +36,10 @@ #include "exprs/table_function/vexplode_numbers.h" #include "exprs/table_function/vexplode_v2.h" #include "exprs/table_function/vjson_each.h" +#include "exprs/vectorized_fn_call.h" +#include "runtime/runtime_state.h" #include "testutil/any_type.h" +#include "testutil/desc_tbl_builder.h" #include "util/jsonb_parser_simd.h" #include "util/jsonb_utils.h" #include "util/jsonb_writer.h" @@ -85,6 +92,92 @@ class TableFunctionTest : public testing::Test { std::vector _column_ids; }; +TEST_F(TableFunctionTest, ExplodeFilePreparesThroughVExpr) { + const DataTypePtr text = make_nullable(std::make_shared()); + const DataTypePtr bigint = make_nullable(std::make_shared()); + const DataTypePtr inline_bytes = make_nullable(std::make_shared()); + const auto output_type = make_nullable(std::make_shared( + DataTypes {text, bigint, bigint, text, text, inline_bytes}, + Strings {"uri", "offset", "size", "content_type", "checksum", "inline"})); + + for (const auto* name : {"explode_file", "explode_file_outer"}) { + SCOPED_TRACE(name); + for (const bool nullable : {false, true}) { + SCOPED_TRACE(nullable); + DataTypePtr input_type = std::make_shared(); + if (nullable) input_type = make_nullable(input_type); + ObjectPool pool; + DescriptorTblBuilder builder(&pool); + builder.declare_tuple() << std::make_tuple(input_type, std::string("f")); + auto* descriptors = builder.build(); + auto* tuple = const_cast(descriptors->get_tuple_descriptor(0)); + RuntimeState state; + state.set_desc_tbl(descriptors); + + TFunction function; + TFunctionName function_name; + function_name.__set_function_name(name); + function.__set_name(function_name); + function.__set_binary_type(TFunctionBinaryType::BUILTIN); + function.__set_arg_types({input_type->to_thrift()}); + function.__set_ret_type(output_type->to_thrift()); + function.__set_scalar_fn(TScalarFunction()); + TExprNode call; + call.__set_node_type(TExprNodeType::FUNCTION_CALL); + call.__set_type(output_type->to_thrift()); + call.__set_is_nullable(true); + call.__set_num_children(1); + call.__set_fn(function); + + TSlotRef slot_ref; + slot_ref.__set_slot_id(tuple->slots()[0]->id()); + slot_ref.__set_tuple_id(tuple->id()); + TExprNode slot; + slot.__set_node_type(TExprNodeType::SLOT_REF); + slot.__set_type(input_type->to_thrift()); + slot.__set_is_nullable(nullable); + slot.__set_num_children(0); + slot.__set_slot_ref(slot_ref); + slot.__set_label("f"); + TExpr tree; + tree.__set_nodes({call, slot}); + + // Real expression preparation resolves the scalar placeholder before the + // lateral-view operator can use the separately registered table function. + VExprContextSPtr context; + auto status = VExpr::create_expr_tree(tree, context); + ASSERT_TRUE(status.ok()) << status; + status = context->prepare(&state, RowDescriptor(tuple, false)); + ASSERT_TRUE(status.ok()) << status; + auto* prepared = dynamic_cast(context->root().get()); + ASSERT_NE(prepared, nullptr); + ASSERT_NE(prepared->_function, nullptr); + EXPECT_TRUE(prepared->_function->get_return_type()->equals(*output_type)); + + // Inspect the implementation's inferred signature separately from the FE plan type. + const auto executable = prepared->_function->prepare(nullptr, Block {}, {}, 0); + const auto* implementation = dynamic_cast(executable.get()); + ASSERT_NE(implementation, nullptr); + const auto inferred_type = implementation->get_return_type( + ColumnsWithTypeAndName {{nullptr, input_type, "f"}}); + ASSERT_TRUE(inferred_type->is_nullable()); + const auto* structure = + check_and_get_data_type(remove_nullable(inferred_type).get()); + ASSERT_NE(structure, nullptr); + ASSERT_EQ(structure->get_elements().size(), 6); + EXPECT_EQ(structure->get_element_names(), + (Strings {"uri", "offset", "size", "content_type", "checksum", "inline"})); + const auto& inline_type = structure->get_element(5); + ASSERT_TRUE(inline_type->is_nullable()); + EXPECT_EQ(remove_nullable(inline_type)->get_primitive_type(), TYPE_VARBINARY); + EXPECT_TRUE(inferred_type->equals(*output_type)); + status = context->open(&state); + ASSERT_TRUE(status.ok()) << status; + context->close(); + } + } +} + TEST_F(TableFunctionTest, vexplode_outer) { init_expr_context(1); VExplodeTableFunction explode_outer; diff --git a/be/test/exprs/vcondition_expr_test.cpp b/be/test/exprs/vcondition_expr_test.cpp index e111f7f65ce208..7ca8e8ccc99aef 100644 --- a/be/test/exprs/vcondition_expr_test.cpp +++ b/be/test/exprs/vcondition_expr_test.cpp @@ -27,12 +27,16 @@ #include #include +#include "core/assert_cast.h" +#include "core/column/column_const.h" #include "core/column/column_nullable.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_file.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_timestamp_ns.h" +#include "exprs/vcase_expr.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" @@ -138,6 +142,87 @@ static double get_float64_value(const ColumnPtr& column, size_t row, bool* is_nu class VConditionExprCoalesceTest : public ::testing::Test {}; +TEST_F(VConditionExprCoalesceTest, FilePreservesSixFieldsAndNulls) { + auto type = std::make_shared(); + auto nullable_type = make_nullable(type); + auto node = make_coalesce_node(TPrimitiveType::STRING, true); + node.type.types.clear(); + type->to_thrift(node.type); + auto expression = VectorizedCoalesceExpr::create_shared(node); + expression->data_type() = nullable_type; + const std::string bytes("\0\xff\x80\n", 4); + File fields {Field::create_field("urn:first"), + Field::create_field(3), + Field::create_field(4), + Field::create_field("application/octet-stream"), + Field::create_field("ETAG:opaque-2"), + Field::create_field(StringView(bytes))}; + auto first = Field::create_field(fields); + fields[0] = Field::create_field("urn:empty"); + fields[5] = Field::create_field(StringView()); + auto empty = Field::create_field(fields); + fields[0] = Field::create_field("urn:no-inline"); + fields[5] = Field(); + auto no_inline = Field::create_field(fields); + const std::vector expected {first, empty, no_inline, Field()}; + for (size_t source = 0; source < 3; ++source) { + auto column = nullable_type->create_column(); + for (size_t row = 0; row < expected.size(); ++row) { + column->insert(row == source ? expected[row] : Field()); + } + expression->add_child(std::make_shared(std::move(column), nullable_type)); + } + VExprContext context(expression); + ColumnPtr result; + auto status = + expression->execute_column_impl(&context, nullptr, nullptr, expected.size(), result); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(expected.size(), result->size()); + EXPECT_TRUE(result->is_null_at(3)); + for (size_t row = 0; row < 3; ++row) { + const auto value = (*result)[row]; + const auto& actual_fields = value.get(); + const auto& expected_fields = expected[row].get(); + for (size_t child = 0; child < 5; ++child) { + EXPECT_EQ(expected_fields[child], actual_fields[child]); + } + EXPECT_EQ(expected_fields[5].is_null(), actual_fields[5].is_null()); + if (!expected_fields[5].is_null()) { + EXPECT_EQ(expected_fields[5].get().to_string_ref().to_string(), + actual_fields[5].get().to_string_ref().to_string()); + } + } +} + +TEST_F(VConditionExprCoalesceTest, FileNonNullableResultUsesConstantFallback) { + auto type = std::make_shared(); + auto nullable_type = make_nullable(type); + auto node = make_coalesce_node(TPrimitiveType::STRING, false); + node.type.types.clear(); + type->to_thrift(node.type); + auto expression = VectorizedCoalesceExpr::create_shared(node); + expression->data_type() = type; + File fields(6); + fields[0] = Field::create_field("urn:present"); + auto first = nullable_type->create_column(); + first->insert(Field::create_field(fields)); + first->insert_default(); + fields[0] = Field::create_field("urn:fallback"); + auto fallback = type->create_column(); + fallback->insert(Field::create_field(fields)); + expression->add_child(std::make_shared(std::move(first), nullable_type)); + expression->add_child( + std::make_shared(ColumnConst::create(std::move(fallback), 2), type)); + VExprContext context(expression); + ColumnPtr result; + auto status = expression->execute_column_impl(&context, nullptr, nullptr, 2, result); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_FALSE(result->is_nullable()); + ASSERT_EQ(2, result->size()); + EXPECT_EQ("urn:present", (*result)[0].get()[0].get()); + EXPECT_EQ("urn:fallback", (*result)[1].get()[0].get()); +} + // Targets the fix: after some rows of col0 are filled, NaN/Inf values in the // same rows of later columns must not pollute the already-filled result via // "value * 0" arithmetic. @@ -409,4 +494,82 @@ TEST_F(VConditionExprCoalesceTest, TimeStampNs) { EXPECT_EQ(values[3].epoch_nanos(), std::numeric_limits::min()); } +TEST(VCaseExprTest, FileBranchesPreserveValuesAndNulls) { + const auto file_type = std::make_shared(); + const auto file_value = [](const std::string& uri) { + const auto bytes = uri + std::string("\0\xff", 2); + return Field::create_field( + File {Field::create_field(uri), Field(), Field(), Field(), + Field::create_field("ETAG:" + uri), + Field::create_field(StringView(bytes))}); + }; + struct Scenario { + bool nullable; + bool constant; + bool has_else; + bool null_branch; + }; + const Scenario scenarios[] {{false, false, true, false}, {false, true, true, false}, + {true, false, true, false}, {true, true, true, false}, + {true, true, true, true}, {true, false, false, false}}; + constexpr size_t rows = 4; + const uint8_t indices[] {0, 1, 1, 0}; + for (const auto& scenario : scenarios) { + SCOPED_TRACE(::testing::Message() << "nullable=" << scenario.nullable << " constant=" + << scenario.constant << " has_else=" << scenario.has_else + << " null_branch=" << scenario.null_branch); + const auto result_type = scenario.nullable ? make_nullable(file_type) : file_type; + TExprNode node; + node.__set_node_type(TExprNodeType::CASE_EXPR); + node.__set_type(file_type->to_thrift()); + node.__set_is_nullable(scenario.nullable); + node.case_expr.__set_has_else_expr(scenario.has_else); + VCaseExpr expression(node); + + std::vector columns {nullptr}; + if (scenario.has_else) { + auto otherwise = file_type->create_column(); + for (size_t row = 0; row < rows; ++row) { + otherwise->insert(file_value("urn:left")); + } + columns[0] = std::move(otherwise); + } + auto branch = result_type->create_column(); + for (size_t row = 0; row < (scenario.constant ? 1 : rows); ++row) { + if (scenario.nullable && (scenario.null_branch || (!scenario.constant && row == 1))) { + branch->insert_default(); + } else { + branch->insert(file_value("urn:right")); + } + } + if (scenario.constant) { + columns.push_back(ColumnConst::create(std::move(branch), rows)); + } else { + columns.push_back(std::move(branch)); + } + auto result = expression._execute_update_result(indices, columns, rows); + columns.clear(); + ASSERT_EQ(result->size(), rows); + for (size_t row = 0; row < rows; ++row) { + const bool expected_null = + indices[row] == 0 ? !scenario.has_else + : scenario.nullable && (scenario.null_branch || + (!scenario.constant && row == 1)); + ASSERT_EQ(result->is_null_at(row), expected_null) << row; + if (expected_null) { + continue; + } + const std::string uri = indices[row] == 0 ? "urn:left" : "urn:right"; + const auto value = (*result)[row]; + const auto& file = value.get(); + EXPECT_EQ(file[0].get(), uri); + EXPECT_TRUE(file[1].is_null()); + EXPECT_TRUE(file[2].is_null()); + EXPECT_TRUE(file[3].is_null()); + EXPECT_EQ(file[4].get(), "ETAG:" + uri); + EXPECT_EQ(file[5].get().str(), uri + std::string("\0\xff", 2)); + } + } +} + } // namespace doris diff --git a/be/test/format/transformer/vcsv_file_transformer_test.cpp b/be/test/format/transformer/vcsv_file_transformer_test.cpp new file mode 100644 index 00000000000000..5ff074d564a329 --- /dev/null +++ b/be/test/format/transformer/vcsv_file_transformer_test.cpp @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "format/transformer/vcsv_transformer.h" +#include "io/fs/file_writer.h" +#include "runtime/runtime_state.h" +#include "testutil/mock/mock_slot_ref.h" + +namespace doris { +namespace { + +class CsvMemoryWriter final : public io::FileWriter { +public: + Status appendv(const Slice* slices, size_t count) override { + for (size_t i = 0; i < count; ++i) data.append(slices[i].data, slices[i].size); + return Status::OK(); + } + Status close(bool non_block = false) override { + _state = State::CLOSED; + return Status::OK(); + } + const io::Path& path() const override { return _path; } + size_t bytes_appended() const override { return data.size(); } + State state() const override { return _state; } + + std::string data; + +private: + io::Path _path = "csv-memory"; + State _state = State::OPENED; +}; + +std::vector file_types() { + const auto file = make_nullable(std::make_shared()); + return {file, std::make_shared(file), + std::make_shared(std::make_shared(), file), + make_nullable(std::make_shared(DataTypes {file}, Strings {"f"}))}; +} +} // namespace + +TEST(VCsvFileTransformerTest, RejectsFileSchemaBeforeWritingEmptyOrNullRows) { + RuntimeState state; + state.set_timezone("UTC"); + for (const auto& type : file_types()) { + SCOPED_TRACE(type->get_name()); + for (bool empty : {false, true}) { + CsvMemoryWriter writer; + const auto expressions = MockSlotRef::create_mock_contexts(DataTypes {type}); + VCSVTransformer transformer(&state, &writer, expressions, false, {}, {}, ",", "\n", + false, TFileCompressType::PLAIN, nullptr); + auto column = type->create_column(); + if (!empty) { + column->insert_default(); + } + Block block {{std::move(column), type, "file"}}; + const auto opened = transformer.open(); + const auto status = opened.ok() ? transformer.write(block) : opened; + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(writer.data.empty()); + } + } +} + +TEST(VCsvFileTransformerTest, OrdinaryStringsRetainCsvOutput) { + RuntimeState state; + state.set_timezone("UTC"); + const auto type = std::make_shared(); + auto column = type->create_column(); + column->insert(Field::create_field("plain")); + CsvMemoryWriter writer; + const auto expressions = MockSlotRef::create_mock_contexts(DataTypes {type}); + VCSVTransformer transformer(&state, &writer, expressions, false, {}, {}, ",", "\n", false, + TFileCompressType::PLAIN, nullptr); + ASSERT_TRUE(transformer.open().ok()); + ASSERT_TRUE(transformer.write(Block {{std::move(column), type, "text"}}).ok()); + EXPECT_EQ(writer.data, "plain\n"); +} +} // namespace doris diff --git a/be/test/format/transformer/vparquet_writer_test.cpp b/be/test/format/transformer/vparquet_writer_test.cpp index 929ebf2d09cf66..700f594ec04458 100644 --- a/be/test/format/transformer/vparquet_writer_test.cpp +++ b/be/test/format/transformer/vparquet_writer_test.cpp @@ -35,15 +35,22 @@ #include "core/column/variant_v2/column_variant_v2.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" #include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/data_type_variant_v2.h" #include "exec/sink/writer/iceberg/viceberg_partition_writer.h" #include "exec/sink/writer/vhive_partition_writer.h" #include "exprs/function/parse/variant_string_parse.h" +#include "exprs/vexpr_context.h" +#include "exprs/vslot_ref.h" #include "format/table/iceberg/schema_parser.h" #include "format/transformer/viceberg_parquet_writer.h" +#include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" @@ -61,9 +68,14 @@ class VParquetWriterTest : public testing::Test { void SetUp() override { _file_path = "./vparquet_transformer_" + UniqueId::gen_uid().to_string() + ".parquet"; _fs = io::global_local_filesystem(); + _previous_pool = ExecEnv::GetInstance()->_arrow_memory_pool; + ExecEnv::GetInstance()->_arrow_memory_pool = arrow::default_memory_pool(); } - void TearDown() override { static_cast(_fs->delete_file(_file_path)); } + void TearDown() override { + ExecEnv::GetInstance()->_arrow_memory_pool = _previous_pool; + static_cast(_fs->delete_file(_file_path)); + } void check_split_offsets(int row_groups, bool dictionary) { const auto previous_row_group_size = config::min_row_group_size; @@ -194,6 +206,7 @@ class VParquetWriterTest : public testing::Test { EXPECT_EQ(ids.Value(11999), 11999); } + arrow::MemoryPool* _previous_pool = nullptr; std::string _file_path; std::shared_ptr _fs; }; @@ -342,6 +355,91 @@ TEST_F(VParquetWriterTest, HiveAndIcebergParquetPageIndexOptions) { } } +TEST_F(VParquetWriterTest, RejectsFileOutputIncludingNestedTypes) { + const auto file = std::make_shared(); + const auto nullable_file = make_nullable(file); + const auto string = make_nullable(std::make_shared()); + const auto array = make_nullable(std::make_shared(nullable_file)); + const auto map = make_nullable(std::make_shared(string, nullable_file)); + const DataTypes types { + file, + nullable_file, + array, + map, + make_nullable(std::make_shared(DataTypes {file}, Strings {"file"})), + make_nullable(std::make_shared(DataTypes {array, map}, + Strings {"files", "by_name"})), + make_nullable(std::make_shared(map))}; + const std::string iceberg_json = + R"({"type":"struct","schema-id":0,"fields":[{"id":1,"name":"f","required":false,"type":"string"}]})"; + const auto iceberg_schema = iceberg::SchemaParser::from_json(iceberg_json); + RuntimeState state; + state.set_timezone("UTC"); + for (const auto& type : types) { + SCOPED_TRACE(type->get_name()); + for (int schema_mode = 0; schema_mode < 3; ++schema_mode) { + SCOPED_TRACE(schema_mode); + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(_file_path, &file_writer).ok()); + const VExprContextSPtrs contexts {std::make_shared( + std::make_shared(0, 0, -1, type, "f"))}; + std::unique_ptr transformer; + if (schema_mode == 1) { + TParquetSchema schema; + schema.__set_schema_column_name("f"); + transformer = std::make_unique(&state, file_writer.get(), contexts, + std::vector {schema}, + false, ParquetFileOptions {}); + } else if (schema_mode == 2) { + transformer = std::make_unique( + &state, file_writer.get(), contexts, std::vector {"f"}, false, + ParquetFileOptions {}, &iceberg_json, *iceberg_schema); + } else { + transformer = std::make_unique(&state, file_writer.get(), contexts, + std::vector {"f"}, + false, ParquetFileOptions {}); + } + const auto status = transformer->open(); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("Parquet output does not support type"), + std::string::npos); + EXPECT_EQ(transformer->written_len(), 0); + EXPECT_TRUE(transformer->close().ok()); + } + } +} + +TEST_F(VParquetWriterTest, OrdinaryScalarAndNestedOutputStillWrites) { + const auto integer = make_nullable(std::make_shared()); + const auto nested = make_nullable(std::make_shared( + DataTypes {make_nullable(std::make_shared(integer)), + make_nullable(std::make_shared( + make_nullable(std::make_shared()), integer))}, + Strings {"items", "by_name"})); + const DataTypes types {integer, nested}; + const VExprContextSPtrs contexts { + std::make_shared(std::make_shared(0, 0, -1, integer, "i")), + std::make_shared(std::make_shared(1, 1, -1, nested, "s"))}; + RuntimeState state; + state.set_timezone("UTC"); + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(_file_path, &file_writer).ok()); + VParquetWriter transformer(&state, file_writer.get(), contexts, + std::vector {"i", "s"}, false, ParquetFileOptions {}); + ASSERT_TRUE(transformer.open().ok()); + Block block; + for (size_t i = 0; i < types.size(); ++i) { + auto column = types[i]->create_column(); + column->insert_default(); + block.insert({std::move(column), types[i], i == 0 ? "i" : "s"}); + } + ASSERT_TRUE(transformer.write(block).ok()); + ASSERT_TRUE(transformer.close().ok()); + const auto reader = ::parquet::ParquetFileReader::OpenFile(_file_path); + EXPECT_EQ(reader->metadata()->num_rows(), 1); + EXPECT_EQ(reader->metadata()->schema()->group_node()->field_count(), 2); +} + TEST_F(VParquetWriterTest, WritesIcebergVariantAndCollectsLogicalMetrics) { auto variant_type = std::make_shared(); auto nullable_variant_type = make_nullable(variant_type); diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index fae07484196479..45e3ce50eae21c 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -32,10 +32,27 @@ #include "util/s3_util.h" #ifdef USE_AZURE +#include + +#include +#include #include #include #include #include + +#include "core/block/block.h" +#include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_string.h" +#include "core/field.h" +#include "core/value/file_value.h" +#include "exec/common/data_gen_functions/vlist_file_tvf.h" +#include "exprs/function/function_to_file.h" +#include "io/fs/s3_file_system.h" +#include "testutil/mock/mock_descriptors.h" +#include "testutil/mock/mock_query_context.h" +#include "testutil/mock/mock_runtime_state.h" #endif namespace doris { @@ -86,6 +103,216 @@ TEST(AzureAuthFactoryTest, AllowsEmptySharedKeyCredentials) { EXPECT_TRUE(result); } +class AzureListTransport : public Azure::Core::Http::HttpTransport { +public: + std::unique_ptr Send(Azure::Core::Http::Request& request, + const Azure::Core::Context&) override { + const auto params = request.GetUrl().GetQueryParameters(); + EXPECT_EQ(Azure::Core::Url::Decode(params.at("prefix")), "tree/"); + const bool hierarchical = params.contains("delimiter"); + if (hierarchical) { + EXPECT_EQ(Azure::Core::Url::Decode(params.at("delimiter")), "/"); + } + const bool next = params.contains("marker"); + if (next) { + EXPECT_EQ(Azure::Core::Url::Decode(params.at("marker")), "next-page"); + } + auto blob = [](const std::string& name) { + return "" + name + + "Mon, 12 Oct 2026 00:00:00 " + "GMT" + ""opaque-2"7" + "BlockBlob"; + }; + _body = "" + "tree/"; + _body += blob(next ? "tree/b.txt" : "tree/a.txt"); + if (!next) { + _body += hierarchical ? "tree/sub/" + : blob("tree/sub/deep.txt"); + } + _body += std::string("") + (next ? "" : "next-page") + + ""; + auto response = std::make_unique( + 1, 1, Azure::Core::Http::HttpStatusCode::Ok, "OK"); + response->SetHeader("content-type", "application/xml"); + response->SetBodyStream(std::make_unique( + reinterpret_cast(_body.data()), _body.size())); + ++requests; + return response; + } + std::string _body; + size_t requests = 0; +}; + +// NOLINTNEXTLINE(readability-function-cognitive-complexity): GTest assertions expand to branches. +TEST(AzureObjStorageClientListTest, NativeHierarchyListingKeepsOnlyDirectBlobsAndPages) { + auto transport = std::make_shared(); + BlobClientOptions options; + options.Transport.Transport = transport; + auto container = std::make_shared( + "https://account.blob.core.windows.net/container", options); + auto client = std::make_shared(container, ObjStorageEndpointInfo {}); + for (const std::string delimiter : {"/", ""}) { + std::vector objects; + std::string token; + do { + auto page = + client->list_objects_page({.prefix = "tree/", .delimiter = delimiter}, token); + ASSERT_TRUE(page.resp.ok()) << page.resp.status.msg; + objects.insert(objects.end(), page.objects.begin(), page.objects.end()); + token = page.has_more ? page.continuation_token : std::string(); + } while (!token.empty()); + ASSERT_EQ(objects.size(), delimiter.empty() ? 3 : 2); + EXPECT_EQ(objects.front().key, "tree/a.txt"); + EXPECT_EQ(objects.back().key, "tree/b.txt"); + if (delimiter.empty()) { + EXPECT_EQ(objects[1].key, "tree/sub/deep.txt"); + } + for (const auto& object : objects) { + ASSERT_TRUE(object.etag.has_value()); + EXPECT_EQ(*object.etag, "\"opaque-2\""); + EXPECT_EQ(file_checksum_from_etag(object.etag), "ETAG:opaque-2"); + } + } + EXPECT_EQ(transport->requests, 4); +} + +class AzureHeadTransport : public Azure::Core::Http::HttpTransport { +public: + std::unique_ptr Send(Azure::Core::Http::Request& request, + const Azure::Core::Context&) override { + EXPECT_EQ(request.GetMethod(), Azure::Core::Http::HttpMethod::Head); + EXPECT_TRUE(request.GetUrl().GetPath().ends_with("container/payload.bin")); + auto response = std::make_unique( + 1, 1, Azure::Core::Http::HttpStatusCode::Ok, "OK"); + response->SetHeader("content-length", "7"); + response->SetHeader("content-type", "text/csv"); + response->SetHeader("etag", "\"opaque-2\""); + response->SetHeader("last-modified", "Mon, 12 Oct 2026 00:00:00 GMT"); + response->SetHeader("x-ms-creation-time", "Mon, 12 Oct 2026 00:00:00 GMT"); + response->SetHeader("x-ms-blob-type", "BlockBlob"); + response->SetHeader("x-ms-server-encrypted", "true"); + response->SetBodyStream(std::make_unique(nullptr, 0)); + ++requests; + return response; + } + size_t requests = 0; +}; + +class AzureFileMetadataTest : public testing::Test { +protected: + static void SetUpTestSuite() { S3ClientFactory::instance(); } + + std::shared_ptr filesystem( + const std::shared_ptr& transport) { + BlobClientOptions options; + options.Transport.Transport = transport; + auto container = std::make_shared( + "https://account.blob.core.windows.net/container", options); + auto fs = std::shared_ptr(new io::S3FileSystem( + S3Conf {.bucket = "container", .prefix = "", .client_conf = {}}, + "azure-file-test")); + fs->_client->_client = + std::make_shared(container, ObjStorageEndpointInfo {}); + return fs; + } +}; + +class ToFileWithAzureFilesystem : public FunctionToFile { +public: + explicit ToFileWithAzureFilesystem(std::shared_ptr filesystem) + : _filesystem(std::move(filesystem)) {} + + Status create_filesystem(const TS3Resource&, const std::string&, + std::shared_ptr* result) const override { + *result = _filesystem; + return Status::OK(); + } + +private: + std::shared_ptr _filesystem; +}; + +TEST_F(AzureFileMetadataTest, HeadMetadataOverridesFilenameInferenceAndProvidesChecksum) { + auto transport = std::make_shared(); + auto fs = filesystem(transport); + const auto head = fs->client_holder()->get()->head_object({.key = "payload.bin"}); + ASSERT_TRUE(head.resp.ok()) << head.resp.status.msg; + EXPECT_EQ(head.file_size, 7); + EXPECT_EQ(head.content_type, "text/csv"); + EXPECT_EQ(head.etag, "\"opaque-2\""); + + auto text = std::make_shared(); + auto file = std::make_shared(); + auto resource = text->create_column_const(1, Field::create_field("resource")); + auto uri = text->create_column(); + uri->insert(Field::create_field("s3://container/payload.bin")); + auto query = MockQueryContext::create(); + query->set_s3_resources({{"resource", TS3Resource {}}}); + MockRuntimeState state; + state._query_ctx = query.get(); + auto context = FunctionContext::create_context(&state, file, {text, text}); + context->set_constant_cols({std::make_shared(resource), nullptr}); + ToFileWithAzureFilesystem to_file(fs); + ASSERT_TRUE(to_file.open(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); + Block block { + {resource, text, "resource"}, {std::move(uri), text, "uri"}, {nullptr, file, "result"}}; + ASSERT_TRUE(to_file.execute(context.get(), block, {0, 1}, 2, 1).ok()); + const auto value = (*block.get_by_position(2).column)[0].get(); + EXPECT_EQ(value[2].get(), 7); + EXPECT_EQ(value[3].get(), "text/csv"); + ASSERT_FALSE(value[4].is_null()); + EXPECT_EQ(value[4].get(), "ETAG:opaque-2"); + EXPECT_EQ(transport->requests, 2); + ASSERT_TRUE(to_file.close(context.get(), FunctionContext::FRAGMENT_LOCAL).ok()); +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity): GTest assertions expand to branches. +TEST_F(AzureFileMetadataTest, ListingEtagsReachFileChecksumsWithoutHeadRequests) { + for (const bool recursive : {false, true}) { + auto transport = std::make_shared(); + ObjectPool pool; + MockRowDescriptor descriptor({std::make_shared(), + make_nullable(std::make_shared())}, + &pool); + descriptor.tuple_desc_map[0]->slots()[0]->_col_name = "path"; + descriptor.tuple_desc_map[0]->slots()[1]->_col_name = "file"; + VListFileTVF listing(0, descriptor.tuple_desc_map[0]); + listing._filesystem = filesystem(transport); + TTVFListFileScanRange params; + params.uri = "s3://container/tree/"; + params.recursive = recursive; + TScanRangeParams range; + range.scan_range.data_gen_scan_range.__set_list_file_params(params); + ASSERT_TRUE(listing.set_scan_ranges({range}).ok()); + MockRuntimeState state; + Block block; + bool eos = false; + size_t rows = 0; + while (!eos) { + ASSERT_TRUE(listing.get_next(&state, &block, &eos).ok()); + for (size_t i = 0; i < block.rows(); ++i) { + const auto field = (*block.get_by_position(1).column)[i]; + if (field.is_null()) { + EXPECT_FALSE(recursive); + EXPECT_EQ((*block.get_by_position(0).column)[i].get(), + "s3://container/tree/sub/"); + continue; + } + const auto value = field.get(); + ASSERT_FALSE(value[4].is_null()); + EXPECT_EQ(value[4].get(), "ETAG:opaque-2"); + } + rows += block.rows(); + block.clear_column_data(); + } + EXPECT_EQ(rows, 3); + EXPECT_EQ(transport->requests, 2); + } +} + TEST(AzureObjStorageClientTlsHelperTest, detects_tls_ca_error) { EXPECT_TRUE(io::is_azure_tls_ca_error_message( "Problem with the SSL CA cert (path? access rights?)")); diff --git a/be/test/io/fs/file_system_stat_test.cpp b/be/test/io/fs/file_system_stat_test.cpp new file mode 100644 index 00000000000000..9239df9562f152 --- /dev/null +++ b/be/test/io/fs/file_system_stat_test.cpp @@ -0,0 +1,201 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "cpp/obj-client/s3_obj_storage_client.h" +#include "io/fs/file_system.h" +#include "io/fs/local_file_system.h" +#include "io/fs/s3_file_system.h" + +namespace doris::io { + +class FileStatS3Client : public Aws::S3::S3Client { +public: + MOCK_METHOD(Aws::S3::Model::HeadObjectOutcome, HeadObject, + (const Aws::S3::Model::HeadObjectRequest&), (const, override)); +}; + +class FileStatStorageClient : public ObjStorageClient { +public: + MOCK_METHOD(ObjStorageUploadResult, create_multipart_upload, (const ObjStoragePath&), + (override)); + MOCK_METHOD(ObjStorageResponse, put_object, (const ObjStoragePath&, std::string_view), + (override)); + MOCK_METHOD(ObjStorageUploadResult, upload_part, + (const ObjStoragePath&, const std::string&, std::string_view, int), (override)); + MOCK_METHOD(ObjStorageResponse, complete_multipart_upload, + (const ObjStoragePath&, const std::string&, + const std::vector&), + (override)); + MOCK_METHOD(ObjStorageHeadResult, head_object, (const ObjStoragePath&), (override)); + MOCK_METHOD(ObjStorageResponse, get_object, + (const ObjStoragePath&, void*, size_t, size_t, size_t*), (override)); + MOCK_METHOD(ObjStorageResponse, delete_objects, + (const ObjStoragePath&, std::vector), (override)); + MOCK_METHOD(ObjStorageResponse, delete_object, (const ObjStoragePath&), (override)); + MOCK_METHOD(std::string, generate_presigned_url, (const ObjStoragePath&, int64_t), (override)); + MOCK_METHOD(ObjStorageResponse, list_objects, (const ObjStoragePath&, std::vector*), + (override)); + MOCK_METHOD(ObjStorageListPageResult, list_objects_page, + (const ObjStoragePath&, std::string_view), (override)); + MOCK_METHOD(ObjStorageCapabilities, capabilities, (), (const, override)); + MOCK_METHOD(ObjStorageResponse, get_lifecycle, (const std::string&, int64_t*), (override)); + MOCK_METHOD(ObjStorageResponse, check_versioning, (const std::string&), (override)); + MOCK_METHOD(ObjStorageResponse, abort_multipart_upload, + (const ObjStoragePath&, const std::string&), (override)); +}; + +class FileSystemStatTest : public testing::Test { +protected: + static void SetUpTestSuite() { S3ClientFactory::instance(); } +}; + +TEST_F(FileSystemStatTest, S3PreservesProviderMetadataAndUriKey) { + const auto client = std::make_shared(); + S3FileSystem fs(S3Conf {.bucket = "resource-bucket", .prefix = "prefix", .client_conf = {}}, + "stat-test"); + fs._client->_client = std::make_shared(client); + Aws::S3::Model::HeadObjectResult result; + result.SetContentLength(42); + result.SetContentType("Image/PNG; custom=Value"); + result.SetETag("\"opaque-2\""); + // Stock HEAD may return checksum headers; basic FILE stat keeps the ETag opaque. + result.SetChecksumSHA256("not-a-file-digest"); + EXPECT_CALL(*client, HeadObject(testing::_)) + .WillOnce([&](const Aws::S3::Model::HeadObjectRequest& request) { + EXPECT_FALSE(request.ChecksumModeHasBeenSet()); + EXPECT_EQ(request.GetBucket(), "resource-bucket"); + EXPECT_EQ(request.GetKey(), "a%2Fb/../c?versionId=AbC"); + return Aws::S3::Model::HeadObjectOutcome(result); + }); + FileStat metadata; + FileStatContext context; + ASSERT_TRUE(fs.stat("s3://resource-bucket/a%2Fb/../c?versionId=AbC", &metadata, &context).ok()); + EXPECT_EQ(metadata.size, 42); + EXPECT_EQ(metadata.content_type, "Image/PNG; custom=Value"); + EXPECT_EQ(metadata.checksum, "ETAG:opaque-2"); +} + +TEST_F(FileSystemStatTest, S3StatAndFileSizeUseTheSameRawKeys) { + const auto client = std::make_shared(); + S3FileSystem fs(S3Conf {.bucket = "bucket", .prefix = "", .client_conf = {}}, "stat-test"); + fs._client->_client = std::make_shared(client); + for (const std::string key : {"dir/a b.txt", "dir/a+b.txt", "dir/中文.txt", "dir/a%20b.txt", + "dir/a%252Fb.txt", "dir/invalid%zz"}) { + EXPECT_CALL(*client, HeadObject(testing::_)) + .Times(2) + .WillRepeatedly([&](const Aws::S3::Model::HeadObjectRequest& request) { + EXPECT_EQ(request.GetKey(), key); + return Aws::S3::Model::HeadObjectOutcome(Aws::S3::Model::HeadObjectResult()); + }); + FileStat metadata; + ASSERT_TRUE(fs.stat("s3://bucket/" + key, &metadata).ok()); + int64_t size = 0; + ASSERT_TRUE(fs.file_size("s3://bucket/" + key, &size).ok()); + } +} + +TEST_F(FileSystemStatTest, S3RelativePathAndAbsentMetadataReplacePriorResult) { + const auto client = std::make_shared(); + S3FileSystem fs(S3Conf {.bucket = "bucket", .prefix = "prefix", .client_conf = {}}, + "stat-test"); + fs._client->_client = std::make_shared(client); + Aws::S3::Model::HeadObjectResult result; + result.SetContentLength(0); + EXPECT_CALL(*client, HeadObject(testing::_)) + .WillOnce([&](const Aws::S3::Model::HeadObjectRequest& request) { + EXPECT_EQ(request.GetKey(), "prefix/empty"); + return Aws::S3::Model::HeadObjectOutcome(result); + }); + FileStat metadata {.size = 42, .content_type = "image/png", .checksum = "ETAG:previous"}; + ASSERT_TRUE(fs.stat("empty", &metadata).ok()); + EXPECT_EQ(metadata.size, 0); + EXPECT_FALSE(metadata.content_type.has_value()); + EXPECT_FALSE(metadata.checksum.has_value()); +} + +TEST_F(FileSystemStatTest, S3FailurePreservesOutputAndStatus) { + const auto client = std::make_shared(); + S3FileSystem fs(S3Conf {.bucket = "bucket", .prefix = "", .client_conf = {}}, "stat-test"); + fs._client->_client = std::make_shared(client); + Aws::S3::S3Error error( + Aws::Client::AWSError(Aws::S3::S3Errors::NO_SUCH_KEY, false)); + error.SetResponseCode(Aws::Http::HttpResponseCode::NOT_FOUND); + EXPECT_CALL(*client, HeadObject(testing::_)).WillOnce(testing::Return(error)); + FileStat metadata {.size = 99}; + const auto status = fs.stat("s3://bucket/missing", &metadata); + EXPECT_EQ(status.code(), ErrorCode::NOT_FOUND); + EXPECT_EQ(metadata.size, 99); +} + +TEST_F(FileSystemStatTest, S3OrdinaryHeadFailureDoesNotRetry) { + auto client = std::make_shared(); + S3FileSystem fs(S3Conf {.bucket = "bucket", .prefix = "", .client_conf = {}}, "stat-test"); + fs._client->_client = client; + ObjStorageHeadResult denied; + denied.resp.status = {ErrorCode::INTERNAL_ERROR, "denied"}; + EXPECT_CALL(*client, head_object(testing::_)).Times(1).WillOnce(testing::Return(denied)); + FileStatContext context; + FileStat metadata {.size = 99}; + EXPECT_EQ(fs.stat("key", &metadata, &context).code(), ErrorCode::INTERNAL_ERROR); + EXPECT_EQ(metadata.size, 99); +} + +TEST_F(FileSystemStatTest, S3CancellationBeforeAndAfterHeadPreservesOutput) { + auto client = std::make_shared(); + S3FileSystem fs(S3Conf {.bucket = "bucket", .prefix = "", .client_conf = {}}, "stat-test"); + fs._client->_client = client; + bool cancelled = true; + FileStatContext context {.is_cancelled = [&] { return cancelled; }}; + FileStat metadata {.size = 99}; + EXPECT_CALL(*client, head_object(testing::_)).Times(0); + EXPECT_EQ(fs.stat("key", &metadata, &context).code(), ErrorCode::CANCELLED); + EXPECT_EQ(metadata.size, 99); + ASSERT_TRUE(testing::Mock::VerifyAndClearExpectations(client.get())); + + for (const bool success : {true, false}) { + cancelled = false; + EXPECT_CALL(*client, head_object(testing::_)).WillOnce([&](const ObjStoragePath&) { + cancelled = true; + ObjStorageHeadResult result {.file_size = 42}; + if (!success) { + result.resp.status = {ErrorCode::INTERNAL_ERROR, "transport error"}; + } + return result; + }); + EXPECT_EQ(fs.stat("key", &metadata, &context).code(), ErrorCode::CANCELLED); + EXPECT_EQ(metadata.size, 99); + } +} + +TEST_F(FileSystemStatTest, UnsupportedBackendDoesNotPretendToHaveMetadata) { + FileStat metadata {.size = 99}; + EXPECT_FALSE(global_local_filesystem()->stat("/unused", &metadata).ok()); + EXPECT_EQ(metadata.size, 99); +} + +} // namespace doris::io diff --git a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp index fce15d9ae75081..ae9594ae611003 100644 --- a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp +++ b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp @@ -72,7 +72,7 @@ class FakeObjStorageClient final : public ObjStorageClient { ObjStorageListPageResult list_objects_page(const ObjStoragePath&, std::string_view) override { ++calls; - return {}; + return page_result; } ObjStorageResponse delete_objects(const ObjStoragePath&, std::vector) override { @@ -116,6 +116,7 @@ class FakeObjStorageClient final : public ObjStorageClient { int create_multipart_upload_provider_calls_per_logical_call = 1; size_t actual_read_size = 4; bool fail_get = false; + ObjStorageListPageResult page_result; }; class RecordingRateLimitPolicy final : public ObjStorageRateLimitPolicy { @@ -265,5 +266,25 @@ TEST(RateLimitedObjStorageClientTest, MultipartCreateUsesOneLogicalAdmission) { EXPECT_EQ(inner_client->create_multipart_upload_provider_calls, 2); } +TEST(RateLimitedObjStorageClientTest, paged_listing_charges_each_provider_request) { + auto inner_client = std::make_shared(); + inner_client->page_result = {.continuation_token = "next", .has_more = true}; + auto policy = std::make_shared(); + RateLimitedObjStorageClient client(inner_client, policy); + const auto first = client.list_objects_page({.bucket = "b", .prefix = "p"}, ""); + ASSERT_TRUE(first.resp.ok()); + EXPECT_TRUE(first.has_more); + EXPECT_EQ(first.continuation_token, "next"); + policy->reject = true; + const auto second = client.list_objects_page({.bucket = "b", .prefix = "p"}, "next"); + expect_rejected(second.resp); + EXPECT_EQ(inner_client->calls, 1); + ASSERT_EQ(policy->requests.size(), 2); + for (const auto& request : policy->requests) { + EXPECT_EQ(request.type, S3RateLimitType::GET); + EXPECT_EQ(request.estimated_bytes, 0); + } +} + } // namespace } // namespace doris diff --git a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp index fb45933ac2ab3e..7c2acdba72df42 100644 --- a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp +++ b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp @@ -17,6 +17,12 @@ #include #include +#include +#include +#include +#include +#include +#include #include #include #include @@ -42,6 +48,8 @@ class MockS3Client : public Aws::S3::S3Client { (const Aws::S3::Model::DeleteObjectRequest& request), (const, override)); MOCK_METHOD(Aws::S3::Model::DeleteObjectsOutcome, DeleteObjects, (const Aws::S3::Model::DeleteObjectsRequest& request), (const, override)); + MOCK_METHOD(Aws::S3::Model::HeadObjectOutcome, HeadObject, + (const Aws::S3::Model::HeadObjectRequest& request), (const, override)); }; class CountingGetRateLimitPolicy final : public ObjStorageRateLimitPolicy { @@ -68,6 +76,39 @@ class S3ObjStorageClientMockTest : public testing::Test { Aws::SDKOptions S3ObjStorageClientMockTest::options {}; +TEST_F(S3ObjStorageClientMockTest, HeadPreservesFileMetadata) { + auto mock = std::make_shared(); + auto client = std::make_shared(mock); + HeadObjectResult result; + result.SetContentLength(42); + result.SetContentType("image/png"); + result.SetETag("\"opaque-2\""); + EXPECT_CALL(*mock, HeadObject(testing::_)).WillOnce(testing::Return(result)); + auto metadata = client->head_object({.bucket = "bucket", .key = "key"}); + ASSERT_TRUE(metadata.resp.ok()); + EXPECT_EQ(metadata.file_size, 42); + EXPECT_EQ(metadata.content_type, "image/png"); + EXPECT_EQ(metadata.etag, "\"opaque-2\""); +} + +TEST_F(S3ObjStorageClientMockTest, HeadMetadataUsesOrdinaryRequestThroughRateLimiter) { + auto mock = std::make_shared(); + size_t get_request_count = 0; + auto client = std::make_shared( + std::make_shared(mock), + std::make_shared(&get_request_count)); + HeadObjectResult result; + result.SetContentLength(42); + result.SetETag("\"opaque-2\""); + EXPECT_CALL(*mock, HeadObject(testing::_)).WillOnce([&](const HeadObjectRequest& request) { + EXPECT_FALSE(request.ChecksumModeHasBeenSet()); + return HeadObjectOutcome(result); + }); + const auto metadata = client->head_object({.bucket = "bucket", .key = "key"}); + ASSERT_TRUE(metadata.resp.ok()); + EXPECT_EQ(metadata.etag, "\"opaque-2\""); +} + TEST_F(S3ObjStorageClientMockTest, list_objects_compatibility) { // If storage only supports ListObjectsV1, s3_obj_storage_client.list_objects // should return an error. @@ -101,6 +142,55 @@ ListObjectsV2Result CreatePageResult(const std::string& nextToken, return result; } +TEST_F(S3ObjStorageClientMockTest, ListPageDelimiterIsOptIn) { + auto mock = std::make_shared(); + std::shared_ptr client = std::make_shared(mock); + EXPECT_CALL(*mock, ListObjectsV2(testing::_)).WillOnce([](const ListObjectsV2Request& request) { + EXPECT_EQ(request.GetDelimiter(), "/"); + EXPECT_EQ(request.GetPrefix(), "dir/"); + EXPECT_EQ(request.GetContinuationToken(), "next"); + return ListObjectsV2Outcome(CreatePageResult("", {"dir/a.txt"}, false)); + }); + const auto direct = client->list_objects_page( + {.bucket = "bucket", .prefix = "dir/", .delimiter = "/"}, "next"); + ASSERT_TRUE(direct.resp.ok()); + ASSERT_EQ(direct.objects.size(), 1); + EXPECT_EQ(direct.objects[0].key, "dir/a.txt"); + EXPECT_CALL(*mock, ListObjectsV2(testing::_)).WillOnce([](const ListObjectsV2Request& request) { + EXPECT_FALSE(request.DelimiterHasBeenSet()); + return ListObjectsV2Outcome(CreatePageResult("", {"dir/deep/a.txt"}, false)); + }); + const auto recursive = client->list_objects_page({.bucket = "bucket", .prefix = "dir/"}, ""); + ASSERT_TRUE(recursive.resp.ok()); + ASSERT_EQ(recursive.objects.size(), 1); + EXPECT_EQ(recursive.objects[0].key, "dir/deep/a.txt"); +} + +TEST_F(S3ObjStorageClientMockTest, ListPagePreservesEtagsWithoutHeadRequests) { + auto mock = std::make_shared>(); + S3ObjStorageClient client(mock); + EXPECT_CALL(*mock, ListObjectsV2(testing::_)).WillOnce([](const auto&) { + ListObjectsV2Result result; + for (const auto& etag : {std::optional("\"opaque-2\""), + std::optional(""), std::optional()}) { + Object object; + object.SetKey("dir/" + std::to_string(result.GetContents().size())); + object.SetSize(7); + if (etag) { + object.SetETag(*etag); + } + result.AddContents(std::move(object)); + } + return ListObjectsV2Outcome(result); + }); + const auto result = client.list_objects_page({.bucket = "bucket", .prefix = "dir/"}, ""); + ASSERT_TRUE(result.resp.ok()); + ASSERT_EQ(result.objects.size(), 3); + EXPECT_EQ(result.objects[0].etag, "\"opaque-2\""); + EXPECT_FALSE(result.objects[1].etag.has_value()); + EXPECT_FALSE(result.objects[2].etag.has_value()); +} + TEST_F(S3ObjStorageClientMockTest, list_objects_with_pagination) { auto mock_s3_client = std::make_shared(); size_t get_request_count = 0; diff --git a/be/test/util/s3_uri_test.cpp b/be/test/util/s3_uri_test.cpp index 93aa2c72eb809d..aab07f7f2cc8cc 100644 --- a/be/test/util/s3_uri_test.cpp +++ b/be/test/util/s3_uri_test.cpp @@ -109,4 +109,38 @@ TEST_F(S3URITest, QueryAndFragment) { EXPECT_EQ("path/to/file", uri1.get_key()); } +TEST_F(S3URITest, ObjectPathsPreserveCompleteKeys) { + for (const std::string key : + {"dir/a://b", "dir/a?b", "dir/a#b", "dir/ a.txt ", "dir/中文+%20.txt", "dir/a?b#c://d"}) { + for (const std::string prefix : {"s3://bucket/", "https://host/bucket/"}) { + S3URI uri(prefix + key); + ASSERT_TRUE(uri.parse(true).ok()); + EXPECT_EQ(uri.get_bucket(), "bucket"); + EXPECT_EQ(uri.get_key(), key); + } + } +} + +TEST_F(S3URITest, DefaultUrlParsingKeepsExistingBehavior) { + S3URI url("s3://bucket/ dir/a?versionId=1#part "); + ASSERT_TRUE(url.parse().ok()); + EXPECT_EQ(url.get_key(), "dir/a"); + S3URI multiple_schemes("s3://bucket/dir/a://b"); + EXPECT_FALSE(multiple_schemes.parse().ok()); +} + +TEST_F(S3URITest, RawBucketRootsHaveEmptyKeys) { + for (const auto* location : + {"s3://bucket", "s3://bucket/", "http://host/bucket", "http://host/bucket/", + "https://host/bucket", "https://host/bucket/"}) { + S3URI uri(location); + ASSERT_TRUE(uri.parse(true).ok()) << location; + EXPECT_EQ(uri.get_bucket(), "bucket"); + EXPECT_TRUE(uri.get_key().empty()); + } + S3URI leading_slash("s3://bucket//"); + ASSERT_TRUE(leading_slash.parse(true).ok()); + EXPECT_EQ(leading_slash.get_key(), "/"); +} + } // end namespace doris diff --git a/common/cpp/obj-client/azure_obj_storage_client.cpp b/common/cpp/obj-client/azure_obj_storage_client.cpp index 7e2f02f243bd3a..2abe410136bd06 100644 --- a/common/cpp/obj-client/azure_obj_storage_client.cpp +++ b/common/cpp/obj-client/azure_obj_storage_client.cpp @@ -294,7 +294,10 @@ ObjStorageHeadResult AzureObjStorageClient::head_object(const ObjStoragePath& op client_bvar::ScopedLatency scoped_latency(client_bvar::s3_head_latency); return _client->GetBlockBlobClient(opts.key).GetProperties().Value; }(); - return {.resp = ObjStorageResponse::OK(), .file_size = properties.BlobSize}; + return {.resp = ObjStorageResponse::OK(), + .file_size = properties.BlobSize, + .content_type = properties.HttpHeaders.ContentType, + .etag = properties.ETag.ToString()}; } catch (Azure::Core::RequestFailedException& e) { if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { return ObjStorageHeadResult { @@ -363,35 +366,40 @@ ObjStorageListPageResult AzureObjStorageClient::list_objects_page( TEST_SYNC_POINT_CALLBACK("AzureObjStorageClient::list_objects", &request); try { - auto response = [&]() { + ObjStorageListPageResult page; + auto make_page = [&](auto response) { + page.has_more = response.NextPageToken.HasValue(); + page.continuation_token = + page.has_more ? response.NextPageToken.Value() : std::string {}; + page.objects.reserve(response.Blobs.size()); + for (auto&& item : response.Blobs) { + DCHECK(item.Name.starts_with(*request.Prefix)) << item.Name << ' ' << *request.Prefix; + page.objects.emplace_back(ObjectMeta { + .key = std::move(item.Name), + .size = item.BlobSize, + .mtime_s = duration_cast(item.Details.LastModified - + SystemClockEpoch) + .count(), + .etag = item.Details.ETag.ToString()}); + } + }; + { client_bvar::ScopedLatency scoped_latency(client_bvar::s3_list_latency); - return _client->ListBlobs(request); - }(); - const bool has_more = response.NextPageToken.HasValue(); - auto next_token = has_more ? response.NextPageToken.Value() : std::string {}; - if (has_more && next_token.empty()) { + if (opts.delimiter.empty()) { + make_page(_client->ListBlobs(request)); + } else { + auto response = _client->ListBlobsByHierarchy(opts.delimiter, request); + page.common_prefixes = std::move(response.BlobPrefixes); + make_page(std::move(response)); + } + } + if (page.has_more && page.continuation_token.empty()) { return { .resp = {.status = {ObjStorageStatus::INTERNAL_ERROR, "Azure list response has an empty continuation token"}, .http_code = 0}, }; } - ObjStorageListPageResult page {.resp = ObjStorageResponse::OK(), - .continuation_token = std::move(next_token), - .has_more = has_more}; - page.objects.reserve(response.Blobs.size()); - for (auto&& item : response.Blobs) { - DCHECK(item.Name.starts_with(*request.Prefix)) << item.Name << ' ' << *request.Prefix; - page.objects.emplace_back(ObjectMeta { - .key = std::move(item.Name), - .size = item.BlobSize, - // `Azure::DateTime` adds the offset of `SystemClockEpoch` to the given Unix timestamp, - // so here we need to subtract this offset to obtain the Unix timestamp of the mtime. - // https://github.com/Azure/azure-sdk-for-cpp/blob/azure-core_1.12.0/sdk/core/azure-core/inc/azure/core/datetime.hpp#L129 - .mtime_s = duration_cast(item.Details.LastModified - - SystemClockEpoch) - .count()}); - } return page; } catch (Azure::Core::RequestFailedException& e) { record_object_request_failed(static_cast(e.StatusCode)); diff --git a/common/cpp/obj-client/azure_obj_storage_client.h b/common/cpp/obj-client/azure_obj_storage_client.h index 29d22b81e08e22..dba54631ea33ce 100644 --- a/common/cpp/obj-client/azure_obj_storage_client.h +++ b/common/cpp/obj-client/azure_obj_storage_client.h @@ -105,7 +105,6 @@ class AzureObjStorageClient final : public ObjStorageClient { return {.max_delete_batch = 256, .max_list_page = 5000}; } -protected: ObjStorageListPageResult list_objects_page(const ObjStoragePath& path, std::string_view continuation_token) override; diff --git a/common/cpp/obj-client/obj_storage_client.h b/common/cpp/obj-client/obj_storage_client.h index 5e93dd1e83980d..9fb4ca9e0582d2 100644 --- a/common/cpp/obj-client/obj_storage_client.h +++ b/common/cpp/obj-client/obj_storage_client.h @@ -53,6 +53,7 @@ struct ObjStoragePath { std::string bucket {}; // blob container in azure std::string key {}; // blob name in azure std::string prefix {}; // for list and recursive delete + std::string delimiter {}; }; struct ObjStorageEndpointInfo { @@ -66,6 +67,7 @@ struct ObjectMeta { std::string key {}; int64_t size {0}; int64_t mtime_s {0}; + std::optional etag {}; }; struct ObjStorageCompletedPart { @@ -131,6 +133,8 @@ struct ObjStorageUploadResult { struct ObjStorageHeadResult { ObjStorageResponse resp = ObjStorageResponse::OK(); long long file_size {0}; + std::optional content_type {}; + std::optional etag {}; }; struct ObjStorageListResult { @@ -141,6 +145,7 @@ struct ObjStorageListResult { struct ObjStorageListPageResult { ObjStorageResponse resp = ObjStorageResponse::OK(); std::vector objects {}; + std::vector common_prefixes {}; std::string continuation_token {}; bool has_more = false; }; @@ -233,7 +238,6 @@ class ObjStorageClient : public std::enable_shared_from_this { virtual ObjStorageResponse abort_multipart_upload(const ObjStoragePath& path, const std::string& upload_id) = 0; -protected: // Fetch at most one page. One call corresponds to exactly one provider request. virtual ObjStorageListPageResult list_objects_page(const ObjStoragePath& opts, std::string_view continuation_token) = 0; diff --git a/common/cpp/obj-client/rate_limited_obj_storage_client.h b/common/cpp/obj-client/rate_limited_obj_storage_client.h index 4c582245756317..2a10b1f2d84eb7 100644 --- a/common/cpp/obj-client/rate_limited_obj_storage_client.h +++ b/common/cpp/obj-client/rate_limited_obj_storage_client.h @@ -72,7 +72,6 @@ class RateLimitedObjStorageClient final : public ObjStorageClient { ObjStorageResponse abort_multipart_upload(const ObjStoragePath& opts, const std::string& upload_id) override; -protected: ObjStorageListPageResult list_objects_page(const ObjStoragePath& opts, std::string_view continuation_token) override; diff --git a/common/cpp/obj-client/s3_obj_storage_client.cpp b/common/cpp/obj-client/s3_obj_storage_client.cpp index 3b9c67b6f3ee20..847a1135f33be7 100644 --- a/common/cpp/obj-client/s3_obj_storage_client.cpp +++ b/common/cpp/obj-client/s3_obj_storage_client.cpp @@ -268,7 +268,13 @@ ObjStorageHeadResult S3ObjStorageClient::head_object(const ObjStoragePath& opts) if (outcome.IsSuccess()) { return {.resp = ObjStorageResponse::OK(), - .file_size = outcome.GetResult().GetContentLength()}; + .file_size = outcome.GetResult().GetContentLength(), + .content_type = outcome.GetResult().GetContentType().empty() + ? std::nullopt + : std::make_optional(outcome.GetResult().GetContentType()), + .etag = outcome.GetResult().GetETag().empty() + ? std::nullopt + : std::make_optional(outcome.GetResult().GetETag())}; } else if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { return {.resp = {.status = ObjStorageStatus::NOT_FOUND}, .file_size = 0}; } else { @@ -331,6 +337,9 @@ ObjStorageListPageResult S3ObjStorageClient::list_objects_page( request.WithBucket(opts.bucket) .WithPrefix(prefix) .WithMaxKeys(static_cast(capabilities().max_list_page)); + if (!opts.delimiter.empty()) { + request.SetDelimiter(opts.delimiter); + } if (!continuation_token.empty()) { request.SetContinuationToken(std::string(continuation_token)); } @@ -390,6 +399,10 @@ ObjStorageListPageResult S3ObjStorageClient::list_objects_page( .continuation_token = result.GetNextContinuationToken(), .has_more = result.GetIsTruncated(), }; + page.common_prefixes.reserve(result.GetCommonPrefixes().size()); + for (const auto& entry : result.GetCommonPrefixes()) { + page.common_prefixes.emplace_back(entry.GetPrefix()); + } const auto& content = result.GetContents(); page.objects.reserve(content.size()); for (const auto& obj : content) { @@ -397,7 +410,10 @@ ObjStorageListPageResult S3ObjStorageClient::list_objects_page( << obj.GetKey() << ' ' << request.GetPrefix(); page.objects.emplace_back(ObjectMeta {.key = obj.GetKey(), .size = obj.GetSize(), - .mtime_s = obj.GetLastModified().Seconds()}); + .mtime_s = obj.GetLastModified().Seconds(), + .etag = obj.GetETag().empty() + ? std::nullopt + : std::make_optional(obj.GetETag())}); } return page; } diff --git a/common/cpp/obj-client/s3_obj_storage_client.h b/common/cpp/obj-client/s3_obj_storage_client.h index 7dd64358f030ed..cae45e9d2b0d8a 100644 --- a/common/cpp/obj-client/s3_obj_storage_client.h +++ b/common/cpp/obj-client/s3_obj_storage_client.h @@ -124,7 +124,6 @@ class S3ObjStorageClient final : public ObjStorageClient { return {.max_delete_batch = 1000, .max_list_page = 1000}; } -protected: ObjStorageListPageResult list_objects_page(const ObjStoragePath& opts, std::string_view continuation_token) override; diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/FileType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/FileType.java new file mode 100644 index 00000000000000..50b98032bb4e60 --- /dev/null +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/FileType.java @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.thrift.TColumnType; +import org.apache.doris.thrift.TTypeDesc; +import org.apache.doris.thrift.TTypeNode; +import org.apache.doris.thrift.TTypeNodeType; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.gson.annotations.SerializedName; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +/** + * Independent FILE identity with six canonical nullable physical children. + * Scan selection is separate metadata and never changes this schema. + */ +public final class FileType extends Type { + public static final int FIELD_COUNT = 6; + // JSON strings can expand sixfold; inline uses padded Base64. MySQL column_length is uint32. + public static final long JSON_DISPLAY_LENGTH = Math.min(0xffff_ffffL, + 6L * (65533 + 1024 + 1024) + 128 + 4L * ((ScalarType.MAX_VARBINARY_LENGTH + 2L) / 3)); + + @SerializedName("fields") + private List fields; + + public FileType(List fields) { + validateFields(fields); + this.fields = canonicalFields(); + } + + public static FileType create() { + return new FileType(canonicalFields()); + } + + private static List canonicalFields() { + return ImmutableList.of( + new StructField("uri", ScalarType.createVarcharType(65533)), + new StructField("offset", Type.BIGINT), + new StructField("size", Type.BIGINT), + new StructField("content_type", ScalarType.createVarcharType(1024)), + new StructField("checksum", ScalarType.createVarcharType(1024)), + new StructField("inline", ScalarType.createVarbinaryType(ScalarType.MAX_VARBINARY_LENGTH))); + } + + /** Return independent field descriptors so callers cannot mutate the fixed schema. */ + public List getFields() { + return canonicalFields(); + } + + public StructField getField(String name) { + String normalized = name.toLowerCase(Locale.ROOT); + for (StructField field : getFields()) { + if (field.getName().equals(normalized)) { + return field; + } + } + return null; + } + + public static void validateFields(List fields) { + Preconditions.checkArgument(fields != null && fields.size() == FIELD_COUNT, + "FILE requires six canonical nullable children"); + List expected = canonicalFields(); + for (int i = 0; i < FIELD_COUNT; i++) { + StructField field = fields.get(i); + StructField canonical = expected.get(i); + Preconditions.checkArgument(field != null + && canonical.getName().equals(field.getName()) + && canonical.getType().equals(field.getType()) + && canonical.getType().getLength() == field.getType().getLength() + && field.getContainsNull(), + "Invalid FILE child at position %s; expected %s", i, canonical.getName()); + } + } + + public void gsonPostProcess() { + validateFields(fields); + fields = canonicalFields(); + } + + public void gsonPreProcess() { + validateFields(fields); + } + + @Override + public PrimitiveType getPrimitiveType() { + return PrimitiveType.FILE; + } + + @Override + protected String toSql(int depth) { + return "FILE"; + } + + @Override + protected String prettyPrint(int lpad) { + return Strings.repeat(" ", lpad) + "FILE"; + } + + @Override + public String toString() { + return "FILE"; + } + + @Override + public boolean matchesType(Type other) { + return equals(other) || (other.isAnyType() && other.matchesType(this)); + } + + @Override + public boolean equals(Object other) { + return other instanceof FileType; + } + + @Override + public int hashCode() { + return FileType.class.hashCode(); + } + + @Override + public Integer getColumnSize() { + return (int) Math.min(Integer.MAX_VALUE, JSON_DISPLAY_LENGTH); + } + + @Override + public int getColumnStringRepSize() { + return (int) Math.min(Integer.MAX_VALUE, JSON_DISPLAY_LENGTH); + } + + @Override + public int getSlotSize() { + return PrimitiveType.FILE.getSlotSize(); + } + + @Override + public void toThrift(TTypeDesc container) { + validateFields(fields); + TTypeNode node = new TTypeNode(TTypeNodeType.FILE); + node.setStructFields(new ArrayList<>()); + container.addToTypes(node); + for (StructField field : fields) { + field.toThrift(container, node); + } + } + + @Override + public TColumnType toColumnTypeThrift() { + return new TColumnType(PrimitiveType.FILE.toThrift()); + } +} diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java index c7f618b6875b72..75bbd4229e0105 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java @@ -81,6 +81,7 @@ public enum PrimitiveType { // 8-byte pointer and 4-byte size and 1 bytes has_null (13 bytes total) // Aligning to 16 bytes total. STRUCT("STRUCT", 16, TPrimitiveType.STRUCT, false), + FILE("FILE", 16, TPrimitiveType.FILE, true), STRING("STRING", 16, TPrimitiveType.STRING, true), VARIANT("VARIANT", 24, TPrimitiveType.VARIANT, false), TEMPLATE("TEMPLATE", -1, TPrimitiveType.INVALID_TYPE, false), @@ -265,6 +266,8 @@ public static PrimitiveType fromThrift(TPrimitiveType tPrimitiveType) { return MAP; case STRUCT: return STRUCT; + case FILE: + return FILE; case VARIANT: return VARIANT; case VARBINARY: @@ -368,7 +371,7 @@ public boolean isStructType() { } public boolean isComplexType() { - return this == ARRAY || this == MAP || this == STRUCT; + return this == ARRAY || this == MAP || this == STRUCT || this == FILE; } public boolean isHllType() { @@ -471,6 +474,8 @@ public MysqlColType toMysqlType() { return MysqlColType.MYSQL_TYPE_JSON; case VARBINARY: return MysqlColType.MYSQL_TYPE_VARSTRING; + case FILE: + return MysqlColType.MYSQL_TYPE_STRING; default: return MysqlColType.MYSQL_TYPE_STRING; } diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java index 22ab1aa14a4118..993309c5eaef98 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java @@ -128,6 +128,7 @@ public long getByteSize() { private String lenStr; public ScalarType(PrimitiveType type) { + Preconditions.checkArgument(type != PrimitiveType.FILE, "FILE requires its independent FileType"); this.type = type; } @@ -1098,4 +1099,12 @@ public boolean getVariantEnableNestedGroup() { return false; // Default to false for backward compatibility. } + public void gsonPostProcess() { + Preconditions.checkArgument(type != PrimitiveType.FILE, "FILE requires its independent FileType"); + } + + public void gsonPreProcess() { + Preconditions.checkArgument(type != PrimitiveType.FILE, "FILE requires its independent FileType"); + } + } diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index 2963ea1d6a1c5e..e72cc7109449df 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -126,6 +126,7 @@ public abstract class Type { public static final StructType GENERIC_STRUCT = new StructType(Lists.newArrayList( new StructField("generic_struct", new ScalarType(PrimitiveType.NULL_TYPE)))); public static final StructType STRUCT = new StructType(); + public static final FileType FILE = FileType.create(); // In the past, variant metadata used the ScalarType type. // Now, we use VariantType, which inherits from ScalarType, as the new metadata storage. public static final VariantType VARIANT = new VariantType(); @@ -225,6 +226,7 @@ public abstract class Type { arraySubTypes.add(ARRAY); arraySubTypes.add(MAP); arraySubTypes.add(STRUCT); + arraySubTypes.add(FILE); mapSubTypes = Lists.newArrayList(); mapSubTypes.add(BOOLEAN); @@ -253,6 +255,7 @@ public abstract class Type { mapSubTypes.add(ARRAY); mapSubTypes.add(MAP); mapSubTypes.add(STRUCT); + mapSubTypes.add(FILE); structSubTypes = Lists.newArrayList(); structSubTypes.add(BOOLEAN); @@ -281,6 +284,7 @@ public abstract class Type { structSubTypes.add(ARRAY); structSubTypes.add(MAP); structSubTypes.add(STRUCT); + structSubTypes.add(FILE); // Before adding a type here, the BE side must implement these three serde // adapters for the type so values falling to the variant sparse path can @@ -473,6 +477,22 @@ public boolean typeContainsPrecision() { return false; } + /** Whether this type or a nested complex type contains FILE. */ + public boolean typeContainsFile() { + if (isStructType()) { + return ((StructType) this).getFields().stream() + .anyMatch(field -> field.getType().typeContainsFile()); + } else if (isMapType()) { + MapType mapType = (MapType) this; + return mapType.getKeyType().typeContainsFile() || mapType.getValueType().typeContainsFile(); + } else if (isArrayType()) { + return ((ArrayType) this).getItemType().typeContainsFile(); + } else if (isAggStateType()) { + return ((AggStateType) this).getSubTypes().stream().anyMatch(Type::typeContainsFile); + } + return isFileType(); + } + /** Whether this type or a nested complex type contains Variant. */ public boolean typeContainsVariant() { if (isVariantType()) { @@ -745,7 +765,7 @@ public boolean isTimeType() { } public boolean isComplexType() { - return isStructType() || isMapType() || isArrayType(); + return isStructType() || isMapType() || isArrayType() || isFileType(); } public boolean isMapType() { @@ -760,6 +780,10 @@ public boolean isAggStateType() { return this instanceof AggStateType; } + public boolean isFileType() { + return this instanceof FileType; + } + public boolean isStructType() { return this instanceof StructType; } @@ -885,7 +909,7 @@ private boolean exceedsMaxNestingDepth(int d) { } return mapType.getValueType().exceedsMaxNestingDepth(d + 1); } else { - Preconditions.checkState(isScalarType() || isAggStateType()); + Preconditions.checkState(isScalarType() || isAggStateType() || isFileType()); } return false; } @@ -954,6 +978,8 @@ public static Type fromPrimitiveType(PrimitiveType type) { return new MapType(); case STRUCT: return new StructType(); + case FILE: + return FILE; case BITMAP: return Type.BITMAP; case QUANTILE_STATE: @@ -1014,6 +1040,8 @@ protected static Pair fromThrift(TTypeDesc col, int nodeIdx) { } else if (scalarType.getType() == TPrimitiveType.VARCHAR) { Preconditions.checkState(scalarType.isSetLen()); type = ScalarType.createVarcharType(scalarType.getLen()); + } else if (scalarType.getType() == TPrimitiveType.FILE) { + throw new IllegalArgumentException("FILE must use a FILE type node with six children"); } else if (scalarType.getType() == TPrimitiveType.HLL) { type = ScalarType.createHllType(); } else if (scalarType.getType() == TPrimitiveType.DECIMALV2) { @@ -1061,6 +1089,30 @@ protected static Pair fromThrift(TTypeDesc col, int nodeIdx) { tmpNodeIdx = valueType.second; break; } + case FILE: { + Preconditions.checkArgument(node.getStructFieldsSize() == FileType.FIELD_COUNT + && !node.isSetContainsNulls(), + "FILE requires six named nullable children"); + List fields = new ArrayList<>(); + ++tmpNodeIdx; + for (int i = 0; i < FileType.FIELD_COUNT; i++) { + TStructField field = node.getStructFields().get(i); + Preconditions.checkArgument(FILE.getFields().get(i).getName().equals(field.getName()), + "FILE children must use canonical names"); + Preconditions.checkArgument(field.isSetContainsNull() && field.isContainsNull(), + "FILE children must be nullable"); + Pair child = fromThrift(col, tmpNodeIdx); + Type childType = child.first; + if (i == FileType.FIELD_COUNT - 1 && childType.isVarbinaryType()) { + TScalarType inlineType = col.getTypes().get(tmpNodeIdx).getScalarType(); + childType = ScalarType.createVarbinaryType(inlineType.isSetLen() ? inlineType.getLen() : -1); + } + fields.add(new StructField(field.getName(), childType, null, true)); + tmpNodeIdx = child.second; + } + type = new FileType(fields); + break; + } case STRUCT: { Preconditions.checkState(tmpNodeIdx + node.getStructFieldsSize() < col.getTypesSize()); ArrayList structFields = Lists.newArrayList(); diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 656f37e522369f..1f79bdb34f517f 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1947,6 +1947,7 @@ primitiveColType | type=IPV6 | type=UUID | type=VARBINARY + | type=FILE | type=VARIANT | type=ALL ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java index ac39e630ce3d1b..9532efba8880eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java @@ -619,6 +619,9 @@ private List checkAndPrepareMaterializedView(CreateMaterializedViewComma } for (Column column : newMVColumns) { + if (column.getType().typeContainsFile()) { + throw new DdlException("FILE columns are not supported in internal tables"); + } // check c.2 if (column.isKey() && column.getType().isFloatingPointType()) { throw new DdlException("Do not support float/double type on key column, you can change it to decimal"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java index 73697843930481..e95e7b1ae83021 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java @@ -232,6 +232,12 @@ public void analyze(List resultExprs, List colLabels, boolean need analyzeForParquetFormat(resultExprs, colLabels); } else if (isOrcFormat()) { analyzeForOrcFormat(resultExprs, colLabels); + } else if (fileFormatProperties instanceof CsvFileFormatProperties) { + for (Expr expr : resultExprs) { + if (expr.getType().typeContainsFile()) { + throw new AnalysisException("CSV OUTFILE does not support FILE"); + } + } } } } @@ -466,6 +472,11 @@ private void checkOrcType(String orcType, String expectType, boolean isEqual, St } private void analyzeForParquetFormat(List resultExprs, List colLabels) throws AnalysisException { + for (Expr expr : resultExprs) { + if (expr.getType().typeContainsFile()) { + throw new AnalysisException("Parquet OUTFILE does not support FILE"); + } + } if (this.parquetSchemas.isEmpty()) { genParquetColumnName(resultExprs, colLabels); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/VarBinaryLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/VarBinaryLiteral.java index 599692cba7ddd0..ae48436899d2a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/VarBinaryLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/VarBinaryLiteral.java @@ -51,6 +51,10 @@ public VarBinaryLiteral(byte[] value) throws AnalysisException { analysisDone(); } + public byte[] getValue() { + return value; + } + protected VarBinaryLiteral(VarBinaryLiteral other) { super(other); this.value = other.value; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index 220586eed679e4..0a375ffec98d74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -215,6 +215,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Field; import org.apache.doris.nereids.trees.expressions.functions.scalar.FindInSet; import org.apache.doris.nereids.trees.expressions.functions.scalar.FirstSignificantSubdomain; +import org.apache.doris.nereids.trees.expressions.functions.scalar.FlGetBinary; import org.apache.doris.nereids.trees.expressions.functions.scalar.Floor; import org.apache.doris.nereids.trees.expressions.functions.scalar.Fmod; import org.apache.doris.nereids.trees.expressions.functions.scalar.Format; @@ -529,6 +530,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ToDate; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToDateV2; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToDays; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ToFile; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToIpv4; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToIpv4OrDefault; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToIpv4OrNull; @@ -800,6 +802,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(Field.class, "field"), scalar(FindInSet.class, "find_in_set"), scalar(FirstSignificantSubdomain.class, "first_significant_subdomain"), + scalar(FlGetBinary.class, "fl_get_binary"), scalar(Floor.class, "floor"), scalar(Fmod.class, "fmod"), scalar(Format.class, "format"), @@ -1125,6 +1128,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(ToDate.class, "to_date"), scalar(ToDateV2.class, "to_datev2"), scalar(ToDays.class, "to_days"), + scalar(ToFile.class, "to_file"), scalar(ToIpv4.class, "to_ipv4"), scalar(ToIpv4OrDefault.class, "to_ipv4_or_default"), scalar(ToIpv4OrNull.class, "to_ipv4_or_null"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java index f764da5718c389..94c488735cc26b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java @@ -20,6 +20,8 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.Explode; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeBitmap; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeBitmapOuter; +import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeFile; +import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeFileOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeJsonArrayDouble; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeJsonArrayDoubleOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeJsonArrayInt; @@ -62,6 +64,8 @@ public class BuiltinTableGeneratingFunctions implements FunctionHelper { public final List tableGeneratingFunctions = ImmutableList.of( tableGenerating(Explode.class, "explode"), + tableGenerating(ExplodeFile.class, "explode_file"), + tableGenerating(ExplodeFileOuter.class, "explode_file_outer"), tableGenerating(ExplodeOuter.class, "explode_outer"), tableGenerating(ExplodeMap.class, "explode_map"), tableGenerating(ExplodeMapOuter.class, "explode_map_outer"), @@ -97,6 +101,7 @@ public class BuiltinTableGeneratingFunctions implements FunctionHelper { .add("explode_json_array_string").add("explode_json_array_json").add("explode_json_array_int_outer") .add("explode_json_array_double_outer").add("explode_json_array_string_outer") .add("explode_json_array_json_outer").add("explode_split").add("explode_split_outer") + .add("explode_file").add("explode_file_outer") .add("json_each").add("json_each_outer") .add("json_each_text").add("json_each_text_outer") .add("posexplode").add("posexplode_outer").build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java index 0ee243b80313fa..53bfa6b25adf65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java @@ -32,6 +32,7 @@ import org.apache.doris.nereids.trees.expressions.functions.table.IndexDiskUsage; import org.apache.doris.nereids.trees.expressions.functions.table.Jobs; import org.apache.doris.nereids.trees.expressions.functions.table.LanceIndexEntries; +import org.apache.doris.nereids.trees.expressions.functions.table.ListFiles; import org.apache.doris.nereids.trees.expressions.functions.table.Local; import org.apache.doris.nereids.trees.expressions.functions.table.MvInfos; import org.apache.doris.nereids.trees.expressions.functions.table.Numbers; @@ -65,6 +66,7 @@ public class BuiltinTableValuedFunctions implements FunctionHelper { tableValued(HudiMeta.class, "hudi_meta"), tableValued(Hdfs.class, "hdfs"), tableValued(HttpStream.class, "http_stream"), + tableValued(ListFiles.class, "list_files"), tableValued(Numbers.class, "numbers"), tableValued(S3.class, "s3"), tableValued(MvInfos.class, "mv_infos"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index b1887458da394f..8041eb029f6014 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -26,6 +26,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.util.SqlUtils; import org.apache.doris.persist.gson.GsonPostProcessable; +import org.apache.doris.persist.gson.GsonPreProcessable; import org.apache.doris.proto.OlapFile; import org.apache.doris.proto.OlapFile.PatternTypePB; import org.apache.doris.thrift.TAggregationType; @@ -34,6 +35,7 @@ import org.apache.doris.thrift.TPatternType; import org.apache.doris.thrift.TPrimitiveType; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -56,7 +58,7 @@ /** * This class represents the column-related metadata. */ -public class Column implements GsonPostProcessable { +public class Column implements GsonPostProcessable, GsonPreProcessable { private static final Logger LOG = LogManager.getLogger(Column.class); public static final String HIDDEN_COLUMN_PREFIX = "__DORIS_"; // NOTE: you should name hidden column start with '__DORIS_' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -181,11 +183,12 @@ public Column() { } public Column(String name, PrimitiveType dataType) { - this(name, ScalarType.createType(dataType), false, null, false, null, ""); + this(name, dataType == PrimitiveType.FILE ? Type.FILE : ScalarType.createType(dataType), + false, null, false, null, ""); } public Column(String name, PrimitiveType dataType, boolean isAllowNull) { - this(name, ScalarType.createType(dataType), isAllowNull); + this(name, dataType == PrimitiveType.FILE ? Type.FILE : ScalarType.createType(dataType), isAllowNull); } public Column(String name, PrimitiveType dataType, int len, int precision, int scale, boolean isAllowNull) { @@ -362,6 +365,11 @@ public void createChildrenColumn(Type type, Column column) { c.setIsAllowNull(field.getContainsNull()); column.addChildrenColumn(c); } + } else if (type.isFileType()) { + for (StructField field : ((FileType) type).getFields()) { + Column child = new Column(field.getName(), field.getType(), true); + column.addChildrenColumn(child); + } } else if (type.isVariantType() && type instanceof VariantType) { // variant may contain predefined structured fields ArrayList fields = ((VariantType) type).getPredefinedFields(); @@ -636,6 +644,7 @@ public String getOnUpdateDefaultValueSql() { } public TColumn toThrift() { + validateFileSchema(); TColumn tColumn = new TColumn(); tColumn.setColumnName(removeNamePrefix(this.name)); @@ -701,6 +710,7 @@ public TColumn toThrift() { } private void setChildrenTColumn(Column children, TColumn tColumn) { + children.validateFileSchema(); TColumn childrenTColumn = new TColumn(); childrenTColumn.setColumnName(children.name); @@ -758,7 +768,7 @@ private void toChildrenThrift(Column column, TColumn tColumn) { tColumn.setChildrenColumn(new ArrayList<>()); setChildrenTColumn(k, tColumn); setChildrenTColumn(v, tColumn); - } else if (column.type.isStructType()) { + } else if (column.type.isStructType() || column.type.isFileType()) { List childrenColumns = column.getChildren(); tColumn.setChildrenColumn(new ArrayList<>()); for (Column children : childrenColumns) { @@ -1241,6 +1251,7 @@ public String getSignatureString(Map typeStringMap) { break; case ARRAY: case MAP: + case FILE: case STRUCT: sb.append(type.toString()); break; @@ -1291,8 +1302,28 @@ public boolean hasDefaultValue() { return defaultValue != null || realDefaultValue != null || defaultValueExprDef != null; } + /** Verify redundant column children agree with the canonical FILE type. */ + public void validateFileSchema() { + if (type.isFileType()) { + Preconditions.checkArgument(children != null && children.size() == FileType.FIELD_COUNT, + "FILE column requires six canonical children"); + List fields = new ArrayList<>(); + for (int i = 0; i < children.size(); i++) { + Column child = children.get(i); + fields.add(new StructField(child.getName(), child.getType(), null, child.isAllowNull())); + } + FileType.validateFields(fields); + } + } + + @Override + public void gsonPreProcess() { + validateFileSchema(); + } + @Override public void gsonPostProcess() throws IOException { + validateFileSchema(); // This just for bugfix. Because when user upgrade from 0.x to 1.1.x, // the length of String type become 1. The reason is not very clear and maybe fixed by #14275. // Here we try to rectify the error string length, by setting all String' length to MAX_STRING_LENGTH diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java index 50448a34bb73aa..897177a186ebe9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java @@ -362,7 +362,7 @@ private static void checkSupportSchemaChangeForComplexType(Type checkType, Type public static void write(DataOutput out, Type type) throws IOException { Preconditions.checkArgument(type.isScalarType() || type.isAggStateType() - || type.isArrayType() || type.isMapType() || type.isStructType(), + || type.isArrayType() || type.isMapType() || type.isStructType() || type.isFileType(), "not support serialize this type " + type.toSql()); Text.writeString(out, GsonUtils.GSON.toJson(type)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java index a1f4688cb66693..a2adefba7c511d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java @@ -65,6 +65,9 @@ public static void checkDistributionColumnType(String columnName, Type type) thr if (type.isStructType()) { throw new DdlException("Struct Type should not be used in distribution column[" + columnName + "]."); } + if (type.typeContainsFile()) { + throw new DdlException("FILE cannot be used in distribution column[" + columnName + "]"); + } if (type.isJsonbType()) { throw new DdlException("JsonType type should not be used in distribution column[" + columnName + "]."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index dbd10e2aa6b7ee..864b35c6999a82 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -19,6 +19,7 @@ import org.apache.doris.backup.Status; import org.apache.doris.common.DdlException; +import org.apache.doris.common.ErrorCode; import org.apache.doris.common.credentials.CloudCredentialWithEndpoint; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.util.PrintableMap; @@ -30,6 +31,10 @@ import org.apache.doris.fs.obj.ObjStorage; import org.apache.doris.fs.obj.RemoteObjects; import org.apache.doris.fs.obj.S3ObjStorage; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TS3Resource; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -103,6 +108,35 @@ public S3Resource(String name) { properties = Maps.newHashMap(); } + /** Resolve an S3 resource and check the current query's USAGE privilege without remote IO. */ + public static S3Resource checkResource(String resourceName, ConnectContext context) { + Resource resource = Env.getCurrentEnv().getResourceMgr().getResource(resourceName); + if (resource == null) { + throw new AnalysisException("Can not find resource: " + resourceName); + } + if (!Env.getCurrentEnv().getAccessManager().checkResourcePriv(context, resourceName, PrivPredicate.USAGE)) { + throw new AnalysisException(ErrorCode.ERR_RESOURCE_ACCESS_DENIED_ERROR.formatErrorMsg( + PrivPredicate.USAGE.getPrivs().toString(), resourceName)); + } + if (resource.getType() != ResourceType.S3) { + throw new AnalysisException("Requires an S3 resource: " + resourceName); + } + return (S3Resource) resource; + } + + /** Reuse storage property conversion when sending S3-compatible configuration to BE. */ + public TS3Resource toThrift() { + readLock(); + try { + Map backendProperties = getCopiedProperties(); + backendProperties.putAll(StorageProperties.createPrimary(new HashMap<>(backendProperties)) + .getBackendConfigProperties()); + return new TS3Resource().setProperties(backendProperties); + } finally { + readUnlock(); + } + } + public String getProperty(String propertyKey) { return properties.get(propertyKey); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java index e67f843acefcdc..cee1d56b188bad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java @@ -323,6 +323,9 @@ int getMysqlTypeLength(Type type) { } return precision; } + case FILE: + // Preserve the uint32 bit pattern for writeInt4, including lengths above INT_MAX. + return (int) org.apache.doris.catalog.FileType.JSON_DISPLAY_LENGTH; case VARBINARY: { return type.getLength(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java index 9844de97f950b5..87a21f3e4f86a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java @@ -103,6 +103,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda; import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ToFile; import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdaf; import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdf; import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdtf; @@ -768,6 +769,12 @@ public Expr visitSearchExpression(SearchExpression searchExpression, return searchPredicate; } + @Override + public Expr visitToFile(ToFile toFile, PlanTranslatorContext context) { + context.getConnectContext().getOrRegisterS3Resource(toFile.getResourceName()); + return visitScalarFunction(toFile, context); + } + @Override public Expr visitScalarFunction(ScalarFunction function, PlanTranslatorContext context) { List arguments = function.getArguments().stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java index 60c770df4eb2ab..893fc33193aa8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java @@ -41,6 +41,7 @@ import org.apache.doris.planner.ScanNode; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.CoordinatorContext; +import org.apache.doris.qe.runtime.ThriftPlansBuilder; import org.apache.doris.service.FrontendOptions; import org.apache.doris.thrift.PaloInternalServiceVersion; import org.apache.doris.thrift.TBrokerFileStatus; @@ -267,6 +268,7 @@ public TPipelineFragmentParams plan(TUniqueId loadId, int fragmentInstanceIdInde TPipelineFragmentParams params = new TPipelineFragmentParams(); params.setProtocolVersion(PaloInternalServiceVersion.V1); + params.setS3Resources(ThriftPlansBuilder.collectS3Resources(connectContext)); params.setFragment(fragment.toThrift()); params.setDescTbl(descriptorTable.toThrift()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index fcad7c8bdca1b0..e8736365dacbd2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -65,6 +65,7 @@ import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue; +import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeFile; import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; import org.apache.doris.nereids.trees.expressions.functions.generator.Unnest; import org.apache.doris.nereids.trees.expressions.functions.scalar.Coalesce; @@ -469,8 +470,11 @@ private LogicalPlan bindGenerate(MatchingContext> ctx) { // 2. the expandColumnsAlias is empty, we should use origin boundSlot if (generate.getExpandColumnAlias() != null && i < generate.getExpandColumnAlias().size() && !CollectionUtils.isEmpty(generate.getExpandColumnAlias().get(i))) { - if (boundSlot.getDataType() instanceof StructType - && generate.getExpandColumnAlias().get(i).size() > 1) { + int aliasCount = generate.getExpandColumnAlias().get(i).size(); + if (boundGenerator instanceof ExplodeFile && aliasCount != 6) { + throw new AnalysisException("EXPLODE_FILE requires exactly six column aliases"); + } + if (boundSlot.getDataType() instanceof StructType && aliasCount > 1) { // if the alias is not empty, we should bind it with struct_element as child expr with alias // element_at(#expand_col#k, #k) as #k // element_at(#expand_col#v, #v) as #v @@ -1163,6 +1167,10 @@ private Plan bindProject(MatchingContext> ctx) { } } project = project.withProjects(boundProjectionsBuilder.build()); + if (project.isDistinct()) { + project.getProjects().forEach(value -> ExpressionUtils.checkFileKey( + value, "SELECT DISTINCT")); + } return rewriteProjectForDisabledFullGroupBy(project).orElse(project); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java index 85045e0ca554a8..9f5841b1fbc321 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java @@ -44,8 +44,10 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalQualify; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; import org.apache.doris.nereids.trees.plans.logical.OutputPrunable; import org.apache.doris.nereids.types.DataType; @@ -53,6 +55,8 @@ import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.PlanUtils; import org.apache.doris.nereids.util.Utils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TResultSinkType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -125,12 +129,15 @@ public CheckAnalysis(boolean hadFillMissingSlots) { public List buildRules() { return ImmutableList.of( RuleType.CHECK_ANALYSIS.build( - any().then(plan -> { + any().thenApply(ctx -> { + Plan plan = ctx.root; checkExpressionInputTypes(plan); + CheckAnalysis.checkFilePlan(plan); checkUnexpectedExpressions(plan); checkDistinctSetOperation(plan); checkAggregateFunction(plan); checkGroupingScalarFunction(plan); + checkArrowFlightResult(plan, ctx.connectContext); return null; }) ), @@ -144,6 +151,35 @@ public List buildRules() { ); } + /** Validate operator keys after expression binding. */ + public static void checkFilePlan(Plan plan) { + if (plan instanceof LogicalSort) { + ((LogicalSort) plan).getOrderKeys() + .forEach(key -> ExpressionUtils.checkFileKey(key.getExpr(), "ORDER BY")); + } + if (plan instanceof LogicalAggregate) { + ((LogicalAggregate) plan).getGroupByExpressions() + .forEach(key -> ExpressionUtils.checkFileKey(key, "GROUP BY")); + } + if (plan instanceof LogicalRepeat) { + ((LogicalRepeat) plan).getGroupByExpressions() + .forEach(key -> ExpressionUtils.checkFileKey(key, "GROUPING SETS")); + } + if (plan instanceof LogicalSetOperation + && (!(plan instanceof LogicalUnion) || ((LogicalSetOperation) plan).getQualifier() != Qualifier.ALL)) { + plan.getOutput().forEach(key -> ExpressionUtils.checkFileKey(key, plan.getType().toString())); + } + } + + private void checkArrowFlightResult(Plan plan, ConnectContext connectContext) { + if (plan instanceof LogicalResultSink + && connectContext.getResultSinkType() == TResultSinkType.ARROW_FLIGHT_PROTOCOL + && plan.getExpressions().stream() + .anyMatch(output -> output.getDataType().typeContainsFile())) { + throw new AnalysisException("Arrow Flight does not support FILE result columns"); + } + } + private void checkUnexpectedExpressions(Plan plan) { checkUnexpectedExpressionTypes(plan, UNEXPECTED_EXPRESSION_TYPE_MAP.get(plan.getClass())); if (hadFillMissingSlots) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java index 9afd85c6152708..48e5abeb3e731d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java @@ -391,6 +391,10 @@ public static boolean check(DataType originalType, DataType targetType, boolean */ public static boolean check(DataType originalType, DataType targetType, boolean isStrictMode, boolean looseAggState) { + if (originalType.typeContainsFile() || targetType.typeContainsFile()) { + return !targetType.isAggStateType() + && (originalType.isNullType() || originalType.equals(targetType)); + } if (originalType.isVariantType() && (targetType instanceof PrimitiveType || targetType.isArrayType())) { // variant could cast to primitive types and array return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 4d6b33a6fcff2c..11f088909fa711 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -70,6 +70,7 @@ import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -78,6 +79,7 @@ import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.types.TimeStampNsType; +import org.apache.doris.nereids.types.VarBinaryType; import org.apache.doris.proto.InternalService; import org.apache.doris.proto.InternalService.PConstantExprResult; import org.apache.doris.proto.Types.PScalarType; @@ -241,7 +243,8 @@ private static boolean shouldSkipFold(Expression expr) { // Frontend can not represent those types if (expr.getDataType().isAggStateType() || expr.getDataType().isObjectType() || expr.getDataType().isVariantType() || expr.getDataType().isTimeType() - || expr.getDataType().isIPv6Type() || expr.getDataType().isJsonType()) { + || expr.getDataType().isIPv6Type() || expr.getDataType().isJsonType() + || expr.getDataType().typeContainsFile()) { return true; } @@ -562,6 +565,11 @@ public static List getResultExpression(DataType type, PValues resultCon Literal literal = new StringLiteral(bytesValue.toStringUtf8()); res.add(literal); } + } else if (type.isVarBinaryType()) { + for (int row = 0; row < resultContent.getBytesValueCount(); row++) { + res.add(new VarBinaryLiteral((VarBinaryType) type, + resultContent.getBytesValue(row).toByteArray())); + } } else if (type.isArrayType()) { ArrayType arrayType = (ArrayType) type; int childCount = resultContent.getChildElementCount(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java index 01eeb8a6927fe7..d5ae911fd94f7e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java @@ -76,6 +76,7 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; @@ -433,6 +434,12 @@ && isFunctionNullCheckPath(context.accessPathBuilder.getPathList())) { return continueCollectAccessPath(first, context); } return visit(elementAt, context); + } else if (first.getDataType().isFileType()) { + // Analysis restricts FILE selectors to constant public field names. Physical FILE + // children stay in their canonical positions even when only one path is requested. + context.accessPathBuilder.addPrefix( + ((Literal) arguments.get(1)).getStringValue().toLowerCase(Locale.ROOT)); + return continueCollectAccessPath(first, context); } else { return visit(elementAt, context); } @@ -975,6 +982,11 @@ private static boolean hasPhysicalNullMap(SlotReference slot) { // backend will throw exception because it can not only access the values without the cast keys, // so we should check whether the map type is changed, if not changed, we can prune the type. private static boolean mapTypeIsChanged(DataType originType, DataType castType, boolean inMap) { + // FILE <-> STRUCT is a value conversion, not a rename of an identical physical tree. + // Validation of such casts needs the complete source, including uri and inline. + if (originType.isFileType() || castType.isFileType()) { + return !originType.equals(castType); + } if (originType.isMapType()) { MapType originMapType = (MapType) originType; MapType castMapType = (MapType) castType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java index 51bdc44b66b9c6..26166c549b7a2d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ToFile; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; @@ -103,6 +104,12 @@ boolean scanMatchShortCircuitCondition(LogicalOlapScan olapScan) { // set short circuit flag and return the original plan private Plan shortCircuit(Plan root, OlapTable olapTable, Set conjuncts, StatementContext statementContext) { + // TO_FILE needs query-level resources and QueryContext, neither of which is available + // in the point-query executor. Decide before point-query-specific pruning and translation. + if (root.anyMatch(plan -> ((Plan) plan).getExpressions().stream() + .anyMatch(expression -> expression.anyMatch(ToFile.class::isInstance)))) { + return root; + } // All key columns in conjuncts Set colNames = Sets.newHashSet(); for (Expression expr : conjuncts) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java index bbdea8c081e3cb..5b8dd93a48d769 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.FileType; import org.apache.doris.nereids.types.MapType; import org.apache.doris.nereids.types.NestedColumnPrunable; import org.apache.doris.nereids.types.NullType; @@ -59,6 +60,7 @@ import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; @@ -272,7 +274,8 @@ private static Map pruneDataType( for (Entry kv : slotIdToAllAccessTree.entrySet()) { Slot slot = kv.getKey(); DataTypeAccessTree accessTree = kv.getValue(); - if (!accessTree.hasOffsetPath() && !accessTree.hasNullPath()) { + if (!accessTree.hasOffsetPath() && !accessTree.hasNullPath() + && !slot.getDataType().typeContainsFile()) { continue; } // Expand both sets so all-path and predicate-path routing stay consistent. @@ -565,7 +568,10 @@ public boolean hasNullPath() { /** pruneCastType */ public DataType pruneCastType(DataTypeAccessTree origin, DataTypeAccessTree cast) { - if (type instanceof StructType) { + if (type.isFileType() || cast.type.isFileType()) { + // FILE conversions validate the complete source and retain the declared target. + return cast.type; + } else if (type instanceof StructType) { Map nameMapping = new LinkedHashMap<>(); List castNames = new ArrayList<>(cast.children.keySet()); int i = 0; @@ -632,7 +638,14 @@ public boolean replacePathByAnotherTree( // visitCast falls back to reading the original value data before evaluating cast. return type.equals(cast.type); } - if (cast.type instanceof StructType) { + if (type.isFileType() || cast.type.isFileType()) { + if (!type.equals(cast.type)) { + return false; + } + String fieldName = path.get(index).toLowerCase(Locale.ROOT); + return children.get(fieldName).replacePathByAnotherTree( + cast.children.get(fieldName), path, index + 1, pathType); + } else if (cast.type instanceof StructType) { List fields = ((StructType) cast.type).getFields(); for (int i = 0; i < fields.size(); i++) { String castFieldName = path.get(index); @@ -678,7 +691,7 @@ public void setAccessByPath(List path, int accessIndex, TAccessPathType accessPartialChild = true; - if (this.type.isStructType()) { + if (this.type.isStructType() || this.type.isFileType()) { String fieldName = path.get(accessIndex).toLowerCase(); DataTypeAccessTree child = children.get(fieldName); if (child != null) { @@ -777,6 +790,10 @@ public static DataTypeAccessTree of(DataType type, TAccessPathType pathType) { } else if (type instanceof MapType) { root.children.put(AccessPathInfo.ACCESS_MAP_KEYS, of(((MapType) type).getKeyType(), pathType)); root.children.put(AccessPathInfo.ACCESS_MAP_VALUES, of(((MapType) type).getValueType(), pathType)); + } else if (type instanceof FileType) { + for (StructField field : ((FileType) type).getFields()) { + root.children.put(field.getName(), of(field.getDataType(), pathType)); + } } return root; } @@ -918,6 +935,14 @@ private static void findMapStarPositions( } else { current = mapType.getValueType(); } + } else if (current.isFileType()) { + StructField field = ((FileType) current).getFields().stream() + .filter(candidate -> candidate.getName().equals(component)) + .findFirst().orElse(null); + if (field == null) { + break; + } + current = field.getDataType(); } else { break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java index f3df68c945342e..06d7ff0f563f8c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java @@ -180,7 +180,11 @@ public TypeCheckResult checkInputDataTypes() { return commonCheckResult; } } - return checkInputDataTypesInternal(); + TypeCheckResult result = checkInputDataTypesInternal(); + if (result.success()) { + ExpressionUtils.checkFileExpression(this); + } + return result; } public int fastChildrenHashCode() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAgg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAgg.java index 393adea6d0e939..9e57a17c560c96 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAgg.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAgg.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.types.MapType; import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.nereids.types.coercion.FollowToAnyDataType; +import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -71,6 +72,11 @@ public MapAgg withDistinctAndChildren(boolean distinct, List childre return new MapAgg(getFunctionParams(distinct, children)); } + @Override + public void checkLegalityBeforeTypeCoercion() { + ExpressionUtils.checkFileKey(getArgument(0), "MAP_AGG key"); + } + @Override public List getSignatures() { return SIGNATURES; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAggV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAggV2.java index 699f62c3684088..2209301660123d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAggV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MapAggV2.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.types.MapType; import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.nereids.types.coercion.FollowToAnyDataType; +import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -71,6 +72,11 @@ public MapAggV2 withDistinctAndChildren(boolean distinct, List child return new MapAggV2(getFunctionParams(distinct, children)); } + @Override + public void checkLegalityBeforeTypeCoercion() { + ExpressionUtils.checkFileKey(getArgument(0), "MAP_AGG key"); + } + @Override public List getSignatures() { return SIGNATURES; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Max.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Max.java index ae144c578db3ac..efbcc3399c7cb2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Max.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Max.java @@ -57,7 +57,8 @@ private Max(NullableAggregateFunctionParams functionParams) { @Override public void checkLegalityBeforeTypeCoercion() { - if (getArgumentType(0).isOnlyMetricType() && !getArgumentType(0).isArrayType()) { + if (getArgumentType(0).typeContainsFile() + || (getArgumentType(0).isOnlyMetricType() && !getArgumentType(0).isArrayType())) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MaxBy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MaxBy.java index 96e6c040577ab0..bd412d1ca4b700 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MaxBy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MaxBy.java @@ -66,7 +66,8 @@ private MaxBy(NullableAggregateFunctionParams functionParams) { @Override public void checkLegalityBeforeTypeCoercion() { - if (getArgumentType(1).isOnlyMetricType() && !getArgumentType(1).isArrayType()) { + if (getArgumentType(1).typeContainsFile() + || (getArgumentType(1).isOnlyMetricType() && !getArgumentType(1).isArrayType())) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Min.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Min.java index 50f44fe514d55c..5832c5ef8b42d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Min.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Min.java @@ -58,7 +58,8 @@ private Min(NullableAggregateFunctionParams functionParams) { @Override public void checkLegalityBeforeTypeCoercion() { - if (getArgumentType(0).isOnlyMetricType() && !getArgumentType(0).isArrayType()) { + if (getArgumentType(0).typeContainsFile() + || (getArgumentType(0).isOnlyMetricType() && !getArgumentType(0).isArrayType())) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MinBy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MinBy.java index b59c1a630620aa..a379fa78041dad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MinBy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MinBy.java @@ -66,7 +66,8 @@ private MinBy(NullableAggregateFunctionParams functionParams) { @Override public void checkLegalityBeforeTypeCoercion() { - if (getArgumentType(1).isOnlyMetricType() && !getArgumentType(1).isArrayType()) { + if (getArgumentType(1).typeContainsFile() + || (getArgumentType(1).isOnlyMetricType() && !getArgumentType(1).isArrayType())) { throw new AnalysisException(Type.OnlyMetricTypeErrorMsg); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeFile.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeFile.java new file mode 100644 index 00000000000000..efcd8a0eff5865 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeFile.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.generator; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.types.FileType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** Expand FILE to its six public nullable fields. */ +public class ExplodeFile extends TableGeneratingFunction implements AlwaysNullable { + public ExplodeFile(Expression argument) { + this("explode_file", argument); + } + + protected ExplodeFile(String name, Expression argument) { + super(name, argument); + } + + @Override + public void checkLegalityBeforeTypeCoercion() { + if (!getArgument(0).getDataType().isFileType() && !getArgument(0).getDataType().isNullType()) { + throw new AnalysisException(getName() + " only accepts FILE"); + } + } + + @Override + public List getSignatures() { + return ImmutableList.of(FunctionSignature.ret(FileType.INSTANCE.publicStructType()).args(FileType.INSTANCE)); + } + + @Override + public ExplodeFile withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new ExplodeFile(children.get(0)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeFileOuter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeFileOuter.java new file mode 100644 index 00000000000000..f4f9a418f780df --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeFileOuter.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.generator; + +import org.apache.doris.nereids.trees.expressions.Expression; + +import com.google.common.base.Preconditions; + +import java.util.List; + +/** Preserve one all-NULL output row for a NULL FILE. */ +public final class ExplodeFileOuter extends ExplodeFile { + public ExplodeFileOuter(Expression argument) { + super("explode_file_outer", argument); + } + + @Override + public ExplodeFileOuter withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new ExplodeFileOuter(children.get(0)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayCompact.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayCompact.java index 2b8e0678073012..8149c970ec3b36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayCompact.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayCompact.java @@ -62,7 +62,7 @@ private ArrayCompact(ScalarFunctionParams functionParams) { @Override public void checkLegalityBeforeTypeCoercion() { DataType argType = ((ArrayType) getArgument(0).getDataType()).getItemType(); - if (argType.isMapType() || argType.isStructType()) { + if (argType.isMapType() || argType.isStructType() || argType.typeContainsFile()) { throw new AnalysisException("array_compact does not support type " + argType.toString() + ", expression is " + toSql()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFunctionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFunctionUtils.java index 49cd49d481c2a1..91d25fa62cad72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFunctionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFunctionUtils.java @@ -34,6 +34,9 @@ static void checkNoVarBinaryArguments(ScalarFunction function) { while (type instanceof ArrayType) { type = ((ArrayType) type).getItemType(); } + if (type.typeContainsFile()) { + throw new AnalysisException(function.getName() + " does not support element type " + type.toSql()); + } if (type.isVarBinaryType()) { throw new AnalysisException(function.getName() + " does not support VARBINARY arguments"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayReverseSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayReverseSort.java index 4def17eda8f188..2fc1cf4ed41383 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayReverseSort.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayReverseSort.java @@ -65,7 +65,7 @@ public void checkLegalityBeforeTypeCoercion() { itemType = ((ArrayType) itemType).getItemType(); } if (itemType.isMapType() || itemType.isStructType() - || itemType.isVariantType() || itemType.isJsonType()) { + || itemType.isVariantType() || itemType.isJsonType() || itemType.typeContainsFile()) { throw new AnalysisException("array_reverse_sort does not support types: " + argType.toSql()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySort.java index 32739c48f36dc2..256f1d63ac4d03 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySort.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySort.java @@ -68,7 +68,7 @@ public void checkLegalityBeforeTypeCoercion() { itemType = ((ArrayType) itemType).getItemType(); } if (itemType.isMapType() || itemType.isStructType() - || itemType.isVariantType() || itemType.isJsonType()) { + || itemType.isVariantType() || itemType.isJsonType() || itemType.typeContainsFile()) { throw new AnalysisException("array_sort does not support types: " + argType.toSql()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySortBy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySortBy.java index 1d0fe2b15f3f59..3fd9c8da1a93f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySortBy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySortBy.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.coercion.AnyDataType; +import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; @@ -68,6 +69,7 @@ public ArraySortBy withChildren(List children) { @Override public void checkLegalityBeforeTypeCoercion() { + ExpressionUtils.checkFileKey(getArgument(1), "array_sortby key"); DataType argType = getArgument(0).getDataType(); if (argType instanceof ArrayType) { // Find the innermost element type for nested arrays diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateMap.java index 2e054c6e06a18e..678d99fd9e38a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateMap.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.MapType; import org.apache.doris.nereids.types.VariantType; +import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.TypeCoercionUtils; import com.google.common.collect.ImmutableList; @@ -89,6 +90,9 @@ public void checkLegalityBeforeTypeCoercion() { for (int i = 0; i < arity(); i++) { DataType childType = getArgument(i).getDataType(); boolean isKey = i % 2 == 0; + if (isKey) { + ExpressionUtils.checkFileKey(getArgument(i), "MAP key"); + } if (isKey && childType.isComplexType()) { throw new AnalysisException( "MAP key type must be a primitive type but get " + childType.toSql()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java index ac09b85c3873da..99394e1795324a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java @@ -32,6 +32,7 @@ import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.FileType; import org.apache.doris.nereids.types.MapType; import org.apache.doris.nereids.types.NullType; import org.apache.doris.nereids.types.StructField; @@ -112,12 +113,19 @@ public boolean foldable() { public void checkLegalityBeforeTypeCoercion() { // Struct field access (the former struct_element) only accepts a constant int/string index, // since the selected field — and therefore the result type — must be known at analysis time. - if (child(0).getDataType() instanceof StructType) { + if (getArgument(0).getDataType() instanceof StructType) { Expression field = getArgument(1); if (!(field instanceof StringLikeLiteral || field instanceof IntegerLikeLiteral)) { throw new AnalysisException("element_at over a struct only allows a constant int or" + " string second parameter: " + this.toSql()); } + } else if (getArgument(0).getDataType().isFileType()) { + if (!(getArgument(1) instanceof StringLikeLiteral)) { + throw new AnalysisException("element_at over FILE requires a constant public field name"); + } + if (((StringLikeLiteral) getArgument(1)).getStringValue().equalsIgnoreCase("inline")) { + throw new AnalysisException("ELEMENT_AT does not support the FILE inline field; use FL_GET_BINARY"); + } } } @@ -127,12 +135,10 @@ public List getSignatures() { if (arg0Type instanceof NullType) { return ImmutableList.of( FunctionSignature.ret(NullType.INSTANCE).args(NullType.INSTANCE, child(1).getDataType())); - } - if (arg0Type instanceof StructType) { + } else if (arg0Type instanceof StructType) { return ImmutableList.of(FunctionSignature.ret(structFieldType((StructType) arg0Type)) .args(arg0Type, child(1).getDataType())); - } - if (arg0Type instanceof VariantType) { + } else if (arg0Type instanceof VariantType) { DataType selectorType = child(1).getDataType(); if (selectorType.isIntegralType() || selectorType.isBooleanType()) { return ImmutableList.of(VARIANT_INDEX_SIGNATURE); @@ -140,6 +146,9 @@ public List getSignatures() { if (selectorType.isStringLikeType()) { return ImmutableList.of(VARIANT_KEY_SIGNATURE); } + } else if (arg0Type.isFileType()) { + return ImmutableList.of(FunctionSignature.ret(structFieldType(((FileType) arg0Type).publicStructType())) + .args(arg0Type, child(1).getDataType())); } return SIGNATURES; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FlGetBinary.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FlGetBinary.java new file mode 100644 index 00000000000000..f9eb190fdef0e5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FlGetBinary.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.functions.NullOrIdenticalSignature; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.FileType; +import org.apache.doris.nereids.types.VarBinaryType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** Returns the binary contents of a FILE. */ +public class FlGetBinary extends ScalarFunction + implements UnaryExpression, NullOrIdenticalSignature, AlwaysNullable { + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(VarBinaryType.INSTANCE).args(FileType.INSTANCE)); + + public FlGetBinary(Expression arg) { + super("fl_get_binary", arg); + } + + private FlGetBinary(ScalarFunctionParams functionParams) { + super(functionParams); + } + + @Override + public FlGetBinary withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new FlGetBinary(getFunctionParams(children)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitFlGetBinary(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToFile.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToFile.java new file mode 100644 index 00000000000000..9ee80d27c78b0f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToFile.java @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.catalog.S3Resource; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.VolatileIdentity; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; +import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.FileType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** Resolve S3 file metadata through a named resource at execution time. */ +public class ToFile extends UniqueFunction + implements BinaryExpression, ExplicitlyCastableSignature, PropagateNullable { + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(FileType.INSTANCE).args(StringType.INSTANCE, StringType.INSTANCE)); + + public ToFile(Expression resource, Expression uri) { + this(VolatileIdentity.newVolatileIdentity(), ImmutableList.of(resource, uri)); + } + + private ToFile(VolatileIdentity identity, List arguments) { + super("to_file", identity, arguments); + } + + private ToFile(UniqueFunctionParams params) { + super(params); + } + + @Override + public void checkLegalityBeforeTypeCoercion() { + getResourceName(); + DataType uriType = getArgument(1).getDataType(); + if (!uriType.isStringLikeType() && !uriType.isNullType()) { + throw new AnalysisException("TO_FILE uri must be string-like or NULL"); + } + // URI syntax, schemes and resource/URI compatibility belong to the BE + // filesystem. Even a literal URI must reach it unchanged. + } + + public String getResourceName() { + Expression resource = getArgument(0); + if (!(resource instanceof StringLikeLiteral)) { + throw new AnalysisException("TO_FILE resource must be a non-NULL string literal"); + } + return ((StringLikeLiteral) resource).getStringValue(); + } + + @Override + public void checkLegalityAfterRewrite() { + // Bind the name and check USAGE without capturing properties in an + // expression that can be retained by PREPARE. + S3Resource.checkResource(getResourceName(), ConnectContext.get()); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitToFile(this, context); + } + + @Override + public ToFile withChildren(List children) { + Preconditions.checkArgument(children.size() == 2); + return new ToFile(getFunctionParams(children)); + } + + @Override + public ToFile withIgnoreUniqueId(boolean ignoreUniqueId) { + return new ToFile(volatileIdentity.withIgnoreUniqueId(ignoreUniqueId), getArguments()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java index 51e746f31a2926..330d71fb67f868 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java @@ -101,7 +101,7 @@ public ToJson withChildren(List children) { @Override public List getSignatures() { DataType firstChildType = child(0).getDataType(); - if (firstChildType.isStructType() || firstChildType.isArrayType() + if (firstChildType.isStructType() || firstChildType.isArrayType() || firstChildType.isFileType() || (firstChildType instanceof VariantType && ((VariantType) firstChildType).isExecutionV2())) { return ImmutableList.of(FunctionSignature.ret(JsonType.INSTANCE).args(firstChildType)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/ListFiles.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/ListFiles.java new file mode 100644 index 00000000000000..6dcd00ff755d81 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/ListFiles.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.table; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Properties; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.FileType; +import org.apache.doris.tablefunction.ListFilesTableValuedFunction; +import org.apache.doris.tablefunction.TableValuedFunctionIf; + +import com.google.common.base.Preconditions; + +import java.util.List; + +/** list_files */ +public class ListFiles extends TableValuedFunction { + public ListFiles(Properties properties) { + super(ListFilesTableValuedFunction.NAME, properties); + } + + @Override + public FunctionSignature customSignature() { + return FunctionSignature.of(FileType.INSTANCE, getArgumentsTypes()); + } + + @Override + protected TableValuedFunctionIf toCatalogFunction() { + try { + return new ListFilesTableValuedFunction(getTVFProperties().getMap()); + } catch (org.apache.doris.common.AnalysisException e) { + throw new AnalysisException("Can not build list_files(): " + e.getMessage(), e); + } + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitListFiles(this, context); + } + + @Override + public ListFiles withChildren(List children) { + Preconditions.checkArgument(children.size() == 1 && children.get(0) instanceof Properties); + return new ListFiles((Properties) children.get(0)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index 77db4cf175d1ed..c8cfd5be36dde7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -233,6 +233,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Field; import org.apache.doris.nereids.trees.expressions.functions.scalar.FindInSet; import org.apache.doris.nereids.trees.expressions.functions.scalar.FirstSignificantSubdomain; +import org.apache.doris.nereids.trees.expressions.functions.scalar.FlGetBinary; import org.apache.doris.nereids.trees.expressions.functions.scalar.Floor; import org.apache.doris.nereids.trees.expressions.functions.scalar.Fmod; import org.apache.doris.nereids.trees.expressions.functions.scalar.Format; @@ -546,6 +547,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ToDate; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToDateV2; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToDays; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ToFile; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToIpv4; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToIpv4OrDefault; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToIpv4OrNull; @@ -1511,6 +1513,10 @@ default R visitFirstSignificantSubdomain(FirstSignificantSubdomain firstSignific return visitScalarFunction(firstSignificantSubdomain, context); } + default R visitFlGetBinary(FlGetBinary function, C context) { + return visitScalarFunction(function, context); + } + default R visitFloor(Floor floor, C context) { return visitScalarFunction(floor, context); } @@ -2603,6 +2609,10 @@ default R visitToDays(ToDays toDays, C context) { return visitScalarFunction(toDays, context); } + default R visitToFile(ToFile toFile, C context) { + return visitScalarFunction(toFile, context); + } + default R visitToIpv4(ToIpv4 toIpv4, C context) { return visitScalarFunction(toIpv4, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java index 2df871311c9a42..e5d2bfe24cde64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.functions.table.HudiMeta; import org.apache.doris.nereids.trees.expressions.functions.table.IndexDiskUsage; import org.apache.doris.nereids.trees.expressions.functions.table.Jobs; +import org.apache.doris.nereids.trees.expressions.functions.table.ListFiles; import org.apache.doris.nereids.trees.expressions.functions.table.Local; import org.apache.doris.nereids.trees.expressions.functions.table.MvInfos; import org.apache.doris.nereids.trees.expressions.functions.table.Numbers; @@ -117,6 +118,10 @@ default R visitLocal(Local local, C context) { return visitTableValuedFunction(local, context); } + default R visitListFiles(ListFiles listFiles, C context) { + return visitTableValuedFunction(listFiles, context); + } + default R visitNumbers(Numbers numbers, C context) { return visitTableValuedFunction(numbers, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java index 7a1c291ae05d6e..bff379841a453a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java @@ -430,12 +430,6 @@ private void checkUdfSupportedTypes() throws AnalysisException { } private void checkUdfSupportedType(Type type, String typePosition) throws AnalysisException { - // Reject bitmap/hll/quantile_state type - if (type.isObjectStored()) { - throw new AnalysisException(String.format( - "%s does not support %s type %s", binaryType, typePosition, type.toSql())); - } - if (type.isArrayType()) { checkUdfSupportedType(((ArrayType) type).getItemType(), typePosition + " element"); } else if (type.isMapType()) { @@ -445,6 +439,9 @@ private void checkUdfSupportedType(Type type, String typePosition) throws Analys for (StructField field : ((StructType) type).getFields()) { checkUdfSupportedType(field.getType(), typePosition + " field " + field.getName()); } + } else if (type.isObjectStored()) { + throw new AnalysisException(String.format( + "%s does not support %s type %s", binaryType, typePosition, type.toSql())); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 6e396bf62412a4..92b1c036eb52f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -327,6 +327,23 @@ private DataType updateCharacterTypeLength(DataType dataType) { } } + /** + * Returns whether the given type may be used as an OLAP key column. + */ + public static boolean isEligibleKeyType(DataType type) { + return !type.isFloatLikeType() + && !type.isStringType() + && !type.isArrayType() + && !type.isBitmapType() + && !type.isHllType() + && !type.isQuantileStateType() + && !type.isJsonType() + && !type.isVariantType() + && !type.isMapType() + && !type.isStructType() + && !type.isFileType(); + } + private void checkKeyColumnType(boolean isOlap) { if (isOlap) { if (type.isFloatLikeType()) { @@ -390,6 +407,10 @@ private void validateInternal(boolean isOlap, Set keysSet, Set c "Cannot use `VARIANT` type column named with a `.` character: " + name); } type.validateDataType(); + if (type.typeContainsFile()) { + throw new AnalysisException(isOlap ? "FILE columns are not supported in internal tables" + : "FILE columns are not supported in external tables"); + } type = updateCharacterTypeLength(type); if (type.isArrayType()) { int depth = 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index ed38140376d729..8b505426711ec7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -159,7 +159,12 @@ public void analyze(ConnectContext ctx) throws Exception { // analyze distribute Map columnMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - columns.forEach(c -> columnMap.put(c.getName(), c)); + columns.forEach(c -> { + if (c.getType().typeContainsFile()) { + throw new AnalysisException("FILE columns are not supported in internal tables"); + } + columnMap.put(c.getName(), c); + }); distribution.updateCols(columns.get(0).getName()); distribution.validate(columnMap, KeysType.DUP_KEYS); refreshInfo.validate(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index 8d499fdcd64e3a..94ef1fd63e7c0e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -607,15 +607,9 @@ public void validate(ConnectContext ctx) { if (Config.random_add_order_by_keys_for_mow && isEnableMergeOnWrite && sortOrderFields.isEmpty() && PropertyAnalyzer.analyzeUseLightSchemaChange(new HashMap<>(properties))) { // exclude columns whose data type can not be order key, see {@link ColumnDefinition#validate} - List orderKeysCandidates = columns.stream().filter(c -> { - DataType type = c.getType(); - return !(type.isFloatLikeType() || type.isStringType() || type.isArrayType() - || type.isBitmapType() || type.isHllType() || type.isQuantileStateType() - || type.isJsonType() - || type.isVariantType() - || type.isMapType() - || type.isStructType()); - }).collect(Collectors.toList()); + List orderKeysCandidates = columns.stream() + .filter(c -> ColumnDefinition.isEligibleKeyType(c.getType())) + .collect(Collectors.toList()); if (orderKeysCandidates.size() > 0) { sortOrderFields = new ArrayList<>(); Random random = new Random(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DistributionDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DistributionDescriptor.java index 35dbf8a8b34c5d..73a4453d1e4bfc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DistributionDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DistributionDescriptor.java @@ -103,6 +103,9 @@ public void validate(Map columnMap, KeysType keysType) }); for (String columnName : cols) { ColumnDefinition columnDefinition = columnMap.get(columnName); + if (columnDefinition.getType().toCatalogDataType().typeContainsFile()) { + throw new AnalysisException("FILE cannot be used in distribution column[" + columnName + "]"); + } if (!columnDefinition.isKey() && (keysType == KeysType.UNIQUE_KEYS || keysType == KeysType.AGG_KEYS)) { throw new AnalysisException("Distribution column[" + columnName + "] is not key column"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java index 782e746247128f..ce4f2c883d7cd4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java @@ -400,6 +400,12 @@ public static DataType convertPrimitiveFromStrings(List types) { case "variant": dataType = VariantType.INSTANCE; break; + case "file": + if (types.size() != 1) { + throw new AnalysisException("FILE does not accept type parameters"); + } + dataType = FileType.INSTANCE; + break; case "varbinary": // Keep declared byte limits in table schemas and nested binary leaves. if (types.size() == 1 || (types.size() == 2 && types.get(1).equals("*"))) { @@ -435,6 +441,7 @@ public static DataType convertFromString(String type) { @Developing // should support map, struct public static DataType fromCatalogType(Type type) { switch (type.getPrimitiveType()) { + case FILE: return FileType.INSTANCE; case BOOLEAN: return BooleanType.INSTANCE; case TINYINT: return TinyIntType.INSTANCE; case SMALLINT: return SmallIntType.INSTANCE; @@ -758,6 +765,25 @@ public boolean isVariantType() { return this instanceof VariantType; } + /** Inspect logical types, including unresolved signature placeholders, for a FILE leaf. */ + public boolean typeContainsFile() { + if (this instanceof ArrayType) { + return ((ArrayType) this).getItemType().typeContainsFile(); + } else if (this instanceof MapType) { + return ((MapType) this).getKeyType().typeContainsFile() + || ((MapType) this).getValueType().typeContainsFile(); + } else if (this instanceof StructType) { + return ((StructType) this).getFields().stream().anyMatch(field -> field.getDataType().typeContainsFile()); + } else if (this instanceof AggStateType) { + return ((AggStateType) this).getSubTypes().stream().anyMatch(DataType::typeContainsFile); + } + return isFileType(); + } + + public boolean isFileType() { + return this instanceof FileType; + } + public boolean isStructType() { return this instanceof StructType; } @@ -970,6 +996,9 @@ private static void validateCatalogDataType(Type catalogType) { } } } + } else if (catalogType.isAggStateType() && catalogType.typeContainsFile()) { + throw new AnalysisException("AGG_STATE does not support FILE or a type containing FILE: " + + catalogType.toSql()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/FileType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/FileType.java new file mode 100644 index 00000000000000..780c71edfa392d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/FileType.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.types; + +import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.types.coercion.ComplexDataType; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** The logical FILE leaf; its fixed physical children do not add SQL nesting depth. */ +public final class FileType extends DataType implements ComplexDataType, NestedColumnPrunable { + public static final FileType INSTANCE = new FileType(); + + private FileType() { + } + + public List getFields() { + return Type.FILE.getFields().stream() + .map(field -> new StructField(field.getName(), DataType.fromCatalogType(field.getType()), true, "")) + .collect(ImmutableList.toImmutableList()); + } + + public StructType publicStructType() { + return new StructType(getFields()); + } + + @Override + public Type toCatalogDataType() { + return Type.FILE; + } + + @Override + public String toSql() { + return "FILE"; + } + + @Override + public int width() { + return Type.FILE.getSlotSize(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index a05ac039fc1a63..a4067f43c21688 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -43,6 +43,7 @@ import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.InPredicate; +import org.apache.doris.nereids.trees.expressions.InSubquery; import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -186,6 +187,27 @@ private static boolean isUnsafeDateArithmetic(Expression expression) { return false; } + /** Reject FILE values in SQL keys that require comparison, ordering, or hashing. */ + public static void checkFileKey(Expression value, String operation) { + if (value.getDataType().typeContainsFile()) { + throw new AnalysisException(operation + " does not support FILE or a type containing FILE: " + + value.getDataType().toSql()); + } + } + + /** Called by the existing recursive expression type check, without a second plan traversal. */ + public static void checkFileExpression(Expression expression) { + if (expression instanceof InSubquery) { + InSubquery in = (InSubquery) expression; + checkFileKey(in.getCompareExpr(), "IN"); + checkFileKey(in.getSubqueryOutput(), "IN"); + } else if (expression instanceof WindowExpression) { + WindowExpression window = (WindowExpression) expression; + window.getPartitionKeys().forEach(key -> checkFileKey(key, "window PARTITION BY")); + window.getOrderKeys().forEach(key -> checkFileKey(key.child(), "window ORDER BY")); + } + } + public static List extractConjunction(Expression expr) { return extract(And.class, expr); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 3f962d19b39926..d55053f36e82dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -1646,6 +1646,7 @@ private static Expression processComparisonPredicateInternal(ComparisonPredicate */ public static Expression processInPredicate(InPredicate inPredicate) { // check + inPredicate.children().forEach(argument -> ExpressionUtils.checkFileKey(argument, "IN")); inPredicate.checkLegalityBeforeTypeCoercion(); checkNoVarbinaryComparison("IN/NOT IN", inPredicate.children()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index 74778d32b4a3ae..7bfeb12bc248b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -61,6 +61,7 @@ import org.apache.doris.analysis.TimeStampNsLiteral; import org.apache.doris.analysis.TimestampArithmeticExpr; import org.apache.doris.analysis.UuidLiteral; +import org.apache.doris.analysis.VarBinaryLiteral; import org.apache.doris.analysis.VariableExpr; import org.apache.doris.analysis.VirtualSlotRef; import org.apache.doris.backup.BackupJob; @@ -79,6 +80,7 @@ import org.apache.doris.catalog.DistributionInfo; import org.apache.doris.catalog.EsResource; import org.apache.doris.catalog.EsTable; +import org.apache.doris.catalog.FileType; import org.apache.doris.catalog.Function; import org.apache.doris.catalog.FunctionGenTable; import org.apache.doris.catalog.HMSResource; @@ -308,6 +310,7 @@ public class GsonUtils { .registerSubtype(ArrayType.class, ArrayType.class.getSimpleName()) .registerSubtype(MapType.class, MapType.class.getSimpleName()) .registerSubtype(StructType.class, StructType.class.getSimpleName()) + .registerSubtype(FileType.class, FileType.class.getSimpleName()) .registerSubtype(AggStateType.class, AggStateType.class.getSimpleName()) .registerSubtype(AnyElementType.class, AnyElementType.class.getSimpleName()) .registerSubtype(AnyStructType.class, AnyStructType.class.getSimpleName()) @@ -348,6 +351,7 @@ public class GsonUtils { .registerSubtype(JsonLiteral.class, JsonLiteral.class.getSimpleName()) .registerSubtype(ArrayLiteral.class, ArrayLiteral.class.getSimpleName()) .registerSubtype(StructLiteral.class, StructLiteral.class.getSimpleName()) + .registerSubtype(VarBinaryLiteral.class, VarBinaryLiteral.class.getSimpleName()) .registerSubtype(NumericLiteralExpr.class, NumericLiteralExpr.class.getSimpleName()) .registerSubtype(PlaceHolderExpr.class, PlaceHolderExpr.class.getSimpleName()) .registerSubtype(CaseExpr.class, CaseExpr.class.getSimpleName()) @@ -960,6 +964,11 @@ public void write(JsonWriter out, T value) throws IOException { if (value instanceof GsonPreProcessable) { ((GsonPreProcessable) value).gsonPreProcess(); } + if (value instanceof FileType) { + ((FileType) value).gsonPreProcess(); + } else if (value instanceof ScalarType) { + ((ScalarType) value).gsonPreProcess(); + } delegate.write(out, value); } @@ -989,6 +998,11 @@ public T read(JsonReader reader) throws IOException { if (obj instanceof GsonPostProcessable) { ((GsonPostProcessable) obj).gsonPostProcess(); } + if (obj instanceof FileType) { + ((FileType) obj).gsonPostProcess(); + } else if (obj instanceof ScalarType) { + ((ScalarType) obj).gsonPostProcess(); + } return obj; } }; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index d356639ef16b3f..e4b9a58b0f4645 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -36,6 +36,7 @@ import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.FunctionRegistry; +import org.apache.doris.catalog.S3Resource; import org.apache.doris.catalog.Type; import org.apache.doris.cloud.qe.ComputeGroupException; import org.apache.doris.cloud.system.CloudSystemInfoService; @@ -82,6 +83,7 @@ import org.apache.doris.system.Backend; import org.apache.doris.task.LoadTaskInfo; import org.apache.doris.thrift.TResultSinkType; +import org.apache.doris.thrift.TS3Resource; import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TUniqueId; import org.apache.doris.transaction.TransactionEntry; @@ -147,6 +149,7 @@ public enum ConnectType { protected volatile LoadTaskInfo streamLoadInfo; protected volatile TUniqueId queryId = null; + private final Map s3Resources = Maps.newLinkedHashMap(); // only be active one time. tell Coordinator to regenerate instance ids for certain query(when retry). protected volatile TUniqueId needRegenerateInstanceId = null; protected volatile AtomicInteger instanceIdGenerator = new AtomicInteger(); @@ -452,6 +455,7 @@ public void resetConnection() throws UserException { clearPreparedQueries(); preparedStatementContextMap.clear(); runningQuery = null; + s3Resources.clear(); queryId = null; lastQueryId = null; setTraceId(null); @@ -897,6 +901,7 @@ public Instant getStartTimeInstant() { } public void setStartTime() { + s3Resources.clear(); startTimeInstant = Instant.now(); startTime = startTimeInstant.toEpochMilli(); returnRows = 0; @@ -1455,6 +1460,18 @@ public void setQueryId(TUniqueId queryId) { } } + /** Resolve each resource once for the current query without retaining it in prepared expressions. */ + public TS3Resource getOrRegisterS3Resource(String resourceName) { + return s3Resources.computeIfAbsent(resourceName, + name -> S3Resource.checkResource(name, this).toThrift()).deepCopy(); + } + + public Map getS3Resources() { + Map resources = Maps.newLinkedHashMap(); + s3Resources.forEach((name, resource) -> resources.put(name, resource.deepCopy())); + return resources; + } + public void resetQueryId() { if (this.queryId != null) { this.lastQueryId = this.queryId.deepCopy(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index dedf36a9136169..1e0c965819fdd0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -82,6 +82,7 @@ import org.apache.doris.proto.Types.PUniqueId; import org.apache.doris.qe.ConnectContext.ConnectType; import org.apache.doris.qe.QueryStatisticsItem.FragmentInstanceInfo; +import org.apache.doris.qe.runtime.ThriftPlansBuilder; import org.apache.doris.resource.BackendSelection; import org.apache.doris.resource.BackendSelectionManager; import org.apache.doris.resource.workloadgroup.QueryQueue; @@ -3619,6 +3620,7 @@ Map toThrift(int backendNum) { // Used for AI Functions params.setAiResources(getNeededAiResources()); + params.setS3Resources(ThriftPlansBuilder.collectS3Resources(context)); res.put(instanceExecParam.host, params); res.get(instanceExecParam.host).setBucketSeqToInstanceIdx(new HashMap()); res.get(instanceExecParam.host).setShuffleIdxToInstanceIdx(new HashMap()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java index 014019345860cc..c16d17e1c06971 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java @@ -71,6 +71,7 @@ import org.apache.doris.thrift.TResourceInfo; import org.apache.doris.thrift.TRuntimeFilterInfo; import org.apache.doris.thrift.TRuntimeFilterParams; +import org.apache.doris.thrift.TS3Resource; import org.apache.doris.thrift.TScanRangeParams; import org.apache.doris.thrift.TTopnFilterDesc; @@ -239,6 +240,10 @@ static Map collectAiResources(ConnectContext connectContext return aiResourceMap; } + public static Map collectS3Resources(ConnectContext context) { + return context == null ? Maps.newLinkedHashMap() : context.getS3Resources(); + } + private static void setParamsForOlapTableSink(List distributedPlans, Map fragmentsGroupByWorker, CoordinatorContext coordinatorContext) { @@ -520,6 +525,7 @@ private static TPipelineFragmentParams fragmentToThriftIfAbsent( // Only used for AI Functions params.setAiResources(collectAiResources(connectContext)); + params.setS3Resources(collectS3Resources(connectContext)); return params; }); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java index 8dfd8b86f1f7c6..2ae8cc53145b1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java @@ -60,7 +60,7 @@ public class ColumnStatistic { public static final Set UNSUPPORTED_TYPE = Sets.newHashSet(Type.HLL, Type.BITMAP, Type.ARRAY, Type.STRUCT, Type.MAP, Type.QUANTILE_STATE, Type.JSONB, Type.VARIANT, Type.TIMEV2, Type.LAMBDA_FUNCTION, - Type.VARBINARY); + Type.VARBINARY, Type.FILE); // ATTENTION: Stats deriving WILL NOT use 'count' field any longer. // Use 'rowCount' field in Statistics if needed. diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ListFilesTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ListFilesTableValuedFunction.java new file mode 100644 index 00000000000000..952d8c2174c51d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ListFilesTableValuedFunction.java @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.FileType; +import org.apache.doris.catalog.S3Resource; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.resource.computegroup.ComputeGroupMgr; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TDataGenFunctionName; +import org.apache.doris.thrift.TDataGenScanRange; +import org.apache.doris.thrift.TS3Resource; +import org.apache.doris.thrift.TScanRange; +import org.apache.doris.thrift.TTVFListFileScanRange; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** Resource-backed S3 directory listing, executed by one backend. */ +public class ListFilesTableValuedFunction extends DataGenTableValuedFunction { + public static final String NAME = "list_files"; + private static final ImmutableSet PROPERTIES = ImmutableSet.of("resource", "uri", "recursive"); + + private final String resourceName; + private final String uri; + private final boolean recursive; + + public ListFilesTableValuedFunction(Map params) throws AnalysisException { + Map properties = new HashMap<>(); + for (Map.Entry entry : params.entrySet()) { + String key = entry.getKey().toLowerCase(Locale.ROOT); + if (!PROPERTIES.contains(key)) { + throw new AnalysisException("'" + entry.getKey() + "' is invalid property"); + } + properties.put(key, entry.getValue()); + } + resourceName = properties.get("resource"); + uri = properties.get("uri"); + if (resourceName == null || resourceName.isEmpty()) { + throw new AnalysisException("list_files resource is required"); + } + if (uri == null) { + throw new AnalysisException("list_files uri is required"); + } + String recursiveValue = properties.getOrDefault("recursive", "true"); + if (!"true".equalsIgnoreCase(recursiveValue) && !"false".equalsIgnoreCase(recursiveValue)) { + throw new AnalysisException("list_files recursive must be true or false"); + } + recursive = Boolean.parseBoolean(recursiveValue); + checkS3Resource(ConnectContext.get()); + } + + private S3Resource checkS3Resource(ConnectContext context) { + return S3Resource.checkResource(resourceName, context); + } + + @Override + public void checkAuth(ConnectContext context) { + checkS3Resource(context); + } + + @Override + public TDataGenFunctionName getDataGenFunctionName() { + return TDataGenFunctionName.LIST_FILE; + } + + @Override + public String getTableName() { + return "ListFilesTableValuedFunction"; + } + + @Override + public List getTableColumns() { + return ImmutableList.of( + new Column("path", Type.STRING, false), + new Column("size", Type.BIGINT, false), + new Column("file", FileType.create(), true)); + } + + @Override + public List getTasks() throws AnalysisException { + ConnectContext context = ConnectContext.get(); + checkS3Resource(context); + TS3Resource resource = context.getOrRegisterS3Resource(resourceName); + List backends = new ArrayList<>(); + for (Backend backend : Env.getCurrentSystemInfo().getBackendsByCurrentCluster().values()) { + if (backend.isAlive()) { + backends.add(backend); + } + } + if (backends.isEmpty()) { + String hints = Config.isCloudMode() ? ComputeGroupMgr.computeGroupNotFoundPromptMsg(null) : ""; + throw new AnalysisException("No Alive backends" + hints); + } + Collections.shuffle(backends); + TTVFListFileScanRange params = new TTVFListFileScanRange().setResource(resource) + .setUri(uri).setRecursive(recursive); + TScanRange range = new TScanRange().setDataGenScanRange(new TDataGenScanRange().setListFileParams(params)); + return ImmutableList.of(new TableValuedFunctionTask(backends.get(0), range)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java index 29b86dbb493463..49d77762c170e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java @@ -47,6 +47,8 @@ public FunctionGenTable getTable() throws AnalysisException { public static TableValuedFunctionIf getTableFunction(String funcName, Map params) throws AnalysisException { switch (funcName.toLowerCase()) { + case ListFilesTableValuedFunction.NAME: + return new ListFilesTableValuedFunction(params); case NumbersTableValuedFunction.NAME: return new NumbersTableValuedFunction(params); case S3TableValuedFunction.NAME: diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/OutFileClauseFileTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/OutFileClauseFileTest.java new file mode 100644 index 00000000000000..04b772f6cb8551 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/OutFileClauseFileTest.java @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.ArrayType; +import org.apache.doris.catalog.MapType; +import org.apache.doris.catalog.StructField; +import org.apache.doris.catalog.StructType; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; + +class OutFileClauseFileTest { + @Test + void orcRejectsTopLevelAndNestedFile() { + boolean original = Config.enable_outfile_to_local; + Config.enable_outfile_to_local = true; + try { + for (Type type : Arrays.asList(Type.FILE, new ArrayType(Type.FILE), + new MapType(Type.STRING, Type.FILE), new StructType(new ArrayList<>( + Collections.singletonList(new StructField("asset", Type.FILE)))))) { + OutFileClause clause = new OutFileClause("file:///tmp/file_", "orc", Collections.emptyMap()); + AnalysisException error = Assertions.assertThrows(AnalysisException.class, + () -> clause.analyze(Collections.singletonList(new SlotRef(type, true)), + Collections.singletonList("payload"), true)); + Assertions.assertEquals("currently orc do not support column type: FILE", error.getDetailMessage()); + } + } finally { + Config.enable_outfile_to_local = original; + } + } + + @Test + void parquetRejectsTopLevelAndNestedFile() { + boolean original = Config.enable_outfile_to_local; + Config.enable_outfile_to_local = true; + try { + for (Type type : Arrays.asList(Type.FILE, new ArrayType(Type.FILE), + new MapType(Type.STRING, Type.FILE), new StructType(new ArrayList<>( + Collections.singletonList(new StructField("asset", new ArrayType(Type.FILE))))))) { + OutFileClause clause = new OutFileClause("file:///tmp/file_", "parquet", Collections.emptyMap()); + AnalysisException error = Assertions.assertThrows(AnalysisException.class, + () -> clause.analyze(Collections.singletonList(new SlotRef(type, true)), + Collections.singletonList("payload"), true)); + Assertions.assertEquals("Parquet OUTFILE does not support FILE", error.getDetailMessage()); + } + } finally { + Config.enable_outfile_to_local = original; + } + } + + @Test + void csvRejectsTopLevelAndNestedFile() { + boolean original = Config.enable_outfile_to_local; + Config.enable_outfile_to_local = true; + try { + for (Type type : Arrays.asList(Type.FILE, new ArrayType(Type.FILE), + new MapType(Type.STRING, Type.FILE))) { + OutFileClause clause = new OutFileClause("file:///tmp/file_", "csv", Collections.emptyMap()); + AnalysisException error = Assertions.assertThrows(AnalysisException.class, + () -> clause.analyze(Collections.singletonList(new SlotRef(type, true)), + Collections.singletonList("payload"), true)); + Assertions.assertTrue(error.getDetailMessage().contains("CSV")); + } + } finally { + Config.enable_outfile_to_local = original; + } + } + + @Test + void parquetRetainsOrdinaryNestedTypes() throws Exception { + boolean original = Config.enable_outfile_to_local; + Config.enable_outfile_to_local = true; + try { + OutFileClause clause = new OutFileClause("file:///tmp/ordinary_", "parquet", Collections.emptyMap()); + clause.analyze(Arrays.asList(new SlotRef(Type.INT, true), + new SlotRef(new ArrayType(Type.STRING), true)), + Arrays.asList("id", "items"), true); + Assertions.assertEquals(2, clause.toSinkOptions().getParquetSchemasSize()); + } finally { + Config.enable_outfile_to_local = original; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java index 91cfc9a8fe4a6c..2cefbfb13d8748 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.plans.commands.CreateFunctionCommand; import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.types.DataType; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.Planner; import org.apache.doris.planner.UnionNode; @@ -218,6 +219,65 @@ public void testCreatePythonFunctionRejectsObjectTypes() throws Exception { "ARRAY unsupported sub-type: bitmap"); } + @Test + public void testCreatePythonFunctionsPreserveFileSignatures() throws Exception { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + createDatabase(ctx, "create database py_file_type_db;"); + ctx.setDatabase("py_file_type_db"); + String properties = " properties('type'='PYTHON_UDF', 'symbol'='evaluate', " + + "'runtime_version'='3.10.2');"; + Database db = Env.getCurrentInternalCatalog().getDbNullable("py_file_type_db"); + int index = 0; + for (String kind : ImmutableList.of("function", "aggregate function", "tables function")) { + for (String fileType : ImmutableList.of("file", "array", "map", + "struct", "array>")) { + Type expectedType = DataType.convertFromString(fileType).toCatalogDataType(); + String returnType = kind.equals("tables function") ? "array" : "int"; + String argumentName = "file_arg_" + index++; + createFunction("create " + kind + " py_file_type_db." + argumentName + "(" + fileType + + ") returns " + returnType + properties, ctx); + Function argumentFunction = findFunction(db, argumentName); + Assert.assertEquals(expectedType, argumentFunction.getArgs()[0]); + Assert.assertEquals(Type.INT, argumentFunction.getReturnType()); + String outputType = kind.equals("tables function") ? "array<" + fileType + ">" : fileType; + String returnName = "file_return_" + index++; + createFunction("create " + kind + " py_file_type_db." + returnName + "(int) returns " + + outputType + properties, ctx); + Function returnFunction = findFunction(db, returnName); + Assert.assertEquals(Type.INT, returnFunction.getArgs()[0]); + Assert.assertEquals(expectedType, returnFunction.getReturnType()); + } + } + for (String fileType : ImmutableList.of("file", "array", "map", "struct")) { + String name = "file_state_" + index++; + createFunction("create aggregate function py_file_type_db." + name + + "(int) returns int intermediate " + fileType + properties, ctx); + AggregateFunction function = (AggregateFunction) findFunction(db, name); + Assert.assertEquals(DataType.convertFromString(fileType).toCatalogDataType(), + function.getIntermediateType()); + } + } + + @Test + public void testCreatePythonFunctionsStillAcceptOrdinaryTypes() throws Exception { + ConnectContext ctx = UtFrameUtils.createDefaultCtx(); + createDatabase(ctx, "create database py_normal_type_db;"); + ctx.setDatabase("py_normal_type_db"); + int index = 0; + for (String kind : ImmutableList.of("function", "aggregate function", "tables function")) { + for (String type : ImmutableList.of("int", "string", "array", "map", + "struct")) { + String outputType = kind.equals("tables function") ? "array<" + type + ">" : type; + String name = "ordinary_" + index++; + createFunction("create " + kind + " py_normal_type_db." + name + "(" + type + ") returns " + + outputType + " properties('type'='PYTHON_UDF', 'symbol'='evaluate', " + + "'runtime_version'='3.10.2');", ctx); + Assert.assertNotNull(findFunction( + Env.getCurrentInternalCatalog().getDbNullable("py_normal_type_db"), name)); + } + } + } + @Test public void testDropFunctionReturnsCurrentGenerationId() throws Exception { ConnectContext ctx = UtFrameUtils.createDefaultCtx(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java index 59953396d29903..17f7436af73be3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java @@ -27,9 +27,15 @@ import org.apache.doris.meta.MetaContext; import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ToFile; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.plans.commands.CreateResourceCommand; import org.apache.doris.nereids.trees.plans.commands.info.CreateResourceInfo; +import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.runtime.ThriftPlansBuilder; +import org.apache.doris.thrift.TS3Resource; import org.apache.doris.thrift.TS3StorageParam; import com.google.common.base.Strings; @@ -43,6 +49,9 @@ import org.junit.Assume; import org.junit.Before; import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -758,4 +767,163 @@ public void testStoredGsEndpointCannotChangePolicyResourceOnAlter() throws Excep ImmutableMap.of(S3Properties.SECRET_KEY, "new-secret"))); Assert.assertEquals(legacy, resource.getCopiedProperties()); } + + @Test + public void queryResourceConfigurationIsSharedWithinExecuteAndRefreshedOnNextExecute() throws Exception { + Env env = Mockito.mock(Env.class); + ResourceMgr manager = Mockito.mock(ResourceMgr.class); + AccessControllerManager access = Mockito.mock(AccessControllerManager.class); + ConnectContext context = Mockito.spy(new ConnectContext()); + S3Resource resource = new S3Resource("s3_resource"); + resource.setProperties(ImmutableMap.builder() + .put("s3.endpoint", "https://oss-cn-beijing.aliyuncs.com") + .put("s3.region", "cn-beijing") + .put("s3.bucket", "bucket") + .put("s3.access_key", "fake-access-key") + .put("s3.secret_key", "first") + .put("s3.session_token", "fake-session-token") + .put("s3.connection.maximum", "77") + .put("use_path_style", "true") + .put("s3_validity_check", "false") + .build()); + Mockito.when(env.getResourceMgr()).thenReturn(manager); + Mockito.when(env.getAccessManager()).thenReturn(access); + Mockito.when(manager.getResource("s3_resource")).thenReturn(resource); + Mockito.when(access.checkResourcePriv(context, "s3_resource", PrivPredicate.USAGE)).thenReturn(true); + try (MockedStatic current = Mockito.mockStatic(Env.class)) { + current.when(Env::getCurrentEnv).thenReturn(env); + context.getOrRegisterS3Resource("s3_resource"); + TS3Resource first = ThriftPlansBuilder.collectS3Resources(context).get("s3_resource"); + assertBackendProperties(first, "https://oss-cn-beijing.aliyuncs.com", "cn-beijing", + "fake-access-key", "first"); + Assertions.assertEquals("fake-session-token", first.getProperties().get(S3Properties.Env.TOKEN)); + Assertions.assertEquals("77", first.getProperties().get(S3Properties.Env.MAX_CONNECTIONS)); + Assertions.assertEquals("true", first.getProperties().get(S3Properties.USE_PATH_STYLE)); + resource.modifyProperties(new HashMap<>(ImmutableMap.of("s3.secret_key", "second", "s3_validity_check", "false"))); + first.getProperties().put("AWS_SECRET_KEY", "mutated wire copy"); + Assertions.assertEquals("first", ThriftPlansBuilder.collectS3Resources(context) + .get("s3_resource").getProperties().get("AWS_SECRET_KEY")); + Assertions.assertEquals("second", resource.toThrift().getProperties().get("AWS_SECRET_KEY")); + context.setStartTime(); + Assertions.assertTrue(context.getS3Resources().isEmpty()); + context.getOrRegisterS3Resource("s3_resource"); + Assertions.assertEquals("second", ThriftPlansBuilder.collectS3Resources(context) + .get("s3_resource").getProperties().get("AWS_SECRET_KEY")); + + Mockito.when(access.checkResourcePriv(context, "s3_resource", PrivPredicate.USAGE)).thenReturn(false); + context.setStartTime(); + Assertions.assertThrows(AnalysisException.class, () -> context.getOrRegisterS3Resource("s3_resource")); + Assertions.assertTrue(context.getS3Resources().isEmpty()); + } + } + + @Test + public void legacyAwsResourcePropertiesReachBackendWithResourceBucket() throws Exception { + S3Resource resource = new S3Resource("legacy_resource"); + resource.setProperties(ImmutableMap.of( + "AWS_ENDPOINT", "https://s3.us-east-1.amazonaws.com", + "AWS_REGION", "us-east-1", + "AWS_BUCKET", "legacy-bucket", + "AWS_ACCESS_KEY", "fake-legacy-access-key", + "AWS_SECRET_KEY", "fake-legacy-secret-key", + "s3_validity_check", "false")); + TS3Resource config = resource.toThrift(); + assertBackendProperties(config, "https://s3.us-east-1.amazonaws.com", "us-east-1", + "fake-legacy-access-key", "fake-legacy-secret-key"); + Assertions.assertEquals("legacy-bucket", config.getProperties().get(S3Properties.BUCKET)); + Assertions.assertEquals("legacy-bucket", config.getProperties().get(S3Properties.Env.BUCKET)); + } + + @Test + public void nativeOssPropertiesAreNormalizedWithoutLosingResourceBucket() { + S3Resource resource = replayS3Resource(ImmutableMap.of( + "oss.endpoint", "https://oss-cn-beijing.aliyuncs.com", + "oss.region", "cn-beijing", + "OSS_BUCKET", "oss-bucket", + "oss.access_key", "fake-oss-access-key", + "oss.secret_key", "fake-oss-secret-key")); + TS3Resource config = resource.toThrift(); + assertBackendProperties(config, "https://oss-cn-beijing.aliyuncs.com", "cn-beijing", + "fake-oss-access-key", "fake-oss-secret-key"); + Assertions.assertEquals("oss-bucket", config.getProperties().get("OSS_BUCKET")); + Assertions.assertFalse(config.getProperties().containsKey("AWS_BUCKET")); + Assertions.assertEquals("oss-bucket", config.getProperties().get("OSS_BUCKET")); + Assertions.assertFalse(config.getProperties().containsKey(S3Properties.Env.BUCKET)); + } + + @Test + public void endpointOnlyReplayedStandardS3ResourceIsNormalizedWithoutRequiringBucket() { + S3Resource resource = replayS3Resource(ImmutableMap.of( + "s3.endpoint", "https://oss-cn-beijing.aliyuncs.com", + "s3.region", "cn-beijing", + "s3.access_key", "fake-access-key", + "s3.secret_key", "fake-secret-key")); + TS3Resource config = resource.toThrift(); + assertBackendProperties(config, "https://oss-cn-beijing.aliyuncs.com", "cn-beijing", + "fake-access-key", "fake-secret-key"); + assertNoBucket(config); + } + + @Test + public void endpointOnlyReplayedLegacyAwsResourceIsNormalizedWithoutRequiringBucket() { + S3Resource resource = replayS3Resource(ImmutableMap.of( + "AWS_ENDPOINT", "https://s3.us-east-1.amazonaws.com", + "AWS_REGION", "us-east-1", + "AWS_ACCESS_KEY", "fake-legacy-access-key", + "AWS_SECRET_KEY", "fake-legacy-secret-key")); + TS3Resource config = resource.toThrift(); + assertBackendProperties(config, "https://s3.us-east-1.amazonaws.com", "us-east-1", + "fake-legacy-access-key", "fake-legacy-secret-key"); + assertNoBucket(config); + } + + private static S3Resource replayS3Resource(Map properties) { + // Image replay accepts stored properties without repeating CREATE RESOURCE's ping checks. + String json = GsonUtils.GSON.toJson(ImmutableMap.of("clazz", "S3Resource", "type", "S3", + "name", "endpoint_only_resource", "properties", properties)); + return (S3Resource) GsonUtils.GSON.fromJson(json, Resource.class); + } + + private static void assertNoBucket(TS3Resource config) { + Assertions.assertFalse(config.getProperties().containsKey(S3Properties.BUCKET)); + Assertions.assertFalse(config.getProperties().containsKey(S3Properties.Env.BUCKET)); + } + + private static void assertBackendProperties(TS3Resource config, String endpoint, String region, + String accessKey, String secretKey) { + Assertions.assertEquals(endpoint, config.getProperties().get(S3Properties.Env.ENDPOINT)); + Assertions.assertEquals(region, config.getProperties().get(S3Properties.Env.REGION)); + Assertions.assertEquals(accessKey, config.getProperties().get(S3Properties.Env.ACCESS_KEY)); + Assertions.assertEquals(secretKey, config.getProperties().get(S3Properties.Env.SECRET_KEY)); + } + + @Test + public void resourceBindingRejectsNonS3AndChecksUsageWithoutCopyingProperties() { + Env env = Mockito.mock(Env.class); + ResourceMgr manager = Mockito.mock(ResourceMgr.class); + AccessControllerManager access = Mockito.mock(AccessControllerManager.class); + ConnectContext context = Mockito.spy(new ConnectContext()); + Resource resource = Mockito.mock(Resource.class); + Mockito.when(env.getResourceMgr()).thenReturn(manager); + Mockito.when(env.getAccessManager()).thenReturn(access); + Mockito.when(manager.getResource("resource")).thenReturn(resource); + Mockito.when(resource.getType()).thenReturn(Resource.ResourceType.JDBC); + Mockito.when(access.checkResourcePriv(context, "resource", PrivPredicate.USAGE)).thenReturn(true); + + try (MockedStatic current = Mockito.mockStatic(Env.class); + MockedStatic connection = Mockito.mockStatic(ConnectContext.class)) { + current.when(Env::getCurrentEnv).thenReturn(env); + connection.when(ConnectContext::get).thenReturn(context); + ToFile function = new ToFile(new StringLiteral("resource"), new StringLiteral("arbitrary:uri")); + function.checkLegalityBeforeTypeCoercion(); + Assertions.assertThrows(AnalysisException.class, function::checkLegalityAfterRewrite); + Mockito.when(resource.getType()).thenReturn(Resource.ResourceType.HDFS); + Assertions.assertThrows(AnalysisException.class, function::checkLegalityAfterRewrite); + Mockito.verify(resource, Mockito.never()).getCopiedProperties(); + Assertions.assertThrows(AnalysisException.class, + () -> S3Resource.checkResource("missing", context)); + Mockito.when(access.checkResourcePriv(context, "resource", PrivPredicate.USAGE)).thenReturn(false); + Assertions.assertThrows(AnalysisException.class, function::checkLegalityAfterRewrite); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/ToFileTranslatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/ToFileTranslatorTest.java new file mode 100644 index 00000000000000..be6296c6b90bc4 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/ToFileTranslatorTest.java @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.glue.translator; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Resource; +import org.apache.doris.catalog.ResourceMgr; +import org.apache.doris.catalog.S3Resource; +import org.apache.doris.catalog.Type; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ToFile; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.runtime.ThriftPlansBuilder; +import org.apache.doris.thrift.PaloInternalServiceVersion; +import org.apache.doris.thrift.TExpr; +import org.apache.doris.thrift.TFunction; +import org.apache.doris.thrift.TPipelineFragmentParams; +import org.apache.doris.thrift.TS3Resource; +import org.apache.doris.thrift.TUniqueId; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TSerializer; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.util.HashMap; +import java.util.Map; + +class ToFileTranslatorTest { + @Test + void s3ResourceReachesQueryParametersWithoutUriValidationOrRewrite() throws Exception { + assertResourceWire("hdfs://OtherHost/a/../b%2f?token=value#fragment"); + } + + private void assertResourceWire(String uri) throws Exception { + Env env = Mockito.mock(Env.class); + ResourceMgr manager = Mockito.mock(ResourceMgr.class); + AccessControllerManager access = Mockito.mock(AccessControllerManager.class); + ConnectContext connection = Mockito.spy(new ConnectContext()); + PlanTranslatorContext translation = Mockito.mock(PlanTranslatorContext.class); + S3Resource resource = Mockito.mock(S3Resource.class); + Mockito.when(resource.toThrift()).thenCallRealMethod(); + Map properties = new HashMap<>(); + String resourceSecretKey = "s3.secret_key"; + String backendSecretKey = "AWS_SECRET_KEY"; + properties.put(resourceSecretKey, "first"); + properties.put("s3.endpoint", "https://s3.us-east-1.amazonaws.com"); + properties.put("s3.region", "us-east-1"); + properties.put("s3.bucket", "bucket"); + properties.put("s3.access_key", "fake-access-key"); + Mockito.when(env.getResourceMgr()).thenReturn(manager); + Mockito.when(env.getAccessManager()).thenReturn(access); + Mockito.when(manager.getResource("resource")).thenReturn(resource); + Mockito.when(access.checkResourcePriv(connection, "resource", PrivPredicate.USAGE)).thenReturn(true); + Mockito.when(resource.getType()).thenReturn(Resource.ResourceType.S3); + Mockito.when(resource.getCopiedProperties()).thenAnswer(ignored -> new HashMap<>(properties)); + Mockito.when(translation.getConnectContext()).thenReturn(connection); + Mockito.when(translation.getStatementContext()) + .thenThrow(new AssertionError("S3 resource translation must not use StatementContext")); + + try (MockedStatic current = Mockito.mockStatic(Env.class)) { + current.when(Env::getCurrentEnv).thenReturn(env); + ToFile expression = new ToFile(new StringLiteral("resource"), new StringLiteral(uri)); + Expr first = ExpressionTranslator.translate(expression, translation); + TExpr wire = first.treeToThrift(); + TFunction function = wire.getNodes().get(0).getFn(); + assertFunctionHasNoResource(function, backendSecretKey); + Map queryResources = ThriftPlansBuilder.collectS3Resources(connection); + Assertions.assertEquals(1, queryResources.size()); + TS3Resource config = queryResources.get("resource"); + Assertions.assertEquals("first", config.getProperties().get(backendSecretKey)); + Assertions.assertEquals(uri, wire.getNodes().get(2).getStringLiteral().getValue()); + TPipelineFragmentParams params = new TPipelineFragmentParams() + .setProtocolVersion(PaloInternalServiceVersion.V1).setQueryId(new TUniqueId(1, 2)) + .setPerExchNumSenders(ImmutableMap.of()).setDestinations(ImmutableList.of()).setS3Resources(queryResources); + TPipelineFragmentParams received = new TPipelineFragmentParams(); + new TDeserializer().deserialize(received, new TSerializer().serialize(params)); + Assertions.assertEquals(queryResources, received.getS3Resources()); + ByteArrayOutputStream persisted = new ByteArrayOutputStream(); + first.getFn().write(new DataOutputStream(persisted)); + Assertions.assertFalse(persisted.toString(java.nio.charset.StandardCharsets.UTF_8.name()) + .contains(backendSecretKey)); + + properties.put(resourceSecretKey, "second"); + config.getProperties().put(backendSecretKey, "mutated wire copy"); + Assertions.assertEquals("first", ThriftPlansBuilder.collectS3Resources(connection) + .get("resource").getProperties().get(backendSecretKey)); + Expr sameQuery = ExpressionTranslator.translate(expression, translation); + assertFunctionHasNoResource(wireSnapshot(sameQuery), backendSecretKey); + Assertions.assertEquals("first", ThriftPlansBuilder.collectS3Resources(connection) + .get("resource").getProperties().get(backendSecretKey)); + Mockito.verify(resource, Mockito.times(1)).getCopiedProperties(); + + TFunction cloned = first.getFn().clone().toThrift(first.getType(), + new Type[] {Type.STRING, Type.STRING}, new Boolean[] {false, false}); + assertFunctionHasNoResource(cloned, backendSecretKey); + + connection.setStartTime(); + Assertions.assertTrue(connection.getS3Resources().isEmpty()); + Expr nextExecute = ExpressionTranslator.translate(expression, translation); + assertFunctionHasNoResource(wireSnapshot(nextExecute), backendSecretKey); + Assertions.assertEquals("second", ThriftPlansBuilder.collectS3Resources(connection) + .get("resource").getProperties().get(backendSecretKey)); + Assertions.assertEquals("first", received.getS3Resources().get("resource") + .getProperties().get(backendSecretKey)); + Mockito.verify(resource, Mockito.times(2)).getCopiedProperties(); + } + } + + private void assertFunctionHasNoResource(TFunction function, String propertyKey) throws Exception { + String bytes = new String(new TSerializer().serialize(function), + java.nio.charset.StandardCharsets.ISO_8859_1); + Assertions.assertFalse(bytes.contains(propertyKey)); + } + + private TFunction wireSnapshot(Expr expression) { + return expression.treeToThrift().getNodes().get(0).getFn(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantRuleOnBETest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantRuleOnBETest.java index 28037322ecc674..2fe4e7e09e96fb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantRuleOnBETest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantRuleOnBETest.java @@ -19,26 +19,45 @@ import org.apache.doris.analysis.ExprId; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.rules.expression.check.CheckCast; import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnBE; import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE; import org.apache.doris.nereids.trees.expressions.Add; import org.apache.doris.nereids.trees.expressions.And; +import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonbParse; import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.MapLiteral; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; import org.apache.doris.nereids.trees.expressions.literal.UuidLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.FileType; +import org.apache.doris.nereids.types.MapType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.types.UuidType; +import org.apache.doris.nereids.types.VarBinaryType; +import org.apache.doris.proto.Types.PGenericType; +import org.apache.doris.proto.Types.PGenericType.TypeId; +import org.apache.doris.proto.Types.PValues; import org.apache.doris.thrift.TExpr; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.protobuf.ByteString; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -88,6 +107,79 @@ void testKeepSupportedSibling() { collectConstants(new And(new IsNull(uuidResult), supported))); } + @Test + void testJsonToFileUsesGenericJsonFoldingRestriction() { + Expression expression = new Cast(new JsonbParse(new StringLiteral( + "{\"uri\":\"s3://bucket/object\",\"offset\":null,\"size\":null," + + "\"content_type\":null,\"checksum\":null,\"inline\":null}")), FileType.INSTANCE); + Assertions.assertTrue(expression.isConstant()); + Assertions.assertTrue(collectConstants(expression).isEmpty()); + } + + @Test + void testUnsupportedFileCastsRemainForTypeChecking() { + StructLiteral fileFields = new StructLiteral(ImmutableList.of(new StringLiteral("urn:x"), + new NullLiteral(BigIntType.INSTANCE), new NullLiteral(BigIntType.INSTANCE), + new NullLiteral(StringType.INSTANCE), new NullLiteral(StringType.INSTANCE), + new NullLiteral(VarBinaryType.INSTANCE)), FileType.INSTANCE.publicStructType()); + List expressions = ImmutableList.of( + new Cast(fileFields, FileType.INSTANCE), + new Cast(new ArrayLiteral(ImmutableList.of(fileFields)), ArrayType.of(FileType.INSTANCE)), + new Cast(new MapLiteral(ImmutableMap.of(new StringLiteral("f"), fileFields)), + MapType.of(StringType.INSTANCE, FileType.INSTANCE)), + new Cast(new StructLiteral(ImmutableList.of(fileFields)), + new StructType(ImmutableList.of(new StructField("f", FileType.INSTANCE, true, ""))))); + for (Expression expression : expressions) { + Assertions.assertTrue(expression.isConstant()); + Assertions.assertFalse(CheckCast.check(expression.child(0).getDataType(), expression.getDataType(), false)); + Assertions.assertThrows(AnalysisException.class, + () -> expression.child(0).checkedCastTo(expression.getDataType())); + Assertions.assertSame(expression, FoldConstantRuleOnFE.VISITOR_INSTANCE.rewrite(expression, context)); + Assertions.assertTrue(collectConstants(expression).isEmpty(), expression.toSql()); + Assertions.assertTrue(collectConstants(new IsNull(expression)).isEmpty(), expression.toSql()); + } + Expression supported = new EqualTo(new Add(new IntegerLiteral(1), new IntegerLiteral(2)), + new IntegerLiteral(3)); + Assertions.assertEquals(ImmutableList.of(supported), + collectConstants(new And(new IsNull(expressions.get(0)), supported))); + } + + @Test + void testFileNullCastsUseNativeLiteralFolding() { + for (DataType type : ImmutableList.of(FileType.INSTANCE, ArrayType.of(FileType.INSTANCE), + MapType.of(StringType.INSTANCE, FileType.INSTANCE), + new StructType(ImmutableList.of(new StructField("f", FileType.INSTANCE, true, ""))))) { + Assertions.assertEquals(new NullLiteral(type), FoldConstantRuleOnFE.VISITOR_INSTANCE.rewrite( + new Cast(new NullLiteral(), type), context)); + } + ArrayType fileArray = ArrayType.of(FileType.INSTANCE); + Cast arrayCast = new Cast(new ArrayLiteral(ImmutableList.of(new NullLiteral())), fileArray); + Assertions.assertFalse(CheckCast.check(arrayCast.child().getDataType(), fileArray, false)); + Assertions.assertSame(arrayCast, FoldConstantRuleOnFE.VISITOR_INSTANCE.rewrite(arrayCast, context)); + } + + @Test + void testDecodeVarbinaryAndStructFromProtobuf() { + byte[] bytes = {0, (byte) 0xff, (byte) 0x80}; + PValues binary = PValues.newBuilder().setType(PGenericType.newBuilder().setId(TypeId.VARBINARY)) + .addBytesValue(ByteString.copyFrom(bytes)).addBytesValue(ByteString.EMPTY) + .addBytesValue(ByteString.EMPTY).setHasNull(true) + .addNullMap(false).addNullMap(true).addNullMap(false).build(); + List expected = ImmutableList.of(new VarBinaryLiteral(bytes), + new NullLiteral(VarBinaryType.INSTANCE), new VarBinaryLiteral(new byte[0])); + Assertions.assertEquals(expected, + FoldConstantRuleOnBE.getResultExpression(VarBinaryType.INSTANCE, binary)); + StructType type = new StructType(ImmutableList.of( + new StructField("payload", VarBinaryType.INSTANCE, true, ""))); + PValues structure = PValues.newBuilder().setType(PGenericType.newBuilder().setId(TypeId.STRUCT)) + .addChildElement(binary).build(); + List decoded = FoldConstantRuleOnBE.getResultExpression(type, structure); + Assertions.assertEquals(expected.size(), decoded.size()); + for (int row = 0; row < expected.size(); row++) { + Assertions.assertEquals(new StructLiteral(ImmutableList.of(expected.get(row)), type), decoded.get(row)); + } + } + private List collectConstants(Expression expression) { Map constants = new HashMap<>(); Map serialized = new HashMap<>(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/FileAccessPathTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/FileAccessPathTest.java new file mode 100644 index 00000000000000..e06ab09c1e070c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/FileAccessPathTest.java @@ -0,0 +1,248 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.analysis.AccessPathInfo; +import org.apache.doris.catalog.Column; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectAccessPathResult; +import org.apache.doris.nereids.rules.rewrite.NestedColumnPruning.DataTypeAccessTree; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.FlGetBinary; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Length; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ToJson; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.FileType; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.MapType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; +import org.apache.doris.thrift.DescriptorsConstants; +import org.apache.doris.thrift.TAccessPathType; +import org.apache.doris.thrift.TColumnAccessPath; +import org.apache.doris.thrift.TDataAccessPath; +import org.apache.doris.thrift.TMetaAccessPath; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Multimap; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; + +public class FileAccessPathTest { + @Test + public void testToJsonNeedsAllSixChildren() { + SlotReference file = slot("f", FileType.INSTANCE, true); + AccessPathInfo info = prune(file, new ToJson(file), field(file, "size")); + Assertions.assertEquals(ImmutableList.of(data("f")), info.getAllAccessPaths()); + Assertions.assertSame(FileType.INSTANCE, info.getPrunedType()); + Assertions.assertEquals(6, ((FileType) info.getPrunedType()).getFields().size()); + } + + @Test + public void testCanonicalChildrenAndGetterPaths() { + SlotReference file = slot("f", FileType.INSTANCE, false); + for (String name : ImmutableList.of("uri", "offset", "size", "content_type", "checksum")) { + AccessPathInfo info = prune(file, field(file, name)); + Assertions.assertEquals(ImmutableList.of(data("f", name)), info.getAllAccessPaths()); + Assertions.assertSame(FileType.INSTANCE, info.getPrunedType()); + } + DataTypeAccessTree tree = DataTypeAccessTree.of(FileType.INSTANCE, TAccessPathType.DATA); + tree.setAccessByPath(ImmutableList.of("size"), 0, TAccessPathType.DATA); + Assertions.assertSame(FileType.INSTANCE, tree.pruneDataType().orElseThrow(IllegalStateException::new)); + Assertions.assertEquals(ImmutableList.of("uri", "offset", "size", "content_type", "checksum", "inline"), + new ArrayList<>(tree.getChildren().keySet())); + Assertions.assertEquals(6, ((FileType) tree.getType()).getFields().size()); + Assertions.assertTrue(FileType.INSTANCE.getFields().stream().allMatch(StructField::isNullable)); + } + + @Test + public void testParentAndChildNullPaths() { + SlotReference file = slot("f", FileType.INSTANCE, true); + for (Expression expression : ImmutableList.of(new IsNull(file), new Not(new IsNull(file)))) { + Assertions.assertEquals(ImmutableList.of(meta("f", "NULL")), prune(file, expression).getAllAccessPaths()); + } + for (Expression expression : ImmutableList.of(new IsNull(field(file, "size")))) { + Assertions.assertEquals(ImmutableList.of(meta("f", "size", "NULL")), + prune(file, expression).getAllAccessPaths()); + } + } + + @Test + public void testFileStringFieldAccessPaths() { + SlotReference file = slot("f", FileType.INSTANCE, true); + for (String name : ImmutableList.of("uri", "content_type", "checksum")) { + Expression getter = field(file, name); + AccessPathInfo info = prune(file, new Length(getter)); + Assertions.assertEquals(ImmutableList.of(meta("f", name, "OFFSET")), info.getAllAccessPaths()); + Assertions.assertSame(FileType.INSTANCE, info.getPrunedType()); + Assertions.assertEquals(ImmutableList.of(meta("f", name, "NULL")), + prune(file, new IsNull(getter)).getAllAccessPaths()); + Assertions.assertEquals(ImmutableList.of(data("f", name)), + prune(file, collect(file, false, true, new Length(getter))).getAllAccessPaths()); + } + } + + @Test + public void testAccessPathsCrossFileIdentityCasts() { + SlotReference file = slot("f", FileType.INSTANCE, true); + Expression identityField = field(new Cast(file, FileType.INSTANCE), "uri"); + Assertions.assertEquals(ImmutableList.of(meta("f", "uri", "OFFSET")), + prune(file, new Length(identityField)).getAllAccessPaths()); + Assertions.assertEquals(ImmutableList.of(meta("f", "uri", "NULL")), + prune(file, new IsNull(identityField)).getAllAccessPaths()); + } + + @Test + public void testNestedFilePaths() { + SlotReference array = slot("a", ArrayType.of(FileType.INSTANCE), true); + Expression item = new ElementAt(array, new IntegerLiteral(1)); + Assertions.assertEquals(ImmutableList.of(data("a", "*", "uri")), + prune(array, field(item, "uri")).getAllAccessPaths()); + Assertions.assertSame(FileType.INSTANCE, ((ArrayType) prune(array, field(item, "uri")) + .getPrunedType()).getItemType()); + Assertions.assertEquals(ImmutableList.of(meta("a", "*", "NULL")), + prune(array, new IsNull(item)).getAllAccessPaths()); + + SlotReference struct = slot("s", new StructType(ImmutableList.of( + new StructField("f", FileType.INSTANCE, true, ""), + new StructField("unused", IntegerType.INSTANCE, true, ""))), true); + Expression nested = field(struct, "f"); + AccessPathInfo info = prune(struct, field(nested, "uri")); + Assertions.assertEquals(ImmutableList.of(data("s", "f", "uri")), info.getAllAccessPaths()); + Assertions.assertSame(FileType.INSTANCE, ((StructType) info.getPrunedType()).getField("f").getDataType()); + Assertions.assertEquals(ImmutableList.of(meta("s", "f", "NULL")), + prune(struct, new IsNull(nested)).getAllAccessPaths()); + } + + @Test + public void testMapLookupKeepsKeysAndRoutesFileValues() { + SlotReference map = slot("m", MapType.of(StringType.INSTANCE, FileType.INSTANCE), true); + Expression value = new ElementAt(map, new StringLiteral("key")); + AccessPathInfo info = prune(map, field(value, "uri")); + Assertions.assertEquals(new TreeSet<>(ImmutableList.of(data("m", "KEYS"), data("m", "VALUES", "uri"))), + new TreeSet<>(info.getAllAccessPaths())); + Assertions.assertSame(FileType.INSTANCE, ((MapType) info.getPrunedType()).getValueType()); + Assertions.assertEquals(new TreeSet<>(ImmutableList.of(data("m", "KEYS"), meta("m", "VALUES", "NULL"))), + new TreeSet<>(prune(map, new IsNull(value)).getAllAccessPaths())); + } + + @Test + public void testFullConsumerDominatesGetterAndMetadata() { + SlotReference file = slot("f", FileType.INSTANCE, true); + AccessPathInfo info = prune(file, file, field(file, "uri"), new Count(file)); + Assertions.assertEquals(ImmutableList.of(data("f")), info.getAllAccessPaths()); + Assertions.assertSame(FileType.INSTANCE, info.getPrunedType()); + + List paths = collect(file, false, false, file); + paths.addAll(collect(file, true, false, new IsNull(field(file, "size")))); + info = prune(file, paths); + Assertions.assertTrue(info.getAllAccessPaths().contains(data("f"))); + Assertions.assertEquals(ImmutableList.of(meta("f", "size", "NULL")), info.getPredicateAccessPaths()); + Assertions.assertSame(FileType.INSTANCE, info.getPrunedType()); + } + + @Test + public void testNestedFullConsumerPreservesFileIdentity() { + SlotReference struct = slot("s", new StructType(ImmutableList.of( + new StructField("f", FileType.INSTANCE, true, ""))), true); + Expression file = field(struct, "f"); + AccessPathInfo info = prune(struct, file, field(file, "size"), new IsNull(file)); + Assertions.assertTrue(info.getAllAccessPaths().contains(data("s", "f"))); + Assertions.assertSame(FileType.INSTANCE, ((StructType) info.getPrunedType()).getField("f").getDataType()); + DataTypeAccessTree tree = DataTypeAccessTree.ofRoot(struct, TAccessPathType.DATA); + for (TColumnAccessPath path : info.getAllAccessPaths()) { + tree.setAccessByPath(path.getType() == TAccessPathType.DATA + ? path.getDataAccessPath().getPath() : path.getMetaAccessPath().getPath(), 0, path.getType()); + } + Assertions.assertTrue(tree.getChildren().get("s").getChildren().get("f").isAccessAll()); + } + + @Test + public void testFileCastsDoNotBecomeStructShapePruning() { + SlotReference file = slot("f", FileType.INSTANCE, true); + Assertions.assertEquals(ImmutableList.of(data("f", "size")), + prune(file, field(new Cast(file, FileType.INSTANCE), "size")).getAllAccessPaths()); + } + + @Test + public void testBinaryAccessNeedsCompleteFile() { + SlotReference file = slot("f", FileType.INSTANCE, true); + Assertions.assertEquals(ImmutableList.of(data("f")), + prune(file, new FlGetBinary(file)).getAllAccessPaths()); + } + + private static ElementAt field(Expression value, String name) { + return new ElementAt(value, new StringLiteral(name)); + } + + private static SlotReference slot(String name, DataType type, boolean nullable) { + return new SlotReference(name, type, nullable).withColumn(new Column(name, type.toCatalogDataType(), nullable)); + } + + private static List collect(SlotReference slot, boolean predicate, + boolean skipMeta, Expression... expressions) { + Multimap paths = ArrayListMultimap.create(); + AccessPathExpressionCollector collector = new AccessPathExpressionCollector(null, paths, predicate, skipMeta); + for (Expression expression : expressions) { + collector.collect(expression); + } + return new ArrayList<>(paths.get(slot.getExprId().asInt())); + } + + private static AccessPathInfo prune(SlotReference slot, Expression... expressions) { + return prune(slot, collect(slot, false, false, expressions)); + } + + private static AccessPathInfo prune(SlotReference slot, List paths) { + Map> inputs = ImmutableMap.of(slot, paths); + Map result = Deencapsulation.invoke( + NestedColumnPruning.class, "pruneDataType", inputs); + return result.get(slot.getExprId().asInt()); + } + + private static TColumnAccessPath data(String... path) { + return new TColumnAccessPath(TAccessPathType.DATA) + .setDataAccessPath(new TDataAccessPath(ImmutableList.copyOf(path))) + .setVersion(DescriptorsConstants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); + } + + private static TColumnAccessPath meta(String... path) { + return new TColumnAccessPath(TAccessPathType.META) + .setMetaAccessPath(new TMetaAccessPath(ImmutableList.copyOf(path))) + .setVersion(DescriptorsConstants.TCOLUMN_ACCESS_PATH_VERSION_TYPED); + } + +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ShortCircuitPointQueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ShortCircuitPointQueryTest.java index 512033b35a5cff..2c75a86660fbdc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ShortCircuitPointQueryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ShortCircuitPointQueryTest.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.S3Resource; import org.apache.doris.catalog.Tablet; import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.doris.RemoteOlapTable; @@ -33,8 +34,11 @@ import org.apache.doris.qe.ConnectContext.ConnectType; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import java.lang.reflect.Field; @@ -207,4 +211,16 @@ void testArrowFlightSqlConnectionDoesNotUseShortCircuit() throws Exception { Assertions.assertTrue(connectContext.getStatementContext().isShortCircuitQuery()); } + @Test + void testToFilePointQueryUsesNormalExecution() { + try (MockedStatic resources = Mockito.mockStatic(S3Resource.class)) { + for (String projection : ImmutableList.of("to_file('resource', v1)", + "element_at(to_file('resource', v1), 'size')")) { + rewrite("select " + projection + " from tbl_point_query where `key` = 1"); + Assertions.assertFalse(connectContext.getStatementContext().isShortCircuitQuery(), projection); + } + } + rewrite("select v1 from tbl_point_query where `key` = 1"); + Assertions.assertTrue(connectContext.getStatementContext().isShortCircuitQuery()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/ExpressionEstimationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/ExpressionEstimationTest.java index 8a2b405c553ab1..989d2cd75b8c06 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/ExpressionEstimationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/ExpressionEstimationTest.java @@ -45,6 +45,7 @@ import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.DateType; import org.apache.doris.nereids.types.DoubleType; +import org.apache.doris.nereids.types.FileType; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; import org.apache.doris.statistics.ColumnStatistic; @@ -63,6 +64,13 @@ class ExpressionEstimationTest { + @Test + public void testFileNullLiteralUsesUnknownStatistics() { + Statistics statistics = new Statistics(1, new HashMap<>()); + Assertions.assertSame(ColumnStatistic.UNKNOWN, + ExpressionEstimation.estimate(new NullLiteral(FileType.INSTANCE), statistics)); + } + // MAX(a) // a belongs to [0, 500] @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java index fca540113aa0ae..e8eacdba70025d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java @@ -28,12 +28,14 @@ import org.apache.doris.nereids.rules.rewrite.MergeProjectable; import org.apache.doris.nereids.rules.rewrite.MergeProjects; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.Lists; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.List; @@ -83,6 +85,27 @@ protected void runBeforeEach() throws Exception { StatementScopeIdGenerator.clear(); } + @Test + public void testDecimal256DivisionViewWithDecimal256Disabled() throws Exception { + boolean previous = connectContext.getSessionVariable().isEnableDecimal256(); + try { + connectContext.getSessionVariable().enableDecimal256 = true; + createTable("CREATE TABLE view_decimal_division (id INT, a DECIMAL(38,10), b DECIMAL(38,10)) " + + "DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 " + + "PROPERTIES ('replication_num' = '1')"); + createView("CREATE VIEW view_decimal256_division AS " + + "SELECT a / b AS quotient FROM view_decimal_division"); + connectContext.getSessionVariable().enableDecimal256 = false; + PlanChecker checker = PlanChecker.from(connectContext) + .analyze("SELECT * FROM view_decimal256_division"); + DecimalV3Type type = (DecimalV3Type) checker.getPlan().getOutput().get(0).getDataType(); + Assertions.assertTrue(type.getPrecision() > DecimalV3Type.MAX_DECIMAL128_PRECISION); + Assertions.assertFalse(connectContext.getSessionVariable().isEnableDecimal256()); + } finally { + connectContext.getSessionVariable().enableDecimal256 = previous; + } + } + @Test public void testTranslateAllCase() throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FlGetBinaryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FlGetBinaryTest.java new file mode 100644 index 00000000000000..086f7b089bdc25 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/FlGetBinaryTest.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.RewriteWhenAnalyze; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.FileType; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.JsonType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.VarBinaryType; +import org.apache.doris.nereids.types.VariantType; +import org.apache.doris.nereids.util.TypeCoercionUtils; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +class FlGetBinaryTest { + @Test + void testNullableBinarySignature() { + for (Expression input : Arrays.asList(new SlotReference("file", FileType.INSTANCE), + new NullLiteral(FileType.INSTANCE), new NullLiteral())) { + Expression function = TypeCoercionUtils.processBoundFunction(new FlGetBinary(input)); + Assertions.assertEquals(VarBinaryType.INSTANCE, function.getDataType()); + Assertions.assertTrue(function.nullable()); + Assertions.assertFalse(function instanceof RewriteWhenAnalyze); + } + } + + @Test + void testRejectsOtherArgumentTypes() { + for (DataType type : Arrays.asList(IntegerType.INSTANCE, StringType.INSTANCE, JsonType.INSTANCE, + VariantType.INSTANCE, FileType.INSTANCE.publicStructType(), ArrayType.of(FileType.INSTANCE))) { + Assertions.assertThrows(AnalysisException.class, + () -> TypeCoercionUtils.processBoundFunction(new FlGetBinary(new SlotReference("value", type)))); + } + } + + @Test + void testFileInlineSelectorRejected() { + for (Expression input : Arrays.asList(new SlotReference("file", FileType.INSTANCE), + new NullLiteral(FileType.INSTANCE))) { + for (String name : Arrays.asList("inline", "INLINE", "InLiNe")) { + AnalysisException error = Assertions.assertThrows(AnalysisException.class, + () -> TypeCoercionUtils.processBoundFunction(new ElementAt(input, new StringLiteral(name)))); + Assertions.assertTrue(error.getMessage().contains("use FL_GET_BINARY")); + } + } + Expression struct = new SlotReference("value", FileType.INSTANCE.publicStructType()); + Assertions.assertEquals(VarBinaryType.INSTANCE, + TypeCoercionUtils.processBoundFunction(new ElementAt(struct, new StringLiteral("inline"))) + .getDataType()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToFileTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToFileTest.java new file mode 100644 index 00000000000000..de91763d3120e1 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToFileTest.java @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; +import org.apache.doris.nereids.types.FileType; +import org.apache.doris.nereids.types.StringType; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class ToFileTest { + @Test + void rejectsResourceExpressionsAndImplicitUriParsing() { + for (Expression resource : ImmutableList.of(NullLiteral.INSTANCE, new IntegerLiteral(1), + new SlotReference("resource", StringType.INSTANCE))) { + Assertions.assertThrows(AnalysisException.class, + () -> new ToFile(resource, new StringLiteral("s3://bucket/key")) + .checkLegalityBeforeTypeCoercion()); + } + Assertions.assertThrows(AnalysisException.class, + () -> new ToFile(new StringLiteral("s3_resource"), new IntegerLiteral(1)) + .checkLegalityBeforeTypeCoercion()); + } + + @Test + void leavesUriValuesUnchangedForTheFilesystem() { + for (String uri : ImmutableList.of("", "relative path", "ftp://host/key#fragment", + "S3://Bucket/a/../b%2f?token=value")) { + StringLiteral argument = new StringLiteral(uri); + ToFile function = new ToFile(new VarcharLiteral("resource"), argument); + function.checkLegalityBeforeTypeCoercion(); + Assertions.assertSame(argument, function.getArgument(1)); + Assertions.assertEquals(uri, ((StringLiteral) function.getArgument(1)).getStringValue()); + } + } + + @Test + void preservesVolatilityAndNullPropagationAcrossRewrites() { + StringLiteral resource = new StringLiteral("resource"); + ToFile function = new ToFile(resource, new StringLiteral("s3://bucket/key")); + Assertions.assertEquals(FileType.INSTANCE, function.getDataType()); + Assertions.assertFalse(function.nullable()); + Assertions.assertFalse(function.foldable()); + Assertions.assertFalse(function.isDeterministic()); + Assertions.assertNotEquals(function, new ToFile(resource, function.getArgument(1))); + ToFile nullable = function.withChildren(ImmutableList.of(resource, NullLiteral.INSTANCE)); + nullable.checkLegalityBeforeTypeCoercion(); + Assertions.assertTrue(nullable.nullable()); + Assertions.assertEquals(function.getVolatileIdentity(), nullable.getVolatileIdentity()); + Assertions.assertThrows(IllegalArgumentException.class, + () -> function.withChildren(ImmutableList.of(resource))); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeDdlTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeDdlTest.java new file mode 100644 index 00000000000000..351448f959045b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeDdlTest.java @@ -0,0 +1,316 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.types; + +import org.apache.doris.alter.MaterializedViewHandler; +import org.apache.doris.analysis.IndexDef; +import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.info.TableNameInfo; +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.rules.expression.check.CheckCast; +import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand; +import org.apache.doris.nereids.trees.plans.commands.CreateMaterializedViewCommand; +import org.apache.doris.nereids.trees.plans.commands.CreateTableLikeCommand; +import org.apache.doris.nereids.trees.plans.commands.info.AddColumnOp; +import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; +import org.apache.doris.nereids.trees.plans.commands.info.DefaultValue; +import org.apache.doris.nereids.trees.plans.commands.info.DistributionDescriptor; +import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition; +import org.apache.doris.nereids.trees.plans.commands.info.PartitionTableInfo; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Optional; + +public class FileTypeDdlTest extends TestWithFeService { + @Override + protected void runBeforeAll() throws Exception { + createDatabase("file_agg_state_ddl"); + connectContext.setDatabase("file_agg_state_ddl"); + createTable("CREATE TABLE ddl_base (id INT) DUPLICATE KEY(id) " + + "DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num'='1')"); + } + + private ColumnDefinition column(String sql, boolean key, AggregateType agg, Optional value) { + DataType type = Assertions.assertDoesNotThrow(() -> DataType.convertFromString(sql)); + return new ColumnDefinition("f", type, key, agg, true, value, ""); + } + + private void validate(ColumnDefinition column, boolean olap, KeysType keys) { + column.validate(olap, Collections.emptySet(), Collections.emptySet(), false, keys); + } + + @Test + public void testInternalTableRejectsFileValueColumns() { + for (String sql : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP", + "ARRAY>", "STRUCT>>")) { + for (KeysType keys : Arrays.asList(KeysType.DUP_KEYS, KeysType.UNIQUE_KEYS, KeysType.AGG_KEYS)) { + AggregateType agg = keys == KeysType.AGG_KEYS ? AggregateType.REPLACE : null; + AnalysisException error = Assertions.assertThrows(AnalysisException.class, + () -> validate(column(sql, false, agg, Optional.empty()), true, keys)); + Assertions.assertEquals("FILE columns are not supported in internal tables", error.getMessage()); + } + } + Assertions.assertThrows(AnalysisException.class, + () -> validate(column("FILE", false, AggregateType.REPLACE_IF_NOT_NULL, Optional.empty()), + true, KeysType.AGG_KEYS)); + } + + @Test + public void testCreateTableAndCtasRejectFileColumns() { + for (String sqlType : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP")) { + for (String ddl : Arrays.asList( + "CREATE TABLE ddl_file (id INT, f " + sqlType + ") DUPLICATE KEY(id) " + + "DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num'='1')", + "CREATE TABLE ddl_ctas DISTRIBUTED BY HASH(id) BUCKETS 1 " + + "PROPERTIES('replication_num'='1') AS SELECT 1 AS id, CAST(NULL AS " + + sqlType + ") AS f")) { + Exception error = Assertions.assertThrows(Exception.class, () -> createTable(ddl)); + Assertions.assertTrue(error.getMessage().contains( + "FILE columns are not supported in internal tables"), error.getMessage()); + } + } + } + + @Test + public void testCreateTableLikeRejectsExistingFileSchema() throws Exception { + createTable("CREATE TABLE ddl_legacy (id INT, f STRING) DUPLICATE KEY(id) " + + "DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num'='1')"); + OlapTable table = (OlapTable) Env.getCurrentInternalCatalog().getDbOrAnalysisException("file_agg_state_ddl") + .getTableOrAnalysisException("ddl_legacy"); + // Simulate a FILE schema restored from metadata created before this DDL restriction. + table.getColumn("f").setType(Type.FILE); + String sql = "CREATE TABLE ddl_copy LIKE ddl_legacy"; + CreateTableLikeCommand command = (CreateTableLikeCommand) new NereidsParser().parseSingle(sql); + Exception error = Assertions.assertThrows(Exception.class, + () -> command.run(connectContext, new StmtExecutor(connectContext, sql))); + Assertions.assertTrue(error.getMessage().contains( + "FILE columns are not supported in internal tables"), error.getMessage()); + } + + @Test + public void testMaterializedViewRejectsFileColumns() throws Exception { + createTable("CREATE TABLE ddl_mv_source (id INT, f STRING, extra INT) DUPLICATE KEY(id) " + + "DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num'='1')"); + OlapTable source = (OlapTable) Env.getCurrentInternalCatalog().getDbOrAnalysisException("file_agg_state_ddl") + .getTableOrAnalysisException("ddl_mv_source"); + for (String sqlType : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP")) { + // Simulate a pre-existing FILE table; MV creation must not bypass the new schema restriction. + source.getColumn("f").setType(DataType.convertFromString(sqlType).toCatalogDataType()); + String syncSql = "CREATE MATERIALIZED VIEW ddl_sync_file AS SELECT id, f FROM ddl_mv_source ORDER BY id"; + createStatementCtx(syncSql); + CreateMaterializedViewCommand sync = (CreateMaterializedViewCommand) + new NereidsParser().parseSingle(syncSql); + sync.validate(connectContext); + Exception syncError = Assertions.assertThrows(Exception.class, + () -> Deencapsulation.invoke(new MaterializedViewHandler(), "checkAndPrepareMaterializedView", + sync, source, Collections.emptyMap())); + Assertions.assertTrue(syncError.getMessage().contains( + "FILE columns are not supported in internal tables"), syncError.getMessage()); + + String asyncSql = "CREATE MATERIALIZED VIEW ddl_async_file BUILD DEFERRED REFRESH COMPLETE ON MANUAL " + + "DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num'='1') " + + "AS SELECT id, f FROM ddl_mv_source"; + createStatementCtx(asyncSql); + CreateMTMVCommand async = (CreateMTMVCommand) new NereidsParser().parseSingle(asyncSql); + Exception asyncError = Assertions.assertThrows(Exception.class, + () -> async.getCreateMTMVInfo().analyze(connectContext)); + Assertions.assertTrue(asyncError.getMessage().contains( + "FILE columns are not supported in internal tables"), asyncError.getMessage()); + } + } + + @Test + public void testRejectKeyExternalAndNonReplaceAggregation() { + for (String sql : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP")) { + ColumnDefinition key = column(sql, true, null, Optional.empty()); + Assertions.assertThrows(Exception.class, () -> validate(key, true, KeysType.DUP_KEYS)); + ColumnDefinition external = column(sql, false, null, Optional.empty()); + Assertions.assertThrows(Exception.class, () -> validate(external, false, KeysType.DUP_KEYS)); + } + ColumnDefinition sum = column("FILE", false, AggregateType.SUM, Optional.empty()); + Assertions.assertThrows(Exception.class, () -> validate(sum, true, KeysType.AGG_KEYS)); + } + + @Test + public void testKeyEligibilityPreservesNestedFileRestrictions() { + for (String sql : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP")) { + Assertions.assertFalse(ColumnDefinition.isEligibleKeyType(DataType.convertFromString(sql))); + } + Assertions.assertTrue(ColumnDefinition.isEligibleKeyType(new AggStateType("sum", + Collections.singletonList(IntegerType.INSTANCE), Collections.singletonList(true)))); + Assertions.assertTrue(ColumnDefinition.isEligibleKeyType(IntegerType.INSTANCE)); + } + + @Test + public void testExternalTableRejectsFileAtNestedLeaf() { + for (String sql : Arrays.asList("ARRAY>", "STRUCT>>")) { + Exception error = Assertions.assertThrows(Exception.class, + () -> validate(column(sql, false, null, Optional.empty()), false, KeysType.DUP_KEYS)); + Assertions.assertEquals("FILE columns are not supported in external tables", error.getMessage()); + } + } + + @Test + public void testDefaultAndMapKeyRestrictions() { + ColumnDefinition nullable = column("FILE", false, null, Optional.of(DefaultValue.NULL_DEFAULT_VALUE)); + Assertions.assertThrows(AnalysisException.class, () -> validate(nullable, true, KeysType.DUP_KEYS)); + ColumnDefinition nonNullDefault = column("FILE", false, null, Optional.of(new DefaultValue("{}"))); + Assertions.assertThrows(Exception.class, () -> validate(nonNullDefault, true, KeysType.DUP_KEYS)); + for (String sql : Arrays.asList("MAP", "MAP,INT>")) { + DataType type = Assertions.assertDoesNotThrow(() -> DataType.convertFromString(sql)); + AnalysisException error = Assertions.assertThrows(AnalysisException.class, type::validateDataType); + Assertions.assertTrue(error.getMessage().startsWith("MAP key type must be a primitive type"), + error.getMessage()); + } + } + + @Test + public void testFileAggStateTypesAndAlterColumnValidation() { + TableNameInfo tableName = new TableNameInfo("file_agg_state_ddl", "ddl_base"); + for (String leaf : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP")) { + String sql = "AGG_STATE"; + DataType state = DataType.convertFromString(sql); + AnalysisException error = Assertions.assertThrows(AnalysisException.class, state::validateDataType); + Assertions.assertTrue(error.getMessage().contains("AGG_STATE does not support FILE"), error.getMessage()); + ColumnDefinition added = column(sql, false, null, Optional.empty()); + AnalysisException alterError = Assertions.assertThrows(AnalysisException.class, + () -> AddColumnOp.validateColumnDef(tableName, added, null, null)); + Assertions.assertTrue(alterError.getMessage().contains("AGG_STATE does not support FILE"), + alterError.getMessage()); + } + Assertions.assertDoesNotThrow(() -> DataType.convertFromString("AGG_STATE").validateDataType()); + for (String sql : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP")) { + AnalysisException error = Assertions.assertThrows(AnalysisException.class, + () -> AddColumnOp.validateColumnDef(tableName, column(sql, false, null, Optional.empty()), + null, null)); + Assertions.assertEquals("FILE columns are not supported in internal tables", error.getMessage()); + } + } + + @Test + public void testNullCastRejectsFileAggStateTargets() { + for (String leaf : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP")) { + String sql = "AGG_STATE"; + DataType state = DataType.convertFromString(sql); + for (boolean strict : Arrays.asList(false, true)) { + Assertions.assertFalse(CheckCast.check(NullType.INSTANCE, state, strict)); + } + Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze("select cast(null as " + sql + ")")); + } + Assertions.assertTrue(CheckCast.check(NullType.INSTANCE, + DataType.convertFromString("AGG_STATE"), true)); + Assertions.assertDoesNotThrow(() -> PlanChecker.from(connectContext) + .analyze("select cast(null as AGG_STATE)")); + } + + @Test + public void testSchemaChangePreservesFileIdentityAndAllowsNullableWidening() { + for (String sql : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP")) { + org.apache.doris.catalog.Type type = DataType.convertFromString(sql).toCatalogDataType(); + Column required = new Column("f", type, false); + Column nullable = new Column("f", type, true); + Assertions.assertDoesNotThrow(() -> required.checkSchemaChangeAllowed(nullable), sql); + Assertions.assertThrows(Exception.class, () -> nullable.checkSchemaChangeAllowed(required), sql); + Column string = new Column("f", org.apache.doris.catalog.Type.STRING, true); + Assertions.assertThrows(Exception.class, () -> nullable.checkSchemaChangeAllowed(string), sql); + Assertions.assertThrows(Exception.class, () -> string.checkSchemaChangeAllowed(nullable), sql); + } + Column file = new Column("f", org.apache.doris.catalog.Type.FILE, true); + for (String sql : Arrays.asList("JSON", "STRUCT")) { + Column other = new Column("f", DataType.convertFromString(sql).toCatalogDataType(), true); + Assertions.assertThrows(Exception.class, () -> file.checkSchemaChangeAllowed(other)); + Assertions.assertThrows(Exception.class, () -> other.checkSchemaChangeAllowed(file)); + } + for (String sql : Arrays.asList("ARRAY", "STRUCT", "MAP")) { + String fileSql = sql.replace("ARRAY", "ARRAY") + .replace("STRUCT", "STRUCT").replace("MAP", "MAP"); + Column nestedFile = new Column("f", DataType.convertFromString(fileSql).toCatalogDataType(), true); + Column nestedString = new Column("f", DataType.convertFromString(sql).toCatalogDataType(), true); + Assertions.assertThrows(Exception.class, () -> nestedFile.checkSchemaChangeAllowed(nestedString)); + Assertions.assertThrows(Exception.class, () -> nestedString.checkSchemaChangeAllowed(nestedFile)); + } + } + + @Test + public void testPartitionRejectsFileThroughComplexTypeRule() { + for (String sql : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP", + "ARRAY>", "ARRAY", "STRUCT", "MAP")) { + ColumnDefinition column = column(sql, true, null, Optional.empty()); + Assertions.assertTrue(column.getType().isComplexType(), sql); + Assertions.assertTrue(column.getType().toCatalogDataType().isComplexType(), sql); + for (boolean external : Arrays.asList(false, true)) { + PartitionTableInfo partition = new PartitionTableInfo(false, PartitionType.RANGE.name(), + null, Collections.singletonList(new UnboundSlot("f"))); + AnalysisException error = Assertions.assertThrows(AnalysisException.class, + () -> partition.validatePartitionInfo("olap", Collections.singletonList(column), + Collections.singletonMap("f", column), + Collections.emptyMap(), connectContext, false, external)); + Assertions.assertTrue(error.getMessage().startsWith("Complex type column can't be partition column"), + error.getMessage()); + } + } + } + + @Test + public void testRejectDistributionClusterAndIndexes() { + for (String sql : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP")) { + ColumnDefinition column = column(sql, false, null, Optional.empty()); + Assertions.assertThrows(Exception.class, () -> column.validate(true, Collections.emptySet(), + Collections.singleton("f"), false, KeysType.DUP_KEYS)); + DistributionDescriptor distribution = new DistributionDescriptor(true, false, 1, + Collections.singletonList("f")); + Assertions.assertThrows(Exception.class, + () -> distribution.validate(Collections.singletonMap("f", column), KeysType.DUP_KEYS)); + Assertions.assertThrows(Exception.class, + () -> HashDistributionInfo.checkDistributionColumnType("f", column.getType().toCatalogDataType())); + Assertions.assertFalse(IndexDefinition.isSupportIdxType(column.getType()), sql); + Assertions.assertFalse(new Column("f", column.getType().toCatalogDataType()).isSupportBloomFilter(), sql); + for (String indexType : Arrays.asList("INVERTED", "BLOOMFILTER", "NGRAM_BF")) { + if (!indexType.equals("BLOOMFILTER")) { + Assertions.assertThrows(Exception.class, () -> new IndexDefinition( + "idx", false, Collections.singletonList("f"), indexType, new HashMap<>(), "") + .checkColumn(column, KeysType.DUP_KEYS, false, TInvertedIndexFileStorageFormat.V2)); + } + Assertions.assertThrows(Exception.class, () -> new IndexDef("idx", false, Collections.singletonList("f"), + IndexDef.IndexType.valueOf(indexType), new HashMap<>(), "") + .checkColumn(new Column("f", column.getType().toCatalogDataType()), KeysType.DUP_KEYS, + false, TInvertedIndexFileStorageFormat.V2)); + } + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java new file mode 100644 index 00000000000000..9172aac9b57046 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java @@ -0,0 +1,799 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.types; + +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Function; +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.catalog.FunctionVolatility; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.exceptions.NotSupportedException; +import org.apache.doris.nereids.rules.analysis.CheckAnalysis; +import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; +import org.apache.doris.nereids.rules.expression.check.CheckCast; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Divide; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.InPredicate; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.ImplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.Udf; +import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue; +import org.apache.doris.nereids.trees.expressions.functions.agg.ArrayAgg; +import org.apache.doris.nereids.trees.expressions.functions.agg.CollectList; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.expressions.functions.agg.MapAgg; +import org.apache.doris.nereids.trees.expressions.functions.agg.MapAggV2; +import org.apache.doris.nereids.trees.expressions.functions.agg.Max; +import org.apache.doris.nereids.trees.expressions.functions.agg.MaxBy; +import org.apache.doris.nereids.trees.expressions.functions.agg.Min; +import org.apache.doris.nereids.trees.expressions.functions.agg.MinBy; +import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeFile; +import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeFileOuter; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayCompact; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayDistinct; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMax; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMin; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayReverseSort; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySortBy; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Coalesce; +import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateMap; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.FlGetBinary; +import org.apache.doris.nereids.trees.expressions.functions.scalar.If; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Nvl; +import org.apache.doris.nereids.trees.expressions.functions.udf.AliasUdf; +import org.apache.doris.nereids.trees.expressions.functions.udf.PythonUdaf; +import org.apache.doris.nereids.trees.expressions.functions.udf.PythonUdf; +import org.apache.doris.nereids.trees.expressions.functions.udf.PythonUdtf; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.MemoTestUtils; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.nereids.util.TypeCoercionUtils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TFunctionBinaryType; +import org.apache.doris.thrift.TResultSinkType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.function.Executable; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Locale; + +public class FileTypeExpressionTest { + private DataType type(String sql) { + return DataType.convertFromString(sql); + } + + private void assertAnalysisError(String expectedMessage, Executable executable, String... context) { + AnalysisException error = Assertions.assertThrows(AnalysisException.class, executable, + String.join(" ", context)); + Assertions.assertTrue(error.getMessage().contains(expectedMessage), + () -> String.join(" ", context) + ": expected " + expectedMessage + "; actual " + error.getMessage()); + } + + @Test + public void testFlightRejectsOnlyFileResultColumns() { + ConnectContext previous = ConnectContext.get(); + try { + ConnectContext flight = MemoTestUtils.createConnectContext(); + flight.setResultSinkType(TResultSinkType.ARROW_FLIGHT_PROTOCOL); + flight.setCurrentUserIdentity(UserIdentity.ROOT); + flight.setRemoteIP("127.0.0.1"); + flight.setEnv(Env.getCurrentEnv()); + flight.setThreadLocalInfo(); + String file = "cast(null as file)"; + for (String value : Arrays.asList(file, "array(" + file + ")", + "named_struct('f', " + file + ")", "map('key', " + file + ")")) { + String sql = "select " + value; + assertAnalysisError("Arrow Flight does not support FILE result columns", + () -> PlanChecker.from(flight).analyze(sql), sql); + Assertions.assertDoesNotThrow( + () -> PlanChecker.from(MemoTestUtils.createConnectContext()).analyze(sql), sql); + flight.setThreadLocalInfo(); + } + for (String sql : Arrays.asList("select 1", "select named_struct('s', 'text')", + "select element_at(" + file + ", 'uri')", + "select to_json(" + file + ")")) { + Assertions.assertDoesNotThrow(() -> PlanChecker.from(flight).analyze(sql), sql); + } + } finally { + if (previous == null) { + ConnectContext.remove(); + } else { + previous.setThreadLocalInfo(); + } + } + } + + @Test + public void testFileChecksDoNotReconstructPersistedDecimalArithmeticTypes() { + ConnectContext previous = ConnectContext.get(); + try { + ConnectContext context = MemoTestUtils.createConnectContext(); + context.getSessionVariable().enableDecimal256 = false; + Divide divide = new Divide( + new SlotReference("wide", DecimalV3Type.createDecimalV3TypeNotCheck256(52, 20)), + new SlotReference("divisor", DecimalV3Type.createDecimalV3Type(38, 10))); + Alias alias = new Alias(divide, "quotient"); + LogicalOneRowRelation relation = new LogicalOneRowRelation(StatementScopeIdGenerator.newRelationId(), + Collections.singletonList(alias)); + Assertions.assertThrows(NotSupportedException.class, divide::getDataType); + Assertions.assertDoesNotThrow(divide::checkInputDataTypes); + Assertions.assertDoesNotThrow(alias::checkInputDataTypes); + Assertions.assertDoesNotThrow(() -> CheckAnalysis.checkFilePlan(relation)); + Assertions.assertFalse(context.getSessionVariable().isEnableDecimal256()); + } finally { + if (previous == null) { + ConnectContext.remove(); + } else { + previous.setThreadLocalInfo(); + } + } + } + + @Test + public void testFileChecksDoNotComputeUncoercedNonFileResults() { + for (String function : Arrays.asList("array_max", "array_min")) { + Assertions.assertDoesNotThrow(() -> PlanChecker.from(MemoTestUtils.createConnectContext()) + .analyze("select " + function + "(null)")); + } + } + + @Test + public void testFunctionsKeepTheirNativeTypeRestrictions() { + for (String expression : Arrays.asList("array_distinct(files)", "array_sort(files)")) { + String expectedMessage; + if (expression.startsWith("array_distinct(")) { + expectedMessage = "array_distinct does not support element type"; + } else { + expectedMessage = "array_sort does not support types: ARRAY"; + } + assertAnalysisError(expectedMessage, + () -> PlanChecker.from(MemoTestUtils.createConnectContext()) + .analyze("select " + expression + " from (select 1 id, cast(null as file) f, " + + "cast(null as array) files) t"), expression); + } + for (String expression : Arrays.asList("first_value(f) over(order by id)", "last_value(f) over(order by id)", + "nth_value(f, 1) over(order by id)", "array_sortby(files, array(id))")) { + Assertions.assertDoesNotThrow(() -> PlanChecker.from(MemoTestUtils.createConnectContext()) + .analyze("select " + expression + " from (select 1 id, cast(null as file) f, " + + "cast(null as array) files) t"), expression); + } + } + + @Test + public void testArrayComparisonFunctionsRejectNestedFile() { + for (String sqlType : Arrays.asList("ARRAY", "ARRAY>", + "ARRAY>", "ARRAY>")) { + Expression array = new SlotReference("files", type(sqlType)); + for (BoundFunction function : Arrays.asList(new ArrayReverseSort(array), new ArrayCompact(array))) { + String expected = function.getName() + " does not support type"; + assertAnalysisError(expected, () -> TypeCoercionUtils.processBoundFunction(function), sqlType); + String sql = "select " + function.getName() + "(files) from (select cast(null as " + + sqlType + ") files) t"; + assertAnalysisError(expected, + () -> PlanChecker.from(MemoTestUtils.createConnectContext()).analyze(sql), sql); + } + } + for (String sqlType : Arrays.asList("ARRAY", "ARRAY>")) { + Expression array = new SlotReference("values", type(sqlType)); + for (BoundFunction function : Arrays.asList(new ArrayReverseSort(array), new ArrayCompact(array))) { + Assertions.assertDoesNotThrow(() -> TypeCoercionUtils.processBoundFunction(function), + function.getName() + " " + sqlType); + } + } + } + + @Test + public void testHigherOrderFunctionsUseNativeSignatures() { + String from = " from (select 1 id, cast(null as file) f, cast(null as array) files) t"; + for (String expression : Arrays.asList("array_map(x -> x, files)", + "array_map(x -> element_at(x,'size'), files)", + "array_filter(x -> element_at(x,'size') > 0, files)", + "array_map(x -> f, array(id))", "array_map((x,y) -> x, array(id), files)")) { + Assertions.assertDoesNotThrow(() -> PlanChecker.from(MemoTestUtils.createConnectContext()) + .analyze("select " + expression + from), expression); + } + for (String expression : Arrays.asList("array_map(x -> x + 1, array(id))", + "array_map(x -> element_at(f,'size') + x, array(id))")) { + Assertions.assertDoesNotThrow(() -> PlanChecker.from(MemoTestUtils.createConnectContext()) + .analyze("select " + expression + from), expression); + } + } + + @Test + public void testAliasFunctionsUseDeclaredSignatures() { + for (String sqlType : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP")) { + AliasUdf alias = new AliasUdf("file_alias", Collections.singletonList(type(sqlType)), + new NullLiteral(), Collections.singletonList("f"), Collections.emptyMap(), new NullLiteral()); + Assertions.assertDoesNotThrow(() -> TypeCoercionUtils.processBoundFunction(alias), sqlType); + } + AliasUdf ordinary = new AliasUdf("ordinary_alias", Collections.singletonList(BigIntType.INSTANCE), + new NullLiteral(), Collections.singletonList("n"), Collections.emptyMap(), new NullLiteral()); + Assertions.assertDoesNotThrow(() -> TypeCoercionUtils.processBoundFunction(ordinary)); + } + + @Test + public void testPublicFieldSelectors() { + Expression file = new SlotReference("f", FileType.INSTANCE); + Assertions.assertEquals(BigIntType.INSTANCE, + TypeCoercionUtils.processBoundFunction(new ElementAt(file, new StringLiteral("SIZE"))).getDataType()); + Assertions.assertTrue(new ElementAt(file, new StringLiteral("uri")).nullable()); + for (String selector : Arrays.asList("inline", "INLINE")) { + assertAnalysisError("use FL_GET_BINARY", + () -> TypeCoercionUtils.processBoundFunction(new ElementAt(file, new StringLiteral(selector)))); + } + for (Expression selector : Arrays.asList(new StringLiteral("path"), + new IntegerLiteral(1), new SlotReference("s", StringType.INSTANCE))) { + assertAnalysisError(selector instanceof StringLiteral + ? "the specified field name path was not found" + : "element_at over FILE requires a constant public field name", + () -> TypeCoercionUtils.processBoundFunction(new ElementAt(file, selector))); + } + } + + @Test + public void testFileJsonCastsAreUnsupported() { + for (boolean strict : Arrays.asList(true, false)) { + for (String wrapper : Arrays.asList("%s", "ARRAY<%s>", "STRUCT", "MAP")) { + DataType file = type(String.format(wrapper, "FILE")); + DataType json = type(String.format(wrapper, "JSON")); + Assertions.assertFalse(CheckCast.check(file, json, strict)); + Assertions.assertFalse(CheckCast.check(json, file, strict)); + Assertions.assertFalse(CheckCast.check(file, JsonType.INSTANCE, strict)); + Assertions.assertFalse(CheckCast.check(JsonType.INSTANCE, file, strict)); + } + } + } + + @Test + public void testToJsonAcceptsFile() { + for (String value : Arrays.asList("cast(null as file)", "array(cast(null as file))", + "named_struct('f', cast(null as file))", "map('f', cast(null as file))")) { + Plan plan = PlanChecker.from(MemoTestUtils.createConnectContext()) + .analyze("select to_json(" + value + ")").getPlan(); + Assertions.assertEquals(JsonType.INSTANCE, plan.getOutput().get(0).getDataType()); + } + } + + @Test + public void testFileCastUsesOrdinaryTypeChecksAndNullLiteralCoercion() { + Assertions.assertFalse(Cast.castNullable(false, FileType.INSTANCE.publicStructType(), FileType.INSTANCE)); + Assertions.assertTrue(Cast.castNullable(true, FileType.INSTANCE.publicStructType(), FileType.INSTANCE)); + Assertions.assertTrue(Cast.castNullable(false, JsonType.INSTANCE, FileType.INSTANCE)); + Assertions.assertFalse(CheckCast.check(JsonType.INSTANCE, FileType.INSTANCE, true)); + Expression converted = TypeCoercionUtils.castIfNotSameType(new NullLiteral(FileType.INSTANCE), + JsonType.INSTANCE); + Assertions.assertTrue(converted instanceof NullLiteral); + Assertions.assertEquals(JsonType.INSTANCE, converted.getDataType()); + assertAnalysisError("can not cast", () -> TypeCoercionUtils.castIfNotSameType( + new SlotReference("s", FileType.INSTANCE.publicStructType()), FileType.INSTANCE)); + Assertions.assertFalse(CheckCast.check(IntegerType.INSTANCE, FileType.INSTANCE, false)); + } + + @Test + public void testFileCastsAreUnsupported() { + String metadata = "uri:STRING,offset:BIGINT,size:BIGINT,content_type:STRING,checksum:STRING"; + String fields = metadata + ",inline:VARBINARY"; + for (boolean strict : Arrays.asList(true, false)) { + for (String source : Arrays.asList("STRUCT<" + fields + ">", + "STRUCT")) { + DataType value = type(source); + Assertions.assertFalse(CheckCast.check(value, FileType.INSTANCE, strict), source); + Assertions.assertFalse(CheckCast.check(FileType.INSTANCE, value, strict), source); + Assertions.assertTrue(TypeCoercionUtils.implicitCast(value, FileType.INSTANCE).isPresent()); + Assertions.assertTrue(TypeCoercionUtils.implicitCast(FileType.INSTANCE, value).isPresent()); + } + for (String source : Arrays.asList("STRING", "JSON", "VARIANT", "INT", "STRUCT", + "STRUCT<" + metadata + ">", + "STRUCT<" + metadata + ",inline:STRING>", + "STRUCT<" + fields + ",extra:STRING>", + "STRUCT<" + metadata + ",payload:VARBINARY>", + "STRUCT")) { + Assertions.assertFalse(CheckCast.check(type(source), FileType.INSTANCE, strict), source); + Assertions.assertFalse(CheckCast.check(FileType.INSTANCE, type(source), strict), source); + } + Assertions.assertFalse(CheckCast.check(type("ARRAY>"), + type("ARRAY"), strict)); + for (String wrapper : Arrays.asList("%s", "ARRAY<%s>", "MAP", "STRUCT", + "ARRAY>")) { + DataType source = type(String.format(wrapper, "STRUCT<" + fields + ">")); + DataType target = type(String.format(wrapper, "FILE")); + Assertions.assertFalse(CheckCast.check(source, target, strict), wrapper); + Assertions.assertFalse(CheckCast.check(target, source, strict), wrapper); + } + Assertions.assertTrue(CheckCast.check(StringType.INSTANCE, type("ARRAY>"), strict)); + Assertions.assertFalse(CheckCast.check(StringType.INSTANCE, type("ARRAY"), strict)); + } + StructType nullableInline = new StructType(Arrays.asList( + new StructField("uri", StringType.INSTANCE, false, ""), + new StructField("offset", BigIntType.INSTANCE, true, ""), + new StructField("size", BigIntType.INSTANCE, true, ""), + new StructField("content_type", StringType.INSTANCE, true, ""), + new StructField("checksum", StringType.INSTANCE, true, ""), + new StructField("inline", NullType.INSTANCE, true, ""))); + Assertions.assertFalse(CheckCast.check(nullableInline, FileType.INSTANCE, true)); + Assertions.assertFalse(CheckCast.check(nullableInline, FileType.INSTANCE, false)); + Assertions.assertFalse(CheckCast.check(FileType.INSTANCE, nullableInline, true)); + StructType requiredInline = new StructType(FileType.INSTANCE.getFields().stream() + .map(field -> field.getName().equals("inline") + ? new StructField("inline", VarBinaryType.INSTANCE, false, "") : field) + .collect(java.util.stream.Collectors.toList())); + Assertions.assertFalse(CheckCast.check(requiredInline, FileType.INSTANCE, true)); + Assertions.assertFalse(CheckCast.check(FileType.INSTANCE, requiredInline, true)); + Assertions.assertFalse(CheckCast.check(FileType.INSTANCE, requiredInline, false)); + Assertions.assertTrue(TypeCoercionUtils.implicitCast(NullType.INSTANCE, FileType.INSTANCE).isPresent()); + Assertions.assertTrue(TypeCoercionUtils.implicitCast(FileType.INSTANCE, FileType.INSTANCE).isPresent()); + } + + @Test + public void testNestedFileCastsRejectStringAndWholeVariantContainers() { + for (boolean strict : Arrays.asList(true, false)) { + for (String container : Arrays.asList("ARRAY", "MAP", "STRUCT", + "ARRAY>", "STRUCT>", "MAP>")) { + DataType nested = type(container); + for (String source : Arrays.asList("STRING", "VARCHAR(20)", "CHAR(20)", "VARIANT")) { + Assertions.assertFalse(CheckCast.check(type(source), nested, strict), source + " -> " + container); + } + Assertions.assertFalse(CheckCast.check(nested, VariantType.INSTANCE, strict), + container + " -> VARIANT"); + Assertions.assertFalse(CheckCast.check(nested, StringType.INSTANCE, strict), + container + " -> STRING"); + Assertions.assertTrue(CheckCast.check(NullType.INSTANCE, nested, strict), "NULL -> " + container); + Assertions.assertTrue(CheckCast.check(nested, nested, strict), container); + } + for (String wrapper : Arrays.asList("ARRAY<%s>", "MAP", "STRUCT")) { + for (String leaf : Arrays.asList("STRING", "VARIANT")) { + Assertions.assertFalse(CheckCast.check(type(String.format(wrapper, leaf)), + type(String.format(wrapper, "ARRAY")), strict), wrapper + leaf); + } + Assertions.assertFalse(CheckCast.check(type(String.format(wrapper, "VARIANT")), + type(String.format(wrapper, "FILE")), strict), wrapper); + Assertions.assertFalse(CheckCast.check(type(String.format(wrapper, "FILE")), + type(String.format(wrapper, "VARIANT")), strict), wrapper); + } + Assertions.assertTrue(CheckCast.check(VariantType.INSTANCE, type("ARRAY"), strict)); + } + } + + @Test + public void testConditionalFunctionsAcceptFileAndRejectMixedTypes() { + Expression file = new SlotReference("f", FileType.INSTANCE); + Expression structured = new SlotReference("s", type("STRUCT")); + assertAnalysisError("Can not find the compatibility function signature:", + () -> TypeCoercionUtils.processBoundFunction(new If(BooleanLiteral.TRUE, file, structured))); + assertAnalysisError("Can not find the compatibility function signature:", + () -> TypeCoercionUtils.processBoundFunction(new Coalesce(structured, file))); + for (DataType valueType : Arrays.asList(FileType.INSTANCE, type("ARRAY"), + type("STRUCT"), type("MAP"))) { + Expression value = new SlotReference("v", valueType); + for (BoundFunction function : Arrays.asList(new Coalesce(new NullLiteral(), value), + new Coalesce(value, new NullLiteral(valueType)), + new Coalesce(new NullLiteral(valueType), new NullLiteral()), + new If(BooleanLiteral.TRUE, value, value), + new If(BooleanLiteral.TRUE, new NullLiteral(), value), + new If(BooleanLiteral.FALSE, new NullLiteral(valueType), new NullLiteral()), + new Nvl(value, new NullLiteral()), new Nvl(new NullLiteral(), value))) { + Assertions.assertEquals(valueType, + TypeCoercionUtils.processBoundFunction(function).getDataType(), function.toSql()); + } + } + } + + @Test + public void testSqlFileValuePropagationAndContainerConstruction() { + String from = " from (select 1 id, cast(null as file) f, cast(null as array) files, " + + "cast(null as struct) holder, cast(null as map) lookup) t"; + for (String expression : Arrays.asList("array(f, null)", "struct(f)", "named_struct('asset', f)", + "map('asset', f)", "array(holder, null)", "map('asset', files)", + "reverse(files)", "array_slice(files, 1, 1)", "array_concat(files, files)", + "map_values(lookup)", "map_keys(lookup)", "array_size(files)", "map_size(lookup)", + "cardinality(files)", "cardinality(lookup)", + "case when id > 0 then f else null end")) { + Assertions.assertDoesNotThrow(() -> PlanChecker.from(MemoTestUtils.createConnectContext()) + .analyze("select " + expression + from), expression); + } + for (String expression : Arrays.asList("map('asset', f) = map('asset', f)", + "array_distinct(array(f))")) { + String expectedMessage; + if (expression.contains(" = ")) { + expectedMessage = "comparison predicate could not contains complex type"; + } else { + expectedMessage = "array_distinct does not support element type"; + } + assertAnalysisError(expectedMessage, + () -> PlanChecker.from(MemoTestUtils.createConnectContext()) + .analyze("select " + expression + from), expression); + } + String file = "cast(null as file)"; + for (String expression : Arrays.asList("coalesce(cast(null as file), " + file + ")", + "if(true, " + file + ", null)", "ifnull(null, " + file + ")", + "fl_get_binary(element_at(array(" + file + ", null), 1))", + "fl_get_binary(element_at(named_struct('asset', " + file + "), 'asset'))", + "fl_get_binary(element_at(map('asset', " + file + "), 'asset'))")) { + Assertions.assertDoesNotThrow(() -> PlanChecker.from(MemoTestUtils.createConnectContext()) + .analyze("select " + expression), expression); + } + } + + @Test + public void testRecursiveComparisonGates() { + for (String sql : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP")) { + Expression value = new SlotReference("v", type(sql)); + assertAnalysisError(value.getDataType().isArrayType() + ? "data type " + sql + " could not used in ComparisonPredicate" + : "comparison predicate could not contains complex type", + () -> TypeCoercionUtils.processComparisonPredicate(new EqualTo(value, value))); + assertAnalysisError("does not support FILE", + () -> TypeCoercionUtils.processInPredicate( + new InPredicate(value, Collections.singletonList(value)))); + assertAnalysisError("does not support FILE", + () -> ExpressionUtils.checkFileKey(value, "ORDER BY")); + } + } + + @Test + public void testCreateMapRejectsFileKeys() { + Expression value = new IntegerLiteral(1); + for (String sqlType : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP")) { + Expression key = new SlotReference("key", type(sqlType)); + assertAnalysisError("MAP key does not support FILE", + () -> new CreateMap(key, value).checkLegalityBeforeTypeCoercion(), sqlType); + assertAnalysisError("MAP key does not support FILE", + () -> new CreateMap(new StringLiteral("ordinary"), value, key, value) + .checkLegalityBeforeTypeCoercion(), sqlType); + } + } + + @Test + public void testCreateMapAllowsFileValues() { + Expression key = new StringLiteral("asset"); + for (String sqlType : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP")) { + Expression value = new SlotReference("value", type(sqlType)); + Assertions.assertDoesNotThrow(() -> new CreateMap(key, value).checkLegalityBeforeTypeCoercion(), sqlType); + } + } + + @Test + public void testCreateMapRejectsComplexKeysAndAllowsPrimitiveKeys() { + Expression value = new IntegerLiteral(1); + for (String sqlType : Arrays.asList("ARRAY", "STRUCT", "MAP")) { + Expression key = new SlotReference("key", type(sqlType)); + assertAnalysisError("MAP key type must be a primitive type", + () -> new CreateMap(key, value).checkLegalityBeforeTypeCoercion(), sqlType); + } + for (String sqlType : Arrays.asList("INT", "BIGINT", "STRING")) { + Expression key = new SlotReference("key", type(sqlType)); + Assertions.assertDoesNotThrow(() -> new CreateMap(key, value).checkLegalityBeforeTypeCoercion(), sqlType); + } + } + + @Test + public void testInPredicateRejectsStructsAndAllowsPrimitiveTypes() { + for (String sqlType : Arrays.asList("STRUCT", "STRUCT>", + "STRUCT", "STRUCT>")) { + Expression value = new SlotReference("s", type(sqlType)); + InPredicate predicate = new InPredicate(value, Collections.singletonList(value)); + assertAnalysisError("in predicate could not contains complex type", + () -> TypeCoercionUtils.processInPredicate(predicate), sqlType); + } + for (String sqlType : Arrays.asList("INT", "BIGINT", "STRING")) { + Expression value = new SlotReference("s", type(sqlType)); + InPredicate predicate = new InPredicate(value, Collections.singletonList(value)); + Assertions.assertSame(predicate, TypeCoercionUtils.processInPredicate(predicate), sqlType); + } + } + + @Test + public void testInPredicateRejectsFileInAnyArgument() { + for (String sqlType : Arrays.asList("FILE", "ARRAY", "STRUCT", + "STRUCT>", "MAP")) { + Expression value = new SlotReference("f", type(sqlType)); + assertAnalysisError("IN does not support FILE", + () -> TypeCoercionUtils.processInPredicate( + new InPredicate(value, Collections.singletonList(value))), sqlType); + assertAnalysisError("IN does not support FILE", + () -> TypeCoercionUtils.processInPredicate( + new InPredicate(new NullLiteral(), Collections.singletonList(value))), sqlType); + } + } + + @Test + public void testElementAtAndExplodeFields() { + Expression file = new SlotReference("f", FileType.INSTANCE); + for (StructField field : FileType.INSTANCE.getFields()) { + if (field.getName().equals("inline")) { + continue; + } + Expression accessed = ExpressionAnalyzer.analyzeFunction(null, null, + new ElementAt(file, new StringLiteral(field.getName().toUpperCase(Locale.ROOT)))); + Assertions.assertTrue(accessed instanceof ElementAt); + Assertions.assertEquals(field.getName().toUpperCase(Locale.ROOT), + ((StringLiteral) accessed.child(1)).getStringValue()); + Assertions.assertEquals(field.getDataType(), accessed.getDataType()); + Assertions.assertTrue(accessed.nullable()); + } + FlGetBinary binary = new FlGetBinary(new SlotReference("required", FileType.INSTANCE, false)); + Assertions.assertEquals(VarBinaryType.INSTANCE, binary.getDataType()); + Assertions.assertTrue(binary.nullable()); + for (BoundFunction explode : Arrays.asList(new ExplodeFile(file), new ExplodeFileOuter(file))) { + Assertions.assertEquals(FileType.INSTANCE.publicStructType(), + TypeCoercionUtils.processBoundFunction(explode).getDataType()); + } + SlotReference struct = new SlotReference("s", FileType.INSTANCE.publicStructType()); + Expression accessed = TypeCoercionUtils.processBoundFunction(new ElementAt(struct, new StringLiteral("INLINE"))); + Assertions.assertTrue(accessed instanceof ElementAt); + Assertions.assertSame(struct, accessed.child(0)); + Assertions.assertEquals(VarBinaryType.INSTANCE, accessed.getDataType()); + Assertions.assertEquals("INLINE", ((StringLiteral) accessed.child(1)).getStringValue()); + } + + @Test + public void testFunctionParameterAndAssignmentBoundaries() { + DataType struct = type("STRUCT"); + for (DataType source : Arrays.asList(struct, JsonType.INSTANCE)) { + Assertions.assertFalse(ImplicitlyCastableSignature.isImplicitlyCastable(FileType.INSTANCE, source)); + Assertions.assertFalse(ExplicitlyCastableSignature.isExplicitlyCastable(FileType.INSTANCE, source)); + } + Assertions.assertEquals(ExplicitlyCastableSignature.isExplicitlyCastable( + StringType.INSTANCE, FileType.INSTANCE.publicStructType()), + ExplicitlyCastableSignature.isExplicitlyCastable(StringType.INSTANCE, FileType.INSTANCE)); + Assertions.assertFalse(ImplicitlyCastableSignature.isImplicitlyCastable( + FileType.INSTANCE, StringType.INSTANCE)); + Assertions.assertTrue(ImplicitlyCastableSignature.isImplicitlyCastable( + type("STRUCT"), type("STRUCT"))); + Assertions.assertTrue(ExplicitlyCastableSignature.isExplicitlyCastable( + type("STRUCT"), type("STRUCT"))); + Assertions.assertTrue(ImplicitlyCastableSignature.isImplicitlyCastable( + type("STRUCT"), type("STRUCT>"))); + assertAnalysisError("can not cast from origin type FILE to target type=TEXT", + () -> TypeCoercionUtils.castIfNotSameType( + new SlotReference("f", FileType.INSTANCE), StringType.INSTANCE)); + } + + @Test + public void testGenericAggregatesUseNativeSignatures() { + Expression key = new IntegerLiteral(1); + for (String sqlType : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP")) { + Expression file = new SlotReference("f", type(sqlType)); + for (BoundFunction function : Arrays.asList(new Count(file), new AnyValue(file), new ArrayAgg(file), + new CollectList(file), new MinBy(file, key), new MaxBy(file, key), new MapAggV2(key, file))) { + Assertions.assertDoesNotThrow(() -> TypeCoercionUtils.processBoundFunction(function), + function.getName() + " " + sqlType); + } + assertAnalysisError("COUNT DISTINCT could not process type", + () -> new Count(true, file).checkLegalityAfterRewrite(), sqlType); + assertAnalysisError("array_distinct does not support element type", + () -> TypeCoercionUtils.processBoundFunction( + new ArrayDistinct(new SlotReference("a", ArrayType.of(file.getDataType())))), sqlType); + } + Expression file = new SlotReference("f", FileType.INSTANCE); + assertAnalysisError("column must use with specific function", + () -> TypeCoercionUtils.processBoundFunction(new Min(file))); + assertAnalysisError("column must use with specific function", + () -> TypeCoercionUtils.processBoundFunction(new MinBy(key, file))); + } + + @Test + public void testFileComparatorFunctionsRejectNestedKeys() { + Expression key = new IntegerLiteral(1); + for (String sqlType : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP")) { + Expression file = new SlotReference("f", type(sqlType)); + for (BoundFunction function : Arrays.asList(new Min(file), new Max(file), + new MinBy(key, file), new MaxBy(key, file))) { + assertAnalysisError("column must use with specific function", + () -> TypeCoercionUtils.processBoundFunction(function), function.getName() + " " + sqlType); + } + Expression files = new SlotReference("files", ArrayType.of(file.getDataType())); + assertAnalysisError("array_sortby key does not support FILE", + () -> TypeCoercionUtils.processBoundFunction(new ArraySortBy( + new SlotReference("values", ArrayType.of(IntegerType.INSTANCE)), files)), sqlType); + for (BoundFunction function : Arrays.asList(new MapAgg(file, key), new MapAggV2(file, key))) { + assertAnalysisError("MAP_AGG key does not support FILE", + () -> TypeCoercionUtils.processBoundFunction(function), sqlType); + } + assertAnalysisError("array_min does not support complex types", + () -> TypeCoercionUtils.processBoundFunction(new ArrayMin(files)), sqlType); + assertAnalysisError("array_max does not support complex types", + () -> TypeCoercionUtils.processBoundFunction(new ArrayMax(files)), sqlType); + } + } + + @Test + public void testGenericAggregateSqlAdmission() { + for (String sqlType : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP")) { + String input = " from (select cast(null as " + sqlType + ") f) t"; + for (String aggregate : Arrays.asList("count(f)", "count(*)", "any_value(f)", "array_agg(f)", + "collect_list(f)", "group_array(f)", "min_by(f, 1)", "max_by(f, 1)", "map_agg(1, f)")) { + Assertions.assertDoesNotThrow(() -> PlanChecker.from(MemoTestUtils.createConnectContext()) + .analyze("select " + aggregate + input), aggregate + " " + sqlType); + } + } + } + + @Test + public void testPythonFunctionsUseDeclaredSignaturesAtBinding() { + for (DataType fileType : Arrays.asList(FileType.INSTANCE, type("ARRAY"), + type("STRUCT"), type("MAP"))) { + for (String kind : Arrays.asList("scalar", "aggregate", "table")) { + Assertions.assertDoesNotThrow(() -> TypeCoercionUtils.processBoundFunction( + pythonFunction(kind, fileType, IntegerType.INSTANCE, IntegerType.INSTANCE))); + Assertions.assertDoesNotThrow(() -> TypeCoercionUtils.processBoundFunction( + (BoundFunction) pythonFunction(kind, fileType, IntegerType.INSTANCE, IntegerType.INSTANCE) + .withChildren(Collections.singletonList(new NullLiteral())))); + Assertions.assertDoesNotThrow(() -> TypeCoercionUtils.processBoundFunction( + pythonFunction(kind, IntegerType.INSTANCE, fileType, IntegerType.INSTANCE))); + } + Assertions.assertDoesNotThrow(() -> TypeCoercionUtils.processBoundFunction( + pythonFunction("aggregate", IntegerType.INSTANCE, IntegerType.INSTANCE, fileType))); + } + } + + @Test + public void testPythonFunctionsStillAcceptOrdinaryTypesAtBinding() { + for (DataType type : Arrays.asList(IntegerType.INSTANCE, StringType.INSTANCE, type("ARRAY"), + type("STRUCT"), type("MAP"))) { + for (String kind : Arrays.asList("scalar", "aggregate", "table")) { + Assertions.assertDoesNotThrow(() -> TypeCoercionUtils.processBoundFunction( + pythonFunction(kind, type, type, type))); + } + Assertions.assertDoesNotThrow(() -> TypeCoercionUtils.processBoundFunction( + pythonFunction("aggregate", type, type, null))); + } + } + + private BoundFunction pythonFunction(String kind, DataType argumentType, DataType returnType, + DataType intermediateType) { + Expression argument = new SlotReference("v", argumentType); + FunctionSignature signature = FunctionSignature.ret( + kind.equals("table") ? ArrayType.of(returnType) : returnType).args(argumentType); + if (kind.equals("aggregate")) { + return new PythonUdaf("py_agg", 1, "db1", TFunctionBinaryType.PYTHON_UDF, + signature, intermediateType, Function.NullableMode.ALWAYS_NULLABLE, + FunctionVolatility.IMMUTABLE, Udf.createVolatileIdentity(FunctionVolatility.IMMUTABLE), + null, "Agg", null, null, null, null, null, null, null, false, "", false, 360, + "3.10.2", "", argument); + } + if (kind.equals("table")) { + return new PythonUdtf("py_table", 1, "db1", TFunctionBinaryType.PYTHON_UDF, + signature, Function.NullableMode.ALWAYS_NULLABLE, FunctionVolatility.IMMUTABLE, + Udf.createVolatileIdentity(FunctionVolatility.IMMUTABLE), + null, "evaluate", null, null, "", false, 360, "3.10.2", "", argument); + } + return new PythonUdf("py_scalar", 1, "db1", TFunctionBinaryType.PYTHON_UDF, + signature, Function.NullableMode.ALWAYS_NULLABLE, FunctionVolatility.IMMUTABLE, + Udf.createVolatileIdentity(FunctionVolatility.IMMUTABLE), + null, "evaluate", null, null, "", false, 360, "3.10.2", "", argument); + } + + @Test + public void testSqlValueTransportAndExplodeAliases() { + for (String sql : Arrays.asList( + "select element_at(cast(null as file), 'uri')", + "select fl_get_binary(cast(null as file))", + "select fl_get_binary(null)", + "select element_at(cast(null as file), 'SIZE')", + "select cast(null as file) union all select null", + "select cast(null as array) union all " + + "select array(cast(null as file))", + "select count(f), count(*) " + + "from (select cast(null as file) f) t", + "select any_value(distinct f) from (select cast(null as file) f) t", + "select element_at(f, 'uri'), count(*) from (select cast(null as file) f) t " + + "group by element_at(f, 'uri')", + "select u, o, s, c, h, b from (select cast(null as file) f) t " + + "lateral view explode_file(f) e as u, o, s, c, h, b", + "select u, o, s, c, h, b from (select cast(null as file) f) t " + + "lateral view explode_file_outer(f) e as u, o, s, c, h, b")) { + Assertions.assertDoesNotThrow( + () -> PlanChecker.from(MemoTestUtils.createConnectContext()).analyze(sql), sql); + } + } + + @Test + public void testInlineAndExplodeSqlOutputSchema() { + Plan getters = PlanChecker.from(MemoTestUtils.createConnectContext()).analyze( + "select fl_get_binary(f), FL_GET_BINARY(f) " + + "from (select cast(null as file) f) t").getPlan(); + Assertions.assertEquals(2, getters.getOutput().size()); + getters.getOutput().forEach(output -> { + Assertions.assertEquals(VarBinaryType.INSTANCE, output.getDataType()); + Assertions.assertTrue(output.nullable()); + }); + for (String function : Arrays.asList("explode_file", "explode_file_outer")) { + Plan expanded = PlanChecker.from(MemoTestUtils.createConnectContext()).analyze( + "select u, o, s, c, h, b from (select cast(null as file) f) t lateral view " + + function + "(f) e as u, o, s, c, h, b").getPlan(); + Assertions.assertEquals(6, expanded.getOutput().size()); + Assertions.assertEquals(VarBinaryType.INSTANCE, expanded.getOutput().get(5).getDataType()); + Assertions.assertTrue(expanded.getOutput().stream().allMatch(Expression::nullable)); + } + } + + @Test + public void testUnionUsesExplicitFileTargets() { + for (String sql : Arrays.asList( + "select cast(null as file) union all select null", + "select null union all select cast(null as file) union all select null")) { + Plan plan = PlanChecker.from(MemoTestUtils.createConnectContext()).analyze(sql).getPlan(); + Assertions.assertEquals(FileType.INSTANCE, plan.getOutput().get(0).getDataType()); + } + Assertions.assertDoesNotThrow(() -> PlanChecker.from(MemoTestUtils.createConnectContext()) + .analyze("select 1 union all select 2.5 union all select null union all select 3")); + assertAnalysisError("Can not find compatible type", () -> PlanChecker.from(MemoTestUtils.createConnectContext()) + .analyze("select cast(null as file) union all select named_struct('uri', 'urn:x', 'offset', " + + "cast(0 as bigint), 'size', cast(0 as bigint), 'content_type', '', 'checksum', '', " + + "'inline', null)")); + } + + @Test + public void testSqlOperatorRejections() { + for (String sql : Arrays.asList( + "select cast(null as file) = cast(null as file)", + "select cast(null as struct) in (cast(null as struct))", + "select f from (select cast(null as file) f) t order by f", + "select f from (select cast(null as file) f) t group by f", + "select cast(null as file) union select cast(null as file)", + "select cast(null as file) intersect select cast(null as file)", + "select min(cast(null as file))", + "select element_at(cast(null as file), 'missing')", + "select u from (select cast(null as file) f) t lateral view explode_file(f) e as u", + "select u from (select cast(null as file) f) t lateral view explode_file(f) e as u, o, s, c, h", + "select u from (select cast(null as file) f) t " + + "lateral view explode_file_outer(f) e as u, o, s, c, h")) { + String expectedMessage = "does not support FILE"; + if (sql.contains("lateral view")) { + expectedMessage = "EXPLODE_FILE requires exactly six column aliases"; + } else if (sql.contains("missing")) { + expectedMessage = "the specified field name missing was not found"; + } else if (sql.contains(" = ")) { + expectedMessage = "comparison predicate could not contains complex type"; + } else if (sql.startsWith("select min(")) { + expectedMessage = "column must use with specific function"; + } + assertAnalysisError(expectedMessage, + () -> PlanChecker.from(MemoTestUtils.createConnectContext()).analyze(sql), sql); + } + } + + @Test + public void testScalarWindowFrameKeepsExistingAnalysis() { + Assertions.assertDoesNotThrow(() -> PlanChecker.from(MemoTestUtils.createConnectContext()).analyze( + "select sum(v) over(order by v rows between unbounded preceding and current row) " + + "from (select 1 v) t")); + Assertions.assertDoesNotThrow(() -> PlanChecker.from(MemoTestUtils.createConnectContext()).analyze( + "select last_value(f) over(order by id rows between unbounded preceding and current row) " + + "from (select 1 id, cast(null as file) f) t")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeImplicitCastTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeImplicitCastTest.java new file mode 100644 index 00000000000000..c3cdc6d71c5cc1 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeImplicitCastTest.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.types; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; +import org.apache.doris.nereids.rules.expression.ExpressionRewrite; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.rules.expression.ExpressionRuleExecutor; +import org.apache.doris.nereids.rules.expression.check.CheckCast; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.commands.insert.InsertUtils; +import org.apache.doris.nereids.util.MemoTestUtils; +import org.apache.doris.nereids.util.TypeCoercionUtils; +import org.apache.doris.qe.ConnectContext; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Collections; + +public class FileTypeImplicitCastTest { + private final NereidsParser parser = new NereidsParser(); + private ConnectContext previousContext; + private CascadesContext context; + private ExpressionAnalyzer analyzer; + + @BeforeEach + public void setUp() { + previousContext = ConnectContext.get(); + context = MemoTestUtils.createCascadesContext("SELECT 1"); + // Exercise the INSERT VALUES analyzer, including its immediate FE constant folding. + analyzer = InsertUtils.buildExprAnalyzer(context.getRewritePlan(), context); + } + + @AfterEach + public void tearDown() { + if (previousContext == null) { + ConnectContext.remove(); + } else { + previousContext.setThreadLocalInfo(); + } + } + + @Test + public void testUnboundNullFileIsCheckedBeforeFolding() { + Expression unbound = parser.parseExpression("coalesce(CAST(NULL AS FILE), CAST(NULL AS FILE))"); + Expression assignment = TypeCoercionUtils.castUnbound(unbound, StringType.INSTANCE); + Assertions.assertThrows(AnalysisException.class, () -> analyzer.analyze(assignment)); + Assertions.assertThrows(AnalysisException.class, + () -> analyzer.analyze(new Cast(unbound, StringType.INSTANCE, true))); + } + + @Test + public void testCheckCastRejectsUnsupportedTargets() { + ExpressionRuleExecutor checks = new ExpressionRuleExecutor(Collections.singletonList( + ExpressionRewrite.bottomUp(new CheckCast()))); + ExpressionRewriteContext rewriteContext = new ExpressionRewriteContext(context); + Expression file = new SlotReference("f", FileType.INSTANCE); + Assertions.assertThrows(AnalysisException.class, + () -> checks.rewrite(new Cast(file, StringType.INSTANCE), rewriteContext)); + Assertions.assertThrows(AnalysisException.class, + () -> checks.rewrite(new Cast(file, StringType.INSTANCE, true), rewriteContext)); + } + +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeTest.java new file mode 100644 index 00000000000000..968f18a4be3678 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeTest.java @@ -0,0 +1,224 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.types; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.MysqlColType; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.mysql.MysqlSerializer; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.thrift.TTypeDesc; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.stream.Collectors; + +public class FileTypeTest { + private DataType fileType() { + return Assertions.assertDoesNotThrow(() -> DataType.convertFromString("FILE")); + } + + @Test + public void testIndependentIdentityAndCanonicalChildren() { + DataType file = fileType(); + Type catalog = file.toCatalogDataType(); + Assertions.assertEquals(DataType.class, file.getClass().getSuperclass()); + Assertions.assertEquals(Type.class, catalog.getClass().getSuperclass()); + Assertions.assertFalse(file.isStructType()); + Assertions.assertFalse(catalog.isStructType()); + Assertions.assertTrue(file.isComplexType()); + Assertions.assertTrue(catalog.isComplexType()); + Assertions.assertEquals("FILE", file.toSql()); + Assertions.assertEquals("FILE", catalog.toSql()); + Assertions.assertEquals(file, DataType.fromCatalogType(catalog)); + Assertions.assertEquals(catalog, Type.fromPrimitiveType(PrimitiveType.valueOf("FILE"))); + Column column = new Column("f", catalog, true); + Assertions.assertEquals(Arrays.asList("uri", "offset", "size", "content_type", "checksum", "inline"), + column.getChildren().stream().map(Column::getName).collect(Collectors.toList())); + Assertions.assertEquals(Arrays.asList("varchar(65533)", "bigint", "bigint", "varchar(1024)", + "varchar(1024)"), column.getChildren().subList(0, 5).stream() + .map(child -> child.getType().toSql()).collect(Collectors.toList())); + Assertions.assertEquals("VARBINARY", column.getChildren().get(5).getDataType().name()); + Assertions.assertTrue(column.getChildren().stream().allMatch(Column::isAllowNull)); + StructType publicStruct = FileType.INSTANCE.publicStructType(); + Assertions.assertEquals(FileType.INSTANCE.getFields(), publicStruct.getFields()); + Assertions.assertEquals(6, publicStruct.getFields().size()); + Assertions.assertEquals(VarBinaryType.INSTANCE, publicStruct.getField("inline").getDataType()); + Assertions.assertTrue(publicStruct.getFields().stream().allMatch(StructField::isNullable)); + Assertions.assertEquals(catalog, new Column("f", PrimitiveType.FILE).getType()); + Assertions.assertFalse(file.acceptsType(StructType.SYSTEM_DEFAULT)); + Assertions.assertFalse(catalog.matchesType(new org.apache.doris.catalog.StructType( + new java.util.ArrayList<>(Type.FILE.getFields())))); + } + + @Test + public void testNestedParsingAndNoParameters() { + fileType(); + for (String sql : Arrays.asList("ARRAY", "MAP", "STRUCT>")) { + DataType type = DataType.convertFromString(sql); + Assertions.assertDoesNotThrow(type::validateDataType); + Assertions.assertEquals(type, DataType.fromCatalogType(type.toCatalogDataType())); + } + Assertions.assertThrows(Exception.class, + () -> new NereidsParser().parseSingle("SELECT CAST(NULL AS FILE)")); + Assertions.assertThrows(Exception.class, + () -> new NereidsParser().parseSingle("SELECT CAST(NULL AS FILE(10))")); + Assertions.assertDoesNotThrow(() -> new NereidsParser().parseSingle("SELECT file FROM files")); + } + + @Test + public void testCanonicalThriftRoundTrip() { + Type type = fileType().toCatalogDataType(); + TTypeDesc wire = type.toThrift(); + Assertions.assertEquals("FILE", wire.getTypes().get(0).getType().name()); + Assertions.assertEquals(7, wire.getTypesSize()); + Assertions.assertFalse(wire.getTypes().get(0).isSetContainsNulls()); + Assertions.assertTrue(wire.getTypes().get(0).getStructFields().stream() + .allMatch(field -> field.isSetContainsNull() && field.isContainsNull())); + Assertions.assertEquals(type, Type.fromThrift(wire)); + TTypeDesc badName = wire.deepCopy(); + badName.getTypes().get(0).getStructFields().get(0).setName("path"); + Assertions.assertThrows(Exception.class, () -> Type.fromThrift(badName)); + TTypeDesc badNullable = wire.deepCopy(); + badNullable.getTypes().get(0).getStructFields().get(0).setContainsNull(false); + Assertions.assertThrows(Exception.class, () -> Type.fromThrift(badNullable)); + TTypeDesc badType = wire.deepCopy(); + badType.getTypes().get(1).getScalarType().setLen(1024); + Assertions.assertThrows(Exception.class, () -> Type.fromThrift(badType)); + TTypeDesc missingChild = wire.deepCopy(); + missingChild.getTypes().remove(6); + Assertions.assertThrows(Exception.class, () -> Type.fromThrift(missingChild)); + TTypeDesc wrongCase = wire.deepCopy(); + wrongCase.getTypes().get(0).getStructFields().get(0).setName("URI"); + Assertions.assertThrows(Exception.class, () -> Type.fromThrift(wrongCase)); + TTypeDesc redundantNulls = wire.deepCopy(); + redundantNulls.getTypes().get(0).setContainsNulls(Collections.nCopies(6, true)); + Assertions.assertThrows(Exception.class, () -> Type.fromThrift(redundantNulls)); + } + + @Test + public void testThriftRestoresVarbinaryLengthOnlyForFileInline() { + for (int length : Arrays.asList(-1, 16, ScalarType.MAX_VARBINARY_LENGTH)) { + TTypeDesc ordinary = ScalarType.createVarbinaryType(length).toThrift(); + Assertions.assertEquals(65533, Type.fromThrift(ordinary).getLength()); + } + TTypeDesc file = Type.FILE.toThrift(); + Assertions.assertEquals(ScalarType.MAX_VARBINARY_LENGTH, + ((org.apache.doris.catalog.FileType) Type.fromThrift(file)).getFields().get(5).getType().getLength()); + TTypeDesc invalidInline = file.deepCopy(); + invalidInline.getTypes().get(6).getScalarType().setLen(16); + Assertions.assertThrows(IllegalArgumentException.class, () -> Type.fromThrift(invalidInline)); + Type holder = new org.apache.doris.catalog.StructType(new ArrayList<>(Arrays.asList( + new org.apache.doris.catalog.StructField("asset", Type.FILE), + new org.apache.doris.catalog.StructField("payload", ScalarType.createVarbinaryType(16))))); + org.apache.doris.catalog.StructType restored = + (org.apache.doris.catalog.StructType) Type.fromThrift(holder.toThrift()); + Assertions.assertEquals(ScalarType.MAX_VARBINARY_LENGTH, + ((org.apache.doris.catalog.FileType) restored.getFields().get(0).getType()) + .getFields().get(5).getType().getLength()); + Assertions.assertEquals(65533, restored.getFields().get(1).getType().getLength()); + } + + @Test + public void testCatalogMetadataAndQueryThrift() throws Exception { + Column column = new Column("f", fileType().toCatalogDataType(), true); + String json = GsonUtils.GSON.toJson(column); + Column restored = GsonUtils.GSON.fromJson(json, Column.class); + Assertions.assertEquals(column.getType(), restored.getType()); + Assertions.assertEquals(6, restored.getChildren().size()); + Assertions.assertEquals(6, restored.toThrift().getChildrenColumnSize()); + Assertions.assertEquals("VARBINARY", restored.toThrift() + .getChildrenColumn().get(5).getColumnType().getType().name()); + Column catalogRestored = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(column), Column.class); + Assertions.assertEquals("FILE", catalogRestored.getType().toSql()); + Assertions.assertEquals(6, catalogRestored.getChildren().size()); + } + + @Test + public void testMysqlMetadata() throws Exception { + Assertions.assertEquals(MysqlColType.MYSQL_TYPE_STRING, PrimitiveType.FILE.toMysqlType()); + Assertions.assertTrue(Type.FILE.getColumnSize() >= 6 * (65533 + 1024 + 1024) + 100); + Assertions.assertEquals(Type.FILE.getColumnSize().intValue(), Type.FILE.getColumnStringRepSize()); + MysqlSerializer serializer = MysqlSerializer.newInstance(); + serializer.writeField("f", Type.FILE); + ByteBuffer packet = serializer.toByteBuffer().order(ByteOrder.LITTLE_ENDIAN); + for (int i = 0; i < 6; i++) { + int length = packet.get() & 0xff; + packet.position(packet.position() + length); + } + Assertions.assertEquals(12, packet.get() & 0xff); + Assertions.assertEquals(33, packet.getShort()); + long declaredLength = Integer.toUnsignedLong(packet.getInt()); + byte[] inline = new byte[1024 * 1024]; + Arrays.fill(inline, (byte) 0xff); + String encodedInline = Base64.getEncoder().encodeToString(inline); + String json = "{\"uri\":null,\"offset\":null,\"size\":null,\"content_type\":null," + + "\"checksum\":null,\"inline\":\"" + encodedInline + "\"}"; + Assertions.assertTrue(declaredLength >= json.getBytes(StandardCharsets.UTF_8).length); + long maxEncodedInline = 4L * ((ScalarType.MAX_VARBINARY_LENGTH + 2L) / 3); + Assertions.assertTrue(declaredLength >= maxEncodedInline + json.length() - encodedInline.length()); + Assertions.assertTrue(declaredLength > Integer.MAX_VALUE); + Assertions.assertTrue(declaredLength <= 0xffff_ffffL); + Assertions.assertEquals(MysqlColType.MYSQL_TYPE_STRING.getCode(), packet.get() & 0xff); + Assertions.assertEquals(0, packet.getShort()); + } + + @Test + public void testLogicalNestingDepthMatchesScalarLeaf() { + Type file = fileType().toCatalogDataType(); + Type scalar = Type.BIGINT; + for (int depth = 0; depth <= Type.MAX_NESTING_DEPTH + 1; depth++) { + Assertions.assertEquals(scalar.exceedsMaxNestingDepth(), file.exceedsMaxNestingDepth()); + file = new org.apache.doris.catalog.ArrayType(file); + scalar = new org.apache.doris.catalog.ArrayType(scalar); + } + } + + @Test + public void testContainsFileAcrossNestedTypes() { + for (String sql : Arrays.asList("FILE", "ARRAY", "STRUCT", "MAP")) { + Type type = DataType.convertFromString(sql).toCatalogDataType(); + Assertions.assertTrue(type.typeContainsFile()); + } + for (String sql : Arrays.asList("BIGINT", "ARRAY", "STRUCT", "MAP")) { + Assertions.assertFalse(DataType.convertFromString(sql).toCatalogDataType().typeContainsFile()); + } + } + + @Test + public void testRejectMalformedNestedColumnMetadata() { + Column nested = new Column("a", new org.apache.doris.catalog.ArrayType(fileType().toCatalogDataType())); + nested.getChildren().get(0).getChildren().remove(5); + Assertions.assertThrows(Exception.class, () -> nested.toThrift()); + Assertions.assertThrows(Exception.class, + () -> GsonUtils.GSON.fromJson("{\"clazz\":\"FileType\"}", Type.class)); + Assertions.assertThrows(Exception.class, () -> GsonUtils.GSON.fromJson( + "{\"clazz\":\"ScalarType\",\"type\":\"FILE\"}", Type.class)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/FileTypePersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/FileTypePersistenceTest.java new file mode 100644 index 00000000000000..7d0aceca8a62f9 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/FileTypePersistenceTest.java @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.persist; + +import org.apache.doris.catalog.ArrayType; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.FileType; +import org.apache.doris.catalog.MapType; +import org.apache.doris.catalog.StructField; +import org.apache.doris.catalog.StructType; +import org.apache.doris.catalog.Type; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; + +public class FileTypePersistenceTest { + @Test + public void testTypeJsonRoundTripPreservesFileIdentity() { + for (Gson gson : Arrays.asList(GsonUtils.GSON)) { + for (Type type : types()) { + Type restored = gson.fromJson(gson.toJson(type, Type.class), Type.class); + Assertions.assertEquals(type.getClass(), restored.getClass()); + Assertions.assertEquals(type, restored); + Assertions.assertTrue(restored.typeContainsFile()); + } + } + } + + @Test + public void testColumnJsonRoundTripPreservesCanonicalSchema() { + for (Gson gson : Arrays.asList(GsonUtils.GSON)) { + for (Type type : types()) { + Column column = new Column("payload", type, true); + Column restored = gson.fromJson(gson.toJson(column), Column.class); + assertColumn(column, restored); + } + } + } + + @Test + public void testMalformedColumnSchemaRejectedOnWriteAndRead() { + for (Gson gson : Arrays.asList(GsonUtils.GSON)) { + Column column = new Column("files", new ArrayType(FileType.create()), true); + JsonObject json = gson.toJsonTree(column).getAsJsonObject(); + json.getAsJsonArray("children").get(0).getAsJsonObject() + .getAsJsonArray("children").remove(5); + Assertions.assertThrows(Exception.class, + () -> gson.fromJson(json, Column.class)); + column.getChildren().get(0).getChildren().remove(5); + Assertions.assertThrows(Exception.class, () -> gson.toJson(column)); + } + } + + private static List types() { + return Arrays.asList(FileType.create(), new ArrayType(FileType.create(), true), + new StructType(new StructField("payload", FileType.create()), + new StructField("grouped", new ArrayType(new MapType(Type.STRING, FileType.create())))), + new MapType(Type.STRING, FileType.create())); + } + + private static void assertColumn(Column expected, Column actual) { + Assertions.assertEquals(expected.getName(), actual.getName()); + Assertions.assertEquals(expected.isAllowNull(), actual.isAllowNull()); + Assertions.assertEquals(expected.getType().getClass(), actual.getType().getClass()); + Assertions.assertEquals(expected.getType(), actual.getType()); + if (expected.getChildren() == null) { + Assertions.assertNull(actual.getChildren()); + return; + } + Assertions.assertNotNull(actual.getChildren()); + Assertions.assertEquals(expected.getChildren().size(), actual.getChildren().size()); + if (expected.getType().isFileType()) { + Assertions.assertEquals(6, actual.getChildren().size()); + Assertions.assertFalse(actual.getType().isStructType()); + List fields = ((FileType) actual.getType()).getFields(); + for (int i = 0; i < fields.size(); i++) { + Column child = actual.getChildren().get(i); + Assertions.assertEquals(fields.get(i).getName(), child.getName()); + Assertions.assertEquals(fields.get(i).getType(), child.getType()); + Assertions.assertTrue(child.isAllowNull()); + } + } + for (int i = 0; i < expected.getChildren().size(); i++) { + assertColumn(expected.getChildren().get(i), actual.getChildren().get(i)); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java index 7ceb825779cda6..3e95889faf2f76 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java @@ -87,6 +87,16 @@ private static Field buildField(TColumnDesc columnDesc) { return Deencapsulation.invoke(FlightSqlSchemaHelper.class, "buildField", DB, TABLE, columnDesc); } + @Test + public void fileHasNoSupportedArrowType() { + Field field = buildField(desc("file", TPrimitiveType.FILE, + desc("uri", TPrimitiveType.VARCHAR), desc("offset", TPrimitiveType.BIGINT), + desc("size", TPrimitiveType.BIGINT), desc("content_type", TPrimitiveType.VARCHAR), + desc("checksum", TPrimitiveType.VARCHAR), desc("inline", TPrimitiveType.VARBINARY))); + Assertions.assertEquals(new ArrowType.Null(), field.getType()); + Assertions.assertTrue(field.getChildren().isEmpty()); + } + /** * BE writes a DATEV2 column as {@code arrow::Date32Type} -- a day number. {@link DateUnit#MILLISECOND} * is date64, a different width and a different meaning, and a client that believes it renders and diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ListFilesTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ListFilesTableValuedFunctionTest.java new file mode 100644 index 00000000000000..0036741ad6c83f --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ListFilesTableValuedFunctionTest.java @@ -0,0 +1,233 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Resource; +import org.apache.doris.catalog.ResourceMgr; +import org.apache.doris.catalog.S3Resource; +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.property.storage.S3Properties; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.expressions.Properties; +import org.apache.doris.nereids.trees.expressions.functions.table.ListFiles; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.thrift.TDataGenFunctionName; +import org.apache.doris.thrift.TTVFListFileScanRange; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class ListFilesTableValuedFunctionTest { + @Test + void exposesPathSizeAndNullableFileColumnsWithoutRemoteIo() throws Exception { + try (Fixture fixture = new Fixture()) { + TableValuedFunctionIf function = TableValuedFunctionIf.getTableFunction("LIST_FILES", fixture.params()); + List columns = function.getTableColumns(); + assertSchema(columns); + Assertions.assertInstanceOf(DataGenTableValuedFunction.class, function); + ListFiles nereids = new ListFiles(new Properties(fixture.params())); + assertSchema(nereids.getTableColumns()); + } + } + + @Test + void requiresResourceAndUriAndRejectsUnknownProperties() { + try (Fixture fixture = new Fixture()) { + assertInvalid(ImmutableMap.of("uri", "s3://bucket/"), "resource"); + assertInvalid(ImmutableMap.of("resource", "s3_resource"), "uri"); + assertInvalid(ImmutableMap.of("resource", "", "uri", "s3://bucket/"), "resource"); + Map params = fixture.params(); + params.put("unknown", "value"); + assertInvalid(params, "unknown"); + } + } + + @Test + void acceptsCaseInsensitiveKeysAndStrictBooleanValues() throws Exception { + try (Fixture fixture = new Fixture()) { + ListFilesTableValuedFunction function = new ListFilesTableValuedFunction( + ImmutableMap.of("RESOURCE", "s3_resource", "URI", "s3://bucket/", "RECURSIVE", "TRUE")); + Assertions.assertTrue(range(function).isRecursive()); + Assertions.assertTrue(range(new ListFilesTableValuedFunction(fixture.params())).isRecursive()); + for (String value : new String[] {"yes", "1", "", " false "}) { + Map params = fixture.params(); + params.put("recursive", value); + assertInvalid(params, "recursive"); + } + } + } + + @Test + void acceptsBucketRootAndPreservesDirectoryUriForBackend() throws Exception { + try (Fixture fixture = new Fixture()) { + for (String uri : new String[] {"s3://bucket", "s3://bucket/", "s3://bucket/dir", "s3://bucket/dir/"}) { + Map params = fixture.params(); + params.put("uri", uri); + Assertions.assertEquals(uri, range(new ListFilesTableValuedFunction(params)).getUri()); + } + } + } + + @Test + void passesUriUnchangedWithoutBucketOrSyntaxChecks() throws Exception { + try (Fixture fixture = new Fixture()) { + for (String uri : new String[] {"", "dir/", "hdfs://bucket/dir/", "s3:///dir/", + "s3://another-bucket/a b+中文%ZZ#fragment"}) { + Map params = fixture.params(); + params.put("uri", uri); + Assertions.assertEquals(uri, range(new ListFilesTableValuedFunction(params)).getUri()); + } + } + } + + @Test + void requiresExistingS3ResourceAndUsage() { + try (Fixture fixture = new Fixture()) { + assertInvalid(ImmutableMap.of("resource", "missing", "uri", "s3://bucket/"), "Can not find resource"); + Mockito.when(fixture.resource.getType()).thenReturn(Resource.ResourceType.HDFS); + assertInvalid(fixture.params(), "S3"); + Mockito.when(fixture.resource.getType()).thenReturn(Resource.ResourceType.S3); + Mockito.when(fixture.access.checkResourcePriv(fixture.context, "s3_resource", PrivPredicate.USAGE)) + .thenReturn(false); + assertInvalid(fixture.params(), "denied"); + } + } + + @Test + void queryConfigurationIsSharedWithToFileAndRefreshesNextExecute() throws Exception { + try (Fixture fixture = new Fixture()) { + fixture.properties.put("s3.secret_key", "prepare"); + ListFilesTableValuedFunction function = new ListFilesTableValuedFunction(fixture.params()); + fixture.properties.put("s3.secret_key", "first execute"); + fixture.context.getOrRegisterS3Resource("s3_resource"); + fixture.properties.put("s3.secret_key", "second execute"); + TTVFListFileScanRange first = range(function); + Assertions.assertEquals("first execute", first.getResource().getProperties().get("AWS_SECRET_KEY")); + first.getResource().getProperties().put("AWS_SECRET_KEY", "mutated wire copy"); + Assertions.assertEquals("first execute", + range(function).getResource().getProperties().get("AWS_SECRET_KEY")); + fixture.context.setStartTime(); + Assertions.assertEquals("second execute", + range(function).getResource().getProperties().get("AWS_SECRET_KEY")); + Mockito.when(fixture.access.checkResourcePriv(fixture.context, "s3_resource", PrivPredicate.USAGE)) + .thenReturn(false); + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, function::getTasks); + } + } + + @Test + void assignsExactlyOneTaskToAnAliveBackendAndRejectsNoAliveBackends() throws Exception { + try (Fixture fixture = new Fixture()) { + Backend dead = Mockito.mock(Backend.class); + Mockito.when(fixture.system.getBackendsByCurrentCluster()) + .thenReturn(ImmutableMap.of(1L, dead, 2L, fixture.backend)); + ListFilesTableValuedFunction function = new ListFilesTableValuedFunction(fixture.params()); + List tasks = function.getTasks(); + Assertions.assertEquals(1, tasks.size()); + Assertions.assertSame(fixture.backend, tasks.get(0).getBackend()); + Assertions.assertEquals(TDataGenFunctionName.LIST_FILE, function.getDataGenFunctionName()); + Assertions.assertFalse(tasks.get(0).getExecParams().getDataGenScanRange().isSetNumbersParams()); + Mockito.when(fixture.system.getBackendsByCurrentCluster()).thenReturn(ImmutableMap.of(1L, dead)); + Assertions.assertThrows(org.apache.doris.common.AnalysisException.class, function::getTasks); + } + } + + private static void assertSchema(List columns) { + Assertions.assertEquals(3, columns.size()); + Assertions.assertEquals("path", columns.get(0).getName()); + Assertions.assertEquals(Type.STRING, columns.get(0).getType()); + Assertions.assertFalse(columns.get(0).isAllowNull()); + Assertions.assertEquals("size", columns.get(1).getName()); + Assertions.assertEquals(Type.BIGINT, columns.get(1).getType()); + Assertions.assertFalse(columns.get(1).isAllowNull()); + Assertions.assertEquals("file", columns.get(2).getName()); + Assertions.assertTrue(columns.get(2).getType().isFileType()); + Assertions.assertTrue(columns.get(2).isAllowNull()); + } + + private static TTVFListFileScanRange range(ListFilesTableValuedFunction function) throws Exception { + return function.getTasks().get(0).getExecParams().getDataGenScanRange().getListFileParams(); + } + + private static void assertInvalid(Map params, String message) { + Exception error = Assertions.assertThrows(Exception.class, + () -> TableValuedFunctionIf.getTableFunction("list_files", params)); + Assertions.assertTrue(error.getMessage().contains(message), error.getMessage()); + } + + private static class Fixture implements AutoCloseable { + final Env env = Mockito.mock(Env.class); + final ResourceMgr manager = Mockito.mock(ResourceMgr.class); + final AccessControllerManager access = Mockito.mock(AccessControllerManager.class); + final ConnectContext context = Mockito.spy(new ConnectContext()); + final S3Resource resource = Mockito.mock(S3Resource.class); + final SystemInfoService system = Mockito.mock(SystemInfoService.class); + final Backend backend = Mockito.mock(Backend.class); + final Map properties = new HashMap<>(); + final MockedStatic current = Mockito.mockStatic(Env.class); + final MockedStatic connection = Mockito.mockStatic(ConnectContext.class); + + Fixture() { + current.when(Env::getCurrentEnv).thenReturn(env); + current.when(Env::getCurrentSystemInfo).thenReturn(system); + connection.when(ConnectContext::get).thenReturn(context); + Mockito.when(env.getResourceMgr()).thenReturn(manager); + Mockito.when(env.getAccessManager()).thenReturn(access); + Mockito.when(manager.getResource("s3_resource")).thenReturn(resource); + Mockito.when(access.checkResourcePriv(context, "s3_resource", PrivPredicate.USAGE)).thenReturn(true); + Mockito.when(resource.getType()).thenReturn(Resource.ResourceType.S3); + Mockito.when(resource.toThrift()).thenCallRealMethod(); + properties.put(S3Properties.BUCKET, "bucket"); + properties.put("s3.endpoint", "https://s3.us-east-1.amazonaws.com"); + properties.put("s3.region", "us-east-1"); + properties.put("s3.access_key", "fake-access-key"); + properties.put("s3.secret_key", "fake-secret-key"); + Mockito.when(resource.getCopiedProperties()).thenAnswer(invocation -> new HashMap<>(properties)); + Mockito.when(context.getSessionVariable()).thenReturn(new SessionVariable()); + Mockito.when(backend.isAlive()).thenReturn(true); + try { + Mockito.when(system.getBackendsByCurrentCluster()).thenReturn(ImmutableMap.of(1L, backend)); + } catch (org.apache.doris.common.AnalysisException e) { + throw new AssertionError(e); + } + } + + Map params() { + return new HashMap<>(ImmutableMap.of("resource", "s3_resource", "uri", "s3://bucket/")); + } + + @Override + public void close() { + connection.close(); + current.close(); + } + } +} diff --git a/gensrc/proto/types.proto b/gensrc/proto/types.proto index 37f1260475ae1b..94950821b3f622 100644 --- a/gensrc/proto/types.proto +++ b/gensrc/proto/types.proto @@ -43,11 +43,11 @@ message PStructField { }; message PTypeNode { - // TTypeNodeType(SCALAR, ARRAY, MAP, STRUCT) + // TTypeNodeType(SCALAR, ARRAY, MAP, STRUCT, VARIANT, FILE) required int32 type = 1; // only set for scalar types optional PScalarType scalar_type = 2; - // only used for structs; has struct_fields.size() corresponding child types + // used for STRUCT and FILE; has struct_fields.size() corresponding child types repeated PStructField struct_fields = 3; // only used for complex types, such as array, map and etc. optional bool contains_null = 4; @@ -124,6 +124,7 @@ message PGenericType { TIMESTAMPTZ = 42; TIMESTAMP_NS = 43; UUID = 44; + FILE = 45; UNKNOWN = 999; } required TypeId id = 2; diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index e8a529b8795752..f41bb2fb586866 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -797,6 +797,7 @@ struct TPipelineFragmentParams { 44: optional list topn_filter_source_node_ids 45: optional map ai_resources 46: optional bool need_notify_close + 47: optional map s3_resources // For cloud 1000: optional bool is_mow_table; diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 4a31a5e9cdac8e..3d51e8520d1990 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -803,6 +803,7 @@ struct TExternalScanRange { enum TDataGenFunctionName { NUMBERS = 0, + LIST_FILE = 1, } // Every table valued function should have a scan range definition to save its @@ -813,8 +814,15 @@ struct TTVFNumbersScanRange { 3: optional i64 constValue } +struct TTVFListFileScanRange { + 1: Types.TS3Resource resource + 2: string uri + 3: bool recursive +} + struct TDataGenScanRange { 1: optional TTVFNumbersScanRange numbers_params + 2: optional TTVFListFileScanRange list_file_params } diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index 04e1533b74be32..243d63aacfc790 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -104,7 +104,8 @@ enum TPrimitiveType { VARBINARY = 43, // represent varbinary type TIMESTAMPTZ = 44, // timestamp with time zone TIMESTAMP_NS = 45, // signed nanoseconds since the Unix epoch - UUID = 46 + UUID = 46, + FILE = 47 } enum TTypeNodeType { @@ -113,6 +114,7 @@ enum TTypeNodeType { MAP = 2, STRUCT = 3, VARIANT = 4, + FILE = 5, } enum TStorageBackendType { @@ -167,7 +169,7 @@ struct TTypeNode { // only set for scalar types 2: optional TScalarType scalar_type - // only used for structs; has struct_fields.size() corresponding child types + // used for STRUCT and FILE; has struct_fields.size() corresponding child types 3: optional list struct_fields // old version used for array @@ -381,6 +383,11 @@ struct TDictFunction { 2: optional i64 version_id } +// S3-compatible configuration for query execution. Credentials are not part of a FILE value. +struct TS3Resource { + 1: optional map properties +} + // Represents a function in the Catalog. struct TFunction { // Fully qualified function name. diff --git a/regression-test/data/datatype_p0/complex_types/test_file_type.out b/regression-test/data/datatype_p0/complex_types/test_file_type.out new file mode 100644 index 00000000000000..d8e839529acfa0 --- /dev/null +++ b/regression-test/data/datatype_p0/complex_types/test_file_type.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !null_values_false -- +\N \N \N \N [null] {"f":null} {"f":null} + +-- !null_values_true -- +\N \N \N \N [null] {"f":null} {"f":null} + diff --git a/regression-test/data/datatype_p0/complex_types/test_file_type_internal_disabled.out b/regression-test/data/datatype_p0/complex_types/test_file_type_internal_disabled.out new file mode 100644 index 00000000000000..2b32146d351295 --- /dev/null +++ b/regression-test/data/datatype_p0/complex_types/test_file_type_internal_disabled.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !count -- +0 + +-- !scalar -- +1 \N \N + diff --git a/regression-test/data/datatype_p0/complex_types/test_file_type_resource.out b/regression-test/data/datatype_p0/complex_types/test_file_type_resource.out new file mode 100644 index 00000000000000..8c70efa6663857 --- /dev/null +++ b/regression-test/data/datatype_p0/complex_types/test_file_type_resource.out @@ -0,0 +1,8 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !constant_null -- +\N + +-- !column_null -- +1 \N \N +2 \N \N + diff --git a/regression-test/data/datatype_p0/complex_types/test_file_type_resource_local.out b/regression-test/data/datatype_p0/complex_types/test_file_type_resource_local.out new file mode 100644 index 00000000000000..f0a9dcaf9431fc --- /dev/null +++ b/regression-test/data/datatype_p0/complex_types/test_file_type_resource_local.out @@ -0,0 +1,22 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !null -- +\N + +-- !head_full -- +{"uri":"s3://file-type-local/full.bin","offset":null,"size":0,"content_type":"image/png","checksum":"ETAG:ignored-etag","inline":null} + +-- !head_etag -- +{"uri":"s3://file-type-local/etag.txt","offset":null,"size":7,"content_type":"text/plain","checksum":"ETAG:Opaque-2","inline":null} + +-- !head_raw -- +{"uri":"s3://file-type-local/raw/a%252Fb.bin?versionId=AbC","offset":null,"size":11,"content_type":"application/octet-stream","checksum":"ETAG:raw-etag","inline":null} + +-- !point_file -- +{"uri":"s3://file-type-local/full.bin","offset":null,"size":0,"content_type":"image/png","checksum":"ETAG:ignored-etag","inline":null} + +-- !point_size -- +7 + +-- !point_null -- +true + diff --git a/regression-test/data/datatype_p0/complex_types/test_fl_get_binary.out b/regression-test/data/datatype_p0/complex_types/test_fl_get_binary.out new file mode 100644 index 00000000000000..1cf0e26110b868 --- /dev/null +++ b/regression-test/data/datatype_p0/complex_types/test_fl_get_binary.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !nulls -- +\N \N + diff --git a/regression-test/data/datatype_p0/complex_types/test_list_files.out b/regression-test/data/datatype_p0/complex_types/test_list_files.out new file mode 100644 index 00000000000000..a21d84150640d3 --- /dev/null +++ b/regression-test/data/datatype_p0/complex_types/test_list_files.out @@ -0,0 +1,102 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !schema -- +path text No false \N NONE +size bigint No false \N NONE +file FILE Yes false \N NONE + +-- !direct -- +a.json 2 false 2 application/json +empty.txt 0 false 0 text/plain +empty/ 0 true \N \N +marker/ 0 true \N \N +sub/ 0 true \N \N + +-- !direct_counts -- +5 2 2 + +-- !recursive -- +a.json 2 false 2 application/json +empty.txt 0 false 0 text/plain +sub/c.png 3 false 3 image/png +sub/deep/d.txt 4 false 4 text/plain + +-- !recursive_counts -- +4 4 9 + +-- !defaults -- +a.json 2 false 2 application/json +empty.txt 0 false 0 text/plain +sub/c.png 3 false 3 image/png +sub/deep/d.txt 4 false 4 text/plain + +-- !defaults_counts -- +4 4 9 + +-- !size_only -- +2 +0 +0 +0 +0 + +-- !path_only -- +a.json +empty.txt +empty/ +marker/ +sub/ + +-- !file_only -- +false +false +true +true +true + +-- !limit -- +2 + +-- !roundtrip -- +a.json true true +empty.txt true true +sub/c.png true true +sub/deep/d.txt true true + +-- !file_json_false -- +a.json true true true true true true +empty.txt true true true true true true +empty/ \N \N \N \N \N true +marker/ \N \N \N \N \N true +sub/ \N \N \N \N \N true + +-- !file_json_true -- +a.json true true true true true true +empty.txt true true true true true true +empty/ \N \N \N \N \N true +marker/ \N \N \N \N \N true +sub/ \N \N \N \N \N true + +-- !no_trailing_slash -- +5 2 2 + +-- !empty_direct -- +empty/deep/ 0 true + +-- !empty_recursive -- +0 + +-- !missing -- +0 + +-- !directory_pages -- +1005 0 0 1005 + +-- !directory_pages_recursive -- +0 + +-- !file_pages -- +1005 1005 1005 1005 + +-- !file_pages_direct -- +1005 1005 1005 + diff --git a/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy b/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy new file mode 100644 index 00000000000000..180ec29594af25 --- /dev/null +++ b/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_file_type") { + def structType = "STRUCT" + def structValue = """NAMED_STRUCT('uri', 'urn:file', 'offset', CAST(0 AS BIGINT), + 'size', CAST(0 AS BIGINT), 'content_type', NULL, 'checksum', NULL, 'inline', X'')""" + def foldMode = sql("SELECT @@enable_fold_constant_by_be")[0][0] + try { + for (def fold in [false, true]) { + sql "SET enable_fold_constant_by_be = ${fold}" + for (def castFunction in ["CAST", "TRY_CAST"]) { + for (def entry : [ + [structValue, "FILE"], + ["ARRAY(${structValue})", "ARRAY"], + ["NAMED_STRUCT('f', ${structValue})", "STRUCT"], + ["MAP('f', ${structValue})", "MAP"], + ["'urn:file'", "FILE"], + ["CAST('{}' AS JSON)", "FILE"], + ["CAST('{}' AS VARIANT)", "FILE"], + ["CAST(NULL AS FILE)", structType], + ["CAST(NULL AS FILE)", "STRING"], + ["CAST(NULL AS FILE)", "JSON"], + ["CAST(NULL AS FILE)", "VARIANT"], + ["ARRAY(CAST(NULL AS FILE))", "ARRAY<${structType}>"], + ["NAMED_STRUCT('f', CAST(NULL AS FILE))", "STRUCT"], + ["MAP('f', CAST(NULL AS FILE))", "MAP"]]) { + test { + sql "SELECT ${castFunction}(${entry[0]} AS ${entry[1]})" + exception "can not cast" + } + } + } + quickTest("null_values_${fold}", """ + SELECT CAST(NULL AS FILE), TO_JSON(CAST(NULL AS FILE)), + ELEMENT_AT(CAST(NULL AS FILE), 'uri'), FL_GET_BINARY(CAST(NULL AS FILE)), + TO_JSON(ARRAY(CAST(NULL AS FILE))), + TO_JSON(NAMED_STRUCT('f', CAST(NULL AS FILE))), + TO_JSON(MAP('f', CAST(NULL AS FILE))) + """) + } + } finally { + sql "SET enable_fold_constant_by_be = ${foldMode}" + } +} diff --git a/regression-test/suites/datatype_p0/complex_types/test_file_type_internal_disabled.groovy b/regression-test/suites/datatype_p0/complex_types/test_file_type_internal_disabled.groovy new file mode 100644 index 00000000000000..5d10b86b4a800a --- /dev/null +++ b/regression-test/suites/datatype_p0/complex_types/test_file_type_internal_disabled.groovy @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_file_type_internal_disabled") { + for (def type : ["FILE", "ARRAY", "STRUCT", "MAP", + "ARRAY>", "STRUCT>>"]) { + for (def model : ["DUPLICATE KEY(id)", "UNIQUE KEY(id)", "AGGREGATE KEY(id)"]) { + def aggregation = model.startsWith("AGGREGATE") ? " REPLACE" : "" + test { + sql """ + CREATE TABLE test_file_type_disabled (id INT, f ${type}${aggregation}) + ${model} DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1") + """ + exception "FILE columns are not supported in internal tables" + } + } + test { + sql """ + CREATE TABLE test_file_type_disabled_ctas + DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES("replication_num"="1") + AS SELECT 1 AS id, CAST(NULL AS ${type}) AS f + """ + exception "FILE columns are not supported in internal tables" + } + } + + sql "DROP TABLE IF EXISTS test_file_type_disabled_base" + sql """ + CREATE TABLE test_file_type_disabled_base (id INT, f STRING) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1") + """ + for (def type : ["FILE", "ARRAY", "STRUCT", "MAP"]) { + test { + sql "ALTER TABLE test_file_type_disabled_base ADD COLUMN added ${type}" + exception "FILE columns are not supported in internal tables" + } + test { + sql "ALTER TABLE test_file_type_disabled_base MODIFY COLUMN f ${type}" + exception "FILE columns are not supported in internal tables" + } + } + for (def type : ["FILE", "ARRAY", "STRUCT", "MAP"]) { + test { + sql """ + CREATE MATERIALIZED VIEW test_file_type_disabled_mv + BUILD DEFERRED REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES("replication_num"="1") + AS SELECT id, CAST(NULL AS ${type}) AS f FROM test_file_type_disabled_base + """ + exception "FILE columns are not supported in internal tables" + } + } + qt_count "SELECT COUNT(f) FROM (SELECT CAST(NULL AS FILE) AS f) t" + sql "DROP TABLE IF EXISTS test_file_type_disabled_scalar" + sql """ + CREATE TABLE test_file_type_disabled_scalar + DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES("replication_num"="1") + AS SELECT 1 AS id, ELEMENT_AT(f, 'uri') AS uri, + HEX(CAST(FL_GET_BINARY(f) AS STRING)) AS inline_hex + FROM (SELECT CAST(NULL AS FILE) AS f) t + """ + qt_scalar "SELECT id, uri, inline_hex FROM test_file_type_disabled_scalar ORDER BY id" + +} diff --git a/regression-test/suites/datatype_p0/complex_types/test_file_type_resource.groovy b/regression-test/suites/datatype_p0/complex_types/test_file_type_resource.groovy new file mode 100644 index 00000000000000..44ebe6d0cfb79d --- /dev/null +++ b/regression-test/suites/datatype_p0/complex_types/test_file_type_resource.groovy @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_file_type_resource") { + // These endpoints are deliberately unavailable: TO_FILE with NULL must not create a remote client or do I/O. + sql "DROP RESOURCE IF EXISTS 'file_type_no_io_s3'" + sql "DROP RESOURCE IF EXISTS 'file_type_no_io_hdfs'" + sql """ + CREATE RESOURCE 'file_type_no_io_s3' PROPERTIES( + "type"="s3", "AWS_ENDPOINT"="http://127.0.0.1:1", "AWS_REGION"="us-east-1", + "AWS_BUCKET"="file-type-no-io", "AWS_ROOT_PATH"="files", + "AWS_ACCESS_KEY"="unused", "AWS_SECRET_KEY"="unused", + "AWS_REQUEST_TIMEOUT_MS"="200", "AWS_CONNECTION_TIMEOUT_MS"="100", + "s3_validity_check"="false") + """ + sql """ + CREATE RESOURCE 'file_type_no_io_hdfs' PROPERTIES( + "type"="hdfs", "fs.defaultFS"="hdfs://127.0.0.1:1", + "hadoop.username"="file_type_test", "ipc.client.connect.timeout"="100", + "ipc.client.connect.max.retries"="0") + """ + qt_constant_null """ + SELECT TO_FILE('file_type_no_io_s3', NULL) + """ + sql "DROP TABLE IF EXISTS test_file_type_null_uris" + sql """ + CREATE TABLE test_file_type_null_uris (id INT NOT NULL, uri STRING NULL) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 2 + PROPERTIES("replication_num"="1") + """ + sql "INSERT INTO test_file_type_null_uris VALUES (1, NULL), (2, NULL)" + qt_column_null """ + SELECT id, TO_FILE('file_type_no_io_s3', uri), + ELEMENT_AT(TO_FILE('file_type_no_io_s3', uri), 'uri') + FROM test_file_type_null_uris ORDER BY id + """ + for (def query in [ + "SELECT TO_FILE(NULL, NULL)", + "SELECT TO_FILE('file_type_no_io_s3', 123)", + "SELECT TO_FILE(uri, NULL) FROM test_file_type_null_uris" + ]) { + test { + sql query + exception "TO_FILE" + } + } + for (def query in [ + "SELECT TO_FILE('file_type_no_io_hdfs', NULL)", + "SELECT TO_FILE('file_type_no_io_hdfs', uri) FROM test_file_type_null_uris" + ]) { + test { + sql query + exception "Requires an S3 resource" + } + } +} diff --git a/regression-test/suites/datatype_p0/complex_types/test_file_type_resource_local.groovy b/regression-test/suites/datatype_p0/complex_types/test_file_type_resource_local.groovy new file mode 100644 index 00000000000000..2d5e055d8baddd --- /dev/null +++ b/regression-test/suites/datatype_p0/complex_types/test_file_type_resource_local.groovy @@ -0,0 +1,203 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import com.sun.net.httpserver.HttpExchange +import com.sun.net.httpserver.HttpHandler +import com.sun.net.httpserver.HttpServer +import java.net.InetAddress +import java.net.InetSocketAddress +import java.net.NetworkInterface +import java.net.URI +import java.net.URLDecoder +import java.sql.PreparedStatement +import java.sql.ResultSet +import java.util.concurrent.CopyOnWriteArrayList + +// Like test_http_tvf, the fixture runs inside the regression JVM. No external S3 service is used. +// JDBC and the mock use loopback; same-host BEs may advertise any local interface address. +suite("test_file_type_resource_local") { + if (!context.config.otherConfigs.get("enableFileTypeLocalResource")?.toString()?.toBoolean()) { + logger.info("Skip test_file_type_resource_local: enableFileTypeLocalResource requires a same-host cluster") + return + } + def backends = sql_return_maparray("SHOW BACKENDS").findAll { it.Alive.toString().toBoolean() } + if (URI.create(context.config.jdbcUrl.substring(5)).host != "127.0.0.1" + || backends.isEmpty() || backends.any { + NetworkInterface.getByInetAddress(InetAddress.getByName(it.Host.toString())) == null + }) { + throw new IllegalStateException("enableFileTypeLocalResource requires JDBC on 127.0.0.1 and all live BEs on local network interfaces") + } + + def requests = new CopyOnWriteArrayList() + def handlerErrors = new CopyOnWriteArrayList() + def heads = [ + "/file-type-local/full.bin": [ + "Content-Length": "0", "Content-Type": "image/png", "ETag": '"ignored-etag"', + "x-amz-checksum-type": "FULL_OBJECT", + "x-amz-checksum-sha256": "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=", + "x-amz-checksum-md5": "1B2M2Y8AsgTpgAmY7PhCfg=="], + "/file-type-local/etag.txt": [ + "Content-Length": "7", "Content-Type": "text/plain", "ETag": '"Opaque-2"', + "x-amz-checksum-type": "COMPOSITE", + "x-amz-checksum-sha256": "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="], + // The raw %252F key is encoded by the S3 SDK when constructing the HTTP request. + "/file-type-local/raw/a%25252Fb.bin%3FversionId=AbC": [ + "Content-Length": "11", "Content-Type": "application/octet-stream", "ETag": '"raw-etag"'] + ] + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0) + server.createContext("/", { HttpExchange exchange -> + try { + def query = [:] + exchange.requestURI.rawQuery?.split("&")?.each { part -> + def pair = part.split("=", 2) + query[URLDecoder.decode(pair[0], "UTF-8")] = + pair.length == 2 ? URLDecoder.decode(pair[1], "UTF-8") : "" + } + def path = exchange.requestURI.rawPath + requests.add([method: exchange.requestMethod, path: path, query: query, + authorization: exchange.requestHeaders.getFirst("Authorization"), + checksumMode: exchange.requestHeaders.getFirst("x-amz-checksum-mode")]) + exchange.responseHeaders.set("x-amz-request-id", "file-type-local-request") + if (exchange.requestMethod == "HEAD") { + def metadata = heads[path] + if (metadata == null) { + exchange.responseHeaders.set("x-amz-error-code", "NoSuchKey") + exchange.sendResponseHeaders(404, -1) + } else { + metadata.each { key, value -> exchange.responseHeaders.set(key, value) } + exchange.responseHeaders.set("Last-Modified", "Mon, 01 Jan 2024 00:00:00 GMT") + exchange.sendResponseHeaders(200, -1) + } + } else { + handlerErrors.add("Unexpected request: ${exchange.requestMethod} ${exchange.requestURI}".toString()) + exchange.sendResponseHeaders(400, -1) + } + } catch (Exception e) { + handlerErrors.add(e.toString()) + } finally { + exchange.close() + } + } as HttpHandler) + server.start() + + try { + sql "DROP RESOURCE IF EXISTS 'file_type_local_s3'" + sql """ + CREATE RESOURCE 'file_type_local_s3' PROPERTIES( + "type"="s3", "AWS_ENDPOINT"="http://127.0.0.1:${server.address.port}", + "AWS_REGION"="us-east-1", "AWS_BUCKET"="file-type-local", "AWS_ROOT_PATH"="unused-root", + "AWS_ACCESS_KEY"="fake-file-type-key", "AWS_SECRET_KEY"="fake-file-type-secret", + "AWS_REQUEST_TIMEOUT_MS"="2000", "AWS_CONNECTION_TIMEOUT_MS"="500", + "use_path_style"="true", "s3_validity_check"="false") + """ + qt_null "SELECT TO_FILE('file_type_local_s3', NULL)" + assertEquals(0, requests.size(), "Resource creation and NULL must not send requests") + + qt_head_full "SELECT TO_FILE('file_type_local_s3', 's3://file-type-local/full.bin')" + qt_head_etag "SELECT TO_FILE('file_type_local_s3', 's3://file-type-local/etag.txt')" + qt_head_raw "SELECT TO_FILE('file_type_local_s3', 's3://file-type-local/raw/a%252Fb.bin?versionId=AbC')" + assertTrue(handlerErrors.isEmpty(), handlerErrors.toString()) + assertEquals(heads.keySet().toList(), requests.collect { it.path }) + assertTrue(requests.every { it.method == "HEAD" && it.checksumMode == null + && it.query.isEmpty() }, requests.toString()) + + requests.clear() + test { + sql "SELECT TO_FILE('file_type_local_s3', 's3://file-type-local/missing.bin')" + exception "failed to head s3 file" + } + assertEquals(1, requests.size(), requests.toString()) + assertEquals("HEAD", requests[0].method) + assertEquals("/file-type-local/missing.bin", requests[0].path) + assertEquals(null, requests[0].checksumMode) + assertTrue(requests[0].query.isEmpty(), requests.toString()) + + sql "DROP TABLE IF EXISTS test_file_type_resource_point_query" + sql """ + CREATE TABLE test_file_type_resource_point_query ( + pk INT NOT NULL, uri STRING NULL + ) UNIQUE KEY(pk) + DISTRIBUTED BY HASH(pk) BUCKETS 1 + PROPERTIES("replication_num"="1", "enable_unique_key_merge_on_write"="true", + "light_schema_change"="true", "store_row_column"="true") + """ + sql """ + INSERT INTO test_file_type_resource_point_query VALUES + (1, 's3://file-type-local/full.bin'), (2, 's3://file-type-local/etag.txt'), + (3, NULL), (4, 's3://file-type-local/missing.bin') + """ + explain { + sql "SELECT uri FROM test_file_type_resource_point_query WHERE pk=1" + contains "SHORT-CIRCUIT" + } + explain { + sql "SELECT TO_FILE('file_type_local_s3', uri) FROM test_file_type_resource_point_query WHERE pk=1" + notContains "SHORT-CIRCUIT" + } + requests.clear() + qt_point_file """ + SELECT TO_FILE('file_type_local_s3', uri) + FROM test_file_type_resource_point_query WHERE pk=1 + """ + qt_point_size """ + SELECT ELEMENT_AT(TO_FILE('file_type_local_s3', uri), 'size') + FROM test_file_type_resource_point_query WHERE pk=2 + """ + qt_point_null """ + SELECT TO_FILE('file_type_local_s3', uri) IS NULL + FROM test_file_type_resource_point_query WHERE pk=3 + """ + test { + sql """SELECT TO_FILE('file_type_local_s3', uri) + FROM test_file_type_resource_point_query WHERE pk=4""" + exception "failed to head s3 file" + } + assertEquals(["/file-type-local/full.bin", "/file-type-local/etag.txt", + "/file-type-local/missing.bin"], requests.collect { it.path }) + assertTrue(handlerErrors.isEmpty(), handlerErrors.toString()) + + requests.clear() + def database = (sql "SELECT DATABASE()")[0][0].toString() + def preparedUrl = getServerPrepareJdbcUrl(context.config.jdbcUrl, database) + connect(context.config.jdbcUser, context.config.jdbcPassword, preparedUrl) { + try (PreparedStatement statement = prepareStatement( + "SELECT ELEMENT_AT(TO_FILE('file_type_local_s3', ?), 'size')")) { + assertTrue(statement instanceof com.mysql.cj.jdbc.ServerPreparedStatement) + statement.setString(1, "s3://file-type-local/etag.txt") + try (ResultSet result = statement.executeQuery()) { + while (result.next()) {} + } + connect(context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) { + sql """ALTER RESOURCE 'file_type_local_s3' PROPERTIES( + "AWS_ACCESS_KEY"="fake-file-type-next-key", "s3_validity_check"="false")""" + } + try (ResultSet result = statement.executeQuery()) { + while (result.next()) {} + } + } + } + assertEquals(["fake-file-type-key", "fake-file-type-next-key"], requests.collect { + def credential = it.authorization =~ /Credential=([^\/]+)\// + assertTrue(credential.find(), "Missing S3 signing credential") + credential.group(1) + }) + assertTrue(handlerErrors.isEmpty(), handlerErrors.toString()) + + } finally { + server.stop(0) + } +} diff --git a/regression-test/suites/datatype_p0/complex_types/test_fl_get_binary.groovy b/regression-test/suites/datatype_p0/complex_types/test_fl_get_binary.groovy new file mode 100644 index 00000000000000..1cc7d933deb0b4 --- /dev/null +++ b/regression-test/suites/datatype_p0/complex_types/test_fl_get_binary.groovy @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_fl_get_binary") { + qt_nulls "SELECT FL_GET_BINARY(NULL), FL_GET_BINARY(CAST(NULL AS FILE))" + for (def field in ["inline", "INLINE", "InLiNe"]) { + test { + sql "SELECT ELEMENT_AT(CAST(NULL AS FILE), '${field}')" + exception "use FL_GET_BINARY" + } + test { + sql "SELECT (CAST(NULL AS FILE))['${field}']" + exception "use FL_GET_BINARY" + } + } + for (def argument in ["1", "'uri'", "CAST('{}' AS JSON)", "CAST('{}' AS VARIANT)", + "NAMED_STRUCT('uri', 'urn:x')", "ARRAY(CAST(NULL AS FILE))"]) { + test { + sql "SELECT FL_GET_BINARY(${argument})" + exception "Can not find the compatibility function signature" + } + } + test { + sql "SELECT FL_GET_BINARY()" + exception "Can not found function" + } + test { + sql "SELECT FL_GET_BINARY(CAST(NULL AS FILE), 'uri')" + exception "Can not found function" + } +} diff --git a/regression-test/suites/datatype_p0/complex_types/test_list_files.groovy b/regression-test/suites/datatype_p0/complex_types/test_list_files.groovy new file mode 100644 index 00000000000000..0728cfb7525b13 --- /dev/null +++ b/regression-test/suites/datatype_p0/complex_types/test_list_files.groovy @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import com.amazonaws.auth.AWSStaticCredentialsProvider +import com.amazonaws.auth.BasicAWSCredentials +import com.amazonaws.client.builder.AwsClientBuilder +import com.amazonaws.services.s3.AmazonS3ClientBuilder +import com.amazonaws.services.s3.model.ObjectMetadata + +suite("test_list_files", "p0,external") { + boolean usePathStyle = getS3Provider().equalsIgnoreCase("S3") + def client = AmazonS3ClientBuilder.standard() + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(getS3Endpoint(), getS3Region())) + .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(getS3AK(), getS3SK()))) + .withPathStyleAccessEnabled(usePathStyle).build() + String bucket = getS3BucketName() + String prefix = "list-files/${UUID.randomUUID()}/" + String root = "s3://${bucket}/${prefix}" + def keys = [] + def put = { String key, String contents -> + byte[] data = contents.getBytes("UTF-8") + def metadata = new ObjectMetadata() + metadata.setContentLength(data.length) + client.putObject(bucket, prefix + key, new ByteArrayInputStream(data), metadata) + keys.add(prefix + key) + } + try { + ["": "", "empty.txt": "", "a.json": "{}", "sub/": "", "sub/c.png": "abc", + "sub/deep/d.txt": "abcd", "empty/": "", "empty/deep/": "", "marker/": "marker"] + .each { key, contents -> put(key, contents) } + sql "DROP RESOURCE IF EXISTS list_files_s3" + sql """CREATE RESOURCE list_files_s3 PROPERTIES( + "type"="s3", "s3.endpoint"="${getS3Endpoint()}", "s3.region"="${getS3Region()}", + "s3.access_key"="${getS3AK()}", "s3.secret_key"="${getS3SK()}", + "s3.bucket"="${bucket}", "s3.root.path"="unused", "use_path_style"="${usePathStyle}")""" + def listing = { String path, String recursive -> + """list_files("resource"="list_files_s3", "uri"="${path}"${recursive})""" + } + String direct = listing(root, ', "recursive"="false"') + String recursive = listing(root, ', "recursive"="true"') + String defaults = listing(root, '') + qt_schema "DESC FUNCTION ${direct}" + for (def entry : [direct: direct, recursive: recursive, defaults: defaults]) { + "qt_${entry.key}" """SELECT substring(path, ${root.length() + 1}), size, + file IS NULL, ELEMENT_AT(file, 'size'), ELEMENT_AT(file, 'content_type') + FROM ${entry.value} ORDER BY path""" + "qt_${entry.key}_counts" "SELECT count(*), count(file), sum(size) FROM ${entry.value}" + } + qt_size_only "SELECT size FROM ${direct} ORDER BY path" + qt_path_only "SELECT substring(path, ${root.length() + 1}) FROM ${direct} ORDER BY path" + qt_file_only "SELECT file IS NULL FROM ${direct} ORDER BY path" + qt_limit "SELECT count(*) FROM (SELECT * FROM ${direct} LIMIT 2) t" + qt_roundtrip """SELECT substring(path, ${root.length() + 1}), + ELEMENT_AT(TO_FILE('list_files_s3', path), 'size') = size, + ELEMENT_AT(file, 'checksum') = ELEMENT_AT(TO_FILE('list_files_s3', path), 'checksum') + FROM ${recursive} ORDER BY path""" + for (def fold in [false, true]) { + quickTest("file_json_${fold}", """ + SELECT /*+ SET_VAR(enable_fold_constant_by_be=${fold}) */ + substring(path, ${root.length() + 1}), + JSON_EXTRACT_STRING(TO_JSON(file), '\$.uri') = path, + JSON_EXTRACT_BIGINT(TO_JSON(file), '\$.size') = size, + JSON_EXTRACT_STRING(TO_JSON(ARRAY(file)), '\$[0].uri') = path, + JSON_EXTRACT_STRING(TO_JSON(NAMED_STRUCT('f', file)), '\$.f.uri') = path, + JSON_EXTRACT_STRING(TO_JSON(MAP('f', file)), '\$.f.uri') = path, + FL_GET_BINARY(file) IS NULL + FROM ${direct} ORDER BY path + """) + } + def fileRows = """ + WITH file_scan AS ( + SELECT number AS id, file AS f + FROM NUMBERS('number'='8') CROSS JOIN ${recursive} + WHERE path = '${root}a.json' + ) + """ + test { + sql "${fileRows} SELECT (SELECT i.f FROM file_scan i WHERE i.id=o.id) FROM file_scan o" + exception "Agg Function any_value(FILE) is not implemented" + } + for (def value in ["ARRAY(i.f)", "NAMED_STRUCT('f', i.f)", "MAP('f', i.f)", + "ELEMENT_AT(i.f, 'uri')"]) { + sql "${fileRows} SELECT (SELECT ${value} FROM file_scan i WHERE i.id=o.id) FROM file_scan o" + } + qt_no_trailing_slash """SELECT count(*), count(file), sum(size) + FROM ${listing(root.substring(0, root.length() - 1), ', "recursive"="false"')}""" + qt_empty_direct """SELECT substring(path, ${root.length() + 1}), size, file IS NULL + FROM ${listing(root + 'empty/', ', "recursive"="false"')} ORDER BY path""" + qt_empty_recursive "SELECT count(*) FROM ${listing(root + 'empty/', '')}" + qt_missing "SELECT count(*) FROM ${listing(root + 'missing/', '')}" + for (int i = 0; i < 1005; ++i) { + put(String.format("pages/d%04d/", i), "") + } + qt_directory_pages """SELECT count(*), count(file), sum(size), + count(distinct path) FROM ${listing(root + 'pages/', ', "recursive"="false"')}""" + qt_directory_pages_recursive "SELECT count(*) FROM ${listing(root + 'pages/', '')}" + for (int i = 0; i < 1005; ++i) { + put(String.format("files/f%04d.txt", i), "x") + } + qt_file_pages """SELECT count(*), count(file), sum(size), count(distinct path) + FROM ${listing(root + 'files/', '')}""" + qt_file_pages_direct """SELECT count(*), count(file), sum(size) + FROM ${listing(root + 'files/', ', "recursive"="false"')}""" + test { + sql """SELECT * FROM list_file("resource"="list_files_s3", "uri"="${root}")""" + exception "Can not found function" + } + test { + sql """SELECT * FROM list_files("resource"="list_files_s3", "uri"="${root}", "recursive"="yes")""" + exception "recursive must be true or false" + } + } finally { + keys.each { key -> client.deleteObject(bucket, key) } + client.shutdown() + } +} diff --git a/regression-test/suites/nereids_p0/variables_persist/test_view_var_persist.groovy b/regression-test/suites/nereids_p0/variables_persist/test_view_var_persist.groovy index 33e3f2773eb71f..e4cd30e3740f23 100644 --- a/regression-test/suites/nereids_p0/variables_persist/test_view_var_persist.groovy +++ b/regression-test/suites/nereids_p0/variables_persist/test_view_var_persist.groovy @@ -51,7 +51,7 @@ suite("test_view_var_persist") { create view v_divide as select a/b from t_decimalv3; set enable_decimal256=false; """ - qt_divide "select * from v_divide;" + qt_divide "select /*+ SET_VAR(enable_fold_constant_by_be=true) */ * from v_divide;" multi_sql """ set enable_decimal256=true;