diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 4d513acbb9dd4c..b38c7cb034674d 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -50,10 +50,12 @@ #include "format/format_common.h" #include "format/table/iceberg_scan_semantics.h" #include "format_v2/column_mapper.h" +#include "format_v2/jni/fluss_jni_reader.h" #include "format_v2/jni/iceberg_sys_table_reader.h" #include "format_v2/jni/jdbc_reader.h" #include "format_v2/jni/max_compute_jni_reader.h" #include "format_v2/jni/trino_connector_jni_reader.h" +#include "format_v2/table/fluss_union_lake_reader.h" #include "format_v2/table/hive_reader.h" #include "format_v2/table/hudi_reader.h" #include "format_v2/table/iceberg_position_delete_sys_table_reader.h" @@ -99,7 +101,11 @@ bool is_supported_table_format(const TFileRangeDesc& range) { return false; } return table_format == "NotSet" || table_format == "tvf" || table_format == "hive" || - table_format == "iceberg" || table_format == "paimon" || table_format == "hudi"; + table_format == "iceberg" || table_format == "paimon" || table_format == "hudi" || + // A lake split of a fluss primary-key table read as its lake plus its log tail. It is the + // paimon sibling's own split, wrapped, so it arrives in whichever form the sibling planned + // it - native Parquet/ORC here, a serialized JNI split below. + table_format == "fluss_union"; } bool is_supported_arrow_table_format(const TFileRangeDesc& range) { @@ -108,7 +114,9 @@ bool is_supported_arrow_table_format(const TFileRangeDesc& range) { bool is_supported_jni_table_format(const TFileRangeDesc& range) { const auto table_format = table_format_name(range); - if (table_format == "paimon") { + // A wrapped lake split is a paimon split in every respect that decides how it is read, so + // whether it can be read here is the paimon answer, asked of the paimon payload it carries. + if (table_format == "paimon" || table_format == "fluss_union") { if (!range.__isset.table_format_params || !range.table_format_params.__isset.paimon_params) { return false; @@ -131,7 +139,8 @@ bool is_supported_jni_table_format(const TFileRangeDesc& range) { (params.file_format == "parquet" || params.file_format == "orc"); } return table_format == "jdbc" || table_format == "iceberg" || table_format == "hudi" || - table_format == "max_compute" || table_format == "trino_connector"; + table_format == "max_compute" || table_format == "trino_connector" || + table_format == "fluss"; } bool is_iceberg_position_deletes_sys_table(const TFileRangeDesc& range) { @@ -403,6 +412,7 @@ Status FileScannerV2::_open_impl(RuntimeState* state) { if (_first_scan_range) { RETURN_IF_ERROR(_create_table_reader_for_format(_current_range, &_table_reader)); DORIS_CHECK(_table_reader != nullptr); + _table_reader_format = table_format_name(_current_range); RETURN_IF_ERROR(_init_expr_ctxes()); RETURN_IF_ERROR(_init_table_reader(_current_range)); } @@ -502,6 +512,24 @@ Status FileScannerV2::_prepare_next_split(bool* eos) { DORIS_CHECK(_table_reader != nullptr); _current_range_path = _current_range.path; + // The reader is chosen by the range's table format, not the node's, because one node can be + // given both: a fluss union read plans its lake half through the paimon connector and its log + // half itself, and both land here as ranges of the same scan. Built once from the first range, + // the reader would then be handed a range of the other format -- which does not fail cleanly. + // It fails as whatever that reader makes of a foreign range, e.g. paimon's reporting an + // unsupported file format for a fluss range that has no paimon parameters at all. Which + // ranges share a scanner is up to the engine's assignment, so the same query succeeds or + // fails by how the ranges happened to be dealt out. + auto table_format = table_format_name(_current_range); + if (table_format != _table_reader_format) { + RETURN_IF_ERROR(_create_table_reader_for_format(_current_range, &_table_reader)); + DORIS_CHECK(_table_reader != nullptr); + _table_reader_format = std::move(table_format); + // Same init the first reader got. The expression contexts are NOT rebuilt: they are + // per-scanner and format-independent, and _init_expr_ctxes is not idempotent. + RETURN_IF_ERROR(_init_table_reader(_current_range)); + } + const auto format_type = get_range_format_type(*_params, _current_range); _init_adaptive_batch_size_state(format_type); if (_block_size_predictor != nullptr) { @@ -608,6 +636,10 @@ Status FileScannerV2::_create_table_reader_for_format( *reader = std::make_unique(); } else if (table_format == "hudi") { *reader = std::make_unique(); + } else if (table_format == "fluss") { + *reader = std::make_unique(); + } else if (table_format == "fluss_union") { + *reader = std::make_unique(); } else if (table_format == "jdbc") { *reader = std::make_unique(); } else if (table_format == "max_compute") { diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index 92edbc1a1817f9..f3a56542022771 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -181,6 +181,10 @@ class FileScannerV2 final : public Scanner { std::string _current_range_path; std::unique_ptr _table_reader; + // The table format _table_reader was built for. A scan node may mix table formats -- a fluss + // union read gives one node its lake half as paimon ranges and its log half as fluss ones -- and + // the reader is format-specific, so it is rebuilt whenever this stops matching the range. + std::string _table_reader_format; std::vector _projected_columns; // File formats without embedded schema, such as CSV, still need the FE slot descriptors in // file-column order. This mirrors old FileScanner::_file_slot_descs and is passed only to diff --git a/be/src/format_v2/jni/fluss_jni_reader.cpp b/be/src/format_v2/jni/fluss_jni_reader.cpp new file mode 100644 index 00000000000000..12f4b545984a3b --- /dev/null +++ b/be/src/format_v2/jni/fluss_jni_reader.cpp @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/jni/fluss_jni_reader.h" + +#include "core/block/block.h" +#include "exprs/vexpr_context.h" +#include "format_v2/column_mapper.h" + +namespace doris::format::fluss { + +Status FlussJniReader::validate_scan_range(const TFileRangeDesc& range) const { + if (!range.__isset.table_format_params) { + return Status::InternalError("missing table_format_params for fluss jni reader"); + } + if (!range.table_format_params.__isset.fluss_params || + range.table_format_params.fluss_params.empty()) { + return Status::InternalError( + "missing fluss_params for fluss jni reader, possibly caused by FE/BE protocol " + "mismatch"); + } + if (_scan_params == nullptr || !_scan_params->__isset.fluss_properties || + _scan_params->fluss_properties.empty()) { + return Status::InternalError( + "missing fluss_properties for fluss jni reader, possibly caused by FE/BE protocol " + "mismatch"); + } + return Status::OK(); +} + +std::string FlussJniReader::connector_class() const { + return "org/apache/doris/fluss/FlussJniScanner"; +} + +Status FlussJniReader::build_scanner_params(std::map* params) const { + DORIS_CHECK(params != nullptr); + DORIS_CHECK(_scan_params != nullptr); + // Scan level first, then the range: the range is the more specific of the two, and it is the one + // that says which bucket and offsets to read. No key is transcribed - both maps are written by FE + // and read by the Java scanner, so naming them here would only add a place to drift. + *params = _scan_params->fluss_properties; + for (const auto& kv : _current_range.table_format_params.fluss_params) { + (*params)[kv.first] = kv.second; + } + return Status::OK(); +} + +Status FlussJniReader::build_jni_columns( + std::vector* columns) const { + DORIS_CHECK(columns != nullptr); + columns->clear(); + columns->reserve(_projected_columns.size()); + for (size_t i = 0; i < _projected_columns.size(); ++i) { + const auto& table_column = _projected_columns[i]; + // A fluss row physically carries its partition column values, but FE declares the partition + // keys as path_partition_keys and ships them per range, so they are constants here. Asking + // the scanner for them would read per row what the split already states once - and would + // read them differently from the legacy JNI path, whose file slots exclude them outright. + if (table_column.is_partition_key && + find_partition_value(table_column, _partition_values) != nullptr) { + continue; + } + columns->push_back({ + .java_name = table_column.name, + .output_index = i, + .output_type = table_column.type, + .transfer_type = table_column.type, + .replace_type = "not_replace", + }); + } + return Status::OK(); +} + +Status FlussJniReader::finalize_jni_block(Block* jni_block, Block* output_block, size_t* rows) { + DORIS_CHECK(jni_block != nullptr); + DORIS_CHECK(output_block != nullptr); + DORIS_CHECK(rows != nullptr); + const auto original_rows = *rows; + + const auto& columns = jni_columns(); + DORIS_CHECK(columns.size() == jni_block->columns()); + for (size_t i = 0; i < columns.size(); ++i) { + const auto& column = columns[i]; + DORIS_CHECK(column.output_index < output_block->columns()); + output_block->get_by_position(column.output_index).type = column.output_type; + output_block->replace_by_position(column.output_index, + jni_block->get_by_position(i).column); + } + + // The columns build_jni_columns() left out: materialized from the range instead of read. + for (size_t i = 0; i < _projected_columns.size(); ++i) { + const auto& table_column = _projected_columns[i]; + const auto* partition_value = find_partition_value(table_column, _partition_values); + if (!table_column.is_partition_key || partition_value == nullptr) { + continue; + } + output_block->get_by_position(i).type = table_column.type; + output_block->replace_by_position( + i, table_column.type->create_column_const(original_rows, *partition_value)); + } + DORIS_CHECK(output_block->rows() == original_rows); + if (!_conjuncts.empty()) { + RETURN_IF_ERROR( + VExprContext::filter_block(_conjuncts, output_block, output_block->columns())); + } + *rows = output_block->rows(); + return Status::OK(); +} + +} // namespace doris::format::fluss diff --git a/be/src/format_v2/jni/fluss_jni_reader.h b/be/src/format_v2/jni/fluss_jni_reader.h new file mode 100644 index 00000000000000..7c1d31e6ddc8c5 --- /dev/null +++ b/be/src/format_v2/jni/fluss_jni_reader.h @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "format_v2/jni/jni_table_reader.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::format::fluss { + +/** + * Reads one fluss scan range through org.apache.doris.fluss.FlussJniScanner. + * + * This layer holds no fluss logic. FE writes two untyped string maps - the scan-level + * `fluss_properties` (connection, table identity, client options) and the per-range `fluss_params` + * (which bucket, which offsets) - and the only decision made here is that the range wins where both + * set a key. + */ +class FlussJniReader final : public format::JniTableReader { +public: + ~FlussJniReader() override = default; + +protected: + std::string connector_class() const override; + Status validate_scan_range(const TFileRangeDesc& range) const override; + Status build_scanner_params(std::map* params) const override; + Status build_jni_columns( + std::vector* columns) const override; + Status finalize_jni_block(Block* jni_block, Block* output_block, size_t* rows) override; +}; + +} // namespace doris::format::fluss diff --git a/be/src/format_v2/table/fluss_union_lake_reader.cpp b/be/src/format_v2/table/fluss_union_lake_reader.cpp new file mode 100644 index 00000000000000..970f54c7d18b39 --- /dev/null +++ b/be/src/format_v2/table/fluss_union_lake_reader.cpp @@ -0,0 +1,524 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/fluss_union_lake_reader.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "core/assert_cast.h" +#include "core/column/column_vector.h" +#include "exprs/vexpr_context.h" +#include "exprs/vslot_ref.h" +#include "format_v2/column_mapper.h" +#include "format_v2/expr/equality_delete_predicate.h" +#include "format_v2/jni/fluss_jni_reader.h" +#include "format_v2/table/paimon_reader.h" +#include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" +#include "runtime/runtime_state.h" + +namespace doris::format::fluss { +namespace { + +// The scan-node properties this reader reads. The fluss connector states that `fluss.union.*` is the +// whole of what BE's C++ side knows about fluss; anything added here has to be added there too. +constexpr const char* PROP_PK_NAMES = "fluss.union.pk_names"; +constexpr const char* PROP_MAX_TAIL_ROWS = "fluss.union.max_tail_rows"; + +// The per-range payload of a wrapped lake split. +constexpr const char* PROP_RANGE_TYPE = "fluss.range_type"; +constexpr const char* PROP_TAIL = "fluss.union.tail"; +constexpr const char* RANGE_TYPE_LAKE_SUPPRESS = "LAKE_SUPPRESS"; + +// The range this reader synthesizes to read the tail. A plain bounded log read, which is what the +// suppression set is: every key the tail touched, whatever it ended up saying about it. +constexpr const char* RANGE_TYPE_LOG = "LOG"; +constexpr const char* PROP_PARTITION_ID = "fluss.partition_id"; +constexpr const char* PROP_BUCKET_ID = "fluss.bucket_id"; +constexpr const char* PROP_LOG_START_OFFSET = "fluss.log_start_offset"; +constexpr const char* PROP_LOG_STOP_OFFSET = "fluss.log_stop_offset"; + +constexpr size_t TAIL_BATCH_ROWS = 4096; + +std::vector split_on(std::string_view value, char separator) { + std::vector parts; + size_t start = 0; + while (true) { + const auto end = value.find(separator, start); + if (end == std::string_view::npos) { + parts.push_back(value.substr(start)); + return parts; + } + parts.push_back(value.substr(start, end - start)); + start = end + 1; + } +} + +void update_counter(RuntimeProfile::Counter* counter, int64_t value) { + if (counter != nullptr) { + COUNTER_UPDATE(counter, value); + } +} + +template +bool parse_integer(std::string_view text, T* value) { + if (text.empty()) { + return false; + } + const auto result = std::from_chars(text.data(), text.data() + text.size(), *value); + return result.ec == std::errc() && result.ptr == text.data() + text.size(); +} + +} // namespace + +Status FlussUnionLakeReader::parse_tail(const std::string& spec, Tail* tail) { + DORIS_CHECK(tail != nullptr); + const auto parts = split_on(spec, ':'); + if (parts.size() != 4) { + return Status::InternalError( + "fluss union read: '{}' is not a log tail of the form " + "partitionId:bucket:start:stop", + spec); + } + // An unpartitioned table leaves the partition segment empty rather than writing a sentinel, so + // that its bucket 0 and a partitioned table's bucket 0 cannot become the same cache entry. + if (!parts[0].empty()) { + int64_t partition_id = 0; + if (!parse_integer(parts[0], &partition_id)) { + return Status::InternalError( + "fluss union read: '{}' has a partition id that is not a " + "number in log tail '{}'", + parts[0], spec); + } + } + Tail parsed; + parsed.partition_id = std::string(parts[0]); + if (!parse_integer(parts[1], &parsed.bucket_id) || + !parse_integer(parts[2], &parsed.start_offset) || + !parse_integer(parts[3], &parsed.stop_offset)) { + return Status::InternalError( + "fluss union read: log tail '{}' has a bucket or offset that is not a number", + spec); + } + if (parsed.start_offset >= parsed.stop_offset) { + // Planning never wraps a lake split whose bucket has nothing left in its log. One arriving + // here means the two halves of this read were bounded by different offsets, and that is a + // duplicated or a missing row either way. + return Status::InternalError( + "fluss union read: a suppressing log tail must contain something, but bucket {} " + "was " + "given [{}, {})", + parsed.bucket_id, parsed.start_offset, parsed.stop_offset); + } + parsed.spec = spec; + *tail = std::move(parsed); + return Status::OK(); +} + +TFileRangeDesc FlussUnionLakeReader::tail_scan_range(const Tail& tail) { + std::map params { + {PROP_RANGE_TYPE, RANGE_TYPE_LOG}, + {PROP_BUCKET_ID, std::to_string(tail.bucket_id)}, + {PROP_LOG_START_OFFSET, std::to_string(tail.start_offset)}, + {PROP_LOG_STOP_OFFSET, std::to_string(tail.stop_offset)}, + }; + if (!tail.partition_id.empty()) { + // Absent rather than -1 on an unpartitioned table: that is how the scanner tells the two + // apart, and fluss subscribes to a bucket of each by a different call. + params.emplace(PROP_PARTITION_ID, tail.partition_id); + } + TTableFormatFileDesc table_format_params; + table_format_params.__set_table_format_type("fluss"); + table_format_params.__set_fluss_params(std::move(params)); + TFileRangeDesc range; + range.__set_table_format_params(std::move(table_format_params)); + range.__set_format_type(TFileFormatType::FORMAT_JNI); + return range; +} + +Status FlussUnionLakeReader::init(format::TableReadOptions&& options) { + RETURN_IF_ERROR(format::TableReader::init(std::move(options))); + RETURN_IF_ERROR(_resolve_union_properties()); + _init_union_profile(); + + VExprContextSPtrs conjuncts; + conjuncts.reserve(_conjuncts.size()); + for (const auto& conjunct : _conjuncts) { + VExprSPtr root; + RETURN_IF_ERROR(format::clone_table_expr_tree(conjunct->root(), &root)); + conjuncts.push_back(VExprContext::create_shared(std::move(root))); + } + _lake_reader = std::make_unique(); + RETURN_IF_ERROR(_lake_reader->init({ + .projected_columns = _projected_columns, + .conjuncts = std::move(conjuncts), + .format = _format, + .scan_params = _scan_params, + .io_ctx = _io_ctx, + .runtime_state = _runtime_state, + .scanner_profile = _scanner_profile, + .file_slot_descs = _file_slot_descs, + // Aggregate pushdown is withheld from the lake half on purpose. A COUNT answered from + // paimon's own file metadata would count the very rows this reader is about to suppress, + // and it would do so without ever producing a block to suppress them from. + .push_down_agg_type = TPushAggOp::type::NONE, + .condition_cache_digest = _condition_cache_digest, + })); + if (_batch_size > 0) { + _lake_reader->set_batch_size(_batch_size); + } + return Status::OK(); +} + +Status FlussUnionLakeReader::_resolve_union_properties() { + if (_scan_params == nullptr || !_scan_params->__isset.fluss_properties) { + return Status::InternalError( + "missing fluss_properties for a fluss union read, possibly caused by FE/BE " + "protocol " + "mismatch"); + } + const auto& properties = _scan_params->fluss_properties; + const auto names_it = properties.find(PROP_PK_NAMES); + if (names_it == properties.end() || names_it->second.empty()) { + return Status::InternalError( + "missing '{}' for a fluss union read: without the key columns the lake rows the " + "log " + "tail supersedes cannot be identified", + PROP_PK_NAMES); + } + for (const auto name : split_on(names_it->second, ',')) { + const auto column = std::ranges::find_if( + _projected_columns, + [&](const format::ColumnDefinition& candidate) { return candidate.name == name; }); + if (column == _projected_columns.end()) { + // FE keeps the key columns in the scan's tuple whenever it plans a union read, so this + // means its planning-time decision and its split-planning decision disagreed. Suppressing + // nothing would return every superseded lake row a second time, silently. + return Status::InternalError( + "fluss union read: key column '{}' is not among the columns this scan " + "projects. " + "The lake rows its log tail supersedes cannot be identified without it", + name); + } + _key_column_indexes.push_back( + cast_set(std::distance(_projected_columns.begin(), column))); + _key_columns.push_back(*column); + } + + const auto rows_it = properties.find(PROP_MAX_TAIL_ROWS); + if (rows_it == properties.end() || + !parse_integer(std::string_view(rows_it->second), &_max_tail_rows) || _max_tail_rows <= 0) { + return Status::InternalError( + "fluss union read: '{}' must be a positive number of rows, but was '{}'", + PROP_MAX_TAIL_ROWS, + rows_it == properties.end() ? std::string("missing") : rows_it->second); + } + return Status::OK(); +} + +void FlussUnionLakeReader::_init_union_profile() { + if (_scanner_profile == nullptr) { + return; + } + static const char* table_profile = file_scan_profile::TABLE_READER; + _suppressed_rows_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "FlussUnionSuppressedRows", TUnit::UNIT, table_profile, 1); + _tail_keys_read_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "FlussUnionTailKeysRead", TUnit::UNIT, table_profile, 1); + _tail_cache_hit_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "FlussUnionTailCacheHitCount", TUnit::UNIT, table_profile, 1); +} + +Status FlussUnionLakeReader::prepare_split(const format::SplitReadOptions& options) { + DORIS_CHECK(_lake_reader != nullptr); + RETURN_IF_ERROR(_lake_reader->prepare_split(options)); + if (_lake_reader->current_split_pruned()) { + // A pruned split returns no rows, so there is nothing to suppress and no reason to spend a + // read of the tail on it. + return Status::OK(); + } + return _prepare_suppression(options); +} + +Status FlussUnionLakeReader::_prepare_suppression(const format::SplitReadOptions& options) { + const auto& range = options.current_range; + if (!range.__isset.table_format_params || !range.table_format_params.__isset.fluss_params) { + return Status::InternalError( + "missing fluss_params on a fluss union lake split, possibly caused by FE/BE " + "protocol " + "mismatch"); + } + const auto& params = range.table_format_params.fluss_params; + const auto type_it = params.find(PROP_RANGE_TYPE); + if (type_it == params.end() || type_it->second != RANGE_TYPE_LAKE_SUPPRESS) { + return Status::InternalError( + "a fluss union lake split must carry '{}={}', but carries '{}'", PROP_RANGE_TYPE, + RANGE_TYPE_LAKE_SUPPRESS, + type_it == params.end() ? std::string("nothing") : type_it->second); + } + const auto tail_it = params.find(PROP_TAIL); + if (tail_it == params.end()) { + return Status::InternalError("missing '{}' on a fluss union lake split", PROP_TAIL); + } + if (_suppression != nullptr && _suppression_tail_spec == tail_it->second) { + // Consecutive splits of one bucket are common; their suppression is the same one. + return Status::OK(); + } + Tail tail; + RETURN_IF_ERROR(parse_tail(tail_it->second, &tail)); + RETURN_IF_ERROR(_load_suppression_keys(options, tail)); + _suppression_tail_spec = tail_it->second; + return Status::OK(); +} + +Status FlussUnionLakeReader::_load_suppression_keys(const format::SplitReadOptions& options, + const Tail& tail) { + if (options.cache == nullptr) { + return Status::InternalError( + "fluss union read: no split cache to hold the keys of log tail '{}'", tail.spec); + } + // Length-prefixed so that no boundary between the fixed prefix and the tail can be reinterpreted + // as part of the tail itself. One scan node reads one table, so the tail alone identifies it. + const auto cache_key = fmt::format("fluss_union_tail:{}:{}", tail.spec.size(), tail.spec); + Status read_status = Status::OK(); + bool cache_hit = false; + auto* cached = options.cache->get( + cache_key, + [&]() -> SuppressionKeys* { + auto keys = std::make_unique(); + read_status = _read_tail_keys(tail, &keys->keys); + if (!read_status.ok()) { + return nullptr; + } + return keys.release(); + }, + &cache_hit); + RETURN_IF_ERROR(read_status); + DORIS_CHECK(cached != nullptr); + if (cache_hit) { + update_counter(_tail_cache_hit_counter, 1); + } else { + update_counter(_tail_keys_read_counter, cast_set(cached->keys.rows())); + } + return _build_suppression_predicate(cached->keys); +} + +Block FlussUnionLakeReader::_empty_key_block() const { + Block block; + for (const auto& column : _key_columns) { + block.insert({column.type->create_column(), column.type, column.name}); + } + return block; +} + +Status FlussUnionLakeReader::_accumulate_tail_keys(const Tail& tail, const NextBatch& next_batch, + Block* keys) { + auto builder = MutableBlock::build_mutable_block(_empty_key_block()); + Block batch = _empty_key_block(); + while (true) { + bool eos = false; + batch.clear_column_data(cast_set(_key_columns.size())); + RETURN_IF_ERROR(next_batch(&batch, &eos)); + if (batch.rows() > 0) { + RETURN_IF_ERROR(builder.merge(batch)); + if (cast_set(builder.rows()) > _max_tail_rows) { + // The same yardstick and the same limit the Java side applies while replaying this + // very range: change log records read, not distinct keys. Both halves of one table + // therefore exceed it together, so which of them reports it is not a coin toss. + return Status::InternalError( + "the log tail of bucket {} holds more than {} change log records, the " + "limit " + "set by 'fluss.union_read.max_tail_rows'. Read the table as pure fluss " + "with " + "'fluss.union_read.mode=disabled', wait for tiering to move the tail into " + "the lake, or raise the limit", + tail.bucket_id, _max_tail_rows); + } + } + if (eos) { + *keys = builder.to_block(); + return Status::OK(); + } + } +} + +Status FlussUnionLakeReader::_read_tail_keys(const Tail& tail, Block* keys) { + DORIS_CHECK(keys != nullptr); +#ifdef BE_TEST + if (_test_tail_reader) { + // Stands in for the network read alone. Everything built on it - the accumulation, the + // limit, the cache - is the same code the real read goes through. + Block canned; + RETURN_IF_ERROR(_test_tail_reader(tail, &canned)); + bool delivered = false; + return _accumulate_tail_keys( + tail, + [&](Block* batch, bool* eos) { + if (!delivered) { + *batch = canned; + delivered = true; + } + *eos = true; + return Status::OK(); + }, + keys); + } +#endif + // The tail is read as an ordinary bounded log range, projected down to the key columns. Every + // record counts, including the deletions: a lake row whose key the tail deleted must disappear + // just as surely as one it updated, and what the tail ended up saying is contributed by its own + // range, not by this read. + FlussJniReader reader; + RETURN_IF_ERROR(reader.init({ + .projected_columns = _key_columns, + .conjuncts = {}, + .format = format::FileFormat::JNI, + .scan_params = _scan_params, + .io_ctx = _io_ctx, + .runtime_state = _runtime_state, + .scanner_profile = _scanner_profile, + })); + // Not the adaptive size the lake half is being read at: this is a bounded pass over key columns + // only, and its batches have nothing to do with how wide the lake's rows turned out to be. + reader.set_batch_size(_runtime_state == nullptr ? TAIL_BATCH_ROWS + : _runtime_state->batch_size()); + + format::SplitReadOptions tail_options; + tail_options.conjuncts = VExprContextSPtrs {}; + tail_options.condition_cache_digest = 0; + tail_options.current_range = tail_scan_range(tail); + tail_options.current_split_format = format::FileFormat::JNI; + + const auto drain = [&]() -> Status { + RETURN_IF_ERROR(reader.prepare_split(tail_options)); + return _accumulate_tail_keys( + tail, [&](Block* batch, bool* eos) { return reader.get_block(batch, eos); }, keys); + }; + const auto status = drain(); + // A fluss connection left open keeps its netty and metadata-updater threads alive for the life + // of the BE process, so the reader is closed on the failing path too. + const auto close_status = reader.close(); + RETURN_IF_ERROR(status); + return close_status; +} + +Status FlussUnionLakeReader::_build_suppression_predicate(const Block& keys) { + std::vector key_positions; + key_positions.reserve(_key_column_indexes.size()); + for (const auto index : _key_column_indexes) { + key_positions.push_back(cast_set(index)); + } + DORIS_CHECK(keys.columns() == key_positions.size()); + auto predicate = std::make_shared(keys, key_positions); + for (size_t i = 0; i < _key_column_indexes.size(); ++i) { + const auto position = key_positions[i]; + // The block this runs against is the table-schema block the lake half returns, so a key + // column is simply the projected column at its own position - already mapped, already of the + // projected type, and therefore comparable with the tail's column of the same name. + predicate->add_child(VSlotRef::create_shared(position, position, -1, _key_columns[i].type, + _key_columns[i].name)); + } + _suppression = VExprContext::create_shared(std::move(predicate)); + RowDescriptor row_desc; + RETURN_IF_ERROR(_suppression->prepare(_runtime_state, row_desc)); + return _suppression->open(_runtime_state); +} + +Status FlussUnionLakeReader::get_block(Block* block, bool* eos) { + DORIS_CHECK(_lake_reader != nullptr); + RETURN_IF_ERROR(_lake_reader->get_block(block, eos)); + return _suppress(block); +} + +Status FlussUnionLakeReader::_suppress(Block* block) { + DORIS_CHECK(block != nullptr); + if (_suppression == nullptr) { + // Every wrapped lake split names the tail that supersedes part of it. Reading one without + // having built that suppression would return the superseded rows a second time. + return Status::InternalError( + "fluss union read: a lake split was read with no log tail bound to it"); + } + const auto rows = block->rows(); + if (rows == 0) { + return Status::OK(); + } + SCOPED_TIMER(_profile.finalize_timer); + int result_column_id = -1; + RETURN_IF_ERROR(_suppression->root()->execute(_suppression.get(), block, &result_column_id)); + DORIS_CHECK(result_column_id >= 0 && result_column_id < cast_set(block->columns())); + const auto& suppressed = + assert_cast(*block->get_by_position(result_column_id).column) + .get_data(); + DORIS_CHECK(suppressed.size() == rows); + IColumn::Filter keep(rows, 1); + int64_t dropped = 0; + for (size_t row = 0; row < rows; ++row) { + keep[row] = suppressed[row] == 0; + dropped += suppressed[row] != 0 ? 1 : 0; + } + block->erase(result_column_id); + if (dropped == 0) { + return Status::OK(); + } + RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(block, keep)); + update_counter(_suppressed_rows_counter, dropped); + return Status::OK(); +} + +bool FlussUnionLakeReader::current_split_pruned() const { + DORIS_CHECK(_lake_reader != nullptr); + return _lake_reader->current_split_pruned(); +} + +bool FlussUnionLakeReader::current_split_uses_metadata_count() const { + DORIS_CHECK(_lake_reader != nullptr); + return _lake_reader->current_split_uses_metadata_count(); +} + +Status FlussUnionLakeReader::abort_split() { + DORIS_CHECK(_lake_reader != nullptr); + return _lake_reader->abort_split(); +} + +Status FlussUnionLakeReader::close() { + _suppression.reset(); + _suppression_tail_spec.clear(); + if (_lake_reader == nullptr) { + return Status::OK(); + } + return _lake_reader->close(); +} + +void FlussUnionLakeReader::set_batch_size(size_t batch_size) { + format::TableReader::set_batch_size(batch_size); + if (_lake_reader != nullptr) { + _lake_reader->set_batch_size(_batch_size); + } +} + +int64_t FlussUnionLakeReader::condition_cache_hit_count() const { + return _lake_reader == nullptr ? 0 : _lake_reader->condition_cache_hit_count(); +} + +} // namespace doris::format::fluss diff --git a/be/src/format_v2/table/fluss_union_lake_reader.h b/be/src/format_v2/table/fluss_union_lake_reader.h new file mode 100644 index 00000000000000..f47f5b856bb5bb --- /dev/null +++ b/be/src/format_v2/table/fluss_union_lake_reader.h @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/block/block.h" +#include "exprs/vexpr_fwd.h" +#include "format_v2/table_reader.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::format::fluss { + +/** + * A lake split of a tiered primary-key fluss table, minus the rows its log tail has superseded. + * + *

A primary-key table whose rows are being tiered into a lake is read as two halves: the lake, as + * the paimon sibling's own splits, and the change log written after the lake snapshot. The halves + * overlap by key rather than by row - a row the lake still holds may since have been updated or + * deleted in the log - so returning both copies would duplicate rows in a way no row count reveals. + * + *

The split of labour: this reader only SUPPRESSES. It reads the keys of one bucket's log tail, + * once per bucket per BE, and drops the lake rows those keys name. What the tail ended up saying + * about each of those keys is contributed separately, by a `PK_TAIL` fluss range replayed on the + * Java side, so every row is produced exactly once and a key the tail ended by deleting is produced + * not at all. + * + *

The lake half is read entirely by the sibling's reader stack. This class owns a + * PaimonHybridReader and forwards every split to it untouched: native ORC/Parquet, serialized JNI + * splits, deletion vectors, schema evolution and split scheduling are all its answer. The wrapping + * range (`fluss_union`) carries the sibling's own payload beside the tail descriptor for exactly + * that reason. + * + *

Where the filter sits. On the block the child returns, in table-schema terms, not inside + * the child's file scan request. FE keeps the primary-key columns in the scan's tuple whenever it + * plans a union read, so the keys are ordinary projected columns of that block and the column mapper + * has already resolved them - by name or by field id, whichever the lake table needs. Pushing the + * predicate into the file request would buy the non-key columns of the suppressed rows not being + * decoded, which is bounded by the size of the tail, and would cost a second place that resolves key + * columns against a file schema plus a separate implementation for the JNI child. + */ +class FlussUnionLakeReader final : public format::TableReader { +public: + /** One bucket's log tail: the half-open offset range the lake snapshot does not cover yet. */ + struct Tail { + /** Empty on an unpartitioned table, which fluss subscribes to by bucket alone. */ + std::string partition_id; + int32_t bucket_id = 0; + int64_t start_offset = 0; + int64_t stop_offset = 0; + /** Exactly what FE wrote. Also the identity of this tail in the scan node's split cache. */ + std::string spec; + }; + + /** + * The keys of one bucket's log tail, one row per change-log record and one column per key + * column. Held by the scan node's split cache so every split of that bucket shares one read. + */ + struct SuppressionKeys { + Block keys; + }; + + ~FlussUnionLakeReader() override = default; + + Status init(format::TableReadOptions&& options) override; + Status prepare_split(const format::SplitReadOptions& options) override; + Status get_block(Block* block, bool* eos) override; + bool current_split_pruned() const override; + bool current_split_uses_metadata_count() const override; + Status abort_split() override; + Status close() override; + void set_batch_size(size_t batch_size) override; + int64_t condition_cache_hit_count() const override; + + /** + * `partitionId:bucket:start:stop`, as FE encoded it. The partition id is empty rather than a + * sentinel on an unpartitioned table, so that its buckets cannot collide with a partitioned + * table's in the cache. + */ + static Status parse_tail(const std::string& spec, Tail* tail); + + /** The fluss range that reads that tail: one bucket's log over `[start, stop)`, keys only. */ + static TFileRangeDesc tail_scan_range(const Tail& tail); + +private: + /** Fills one batch of tail keys and says whether the tail has been read to its end. */ + using NextBatch = std::function; + + Status _resolve_union_properties(); + Status _prepare_suppression(const format::SplitReadOptions& options); + Status _load_suppression_keys(const format::SplitReadOptions& options, const Tail& tail); + Status _accumulate_tail_keys(const Tail& tail, const NextBatch& next_batch, Block* keys); + Status _read_tail_keys(const Tail& tail, Block* keys); + Status _build_suppression_predicate(const Block& keys); + Status _suppress(Block* block); + Block _empty_key_block() const; + void _init_union_profile(); + + std::unique_ptr _lake_reader; + + /** Positions of the key columns in the projected block, in the order FE listed them. */ + std::vector _key_column_indexes; + /** The projected column descriptions of those same columns, reused rather than rebuilt. */ + std::vector _key_columns; + int64_t _max_tail_rows = 0; + + /** The tail the current predicate was built from; a split of the same bucket reuses it. */ + std::string _suppression_tail_spec; + VExprContextSPtr _suppression; + + RuntimeProfile::Counter* _suppressed_rows_counter = nullptr; + RuntimeProfile::Counter* _tail_keys_read_counter = nullptr; + RuntimeProfile::Counter* _tail_cache_hit_counter = nullptr; + +#ifdef BE_TEST + // Reading the tail needs a fluss cluster. Tests stand in for that one step and drive everything + // built on top of it - the cache, the limit, and the suppression itself. + std::function _test_tail_reader; +#endif +}; + +} // namespace doris::format::fluss diff --git a/be/src/service/CMakeLists.txt b/be/src/service/CMakeLists.txt index b9bcef0b7b821a..64ed8651417748 100644 --- a/be/src/service/CMakeLists.txt +++ b/be/src/service/CMakeLists.txt @@ -49,6 +49,24 @@ if (${MAKE_TEST} STREQUAL "OFF" AND ${BUILD_BENCHMARK} STREQUAL "OFF") # This permits libraries loaded by dlopen to link to the symbols in the program. set_target_properties(doris_be PROPERTIES ENABLE_EXPORTS 1) + # ...but not the symbols of the RocksDB we link statically. Exporting those makes this + # executable the definition every later-loaded library binds to, and a JNI library that + # carries its own RocksDB then runs half on ours: the fluss scanner bundles frocksdbjni, + # whose librocksdbjni.so defines 2576 rocksdb symbols under names identical to ours but + # was built against the pre-C++11 libstdc++ string ABI. Objects laid out by one and used + # by the other yield a garbage length, an std::bad_alloc that escapes the JNI frame, and + # an aborted BE. Hiding this archive lets that library bind to its own copy. + # + # Scoped to the archive rather than dropping ENABLE_EXPORTS: what needs the exports is + # native UDFs (runtime/user_function_cache.cpp dlopens them), and those use the Doris UDF + # ABI, which has nothing to do with RocksDB. Crash stacks do not need it either -- they are + # symbolized from debug info, which is why they name even anonymous-namespace functions. + # + # The same library also duplicates zstd, lz4, snappy, bzip2 and zlib symbols. Those are C + # ABIs, stable across versions and layout-free, so they are left alone until something + # shows otherwise -- unlike RocksDB, whose C++ objects are what actually corrupt. + target_link_options(doris_be PRIVATE "-Wl,--exclude-libs,librocksdb.a") + target_link_libraries(doris_be ${DORIS_LINK_LIBS} ) diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 353c08043adef4..9c924b65f30294 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -318,6 +318,8 @@ TEST(FileScannerV2Test, SupportedFormatMatrix) { {"hive", TFileFormatType::FORMAT_PARQUET, std::nullopt, true}, {"iceberg", TFileFormatType::FORMAT_PARQUET, std::nullopt, true}, {"paimon", TFileFormatType::FORMAT_PARQUET, std::nullopt, true}, + {"fluss_union", TFileFormatType::FORMAT_PARQUET, std::nullopt, true}, + {"fluss_union", TFileFormatType::FORMAT_ORC, std::nullopt, true}, {"hudi", TFileFormatType::FORMAT_PARQUET, std::nullopt, true}, {"jdbc", TFileFormatType::FORMAT_PARQUET, std::nullopt, false}, {"", TFileFormatType::FORMAT_JNI, std::nullopt, false}, @@ -475,6 +477,35 @@ TEST(FileScannerV2Test, JniCompatibilityShapesUseV2Scanner) { EXPECT_TRUE(FileScannerV2::is_supported(params, legacy_paimon_jni_range_without_reader_type())); } +// Scenario: a lake split of a fluss primary-key table read as its lake plus its log tail. It is the +// paimon sibling's own split with a suppression descriptor attached, so whether V2 can read it is +// the paimon answer, asked of the paimon payload it carries - not a separate rule that could start +// disagreeing with paimon's after the sibling changes how it plans a split. +TEST(FileScannerV2Test, WrappedLakeSplitsAreSupportedWhereverThePaimonOnesAre) { + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_JNI); + + auto serialized_split = legacy_paimon_jni_range_without_reader_type(); + serialized_split.table_format_params.__set_table_format_type("fluss_union"); + EXPECT_TRUE(FileScannerV2::is_supported(params, serialized_split)); + + auto native_file_as_jni = range_with_format("fluss_union", TFileFormatType::FORMAT_JNI); + TPaimonFileDesc paimon_params; + paimon_params.__set_file_format("orc"); + native_file_as_jni.table_format_params.__set_paimon_params(paimon_params); + EXPECT_TRUE(FileScannerV2::is_supported(params, native_file_as_jni)); + + // No paimon payload at all is not a split this reader can hand to the sibling. + EXPECT_FALSE(FileScannerV2::is_supported( + params, range_with_format("fluss_union", TFileFormatType::FORMAT_JNI))); + + // The C++ paimon reader stays on the V1 fallback, which has no fluss_union branch: a clean + // refusal rather than a lake half read without its suppression. + auto cpp_split = paimon_cpp_jni_range(); + cpp_split.table_format_params.__set_table_format_type("fluss_union"); + EXPECT_FALSE(FileScannerV2::is_supported(params, cpp_split)); +} + TEST(FileScannerV2Test, FailedTableReaderCloseCanBeRetriedThroughScanner) { RuntimeState state {TQueryOptions(), TQueryGlobals()}; RuntimeProfile profile("file_scanner_v2_close_retry"); diff --git a/be/test/format_v2/jni/fluss_jni_reader_test.cpp b/be/test/format_v2/jni/fluss_jni_reader_test.cpp new file mode 100644 index 00000000000000..2707c780b3aede --- /dev/null +++ b/be/test/format_v2/jni/fluss_jni_reader_test.cpp @@ -0,0 +1,234 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/jni/fluss_jni_reader.h" + +#include + +#include +#include +#include + +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "format_v2/table_reader.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::format::fluss { +namespace { + +TFileRangeDesc make_fluss_range(std::map fluss_params) { + TFileRangeDesc range; + TTableFormatFileDesc table_format_params; + table_format_params.__set_table_format_type("fluss"); + table_format_params.__set_fluss_params(std::move(fluss_params)); + range.__set_table_format_params(std::move(table_format_params)); + return range; +} + +TFileScanRangeParams make_scan_params(std::map fluss_properties) { + TFileScanRangeParams scan_params; + scan_params.__set_fluss_properties(std::move(fluss_properties)); + return scan_params; +} + +Status init_reader(FlussJniReader* reader, TFileScanRangeParams* scan_params, + std::vector projected_columns = {}) { + return reader->init({ + .projected_columns = std::move(projected_columns), + .conjuncts = {}, + .format = FileFormat::JNI, + .scan_params = scan_params, + .io_ctx = nullptr, + .runtime_state = nullptr, + .scanner_profile = nullptr, + }); +} + +Status build_params(FlussJniReader* reader, const TFileRangeDesc& range, + std::map* params) { + reader->_current_range = range; + return reader->build_scanner_params(params); +} + +ColumnDefinition make_column(const std::string& name, DataTypePtr type, bool is_partition_key) { + ColumnDefinition column; + column.name = name; + column.type = std::move(type); + column.is_partition_key = is_partition_key; + return column; +} + +// Both maps reach the Java scanner as one. Dropping either half is invisible in C++ - the scanner +// would just report a missing key - so the merge itself is what has to be pinned here. +TEST(FlussJniReaderTest, MergesScanLevelAndRangeLevelParams) { + auto scan_params = make_scan_params({{"fluss.db_name", "db"}, + {"fluss.table_name", "log_basic"}, + {"fluss.client.bootstrap.servers", "host:9123"}}); + auto range = make_fluss_range({{"fluss.range_type", "LOG"}, + {"fluss.bucket_id", "3"}, + {"fluss.log_start_offset", "-2"}, + {"fluss.log_stop_offset", "17"}}); + + FlussJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + ASSERT_TRUE(reader.validate_scan_range(range).ok()); + + std::map params; + ASSERT_TRUE(build_params(&reader, range, ¶ms).ok()); + EXPECT_EQ(params["fluss.db_name"], "db"); + EXPECT_EQ(params["fluss.table_name"], "log_basic"); + EXPECT_EQ(params["fluss.client.bootstrap.servers"], "host:9123"); + EXPECT_EQ(params["fluss.range_type"], "LOG"); + EXPECT_EQ(params["fluss.bucket_id"], "3"); + EXPECT_EQ(params["fluss.log_start_offset"], "-2"); + EXPECT_EQ(params["fluss.log_stop_offset"], "17"); + EXPECT_EQ(params.size(), 7); +} + +// The range is the specific half of the pair. If the scan level ever won, every bucket of a scan +// would read the same one - a query that returns wrong rows rather than failing. +TEST(FlussJniReaderTest, RangeLevelParamWinsOverScanLevelParam) { + auto scan_params = make_scan_params({{"fluss.db_name", "db"}, {"fluss.bucket_id", "0"}}); + auto range = make_fluss_range({{"fluss.range_type", "LOG"}, {"fluss.bucket_id", "7"}}); + + FlussJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + + std::map params; + ASSERT_TRUE(build_params(&reader, range, ¶ms).ok()); + EXPECT_EQ(params["fluss.bucket_id"], "7"); +} + +TEST(FlussJniReaderTest, RejectsRangeWithoutFlussParams) { + auto scan_params = make_scan_params({{"fluss.db_name", "db"}}); + + FlussJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + + TFileRangeDesc no_table_format; + EXPECT_FALSE(reader.validate_scan_range(no_table_format).ok()); + + TFileRangeDesc no_fluss_params; + TTableFormatFileDesc table_format_params; + table_format_params.__set_table_format_type("fluss"); + no_fluss_params.__set_table_format_params(std::move(table_format_params)); + EXPECT_FALSE(reader.validate_scan_range(no_fluss_params).ok()); + + EXPECT_FALSE(reader.validate_scan_range(make_fluss_range({})).ok()); +} + +TEST(FlussJniReaderTest, RejectsScanWithoutFlussProperties) { + auto range = make_fluss_range({{"fluss.range_type", "LOG"}}); + + TFileScanRangeParams empty_scan_params; + FlussJniReader reader; + ASSERT_TRUE(init_reader(&reader, &empty_scan_params).ok()); + EXPECT_FALSE(reader.validate_scan_range(range).ok()); + + auto empty_properties = make_scan_params({}); + FlussJniReader reader_with_empty_properties; + ASSERT_TRUE(init_reader(&reader_with_empty_properties, &empty_properties).ok()); + EXPECT_FALSE(reader_with_empty_properties.validate_scan_range(range).ok()); +} + +// A partition column is a constant of the range, not something the scanner reads. Asking for it +// would also disagree with the paimon half of a union read, whose partition columns come from the +// same per-range values - the two halves of one scan node have to lay their blocks out alike. +TEST(FlussJniReaderTest, LeavesPartitionColumnsToTheRangeAndKeepsOutputPositions) { + auto scan_params = make_scan_params({{"fluss.db_name", "db"}}); + std::vector projected_columns { + make_column("id", std::make_shared(), false), + make_column("dt", std::make_shared(), true), + make_column("name", std::make_shared(), false)}; + + FlussJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params, std::move(projected_columns)).ok()); + reader._partition_values.emplace("dt", + Field::create_field(std::string("20260101"))); + + std::vector columns; + ASSERT_TRUE(reader.build_jni_columns(&columns).ok()); + ASSERT_EQ(columns.size(), 2); + EXPECT_EQ(columns[0].java_name, "id"); + EXPECT_EQ(columns[0].output_index, 0); + EXPECT_EQ(columns[1].java_name, "name"); + // Its position in the output block, not its position among the columns the scanner returns: + // the two differ by every partition column dropped before it. + EXPECT_EQ(columns[1].output_index, 2); +} + +// A partition column the range has no value for would otherwise leave a hole in the output block. +TEST(FlussJniReaderTest, StillReadsAPartitionColumnTheRangeDidNotCarry) { + auto scan_params = make_scan_params({{"fluss.db_name", "db"}}); + std::vector projected_columns { + make_column("id", std::make_shared(), false), + make_column("dt", std::make_shared(), true)}; + + FlussJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params, std::move(projected_columns)).ok()); + + std::vector columns; + ASSERT_TRUE(reader.build_jni_columns(&columns).ok()); + ASSERT_EQ(columns.size(), 2); + EXPECT_EQ(columns[1].java_name, "dt"); + EXPECT_EQ(columns[1].output_index, 1); +} + +// A reader whose projection nobody planned. +// +// Every projection so far arrived from FE as slot descriptors: the query asked for these columns, +// the engine turned them into ColumnDefinitions, the reader read them. A union read of a primary-key +// table needs one that did not come from a query at all - to suppress the lake rows a newer log +// record has replaced, BE first has to read the key columns of that log tail, and those keys are +// needed whether or not the query happens to project them. +// +// What this pins is that a projection assembled here, from names and types alone, drives the reader +// exactly like a planned one: the scanner is asked for those columns, in that order, with those +// types. Everything downstream (the JNI block template, and the Java scanner's own projection) is +// derived from this pair of strings, so if they are right the rest follows. +TEST(FlussJniReaderTest, ReadsAProjectionBuiltHereRatherThanPlannedByFe) { + auto scan_params = make_scan_params({{"fluss.db_name", "db"}}); + // No slot descriptors, no tuple, no query: three key columns named and typed on the spot. + std::vector synthetic_keys { + make_column("id", std::make_shared(), false), + make_column("name", std::make_shared(), false), + make_column("amount", std::make_shared(), false)}; + + FlussJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params, std::move(synthetic_keys)).ok()); + + ASSERT_TRUE(reader.build_jni_columns(&reader._jni_columns).ok()); + ASSERT_EQ(reader._jni_columns.size(), 3); + reader._prepare_jni_scanner_schema(); + + // The two strings the Java scanner projects by. A synthetic projection that produced anything + // else here would read the wrong columns, or fail to open, at the first union read of a + // primary-key table - and nothing between here and there would notice. + EXPECT_EQ(reader._scanner_params["required_fields"], "id,name,amount"); + EXPECT_EQ(reader._scanner_params["columns_types"], "int#string#bigint"); + + // The block C++ receives them into. Its width and types must match the projection, since + // finalize_jni_block moves these columns into the output block by position. + ASSERT_EQ(reader._jni_block_template.columns(), 3); + EXPECT_EQ(reader._jni_block_template.get_by_position(0).name, "id"); + EXPECT_EQ(reader._jni_block_template.get_by_position(2).name, "amount"); + EXPECT_TRUE(reader._jni_block_template.get_by_position(2).type->equals(DataTypeInt64 {})); +} + +} // namespace +} // namespace doris::format::fluss diff --git a/be/test/format_v2/table/fluss_union_lake_reader_test.cpp b/be/test/format_v2/table/fluss_union_lake_reader_test.cpp new file mode 100644 index 00000000000000..46356c0090a284 --- /dev/null +++ b/be/test/format_v2/table/fluss_union_lake_reader_test.cpp @@ -0,0 +1,593 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/fluss_union_lake_reader.h" + +#include + +#include +#include +#include +#include +#include + +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "format/format_common.h" +#include "format_v2/table_reader.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris::format::fluss { +namespace { + +using Tail = FlussUnionLakeReader::Tail; + +// The lake half, stood in for. Everything about how a paimon split is read is the sibling's answer; +// what this reader adds is only what happens to the block afterwards, which is what these tests are. +class RecordingLakeReader final : public format::TableReader { +public: + Status prepare_split(const format::SplitReadOptions& options) override { + prepared_ranges.push_back(options.current_range); + _current_split_pruned = prune_next_split; + return Status::OK(); + } + + Status get_block(Block* block, bool* eos) override { + if (blocks.empty()) { + *eos = true; + return Status::OK(); + } + *eos = false; + *block = std::move(blocks.front()); + blocks.erase(blocks.begin()); + return Status::OK(); + } + + Status close() override { + closed = true; + return Status::OK(); + } + + std::vector prepared_ranges; + std::vector blocks; + bool prune_next_split = false; + bool closed = false; +}; + +DataTypePtr int_type() { + return std::make_shared(); +} + +DataTypePtr string_type() { + return std::make_shared(); +} + +format::ColumnDefinition make_column(const std::string& name, DataTypePtr type) { + format::ColumnDefinition column; + column.name = name; + column.type = std::move(type); + return column; +} + +// id (int) / name (string) / amount (int); the primary key is (id, name) unless stated otherwise. +std::vector projected_columns() { + return {make_column("id", int_type()), make_column("name", string_type()), + make_column("amount", int_type())}; +} + +TFileScanRangeParams make_scan_params(std::map fluss_properties) { + TFileScanRangeParams scan_params; + scan_params.__set_fluss_properties(std::move(fluss_properties)); + return scan_params; +} + +TFileScanRangeParams union_scan_params(const std::string& pk_names = "id,name", + const std::string& max_tail_rows = "2000000") { + return make_scan_params({{"fluss.db_name", "db"}, + {"fluss.table_name", "lake_pk"}, + {"fluss.union.pk_names", pk_names}, + {"fluss.union.max_tail_rows", max_tail_rows}}); +} + +TFileRangeDesc wrapped_lake_split(const std::string& tail) { + TTableFormatFileDesc table_format_params; + table_format_params.__set_table_format_type("fluss_union"); + table_format_params.__set_fluss_params( + {{"fluss.range_type", "LAKE_SUPPRESS"}, {"fluss.union.tail", tail}}); + TFileRangeDesc range; + range.__set_table_format_params(std::move(table_format_params)); + range.__set_format_type(TFileFormatType::FORMAT_PARQUET); + return range; +} + +Status init_reader(FlussUnionLakeReader* reader, TFileScanRangeParams* scan_params, + std::vector columns = projected_columns()) { + return reader->init({ + .projected_columns = std::move(columns), + .conjuncts = {}, + .format = format::FileFormat::PARQUET, + .scan_params = scan_params, + .io_ctx = nullptr, + .runtime_state = nullptr, + .scanner_profile = nullptr, + }); +} + +// Replaces the paimon child after init(), which is where the real one is built and asked what it +// makes of the scan's options. +RecordingLakeReader* install_lake_reader(FlussUnionLakeReader* reader) { + auto lake_reader = std::make_unique(); + auto* raw = lake_reader.get(); + reader->_lake_reader = std::move(lake_reader); + return raw; +} + +Block make_block(const std::vector& ids, const std::vector& names, + const std::vector& amounts) { + auto id_column = ColumnInt32::create(); + auto name_column = ColumnString::create(); + auto amount_column = ColumnInt32::create(); + for (size_t row = 0; row < ids.size(); ++row) { + id_column->insert_value(ids[row]); + name_column->insert_data(names[row].data(), names[row].size()); + amount_column->insert_value(amounts[row]); + } + Block block; + block.insert({std::move(id_column), int_type(), "id"}); + block.insert({std::move(name_column), string_type(), "name"}); + block.insert({std::move(amount_column), int_type(), "amount"}); + return block; +} + +// The tail as this reader receives it: one row per change log record, key columns only, in the order +// FE listed the key. +Block make_key_block(const std::vector& ids, const std::vector& names) { + auto id_column = ColumnInt32::create(); + auto name_column = ColumnString::create(); + for (size_t row = 0; row < ids.size(); ++row) { + id_column->insert_value(ids[row]); + name_column->insert_data(names[row].data(), names[row].size()); + } + Block block; + block.insert({std::move(id_column), int_type(), "id"}); + block.insert({std::move(name_column), string_type(), "name"}); + return block; +} + +std::vector ids_of(const Block& block) { + std::vector ids; + const auto& column = assert_cast(*block.get_by_position(0).column); + for (size_t row = 0; row < column.size(); ++row) { + ids.push_back(column.get_data()[row]); + } + return ids; +} + +// ------------------------------------------------------------------ the tail descriptor + +// The four fields are how a split says which slice of which bucket's log may contradict it. A tail +// read from the wrong bucket suppresses rows nothing has superseded and leaves the superseded ones. +TEST(FlussUnionLakeReaderTest, ParsesTheTailOfBothPartitionedAndUnpartitionedTables) { + Tail partitioned; + ASSERT_TRUE(FlussUnionLakeReader::parse_tail("1234:3:17:42", &partitioned).ok()); + EXPECT_EQ(partitioned.partition_id, "1234"); + EXPECT_EQ(partitioned.bucket_id, 3); + EXPECT_EQ(partitioned.start_offset, 17); + EXPECT_EQ(partitioned.stop_offset, 42); + EXPECT_EQ(partitioned.spec, "1234:3:17:42"); + + Tail unpartitioned; + ASSERT_TRUE(FlussUnionLakeReader::parse_tail(":3:0:9", &unpartitioned).ok()); + // Empty rather than a sentinel number, so that bucket 3 of an unpartitioned table and bucket 3 + // of some partition cannot become one entry in the cache that holds their keys. + EXPECT_TRUE(unpartitioned.partition_id.empty()); + EXPECT_EQ(unpartitioned.bucket_id, 3); +} + +TEST(FlussUnionLakeReaderTest, RefusesATailItCannotReadInFull) { + Tail tail; + EXPECT_FALSE(FlussUnionLakeReader::parse_tail("3:0:9", &tail).ok()); + EXPECT_FALSE(FlussUnionLakeReader::parse_tail("1:3:0:9:extra", &tail).ok()); + EXPECT_FALSE(FlussUnionLakeReader::parse_tail("::0:9", &tail).ok()); + EXPECT_FALSE(FlussUnionLakeReader::parse_tail(":bucket:0:9", &tail).ok()); + EXPECT_FALSE(FlussUnionLakeReader::parse_tail("x:3:0:9", &tail).ok()); + EXPECT_FALSE(FlussUnionLakeReader::parse_tail(":3:0:", &tail).ok()); +} + +// An empty range is never planned: FE leaves a bucket whose lake has caught up with its log +// unwrapped. One arriving here means the two halves were bounded by different offsets, which +// duplicates or loses rows - and suppressing nothing would look exactly like working. +TEST(FlussUnionLakeReaderTest, RefusesATailThatSupersedesNothing) { + Tail tail; + EXPECT_FALSE(FlussUnionLakeReader::parse_tail(":3:9:9", &tail).ok()); + EXPECT_FALSE(FlussUnionLakeReader::parse_tail(":3:10:9", &tail).ok()); +} + +// The range that reads the tail is synthesized here, not planned by FE, so nothing downstream would +// notice it naming the wrong bucket or the wrong offsets. Pinned as a whole map for that reason. +TEST(FlussUnionLakeReaderTest, ReadsTheTailAsABoundedLogRangeOfThatBucket) { + Tail tail; + ASSERT_TRUE(FlussUnionLakeReader::parse_tail("77:3:17:42", &tail).ok()); + const auto range = FlussUnionLakeReader::tail_scan_range(tail); + ASSERT_TRUE(range.__isset.table_format_params); + EXPECT_EQ(range.table_format_params.table_format_type, "fluss"); + EXPECT_EQ(range.format_type, TFileFormatType::FORMAT_JNI); + const std::map expected {{"fluss.range_type", "LOG"}, + {"fluss.partition_id", "77"}, + {"fluss.bucket_id", "3"}, + {"fluss.log_start_offset", "17"}, + {"fluss.log_stop_offset", "42"}}; + EXPECT_EQ(range.table_format_params.fluss_params, expected); +} + +// LOG, not PK_TAIL. The suppression set is every key the tail touched, including the ones it ended +// by deleting: a lake row whose key was deleted has to disappear too, and PK_TAIL - which returns +// the surviving state - would not name it. +TEST(FlussUnionLakeReaderTest, ReadsTheTailAsAWholeChangeLogNotAsItsSurvivingState) { + Tail tail; + ASSERT_TRUE(FlussUnionLakeReader::parse_tail(":0:0:5", &tail).ok()); + const auto range = FlussUnionLakeReader::tail_scan_range(tail); + EXPECT_EQ(range.table_format_params.fluss_params.at("fluss.range_type"), "LOG"); +} + +// An unpartitioned table's bucket is subscribed to by a different fluss call than a partitioned +// one's, chosen by whether the range carries a partition id at all. +TEST(FlussUnionLakeReaderTest, LeavesOutThePartitionIdOfAnUnpartitionedTable) { + Tail tail; + ASSERT_TRUE(FlussUnionLakeReader::parse_tail(":3:0:9", &tail).ok()); + const auto range = FlussUnionLakeReader::tail_scan_range(tail); + EXPECT_EQ(range.table_format_params.fluss_params.count("fluss.partition_id"), 0); +} + +// ------------------------------------------------------------------ the key columns + +// The key columns are the projected ones, reused. FE keeps them in the scan's tuple for this read, +// which is what lets the two halves be compared at all: same tuple, same types, already mapped. +TEST(FlussUnionLakeReaderTest, TakesItsKeyColumnsFromTheProjectionInTheOrderFeListedThem) { + auto scan_params = union_scan_params("name,id"); + FlussUnionLakeReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + + // The key order is FE's, not the projection's: the tail's key block arrives in that same order. + ASSERT_EQ(reader._key_column_indexes.size(), 2); + EXPECT_EQ(reader._key_column_indexes[0], 1); + EXPECT_EQ(reader._key_column_indexes[1], 0); + ASSERT_EQ(reader._key_columns.size(), 2); + EXPECT_EQ(reader._key_columns[0].name, "name"); + EXPECT_TRUE(reader._key_columns[0].type->equals(DataTypeString {})); + EXPECT_EQ(reader._key_columns[1].name, "id"); +} + +// The one thing this reader cannot work around. FE promises the key columns are projected whenever +// it plans a union read; if that promise is broken, suppressing nothing returns every superseded +// lake row a second time, and no count or assertion downstream would show it. +TEST(FlussUnionLakeReaderTest, RefusesAScanThatDoesNotProjectAKeyColumn) { + auto scan_params = union_scan_params("id,name"); + FlussUnionLakeReader reader; + const auto status = + init_reader(&reader, &scan_params, + {make_column("id", int_type()), make_column("amount", int_type())}); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("name"), std::string::npos); +} + +TEST(FlussUnionLakeReaderTest, RefusesAScanWithoutTheUnionProperties) { + TFileScanRangeParams no_properties; + FlussUnionLakeReader without_properties; + EXPECT_FALSE(init_reader(&without_properties, &no_properties).ok()); + + auto no_names = make_scan_params({{"fluss.db_name", "db"}}); + FlussUnionLakeReader without_names; + EXPECT_FALSE(init_reader(&without_names, &no_names).ok()); + + auto no_limit = make_scan_params({{"fluss.union.pk_names", "id"}}); + FlussUnionLakeReader without_limit; + EXPECT_FALSE(init_reader(&without_limit, &no_limit).ok()); + + auto zero_limit = union_scan_params("id", "0"); + FlussUnionLakeReader with_zero_limit; + EXPECT_FALSE(init_reader(&with_zero_limit, &zero_limit).ok()); + + auto negative_limit = union_scan_params("id", "-1"); + FlussUnionLakeReader with_negative_limit; + EXPECT_FALSE(init_reader(&with_negative_limit, &negative_limit).ok()); +} + +// A COUNT answered from paimon's file metadata would count the rows this reader exists to remove, +// and would do it without ever producing a block to remove them from. +TEST(FlussUnionLakeReaderTest, WithholdsAggregatePushdownFromTheLakeHalf) { + auto scan_params = union_scan_params(); + FlussUnionLakeReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns(), + .conjuncts = {}, + .format = format::FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = nullptr, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + }) + .ok()); + ASSERT_TRUE(reader._lake_reader != nullptr); + EXPECT_EQ(reader._lake_reader->_push_down_agg_type, TPushAggOp::type::NONE); + EXPECT_EQ(reader._push_down_agg_type, TPushAggOp::type::COUNT); +} + +// ------------------------------------------------------------------ the suppression + +struct SuppressionFixture { + FlussUnionLakeReader reader; + TFileScanRangeParams scan_params = union_scan_params(); + ShardedKVCache cache {2}; + RecordingLakeReader* lake = nullptr; + std::vector tails_read; + + Status open(Block keys) { + RETURN_IF_ERROR(init_reader(&reader, &scan_params)); + lake = install_lake_reader(&reader); + reader._test_tail_reader = [this, keys](const Tail& tail, Block* out) mutable { + tails_read.push_back(tail.spec); + *out = keys; + return Status::OK(); + }; + return Status::OK(); + } + + format::SplitReadOptions split(const std::string& tail) { + format::SplitReadOptions options; + options.cache = &cache; + options.current_range = wrapped_lake_split(tail); + return options; + } +}; + +// The whole point, stated once: a lake row whose key the tail touched is gone, and every other lake +// row is untouched. What the tail ended up saying about those keys arrives separately. +TEST(FlussUnionLakeReaderTest, DropsTheLakeRowsTheTailNamesAndKeepsTheRest) { + SuppressionFixture fixture; + ASSERT_TRUE(fixture.open(make_key_block({2, 4}, {"b", "d"})).ok()); + ASSERT_TRUE(fixture.reader.prepare_split(fixture.split(":0:10:20")).ok()); + fixture.lake->blocks.push_back( + make_block({1, 2, 3, 4}, {"a", "b", "c", "d"}, {10, 20, 30, 40})); + + Block block = make_block({}, {}, {}); + bool eos = false; + ASSERT_TRUE(fixture.reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(ids_of(block), (std::vector {1, 3})); + // The block still has the shape its consumer expects; only rows went away. + EXPECT_EQ(block.columns(), 3); +} + +// The same key column values in different rows are not the same key. Comparing one column of a +// composite key would drop rows nothing superseded. +TEST(FlussUnionLakeReaderTest, MatchesOnTheWholeKeyNotOnOneColumnOfIt) { + SuppressionFixture fixture; + ASSERT_TRUE(fixture.open(make_key_block({1}, {"b"})).ok()); + ASSERT_TRUE(fixture.reader.prepare_split(fixture.split(":0:10:20")).ok()); + // (1, "a") shares its id with the tail's key and (2, "b") shares its name; neither is that key. + fixture.lake->blocks.push_back(make_block({1, 2, 1}, {"a", "b", "b"}, {10, 20, 30})); + + Block block = make_block({}, {}, {}); + bool eos = false; + ASSERT_TRUE(fixture.reader.get_block(&block, &eos).ok()); + EXPECT_EQ(ids_of(block), (std::vector {1, 2})); +} + +TEST(FlussUnionLakeReaderTest, LeavesASplitWhoseKeysTheTailNeverTouchedAlone) { + SuppressionFixture fixture; + ASSERT_TRUE(fixture.open(make_key_block({7, 8}, {"g", "h"})).ok()); + ASSERT_TRUE(fixture.reader.prepare_split(fixture.split(":0:10:20")).ok()); + fixture.lake->blocks.push_back(make_block({1, 2}, {"a", "b"}, {10, 20})); + + Block block = make_block({}, {}, {}); + bool eos = false; + ASSERT_TRUE(fixture.reader.get_block(&block, &eos).ok()); + EXPECT_EQ(ids_of(block), (std::vector {1, 2})); +} + +// One bucket's tail is one read, however many of its splits this BE was given. The read is a network +// round trip to fluss; doing it per split would multiply it by the number of files in the bucket. +TEST(FlussUnionLakeReaderTest, ReadsOneBucketsTailOnceHoweverManySplitsItHas) { + SuppressionFixture fixture; + ASSERT_TRUE(fixture.open(make_key_block({2}, {"b"})).ok()); + ASSERT_TRUE(fixture.reader.prepare_split(fixture.split(":0:10:20")).ok()); + ASSERT_TRUE(fixture.reader.prepare_split(fixture.split(":0:10:20")).ok()); + EXPECT_EQ(fixture.tails_read, (std::vector {":0:10:20"})); +} + +// Two buckets, two tails, and each split suppressed by its own. Binding them by bucket is what makes +// the tail small; getting the binding wrong is invisible unless the two tails differ. +TEST(FlussUnionLakeReaderTest, SuppressesEachBucketWithItsOwnTail) { + FlussUnionLakeReader reader; + auto scan_params = union_scan_params(); + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + auto* lake = install_lake_reader(&reader); + ShardedKVCache cache {2}; + std::vector tails_read; + reader._test_tail_reader = [&](const Tail& tail, Block* out) { + tails_read.push_back(tail.spec); + *out = tail.bucket_id == 0 ? make_key_block({1}, {"a"}) : make_key_block({2}, {"b"}); + return Status::OK(); + }; + const auto split = [&](const std::string& tail) { + format::SplitReadOptions options; + options.cache = &cache; + options.current_range = wrapped_lake_split(tail); + return options; + }; + + ASSERT_TRUE(reader.prepare_split(split(":0:10:20")).ok()); + lake->blocks.push_back(make_block({1, 2}, {"a", "b"}, {10, 20})); + Block bucket0 = make_block({}, {}, {}); + bool eos = false; + ASSERT_TRUE(reader.get_block(&bucket0, &eos).ok()); + EXPECT_EQ(ids_of(bucket0), (std::vector {2})); + + ASSERT_TRUE(reader.prepare_split(split(":1:30:40")).ok()); + lake->blocks.push_back(make_block({1, 2}, {"a", "b"}, {10, 20})); + Block bucket1 = make_block({}, {}, {}); + ASSERT_TRUE(reader.get_block(&bucket1, &eos).ok()); + EXPECT_EQ(ids_of(bucket1), (std::vector {1})); + EXPECT_EQ(tails_read, (std::vector {":0:10:20", ":1:30:40"})); +} + +// The limit exists because a BE is a long-lived process shared by every query on it. Both halves of +// one table count the same thing - change log records over the same offsets - so a table that is +// over the limit is over it on both sides rather than in whichever half ran first. +TEST(FlussUnionLakeReaderTest, RefusesATailLargerThanTheConfiguredLimit) { + SuppressionFixture over_limit; + over_limit.scan_params = union_scan_params("id,name", "2"); + ASSERT_TRUE(over_limit.open(make_key_block({1, 2, 3}, {"a", "b", "c"})).ok()); + const auto status = over_limit.reader.prepare_split(over_limit.split(":0:10:20")); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("max_tail_rows"), std::string::npos); + + SuppressionFixture at_limit; + at_limit.scan_params = union_scan_params("id,name", "3"); + ASSERT_TRUE(at_limit.open(make_key_block({1, 2, 3}, {"a", "b", "c"})).ok()); + EXPECT_TRUE(at_limit.reader.prepare_split(at_limit.split(":0:10:20")).ok()); +} + +// A tail that reaches this BE in several batches is one tail. Accumulating it batch by batch is +// what keeps the limit from being reached only after the whole of an oversized tail is in memory. +TEST(FlussUnionLakeReaderTest, AccumulatesATailThatArrivesInSeveralBatches) { + FlussUnionLakeReader reader; + auto scan_params = union_scan_params(); + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + Tail tail; + ASSERT_TRUE(FlussUnionLakeReader::parse_tail(":0:10:20", &tail).ok()); + + std::vector batches {make_key_block({1}, {"a"}), make_key_block({}, {}), + make_key_block({3}, {"c"})}; + size_t delivered = 0; + Block keys; + ASSERT_TRUE(reader._accumulate_tail_keys( + tail, + [&](Block* batch, bool* eos) { + if (delivered < batches.size()) { + *batch = batches[delivered++]; + } + *eos = delivered >= batches.size(); + return Status::OK(); + }, + &keys) + .ok()); + EXPECT_EQ(keys.rows(), 2); + EXPECT_EQ(ids_of(keys), (std::vector {1, 3})); +} + +// A split that says nothing about which tail supersedes it cannot be read: reading it as-is returns +// the superseded rows a second time. +TEST(FlussUnionLakeReaderTest, RefusesALakeSplitWithNoTailBoundToIt) { + FlussUnionLakeReader reader; + auto scan_params = union_scan_params(); + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + install_lake_reader(&reader); + ShardedKVCache cache {2}; + + const auto prepare = [&](TFileRangeDesc range) { + format::SplitReadOptions options; + options.cache = &cache; + options.current_range = std::move(range); + return reader.prepare_split(options); + }; + + TFileRangeDesc no_params; + EXPECT_FALSE(prepare(no_params).ok()); + + TTableFormatFileDesc empty_fluss_params; + empty_fluss_params.__set_table_format_type("fluss_union"); + empty_fluss_params.__set_fluss_params({}); + TFileRangeDesc without_tail; + without_tail.__set_table_format_params(empty_fluss_params); + EXPECT_FALSE(prepare(without_tail).ok()); + + TTableFormatFileDesc wrong_type; + wrong_type.__set_table_format_type("fluss_union"); + wrong_type.__set_fluss_params({{"fluss.range_type", "LOG"}, {"fluss.union.tail", ":0:10:20"}}); + TFileRangeDesc not_a_suppression; + not_a_suppression.__set_table_format_params(wrong_type); + EXPECT_FALSE(prepare(not_a_suppression).ok()); + + TTableFormatFileDesc no_tail_key; + no_tail_key.__set_table_format_type("fluss_union"); + no_tail_key.__set_fluss_params({{"fluss.range_type", "LAKE_SUPPRESS"}}); + TFileRangeDesc suppression_without_tail; + suppression_without_tail.__set_table_format_params(no_tail_key); + EXPECT_FALSE(prepare(suppression_without_tail).ok()); +} + +// Reading a block with no suppression built would return the superseded rows. This is the state that +// a future refactor could reach by accident, so it fails rather than passing the block through. +TEST(FlussUnionLakeReaderTest, RefusesToReturnABlockWithNoSuppressionBuilt) { + FlussUnionLakeReader reader; + auto scan_params = union_scan_params(); + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + auto* lake = install_lake_reader(&reader); + lake->blocks.push_back(make_block({1}, {"a"}, {10})); + + Block block = make_block({}, {}, {}); + bool eos = false; + EXPECT_FALSE(reader.get_block(&block, &eos).ok()); +} + +// A pruned split returns no rows, so its bucket's tail is not worth a network round trip. +TEST(FlussUnionLakeReaderTest, DoesNotReadTheTailOfAPrunedSplit) { + SuppressionFixture fixture; + ASSERT_TRUE(fixture.open(make_key_block({1}, {"a"})).ok()); + fixture.lake->prune_next_split = true; + ASSERT_TRUE(fixture.reader.prepare_split(fixture.split(":0:10:20")).ok()); + EXPECT_TRUE(fixture.tails_read.empty()); + EXPECT_TRUE(fixture.reader.current_split_pruned()); +} + +// Every split reaches the sibling exactly as it was planned - its path, its byte extent, its paimon +// payload. Rewriting any of it here would reshape a read this reader has no business reshaping. +TEST(FlussUnionLakeReaderTest, ForwardsTheSplitToTheSiblingUntouched) { + SuppressionFixture fixture; + ASSERT_TRUE(fixture.open(make_key_block({1}, {"a"})).ok()); + auto range = wrapped_lake_split(":0:10:20"); + range.__set_path("/warehouse/db.db/lake_pk/bucket-0/data-1.parquet"); + range.__set_start_offset(4); + range.__set_size(1024); + format::SplitReadOptions options; + options.cache = &fixture.cache; + options.current_range = range; + ASSERT_TRUE(fixture.reader.prepare_split(options).ok()); + + ASSERT_EQ(fixture.lake->prepared_ranges.size(), 1); + EXPECT_EQ(fixture.lake->prepared_ranges[0], range); +} + +TEST(FlussUnionLakeReaderTest, ClosesTheLakeHalf) { + SuppressionFixture fixture; + ASSERT_TRUE(fixture.open(make_key_block({1}, {"a"})).ok()); + ASSERT_TRUE(fixture.reader.prepare_split(fixture.split(":0:10:20")).ok()); + ASSERT_TRUE(fixture.reader.close().ok()); + EXPECT_TRUE(fixture.lake->closed); + EXPECT_TRUE(fixture.reader._suppression == nullptr); +} + +} // namespace +} // namespace doris::format::fluss diff --git a/build.sh b/build.sh index 50539178292547..d76c443ceb4245 100755 --- a/build.sh +++ b/build.sh @@ -726,7 +726,7 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then # Connector API, SPI, and plugin modules (loaded at runtime as plugins) modules+=("fe-connector/fe-connector-api") modules+=("fe-connector/fe-connector-spi") - for _conn_mod in es jdbc maxcompute trino hms hive paimon hudi iceberg; do + for _conn_mod in es jdbc maxcompute trino hms hive paimon hudi iceberg fluss; do if [[ -d "${DORIS_HOME}/fe/fe-connector/fe-connector-${_conn_mod}" ]]; then modules+=("fe-connector/fe-connector-${_conn_mod}") fi @@ -746,6 +746,7 @@ if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 ]]; then modules+=("be-java-extensions/java-udf") modules+=("be-java-extensions/jdbc-scanner") modules+=("be-java-extensions/paimon-scanner") + modules+=("be-java-extensions/fluss-scanner") modules+=("be-java-extensions/trino-connector-scanner") modules+=("be-java-extensions/max-compute-connector") # lakesoul-scanner has been deprecated @@ -1070,7 +1071,7 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then # Deploy connector provider plugins as independent plugin directories. # Each sub-directory is one connector backend loaded at runtime by ConnectorPluginManager. CONN_PLUGIN_DIR="${DORIS_OUTPUT}/fe/plugins/connector" - for conn_module in es jdbc maxcompute trino hms hive paimon hudi iceberg; do + for conn_module in es jdbc maxcompute trino hms hive paimon hudi iceberg fluss; do conn_plugin_target="${CONN_PLUGIN_DIR}/${conn_module}" conn_module_dir="${DORIS_HOME}/fe/fe-connector/fe-connector-${conn_module}" if [ ! -d "${conn_module_dir}" ]; then @@ -1196,6 +1197,7 @@ EOF extensions_modules+=("jdbc-scanner") extensions_modules+=("hadoop-hudi-scanner") extensions_modules+=("paimon-scanner") + extensions_modules+=("fluss-scanner") extensions_modules+=("trino-connector-scanner") extensions_modules+=("max-compute-connector") # lakesoul-scanner has been deprecated diff --git a/docker/thirdparties/docker-compose/fluss/README.md b/docker/thirdparties/docker-compose/fluss/README.md new file mode 100644 index 00000000000000..249a4c6537badb --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/README.md @@ -0,0 +1,159 @@ + + +# Fluss regression environment + +Stack: ZooKeeper, a fluss coordinator server, one fluss tablet server, and a +Flink cluster (jobmanager, taskmanager, sql-client). The sql-client container +builds the fixtures once and then idles; its healthcheck only turns green after +every statement succeeded, so `--wait` gates on the fixtures being complete. + +The fluss cluster is lakehouse-enabled: `datalake.format: paimon` with a +filesystem warehouse under `data/paimon`, and the sql-client container runs the +fluss lake tiering service as a Flink job while building the fixtures. + +## Prerequisite: a built fluss checkout + +Fluss 1.0 is not released yet, so neither image can be pulled. `build-images.sh` +builds both from a local source tree, which must be packaged first: + +```bash +git clone https://github.com/apache/fluss.git +mvn -f fluss/pom.xml \ + -pl fluss-dist,fluss-flink/fluss-flink-1.20,fluss-flink/fluss-flink-tiering,fluss-lake/fluss-lake-paimon \ + -am package -DskipTests +``` + +The paimon runtime the tiering job needs (`paimon-flink-1.20`) is resolved from +the local maven repository, or downloaded from Maven Central when it is missing. + +When fluss 1.0 ships, this step and `build-images.sh` are replaced by the +official `apache/fluss` and Flink images. + +## Start / stop + +```bash +CONTAINER_UID=doris-e2e-- FLUSS_SOURCE_DIR=/path/to/fluss \ + bash docker/thirdparties/run-thirdparties-docker.sh -c fluss + +CONTAINER_UID=doris-e2e-- bash docker/thirdparties/run-thirdparties-docker.sh -c fluss --stop +``` + +`build-images.sh` reuses images that already exist; set +`FLUSS_DOCKER_REUSE_IMAGES=0` to force a rebuild after changing the fluss +checkout. + +Then enable the suites in `regression-test/conf/regression-conf.groovy`: + +```groovy +enableFlussTest=true +``` + +## Ports and paths + +| What | Host port | +|---|---| +| ZooKeeper | 22181 | +| fluss coordinator server | 19123 | +| fluss tablet server | 19124 | +| Flink jobmanager UI | 18085 | + +The servers advertise `:`, because Doris FE/BE run on +the host rather than inside the compose network. + +Two directories are bind mounted at the same absolute path inside the containers +and on the host, because Doris reads the files in them directly and the path +string is recorded rather than translated: + +| Directory | Written by | Read by | +|---|---|---| +| `data/remote` (`remote.data.dir`) | fluss servers | Doris BE — kv snapshots, remote log segments | +| `data/paimon` (`datalake.paimon.warehouse`) | the tiering job | Doris FE/BE through the paimon connector | + +## Fixtures + +The fixtures recreate database `fluss_test` from scratch on every start: + +| Table | Shape | +|---|---| +| `log_basic` | log table, 3 rows, table and column comments | +| `log_types` | log table, one column per mapped fluss type, plus an all-NULL row | +| `log_part` | log table partitioned by `dt`, partitions `20260101`, `20260102`, `20260103` | +| `log_empty` | log table with no rows at all (planning must emit zero scan ranges) | +| `log_nested` | log table whose complex types are nested inside complex types, plus rows with NULLs at every level | +| `log_time` | log table carrying a fluss TIME column, the one type Doris cannot represent | +| `part_types` | log table partitioned by one column of every type that survives fluss's partition naming (STRING, CHAR, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, DATE, BINARY) | +| `part_ts` | log table partitioned by a TIMESTAMP, whose value fluss rewrites into the partition name and nothing can read back | +| `pk_basic` | primary-key table, one updated row and one deleted row | +| `pk_types` | primary-key table with the same type coverage as `log_types` | +| `pk_part` | primary-key table partitioned by `dt`, with an update and a delete inside a partition | +| `pk_nested` | primary-key table with the same nesting as `log_nested`, in the kv row format | +| `pk_empty` | primary-key table with no rows and therefore no kv snapshot | +| `lake_log` | lake table, 4 rows tiered + 2 in the log, 3 buckets (some bucket has no tail) | +| `lake_cold` | lake table read entirely from the lake — no log tail at all | +| `lake_types` | lake table with the full type coverage; non-NULL rows tiered, the all-NULL row in the log | +| `lake_part` | lake table partitioned by `dt`; only `20260101` has a log tail | +| `lake_pk` | primary-key lake table, one bucket; its tail updates one tiered row, deletes another and adds a key the lake never saw | +| `lake_pk_multi` | primary-key lake table over 3 buckets; the tail reaches some buckets and not others, which is what makes per-bucket binding observable | +| `lake_pk_part` | primary-key lake table partitioned by `dt`: `20260101` is lake + tail, `20260102` is lake only, `20260103` was written after tiering stopped so the lake has never seen it | +| `lake_pk_cold` | primary-key lake table read entirely from the lake — no tail, so nothing to merge | +| `lake_nested` | lake table with nested complex types; the populated row is tiered, the all-NULL row stays in the log | +| `lake_empty` | lake table nothing was ever written to, so tiering has never committed and there is no snapshot to read | +| `lake_part_int` | lake table partitioned by an INT; one partition has a tail, the other does not | +| `lake_pk_part_int` | primary-key lake table partitioned by an INT — the halves cannot be matched by a non-STRING partition value, so it falls back to the fluss-only read | +| `big_log` | lake table, 100000 rows tiered + 1000 in the log (ids 1..101000) | +| `big_pk` | primary-key lake table, 100000 keys tiered; the tail updates 500, adds 500 and deletes 5 | + +There is deliberately no deletion-vector fixture. Fluss does forward a +`paimon.deletion-vectors.enabled` table property into the paimon table it +creates, but its tiering service writes no deletion vector index, and paimon then +reads such a table as empty — see the note in `sql/init.sql`. + +### Lake tables are frozen half in, half out + +Building them takes three steps (`scripts/run-init-sql.sh`): + +1. `sql/init.sql` writes the rows that belong in paimon, tiering service running; +2. `sql/lake-row-counts.sql` is polled until paimon holds every one of them; +3. the tiering job is cancelled, and only then does `sql/init-lake-tail.sql` + write the rows that must stay in the fluss log. + +Both the counting and the cancelling are load-bearing. Left running, the tiering +service would keep consuming the tail, and a suite asserting that a table is read +as "lake plus log" would quietly become one asserting "lake only" — passing or +failing by how long the environment had been up. And waiting for *a* paimon +snapshot rather than for the *row counts* would freeze some fixtures half-tiered, +which is the same flakiness one step earlier. + +### Primary-key tables come with a kv snapshot + +The server takes kv snapshots every ten seconds here rather than every ten +minutes, and startup does not report the environment ready until each +primary-key table has one on disk (`wait_for_kv_snapshots` in +`scripts/run-init-sql.sh`). + +That is not tuning: Doris BE reads those snapshot files directly, from the +host, at the path this container wrote them to — the directory is bind mounted +at the same absolute path on both sides — and nothing but an end-to-end run +covers that. A primary-key table with no snapshot is read by replaying its +whole change log instead, which is equally correct and takes a different code +path, so without the wait the interesting path would only be exercised by luck. + +Short intervals do not pile up files: a tablet whose log has not advanced since +its last snapshot is skipped, and the fixtures stop writing when init ends. diff --git a/docker/thirdparties/docker-compose/fluss/build-images.sh b/docker/thirdparties/docker-compose/fluss/build-images.sh new file mode 100755 index 00000000000000..dc82ac2064907e --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/build-images.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################################################ +# Builds the two images the fluss regression stack runs on. +# +# Fluss 1.0 is not released, so neither image can be pulled: both are built +# from a local fluss source checkout that has already been packaged. Once +# fluss 1.0 ships, this script is replaced by pulling the official images. +# +# Required: +# FLUSS_SOURCE_DIR path to a built fluss checkout +# Optional: +# FLUSS_VERSION fluss version in that checkout (default 1.0-SNAPSHOT) +# FLINK_BASE_IMAGE base Flink image (default flink:1.20.0-scala_2.12-java17) +# FLUSS_FLINK_CONNECTOR_MODULE fluss connector module matching the base image +# FLUSS_DOCKER_REUSE_IMAGES 1 = skip an image whose tag already exists +# (decided per image; delete a tag to rebuild just it) +# MAVEN_REPO local maven repository (default ~/.m2/repository) +################################################################ + +set -eo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +# Image tags and the paimon version live in fluss.env.tpl so the compose file and +# this script cannot drift apart. The template's other entries reference +# variables that are empty here; only those three literals are read. +# shellcheck source=/dev/null +. "${SCRIPT_DIR}/fluss.env.tpl" + +FLUSS_VERSION="${FLUSS_VERSION:-1.0-SNAPSHOT}" +FLINK_BASE_IMAGE="${FLINK_BASE_IMAGE:-flink:1.20.0-scala_2.12-java17}" +FLUSS_FLINK_CONNECTOR_MODULE="${FLUSS_FLINK_CONNECTOR_MODULE:-fluss-flink-1.20}" +MAVEN_REPO="${MAVEN_REPO:-${HOME}/.m2/repository}" +# Both fluss and paimon name their Flink artifacts after the Flink minor version, +# so deriving it from the fluss module keeps the paimon jar in step with the base +# image whenever the module is overridden. +FLINK_MINOR_VERSION="${FLUSS_FLINK_CONNECTOR_MODULE##*-}" + +# Resolves a maven artifact into a directory: the local repository first, so a +# machine that has already built Doris or fluss needs no network, then central. +resolve_maven_jar() { + local group_path="$1" artifact="$2" version="$3" dest_dir="$4" + local jar="${artifact}-${version}.jar" + local local_path="${MAVEN_REPO}/${group_path}/${artifact}/${version}/${jar}" + + mkdir -p "${dest_dir}" + if [[ -f "${local_path}" ]]; then + cp "${local_path}" "${dest_dir}/" + echo " ${jar} (from ${MAVEN_REPO})" + return 0 + fi + local url="https://repo1.maven.org/maven2/${group_path}/${artifact}/${version}/${jar}" + echo " ${jar} (downloading ${url})" + if ! curl -fsSL -o "${dest_dir}/${jar}" "${url}"; then + echo "ERROR: could not resolve ${jar} locally or from central" >&2 + return 1 + fi +} + +if [[ -z "${DOCKER_USE_SUDO+x}" ]]; then + if [[ "$(uname -s)" == "Darwin" ]]; then + DOCKER_USE_SUDO=0 + else + DOCKER_USE_SUDO=1 + fi +fi + +docker_cli() { + if [[ "${DOCKER_USE_SUDO}" -eq 1 ]]; then + sudo docker "$@" + else + docker "$@" + fi +} + +image_exists() { + docker_cli image inspect "$1" >/dev/null 2>&1 +} + +# Reuse is decided per image, because the two are built from different inputs: +# the server image from fluss-dist, the flink image from the connector, paimon +# and the tiering jar. Rebuilding one to pick up a change in the other wastes +# minutes, and -- since the base images have to be pulled -- fails outright on a +# machine that can reach the local checkout but not a registry. +should_build() { + local image="$1" + if [[ "${FLUSS_DOCKER_REUSE_IMAGES}" == "1" ]] && image_exists "${image}"; then + echo "Reusing existing image ${image}" + return 1 + fi + return 0 +} + +BUILD_SERVER=0 +BUILD_FLINK=0 +should_build "${FLUSS_SERVER_IMAGE}" && BUILD_SERVER=1 +should_build "${FLUSS_FLINK_IMAGE}" && BUILD_FLINK=1 + +if ((BUILD_SERVER == 0 && BUILD_FLINK == 0)); then + exit 0 +fi + +if [[ -z "${FLUSS_SOURCE_DIR}" ]]; then + echo "ERROR: FLUSS_SOURCE_DIR is not set." >&2 + echo " Clone https://github.com/apache/fluss, build it, and point" >&2 + echo " FLUSS_SOURCE_DIR at the checkout." >&2 + exit 1 +fi + +DIST_DIR="${FLUSS_SOURCE_DIR}/fluss-dist/target/fluss-${FLUSS_VERSION}-bin/fluss-${FLUSS_VERSION}" +CONNECTOR_JAR="${FLUSS_SOURCE_DIR}/fluss-flink/${FLUSS_FLINK_CONNECTOR_MODULE}/target/${FLUSS_FLINK_CONNECTOR_MODULE}-${FLUSS_VERSION}.jar" +# The lake half of the environment: the tiering job that moves fluss data into +# paimon, and the fluss-side paimon writer it loads. +TIERING_JAR="${FLUSS_SOURCE_DIR}/fluss-flink/fluss-flink-tiering/target/fluss-flink-tiering-${FLUSS_VERSION}.jar" +LAKE_PAIMON_JAR="${FLUSS_SOURCE_DIR}/fluss-lake/fluss-lake-paimon/target/fluss-lake-paimon-${FLUSS_VERSION}.jar" + +MISSING=() +[[ -d "${DIST_DIR}" ]] || MISSING+=("${DIST_DIR}") +for jar in "${CONNECTOR_JAR}" "${TIERING_JAR}" "${LAKE_PAIMON_JAR}"; do + [[ -f "${jar}" ]] || MISSING+=("${jar}") +done + +if ((${#MISSING[@]} > 0)); then + echo "ERROR: fluss build output is missing:" >&2 + printf ' %s\n' "${MISSING[@]}" >&2 + echo " Build them with:" >&2 + echo " mvn -f ${FLUSS_SOURCE_DIR}/pom.xml -pl fluss-dist,fluss-flink/${FLUSS_FLINK_CONNECTOR_MODULE},fluss-flink/fluss-flink-tiering,fluss-lake/fluss-lake-paimon -am package -DskipTests" >&2 + exit 1 +fi + +BUILD_CONTEXT="$(mktemp -d)" +trap 'rm -rf "${BUILD_CONTEXT}"' EXIT + +if ((BUILD_SERVER == 1)); then + echo "Building ${FLUSS_SERVER_IMAGE} from ${DIST_DIR}" + mkdir -p "${BUILD_CONTEXT}/server" + cp -r "${DIST_DIR}" "${BUILD_CONTEXT}/server/build-target" + cp "${FLUSS_SOURCE_DIR}/docker/fluss/Dockerfile" "${BUILD_CONTEXT}/server/Dockerfile" + cp "${FLUSS_SOURCE_DIR}/docker/fluss/docker-entrypoint.sh" "${BUILD_CONTEXT}/server/docker-entrypoint.sh" + docker_cli build -t "${FLUSS_SERVER_IMAGE}" "${BUILD_CONTEXT}/server" +fi + +if ((BUILD_FLINK == 0)); then + exit 0 +fi + +echo "Building ${FLUSS_FLINK_IMAGE} from ${FLINK_BASE_IMAGE}" +mkdir -p "${BUILD_CONTEXT}/flink/lib" "${BUILD_CONTEXT}/flink/opt" +cp "${CONNECTOR_JAR}" "${BUILD_CONTEXT}/flink/lib/" +echo " $(basename "${CONNECTOR_JAR}")" +# Paimon runtime for the tiering job. fluss-lake-paimon is only the fluss->paimon +# writer: it carries no paimon of its own, so paimon-flink (which bundles paimon +# core) has to sit next to it, and paimon in turn builds every CatalogContext +# around a hadoop Configuration -- a plain directory warehouse still needs hadoop +# present. Same three jars upstream's quickstart image activates for paimon, +# minus paimon-s3: this warehouse is a bind-mounted directory. +cp "${LAKE_PAIMON_JAR}" "${BUILD_CONTEXT}/flink/lib/" +echo " $(basename "${LAKE_PAIMON_JAR}")" +resolve_maven_jar "org/apache/paimon" "paimon-flink-${FLINK_MINOR_VERSION}" \ + "${FLUSS_PAIMON_VERSION}" "${BUILD_CONTEXT}/flink/lib" +resolve_maven_jar "io/trino/hadoop" "hadoop-apache" \ + "${FLUSS_HADOOP_APACHE_VERSION}" "${BUILD_CONTEXT}/flink/lib" +cp "${TIERING_JAR}" "${BUILD_CONTEXT}/flink/opt/" +echo " opt/$(basename "${TIERING_JAR}")" +cp "${SCRIPT_DIR}/flink/Dockerfile" "${BUILD_CONTEXT}/flink/Dockerfile" +docker_cli build --build-arg "FLINK_BASE_IMAGE=${FLINK_BASE_IMAGE}" \ + -t "${FLUSS_FLINK_IMAGE}" "${BUILD_CONTEXT}/flink" + +echo "Built ${FLUSS_FLINK_IMAGE}" diff --git a/docker/thirdparties/docker-compose/fluss/flink/Dockerfile b/docker/thirdparties/docker-compose/fluss/flink/Dockerfile new file mode 100644 index 00000000000000..346024e9cc8011 --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/flink/Dockerfile @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Flink image used to prepare the fluss regression data: stock Flink plus the +# fluss connector, the paimon runtime and the lake tiering job. build-images.sh +# assembles both directories, lib/ from a local fluss checkout (fluss 1.0 is not +# released yet) and paimon from the maven repository. +ARG FLINK_BASE_IMAGE=flink:1.20.0-scala_2.12-java17 +FROM ${FLINK_BASE_IMAGE} + +COPY lib/ /opt/flink/lib/ + +# The tiering job is submitted with `flink run`, so it is a job artifact rather +# than a cluster dependency: it belongs in opt/, not lib/. Upstream's quickstart +# image puts it in the same place. +COPY opt/ /opt/flink/opt/ diff --git a/docker/thirdparties/docker-compose/fluss/fluss.env.tpl b/docker/thirdparties/docker-compose/fluss/fluss.env.tpl new file mode 100644 index 00000000000000..8dc2c07f302b5d --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/fluss.env.tpl @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Rendered to fluss.env by run-thirdparties-docker.sh (envsubst). +# build-images.sh also sources this template directly, for the image tags and +# the paimon version, so keep those lines free of variable references. + +DOCKER_FLUSS_ZOOKEEPER_EXTERNAL_PORT=22181 +DOCKER_FLUSS_COORDINATOR_EXTERNAL_PORT=19123 +DOCKER_FLUSS_TABLET_EXTERNAL_PORT=19124 +DOCKER_FLUSS_FLINK_JOBMANAGER_EXTERNAL_PORT=18085 + +# Fluss 1.0 is not released yet, so there is no published image to pull. +# build-images.sh builds both tags from a local fluss source checkout. +FLUSS_SERVER_IMAGE=doris-fluss-server:1.0-SNAPSHOT-local +FLUSS_FLINK_IMAGE=doris-fluss-flink:1.20.0-fluss-1.0-SNAPSHOT-local + +# Address the fluss servers advertise to clients. Doris FE/BE run on the host, +# so the servers must hand out the host address plus the published ports, not +# their in-container hostnames. +FLUSS_HOST_IP=${IP_HOST} + +# remote.data.dir holds remote log segments and kv snapshots. Doris BE reads +# those files directly (primary-key table reads), so the directory is bind +# mounted at the SAME absolute path inside the containers and on the host. +FLUSS_REMOTE_DATA_DIR=${FLUSS_COMPOSE_DIR}/data/remote + +# The paimon warehouse the tiering service writes the lake tables into. Doris +# reads it from the host through the paimon connector, so -- exactly like +# remote.data.dir above -- the same absolute path has to resolve on both sides: +# the warehouse location is recorded in fluss table properties and handed to +# paimon verbatim, with nothing left to rewrite a container path into a host one. +# +# Two spellings, and both are needed. The bind mount is a plain path, while the +# warehouse Doris is told about must carry the file:// scheme: a location with no +# scheme is read as HDFS (StorageRegistry.fromScheme defaults a blank scheme to +# HDFS), and every data-file path paimon recorded then fails to normalize with +# "Unsupported schema: null" at scan time -- long after catalog creation. +FLUSS_PAIMON_WAREHOUSE_DIR=${FLUSS_COMPOSE_DIR}/data/paimon +FLUSS_PAIMON_WAREHOUSE=file://${FLUSS_COMPOSE_DIR}/data/paimon + +# Paimon build the flink image carries, matched to the one fluss-lake-paimon was +# compiled against (fluss-dist ships paimon-bundle at this version) and to Doris's +# own paimon.version, so all three read the same table format. +FLUSS_PAIMON_VERSION=1.3.1 + +# Paimon builds its CatalogContext around a hadoop Configuration whatever the +# catalog is, so even a plain directory warehouse needs hadoop on the classpath; +# without it the tiering job dies with NoClassDefFoundError the first time it +# writes. Upstream's quickstart image carries the same repackaged jar. +FLUSS_HADOOP_APACHE_VERSION=3.3.5-1 diff --git a/docker/thirdparties/docker-compose/fluss/fluss.yaml.tpl b/docker/thirdparties/docker-compose/fluss/fluss.yaml.tpl new file mode 100644 index 00000000000000..5c751e8f96d486 --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/fluss.yaml.tpl @@ -0,0 +1,252 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3" + +networks: + doris--fluss--network: + ipam: + driver: default + config: + - subnet: 168.52.0.0/24 + +services: + doris--fluss-zookeeper: + image: zookeeper:3.9.2 + container_name: doris--fluss-zookeeper + hostname: doris--fluss-zookeeper + restart: always + ports: + - ${DOCKER_FLUSS_ZOOKEEPER_EXTERNAL_PORT}:2181 + healthcheck: + test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/2181' >/dev/null 2>&1"] + interval: 5s + timeout: 10s + retries: 60 + networks: + - doris--fluss--network + + doris--fluss-coordinator: + image: ${FLUSS_SERVER_IMAGE} + container_name: doris--fluss-coordinator + hostname: doris--fluss-coordinator + command: coordinatorServer + # The image chowns all of /opt/fluss to uid 9999 but never declares USER, so + # it runs as root unless told otherwise -- and then the paimon table + # directories it creates in the shared warehouse belong to root, while the + # tiering job writing into them is the flink image's uid 9999. Same uid on + # both sides, and the two can share the warehouse. + user: "9999:9999" + depends_on: + doris--fluss-zookeeper: + condition: service_healthy + ports: + - ${DOCKER_FLUSS_COORDINATOR_EXTERNAL_PORT}:9123 + environment: + # Doris FE/BE run on the host, so the servers must advertise the host + # address plus the published port: everything (host clients, the flink + # containers and the servers among themselves) then talks over it. + - | + FLUSS_PROPERTIES= + zookeeper.address: doris--fluss-zookeeper:2181 + bind.listeners: FLUSS://0.0.0.0:9123 + advertised.listeners: FLUSS://${FLUSS_HOST_IP}:${DOCKER_FLUSS_COORDINATOR_EXTERNAL_PORT} + remote.data.dir: ${FLUSS_REMOTE_DATA_DIR} + default.bucket.number: 3 + default.replication.factor: 1 + # Lakehouse storage. The coordinator creates the paimon table when a + # datalake-enabled fluss table is created, so it needs the warehouse + # too, not just the tiering job. The plugin jars are already in the + # image: fluss-dist ships plugins/paimon (fluss-lake-paimon + + # paimon-bundle + shaded hadoop). + # + # These three settings are also what makes the tables READABLE by Doris: + # the coordinator copies its datalake.paimon.* config into every lake + # table's properties under a table. prefix, and that copy is the only + # place the fluss connector learns where the warehouse is. + datalake.enabled: true + datalake.format: paimon + datalake.paimon.metastore: filesystem + datalake.paimon.warehouse: ${FLUSS_PAIMON_WAREHOUSE} + volumes: + - ${FLUSS_REMOTE_DATA_DIR}:${FLUSS_REMOTE_DATA_DIR} + - ${FLUSS_PAIMON_WAREHOUSE_DIR}:${FLUSS_PAIMON_WAREHOUSE_DIR} + healthcheck: + test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/9123' >/dev/null 2>&1"] + interval: 5s + timeout: 10s + retries: 60 + networks: + - doris--fluss--network + + doris--fluss-tablet-server: + image: ${FLUSS_SERVER_IMAGE} + container_name: doris--fluss-tablet-server + hostname: doris--fluss-tablet-server + command: tabletServer + # Same uid as the coordinator and the flink containers; see there. + user: "9999:9999" + depends_on: + doris--fluss-coordinator: + condition: service_healthy + ports: + - ${DOCKER_FLUSS_TABLET_EXTERNAL_PORT}:9123 + environment: + - | + FLUSS_PROPERTIES= + zookeeper.address: doris--fluss-zookeeper:2181 + bind.listeners: FLUSS://0.0.0.0:9123 + advertised.listeners: FLUSS://${FLUSS_HOST_IP}:${DOCKER_FLUSS_TABLET_EXTERNAL_PORT} + tablet-server.id: 0 + data.dir: /tmp/fluss/data + remote.data.dir: ${FLUSS_REMOTE_DATA_DIR} + default.bucket.number: 3 + default.replication.factor: 1 + # The fixtures write kilobytes, but the guard measures the whole host + # disk: on a build machine that happens to sit above the threshold it + # rejects every write, and the client retries ~2^31 times instead of + # failing, so the environment hangs instead of reporting anything. + server.data-disk.write-limit-ratio: 1.0 + # Ten minutes by default, which would leave the primary-key fixtures + # with no kv snapshot for as long as the suites take to run: they would + # then be read by replaying the change log, and the path where Doris BE + # reads a snapshot FILE this container wrote -- the one thing only an + # end-to-end run can check -- would never be exercised. Short enough + # that init can wait for it; this does not pile up files, because a + # tablet whose log has not advanced since its last snapshot is skipped + # (KvTabletSnapshotTarget), and the fixtures stop writing after init. + kv.snapshot.interval: 10s + # Same lakehouse settings as the coordinator: a tablet server reads them + # to decide the key encoding and bucketing a datalake table uses. + datalake.enabled: true + datalake.format: paimon + datalake.paimon.metastore: filesystem + datalake.paimon.warehouse: ${FLUSS_PAIMON_WAREHOUSE} + volumes: + - ${FLUSS_REMOTE_DATA_DIR}:${FLUSS_REMOTE_DATA_DIR} + - ${FLUSS_PAIMON_WAREHOUSE_DIR}:${FLUSS_PAIMON_WAREHOUSE_DIR} + healthcheck: + test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/9123' >/dev/null 2>&1"] + interval: 5s + timeout: 10s + retries: 60 + networks: + - doris--fluss--network + + doris--fluss-jobmanager: + image: ${FLUSS_FLINK_IMAGE} + container_name: doris--fluss-jobmanager + hostname: doris--fluss-jobmanager + command: jobmanager + ports: + - ${DOCKER_FLUSS_FLINK_JOBMANAGER_EXTERNAL_PORT}:8081 + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: doris--fluss-jobmanager + rest.address: doris--fluss-jobmanager + rest.bind-address: 0.0.0.0 + # The tiering job runs on this cluster and writes the paimon warehouse; the + # jobmanager builds the job graph, which opens the lake catalog. + volumes: + - ${FLUSS_PAIMON_WAREHOUSE_DIR}:${FLUSS_PAIMON_WAREHOUSE_DIR} + # Tiering a primary-key table reads the kv snapshot FILES rather than the + # change log, straight out of remote.data.dir -- the same way Doris BE reads + # them. Without this mount a log table tiers and a primary-key one fails + # with FileNotFoundException for a file that plainly exists on the host. + - ${FLUSS_REMOTE_DATA_DIR}:${FLUSS_REMOTE_DATA_DIR}:ro + healthcheck: + test: ["CMD-SHELL", "curl -sf http://127.0.0.1:8081/overview >/dev/null"] + interval: 5s + timeout: 10s + retries: 60 + networks: + - doris--fluss--network + + doris--fluss-taskmanager: + image: ${FLUSS_FLINK_IMAGE} + container_name: doris--fluss-taskmanager + hostname: doris--fluss-taskmanager + command: taskmanager + depends_on: + doris--fluss-jobmanager: + condition: service_healthy + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: doris--fluss-jobmanager + taskmanager.numberOfTaskSlots: 4 + taskmanager.memory.process.size: 2048m + taskmanager.memory.task.off-heap.size: 128m + # Writes and commits the paimon files the tiering job produces, and reads the + # kv snapshots it tiers a primary-key table from (see the jobmanager). + volumes: + - ${FLUSS_PAIMON_WAREHOUSE_DIR}:${FLUSS_PAIMON_WAREHOUSE_DIR} + - ${FLUSS_REMOTE_DATA_DIR}:${FLUSS_REMOTE_DATA_DIR}:ro + # The taskmanager RPC port is ephemeral, so health means "registered with + # the jobmanager": that is also exactly what submitting a job needs. + healthcheck: + test: ["CMD-SHELL", "curl -sf http://doris--fluss-jobmanager:8081/taskmanagers | grep -q '\"id\"'"] + interval: 5s + timeout: 10s + retries: 60 + networks: + - doris--fluss--network + + # One-shot data preparation: runs sql/init.sql through the Flink SQL client, + # then keeps running so that `compose up --wait` has a healthy service to + # gate on. The marker file only appears when every statement succeeded. + doris--fluss-sql-client: + image: ${FLUSS_FLINK_IMAGE} + container_name: doris--fluss-sql-client + hostname: doris--fluss-sql-client + depends_on: + doris--fluss-tablet-server: + condition: service_healthy + doris--fluss-taskmanager: + condition: service_healthy + # Runs as a command, not as an entrypoint override: the image entrypoint is + # what turns FLINK_PROPERTIES into the config the SQL client submits with. + command: ["/opt/fluss-scripts/run-init-sql.sh"] + environment: + - FLUSS_BOOTSTRAP_SERVERS=doris--fluss-coordinator:9123 + - FLUSS_JOBMANAGER_HOST=doris--fluss-jobmanager + # Read-only, and only so that init can wait for the kv snapshots to be + # written before declaring the environment ready. + - FLUSS_REMOTE_DATA_DIR=${FLUSS_REMOTE_DATA_DIR} + # This container also submits the tiering job and waits for it to commit, + # so it needs both the warehouse path and the paimon database naming. + - FLUSS_PAIMON_WAREHOUSE=${FLUSS_PAIMON_WAREHOUSE} + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: doris--fluss-jobmanager + rest.address: doris--fluss-jobmanager + volumes: + - ./sql:/opt/fluss-sql:ro + - ./scripts:/opt/fluss-scripts:ro + - ${FLUSS_REMOTE_DATA_DIR}:${FLUSS_REMOTE_DATA_DIR}:ro + # Writable, not read-only like the one above: `flink run` builds the + # tiering job graph in this container, and building it opens the lake + # catalog, which creates the warehouse directory if it is not there yet. + - ${FLUSS_PAIMON_WAREHOUSE_DIR}:${FLUSS_PAIMON_WAREHOUSE_DIR} + healthcheck: + test: ["CMD-SHELL", "test -f /tmp/fluss-init/SUCCESS"] + interval: 5s + timeout: 10s + retries: 120 + networks: + - doris--fluss--network diff --git a/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh b/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh new file mode 100755 index 00000000000000..b06bef5f8e4032 --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/scripts/run-init-sql.sh @@ -0,0 +1,306 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################################################ +# Entrypoint of the fluss sql-client container: creates the regression +# fixtures through the Flink SQL client, then idles so that the compose +# healthcheck has something to gate on. The SUCCESS marker is written only +# after every statement succeeded. +# +# The lake fixtures are built in three steps, because what makes them useful is +# WHERE each row ends up: +# 1. init.sql writes the rows that belong in paimon, with the tiering service +# running; +# 2. the tiering service is stopped once it has committed all of them; +# 3. init-lake-tail.sql writes the rows that must stay in the fluss log. +# Stopping the service is the point: it freezes the division between the two +# halves. Left running, it would keep consuming the tail, and a suite asserting +# that a table is read as "lake plus log" would slowly turn into one asserting +# "lake only" -- passing or failing by how long the environment had been up. +################################################################ + +set -eo pipefail + +MARKER_DIR=/tmp/fluss-init +SQL_TEMPLATE=/opt/fluss-sql/init.sql +LAKE_TAIL_TEMPLATE=/opt/fluss-sql/init-lake-tail.sql +LAKE_COUNTS_TEMPLATE=/opt/fluss-sql/lake-row-counts.sql +JOBMANAGER_PORT=8081 +WAIT_SECONDS=180 +SQL_TIMEOUT_SECONDS=900 +ATTEMPTS=3 +# Primary-key fixtures whose buckets must have been snapshotted before the +# environment counts as ready. See wait_for_kv_snapshots. +# +# pk_empty is deliberately absent: nothing was ever written to it, so no bucket +# will ever be snapshotted and waiting would hang on the state it exists to have. +SNAPSHOT_TABLES=(pk_basic pk_types pk_part pk_nested lake_pk lake_pk_multi lake_pk_part + lake_pk_cold lake_pk_part_int big_pk) +SNAPSHOT_WAIT_SECONDS=180 + +# What each lake fixture must hold in paimon before the tail is written -- the +# row counts init.sql writes, merged where the table has a primary key. Keep in +# step with init.sql; a mismatch here stalls the environment instead of +# corrupting a fixture, and the timeout prints both sides. +LAKE_EXPECTED_ROWS=( + "lake_log=4" + "lake_cold=3" + "lake_types=1" + "lake_part=3" + "lake_pk=3" + "lake_pk_multi=9" + "lake_pk_part=4" + "lake_pk_cold=3" + "lake_nested=1" + "lake_part_int=3" + "lake_pk_part_int=4" + "big_log=100000" + "big_pk=100000" +) +# The two large fixtures put 200000 rows through the tiering service, which is +# most of what this wait is now for; the small ones commit within a round. +LAKE_TIERING_WAIT_SECONDS=900 +TIERING_JAR_GLOB='/opt/flink/opt/fluss-flink-tiering-*.jar' +FLINK_BIN=/opt/flink/bin/flink + +rm -rf "${MARKER_DIR}" +mkdir -p "${MARKER_DIR}" + +wait_for_jobmanager() { + local waited=0 + while ! (exec 3<>"/dev/tcp/${FLUSS_JOBMANAGER_HOST}/${JOBMANAGER_PORT}") >/dev/null 2>&1; do + if ((waited >= WAIT_SECONDS)); then + echo "ERROR: jobmanager ${FLUSS_JOBMANAGER_HOST}:${JOBMANAGER_PORT} not reachable after ${WAIT_SECONDS}s" >&2 + return 1 + fi + sleep 2 + waited=$((waited + 2)) + done +} + +wait_for_jobmanager + +# The bootstrap address and the warehouse path are only known at compose time, +# and the Flink SQL client does not expand environment variables inside SQL +# files. +sed "s|__FLUSS_BOOTSTRAP_SERVERS__|${FLUSS_BOOTSTRAP_SERVERS}|g" \ + "${SQL_TEMPLATE}" >"${MARKER_DIR}/init.sql" +sed "s|__FLUSS_BOOTSTRAP_SERVERS__|${FLUSS_BOOTSTRAP_SERVERS}|g" \ + "${LAKE_TAIL_TEMPLATE}" >"${MARKER_DIR}/init-lake-tail.sql" +sed "s|__FLUSS_PAIMON_WAREHOUSE__|${FLUSS_PAIMON_WAREHOUSE}|g" \ + "${LAKE_COUNTS_TEMPLATE}" >"${MARKER_DIR}/lake-row-counts.sql" + +# Cancels every job on the cluster. This cluster runs nothing but the tiering +# service, and a retry must not leave the previous attempt's job consuming the +# database the next attempt is about to drop and recreate. +cancel_all_jobs() { + local ids + ids="$("${FLINK_BIN}" list -r 2>/dev/null | grep -oE '[0-9a-f]{32}' || true)" + local id + for id in ${ids}; do + echo "Cancelling flink job ${id}" + "${FLINK_BIN}" cancel "${id}" >/dev/null 2>&1 || true + done +} + +# Submits the fluss -> paimon tiering service. Detached, because it is a +# streaming job that has to keep running while init.sql writes. +start_tiering_job() { + local jar + # shellcheck disable=SC2086 + jar="$(ls ${TIERING_JAR_GLOB} 2>/dev/null | head -n 1)" + if [[ -z "${jar}" ]]; then + echo "ERROR: no tiering jar matching ${TIERING_JAR_GLOB}" >&2 + return 1 + fi + echo "Submitting tiering service from ${jar}" + # The lake settings repeat the ones the fluss servers carry. They have to: + # the servers use them to create the paimon table, this job uses them to + # write it, and neither reads the other's configuration. + "${FLINK_BIN}" run -d "${jar}" \ + --fluss.bootstrap.servers "${FLUSS_BOOTSTRAP_SERVERS}" \ + --datalake.format paimon \ + --datalake.paimon.metastore filesystem \ + --datalake.paimon.warehouse "${FLUSS_PAIMON_WAREHOUSE}" +} + +# Waits until paimon holds every row init.sql wrote to a lake table, by counting +# them in the warehouse itself (sql/lake-row-counts.sql) rather than by checking +# that some snapshot exists. Tiering commits what it has consumed so far, so a +# snapshot proves only that it started; freezing a fixture half-tiered would +# leave a table meant to be lake-only with a log tail on some runs and not +# others. +wait_for_lake_rows() { + local waited=0 + local log="${MARKER_DIR}/lake-row-counts.log" + local expected missing + while :; do + missing="" + if timeout "${SQL_TIMEOUT_SECONDS}" /opt/flink/bin/sql-client.sh \ + -f "${MARKER_DIR}/lake-row-counts.sql" >"${log}" 2>&1; then + for expected in "${LAKE_EXPECTED_ROWS[@]}"; do + grep -qF "LAKEROWS:${expected}" "${log}" || missing="${missing} ${expected}" + done + if [[ -z "${missing}" ]]; then + echo "Tiered to paimon:${LAKE_EXPECTED_ROWS[*]}" + return 0 + fi + else + missing=" (count query failed)" + fi + if ((waited >= LAKE_TIERING_WAIT_SECONDS)); then + echo "ERROR: tiering did not reach the expected row counts after ${LAKE_TIERING_WAIT_SECONDS}s:${missing}" >&2 + echo "ERROR: last count output follows" >&2 + cat "${log}" >&2 || true + return 1 + fi + sleep 10 + waited=$((waited + 10)) + done +} + +# Waits until every primary-key fixture has a kv snapshot on disk. +# +# Doris BE reads those files directly, from the host, at the path this container +# wrote them to -- a bind mount at the same absolute path on both sides. Nothing +# but an end-to-end run covers that, and without this wait it would only be +# covered by luck: with no snapshot a primary-key table is read by replaying its +# whole change log, which is equally correct and takes a different code path +# entirely. Baking the snapshot into the environment makes every later suite +# exercise the file-reading path instead of racing the ten-second interval. +# +# What it proves is that a snapshot was taken, not that the coordinator has +# committed it -- completion is registered in ZooKeeper, and the directory here +# is created before the upload. That gap is milliseconds, and losing it costs a +# suite nothing: the read falls back to the change log and still returns the +# right rows. Never seeing a snapshot at all is the real problem, and that is +# what the timeout reports. +wait_for_kv_snapshots() { + local waited=0 + local table + local missing + while :; do + missing="" + for table in "${SNAPSHOT_TABLES[@]}"; do + # Two shapes, because a partition sits between the table and the + # bucket ({partitionName}-p{partitionId}); the trailing /* requires + # the snapshot directory to hold a file, not merely to exist. + # {remote.data.dir}/kv/{db}/{table}-{tableId}/{bucket}/snap-{id}/ + # {remote.data.dir}/kv/{db}/{table}-{tableId}/{partition}/{bucket}/snap-{id}/ + local root="${FLUSS_REMOTE_DATA_DIR}/kv/fluss_test/${table}-" + if ! compgen -G "${root}*/*/snap-*/*" >/dev/null 2>&1 \ + && ! compgen -G "${root}*/*/*/snap-*/*" >/dev/null 2>&1; then + missing="${missing} ${table}" + fi + done + if [[ -z "${missing}" ]]; then + echo "Kv snapshots present for:${SNAPSHOT_TABLES[*]}" + return 0 + fi + if ((waited >= SNAPSHOT_WAIT_SECONDS)); then + echo "ERROR: no kv snapshot after ${SNAPSHOT_WAIT_SECONDS}s for:${missing}" >&2 + echo "ERROR: expected under ${FLUSS_REMOTE_DATA_DIR}/kv/fluss_test/" >&2 + ls -R "${FLUSS_REMOTE_DATA_DIR}/kv" >&2 2>/dev/null || true + return 1 + fi + sleep 5 + waited=$((waited + 5)) + done +} + +run_sql() { + local sql="$1" + local log="$2" + local status=0 + + # Timeout, because a write that the servers keep rejecting is retried by the + # fluss client practically forever: without it the container just hangs. + timeout "${SQL_TIMEOUT_SECONDS}" /opt/flink/bin/sql-client.sh -f "${sql}" 2>&1 | tee "${log}" + status="${PIPESTATUS[0]}" + if ((status != 0)); then + return "${status}" + fi + # The SQL client stops at the first failing statement but still exits 0, so + # the only honest completion signal is its own output. + if grep -q '\[ERROR\]' "${log}"; then + return 1 + fi + return 0 +} + +# Removes the paimon side of the lake tables, which is what makes a retry +# possible at all. init.sql drops its fluss database, but that leaves the paimon +# tables where they are -- and fluss refuses to create a lake table whose paimon +# table already exists and holds rows. Without this, an attempt that failed after +# creating one lake table dooms every attempt after it, and the environment comes +# up "failed after 3 attempts" with the real cause three hundred lines up. +# +# Deleting the directory IS dropping the database here: the warehouse is a +# filesystem catalog, mounted writable for exactly this kind of work, and the +# host script empties the same directory before the containers start. +drop_lake_warehouse() { + local warehouse="${FLUSS_PAIMON_WAREHOUSE#file://}" + if [[ -d "${warehouse}/fluss_test.db" ]]; then + echo "Removing the paimon side of the previous attempt: ${warehouse}/fluss_test.db" + rm -rf "${warehouse}/fluss_test.db" + fi +} + +run_attempt() { + local attempt="$1" + + # A previous attempt's tiering job would still be consuming the database + # init.sql is about to drop. + cancel_all_jobs + drop_lake_warehouse + start_tiering_job || return 1 + + run_sql "${MARKER_DIR}/init.sql" "${MARKER_DIR}/init-attempt-${attempt}.log" || return 1 + + echo "Fluss init SQL finished; waiting for the tiering service to commit" + wait_for_lake_rows || return 1 + + # Everything meant for the lake is in the lake. Stop tiering BEFORE writing + # the tail, so the tail stays in the fluss log for good. + cancel_all_jobs + + run_sql "${MARKER_DIR}/init-lake-tail.sql" \ + "${MARKER_DIR}/init-lake-tail-attempt-${attempt}.log" || return 1 + return 0 +} + +# init.sql drops and recreates its database up front, so a retry always starts +# from the same state. Retries exist because the tablet server may still be +# registering with the coordinator when the ports are already open. +for ((attempt = 1; attempt <= ATTEMPTS; attempt++)); do + echo "Running fluss init SQL (attempt ${attempt}/${ATTEMPTS})" + if run_attempt "${attempt}"; then + echo "Fluss fixtures written; waiting for kv snapshots" + if ! wait_for_kv_snapshots; then + exit 1 + fi + touch "${MARKER_DIR}/SUCCESS" + echo "Fluss environment ready" + exec tail -f /dev/null + fi + echo "Fluss init SQL failed on attempt ${attempt}" >&2 + sleep 10 +done + +echo "ERROR: fluss init SQL failed after ${ATTEMPTS} attempts" >&2 +exit 1 diff --git a/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql b/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql new file mode 100644 index 00000000000000..cacb596b9c41b6 --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/sql/init-lake-tail.sql @@ -0,0 +1,175 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- The second half of the lake fixtures: the rows that must stay in the fluss +-- log rather than reach paimon. scripts/run-init-sql.sh runs this only after +-- the tiering service has committed everything init.sql wrote AND has been +-- stopped, so these rows stay where they land and a union read has both halves +-- to stitch for as long as the environment lives. +-- +-- lake_cold gets nothing on purpose: it is the fixture for a table the lake +-- already holds in full. + +SET 'table.dml-sync' = 'true'; +SET 'parallelism.default' = '2'; + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = '__FLUSS_BOOTSTRAP_SERVERS__' +); + +USE CATALOG fluss_catalog; +USE fluss_test; + +-- Two rows over three buckets: at least one bucket keeps an empty tail. +INSERT INTO lake_log VALUES + (5, 'hot5', CAST(5.50 AS DECIMAL(10, 2))), + (6, 'hot6', CAST(6.60 AS DECIMAL(10, 2))); + +-- The all-NULL row, so the union read covers NULLs arriving from the log half +-- while the non-NULL values of every type come from the lake half. +INSERT INTO lake_types VALUES + ( + 2, + CAST(NULL AS BOOLEAN), + CAST(NULL AS TINYINT), + CAST(NULL AS SMALLINT), + CAST(NULL AS INT), + CAST(NULL AS BIGINT), + CAST(NULL AS FLOAT), + CAST(NULL AS DOUBLE), + CAST(NULL AS DECIMAL(20, 4)), + CAST(NULL AS CHAR(5)), + CAST(NULL AS STRING), + CAST(NULL AS BINARY(3)), + CAST(NULL AS BYTES), + CAST(NULL AS DATE), + CAST(NULL AS TIMESTAMP(6)), + CAST(NULL AS TIMESTAMP_LTZ(3)), + CAST(NULL AS ARRAY), + CAST(NULL AS MAP), + CAST(NULL AS ROW) + ); + +-- Only 20260101 gets a tail; 20260102 stays entirely in the lake. +INSERT INTO lake_part VALUES + (4, 'lp1c', '20260101'); + +-- The tail carries one of each way a change log can disagree with the lake it +-- follows: row 3 is updated, row 1 is deleted, and row 4 exists only here. All +-- three are what a merge of the two halves has to get right, and the fluss-only +-- read of this table -- which is the whole table, because fluss keeps a +-- primary-key table's state in full -- is the answer that merge must reproduce. +INSERT INTO lake_pk VALUES + (3, 'lp3-hot'), + (4, 'lp4-hot'); + +-- Three keys out of nine, so the tail reaches some buckets and not others -- +-- which is the whole point of this table. Key 10 is new here, so it also lands +-- in a bucket the lake already holds rows for and must be added to that bucket +-- rather than replacing anything in it. +INSERT INTO lake_pk_multi VALUES + (2, 'm2-hot'), + (10, 'm10-hot'); + +-- 20260101 gets a tail, 20260102 keeps none, and 20260103 is created here -- +-- after tiering stopped -- so the lake never sees it. +INSERT INTO lake_pk_part VALUES + (1, 'pp1a-hot', '20260101'); + +INSERT INTO lake_pk_part VALUES + (5, 'pp3a', '20260103'); + +-- The all-NULL row of the nested fixture: every nested column arrives from the +-- log half while its populated counterpart comes through paimon. +INSERT INTO lake_nested VALUES + ( + 2, + CAST(NULL AS ARRAY>), + CAST(NULL AS ARRAY>), + CAST(NULL AS MAP>), + CAST(NULL AS MAP>), + CAST(NULL AS ROW, r_map MAP, r_row ROW>) + ); + +-- A tail in one INT-named partition and none in the other. +INSERT INTO lake_part_int VALUES + (4, 'li1c', 1); + +-- A tail on the primary-key table whose partition column is an INT. It exists so +-- that falling back to the fluss-only read is a real choice rather than a +-- distinction without a difference: were the halves merged by key here, this row +-- would be the one the merge got wrong. +INSERT INTO lake_pk_part_int VALUES + (1, 'pi1a-hot', 1); + +-- The large tails. 500 updated keys, 500 new ones and five deletions, against a +-- lake of 100000 -- so the suppression set is a thousand-odd keys wide and the +-- rows it must drop are scattered through every bucket rather than sitting at +-- the front of one. +CREATE TEMPORARY TABLE tail_seq (id INT) WITH ( + 'connector' = 'datagen', + 'fields.id.kind' = 'sequence', + 'fields.id.start' = '1', + 'fields.id.end' = '500' +); + +CREATE TEMPORARY TABLE tail_seq_log (id INT) WITH ( + 'connector' = 'datagen', + 'fields.id.kind' = 'sequence', + 'fields.id.start' = '100001', + 'fields.id.end' = '101000' +); + +-- 1000 rows appended after the lake stopped: the log half of the big log table. +-- Ids stay contiguous with the lake half (1..101000), and every derived column +-- is the same function of the id on both sides, so an aggregate over the union +-- has a closed form and a missing or duplicated half is arithmetic, not opinion. +INSERT INTO big_log +SELECT id, CONCAT('hot', CAST(id AS STRING)), CAST(id AS DECIMAL(10, 2)), MOD(id, 7) +FROM tail_seq_log; + +-- 500 keys the lake already holds, written again with a different name: each one +-- must suppress exactly the lake row it replaces. +INSERT INTO big_pk +SELECT id, CONCAT('hot', CAST(id AS STRING)), MOD(id, 7) FROM tail_seq; + +-- 500 keys the lake has never seen, in buckets it does hold rows for: they are +-- added, not substituted for anything. +INSERT INTO big_pk +SELECT 100000 + id, CONCAT('new', CAST(id AS STRING)), MOD(100000 + id, 7) FROM tail_seq; + +-- Deletions of rows the lake holds. They are the case a suppression set has to +-- cover but a merge of surviving rows cannot: the key is gone from the tail's +-- own result, and the lake row it removes would otherwise stay. +-- +-- The big table's five are spelled out one at a time because a fluss DELETE +-- resolves a full primary key, not a range; they are spread across the key space +-- so they cannot all land in one bucket. +SET 'execution.runtime-mode' = 'batch'; +DELETE FROM lake_pk WHERE id = 1; +DELETE FROM lake_pk_multi WHERE id = 7; +DELETE FROM lake_pk_part WHERE id = 2 AND dt = '20260101'; +DELETE FROM big_pk WHERE id = 7; +DELETE FROM big_pk WHERE id = 19999; +DELETE FROM big_pk WHERE id = 50000; +DELETE FROM big_pk WHERE id = 77777; +DELETE FROM big_pk WHERE id = 99999; +SET 'execution.runtime-mode' = 'streaming'; + +-- lake_pk_cold gets nothing: it is the fixture for a primary-key table the lake +-- already holds in full, where planning must wrap no split and read no tail. diff --git a/docker/thirdparties/docker-compose/fluss/sql/init.sql b/docker/thirdparties/docker-compose/fluss/sql/init.sql new file mode 100644 index 00000000000000..a01b4781d30d7b --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/sql/init.sql @@ -0,0 +1,928 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Regression fixtures for the fluss catalog. Data is static: the suites read +-- it, they never write. __FLUSS_BOOTSTRAP_SERVERS__ is substituted by +-- scripts/run-init-sql.sh. +-- +-- This is the first of two scripts. Everything a lake table should hold in +-- PAIMON is written here; init-lake-tail.sql then writes the rows that must +-- stay in the fluss log. scripts/run-init-sql.sh stops the tiering service in +-- between, which is what freezes the split -- otherwise the tail would drift +-- into the lake at the next tiering round and every assertion about how the +-- two halves divide would decay into "it depends when you ran it". + +SET 'table.dml-sync' = 'true'; +SET 'parallelism.default' = '2'; + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = '__FLUSS_BOOTSTRAP_SERVERS__' +); + +USE CATALOG fluss_catalog; + +-- Recreated from scratch so that rerunning this script is idempotent. +DROP DATABASE IF EXISTS fluss_test CASCADE; +CREATE DATABASE fluss_test; +USE fluss_test; + +-- --------------------------------------------------------------------------- +-- log_basic: plain append-only table, carries table and column comments so the +-- catalog suite can check that they survive the metadata mapping. +-- --------------------------------------------------------------------------- +CREATE TABLE log_basic ( + id INT COMMENT 'row id', + name STRING COMMENT 'row name', + price DECIMAL(10, 2) +) COMMENT 'fluss log table for regression' +WITH ( + 'bucket.num' = '3' +); + +INSERT INTO log_basic VALUES + (1, 'alice', CAST(10.10 AS DECIMAL(10, 2))), + (2, 'bob', CAST(20.20 AS DECIMAL(10, 2))), + (3, 'carol', CAST(30.30 AS DECIMAL(10, 2))); + +-- --------------------------------------------------------------------------- +-- log_types: one column per fluss data type that the connector maps, plus an +-- all-NULL row. TIME is deliberately absent: it has no Doris counterpart and +-- gets its own negative fixture later. +-- --------------------------------------------------------------------------- +CREATE TABLE log_types ( + id INT, + f_boolean BOOLEAN, + f_tinyint TINYINT, + f_smallint SMALLINT, + f_int INT, + f_bigint BIGINT, + f_float FLOAT, + f_double DOUBLE, + f_decimal DECIMAL(20, 4), + f_char CHAR(5), + f_string STRING, + f_binary BINARY(3), + f_bytes BYTES, + f_date DATE, + f_timestamp TIMESTAMP(6), + f_timestamp_ltz TIMESTAMP_LTZ(3), + f_array ARRAY, + f_map MAP, + f_row ROW +) WITH ( + 'bucket.num' = '1' +); + +INSERT INTO log_types VALUES + ( + 1, + TRUE, + CAST(1 AS TINYINT), + CAST(2 AS SMALLINT), + 3, + CAST(4 AS BIGINT), + CAST(1.5 AS FLOAT), + CAST(2.5 AS DOUBLE), + CAST(123.4567 AS DECIMAL(20, 4)), + CAST('char1' AS CHAR(5)), + 'string1', + CAST(X'010203' AS BINARY(3)), + CAST(X'0a0b' AS BYTES), + DATE '2026-01-01', + TIMESTAMP '2026-01-01 01:02:03.456789', + CAST(TIMESTAMP '2026-01-01 01:02:03.456' AS TIMESTAMP_LTZ(3)), + ARRAY[1, 2, 3], + MAP['k1', 1, 'k2', 2], + CAST(ROW(1, 'nested1') AS ROW) + ), + ( + 2, + FALSE, + CAST(-1 AS TINYINT), + CAST(-2 AS SMALLINT), + -3, + CAST(-4 AS BIGINT), + CAST(-1.5 AS FLOAT), + CAST(-2.5 AS DOUBLE), + CAST(-123.4567 AS DECIMAL(20, 4)), + CAST('char2' AS CHAR(5)), + 'string2', + CAST(X'040506' AS BINARY(3)), + CAST(X'0c0d' AS BYTES), + DATE '2026-01-02', + TIMESTAMP '2026-01-02 01:02:03.456789', + CAST(TIMESTAMP '2026-01-02 01:02:03.456' AS TIMESTAMP_LTZ(3)), + ARRAY[4, 5], + MAP['k3', 3], + CAST(ROW(2, 'nested2') AS ROW) + ), + ( + 3, + CAST(NULL AS BOOLEAN), + CAST(NULL AS TINYINT), + CAST(NULL AS SMALLINT), + CAST(NULL AS INT), + CAST(NULL AS BIGINT), + CAST(NULL AS FLOAT), + CAST(NULL AS DOUBLE), + CAST(NULL AS DECIMAL(20, 4)), + CAST(NULL AS CHAR(5)), + CAST(NULL AS STRING), + CAST(NULL AS BINARY(3)), + CAST(NULL AS BYTES), + CAST(NULL AS DATE), + CAST(NULL AS TIMESTAMP(6)), + CAST(NULL AS TIMESTAMP_LTZ(3)), + CAST(NULL AS ARRAY), + CAST(NULL AS MAP), + CAST(NULL AS ROW) + ); + +-- --------------------------------------------------------------------------- +-- log_part: partitioned append-only table. Partitions are created on write +-- (fluss dynamic partitioning), so the partition set is exactly the one below. +-- --------------------------------------------------------------------------- +CREATE TABLE log_part ( + id INT, + name STRING, + dt STRING +) PARTITIONED BY (dt) +WITH ( + 'bucket.num' = '2' +); + +INSERT INTO log_part VALUES + (1, 'p1a', '20260101'), + (2, 'p1b', '20260101'), + (3, 'p2a', '20260102'), + (4, 'p3a', '20260103'); + +-- --------------------------------------------------------------------------- +-- log_empty: never written to. Every bucket's latest offset is 0, so planning +-- must emit no scan range at all rather than ranges that read nothing. +-- --------------------------------------------------------------------------- +CREATE TABLE log_empty ( + id INT, + name STRING +) WITH ( + 'bucket.num' = '2' +); + +-- --------------------------------------------------------------------------- +-- log_nested: complex types nested inside complex types. log_types covers one +-- column per type but never more than one level, and a decoder can be right +-- about a MAP and wrong about the MAP> beside it -- +-- the element decoder is chosen per level, and only a nested column asks for it +-- to be chosen twice. Every combination of the three constructors is here, plus +-- an all-NULL row, because a null at the outer level and a null element inside a +-- present collection are different things to get wrong. +-- --------------------------------------------------------------------------- +CREATE TABLE log_nested ( + id INT, + f_arr_arr ARRAY>, + f_arr_map ARRAY>, + f_arr_row ARRAY>, + f_map_arr MAP>, + f_map_row MAP>, + f_row_deep ROW, r_map MAP, r_row ROW>, + f_arr_arr_arr ARRAY>> +) WITH ( + 'bucket.num' = '1' +); + +INSERT INTO log_nested VALUES + ( + 1, + ARRAY[ARRAY[1, 2], ARRAY[3]], + ARRAY[MAP['a', 1], MAP['b', 2]], + ARRAY[CAST(ROW(1, 'x') AS ROW), CAST(ROW(2, 'y') AS ROW)], + MAP['k1', ARRAY[1, 2], 'k2', ARRAY[3]], + MAP['k1', CAST(ROW(9, 'z') AS ROW)], + CAST(ROW(1, ARRAY[7, 8], MAP['m', 3], ROW(5, 'deep')) + AS ROW, r_map MAP, r_row ROW>), + ARRAY[ARRAY[ARRAY[1], ARRAY[2, 3]], ARRAY[ARRAY[4]]] + ), + ( + 2, + ARRAY[CAST(NULL AS ARRAY)], + ARRAY[CAST(NULL AS MAP)], + ARRAY[CAST(NULL AS ROW)], + MAP['k1', CAST(NULL AS ARRAY)], + MAP['k1', CAST(NULL AS ROW)], + CAST(ROW(CAST(NULL AS INT), CAST(NULL AS ARRAY), CAST(NULL AS MAP), + CAST(NULL AS ROW)) + AS ROW, r_map MAP, r_row ROW>), + ARRAY[CAST(NULL AS ARRAY>)] + ), + ( + 3, + CAST(NULL AS ARRAY>), + CAST(NULL AS ARRAY>), + CAST(NULL AS ARRAY>), + CAST(NULL AS MAP>), + CAST(NULL AS MAP>), + CAST(NULL AS ROW, r_map MAP, r_row ROW>), + CAST(NULL AS ARRAY>>) + ); + +-- --------------------------------------------------------------------------- +-- pk_nested: the same nesting in the kv row format a primary-key table stores. +-- The two formats have separate readers for every constructor, so covering one +-- says nothing about the other. +-- --------------------------------------------------------------------------- +CREATE TABLE pk_nested ( + id INT NOT NULL, + f_arr_arr ARRAY>, + f_arr_row ARRAY>, + f_map_arr MAP>, + f_map_row MAP>, + f_row_deep ROW, r_map MAP, r_row ROW>, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '1' +); + +INSERT INTO pk_nested VALUES + ( + 1, + ARRAY[ARRAY[1, 2], ARRAY[3]], + ARRAY[CAST(ROW(1, 'x') AS ROW)], + MAP['k1', ARRAY[1, 2]], + MAP['k1', CAST(ROW(9, 'z') AS ROW)], + CAST(ROW(1, ARRAY[7, 8], MAP['m', 3], ROW(5, 'deep')) + AS ROW, r_map MAP, r_row ROW>) + ), + ( + 2, + CAST(NULL AS ARRAY>), + CAST(NULL AS ARRAY>), + CAST(NULL AS MAP>), + CAST(NULL AS MAP>), + CAST(NULL AS ROW, r_map MAP, r_row ROW>) + ); + +-- The update is what makes this a primary-key fixture rather than a second log +-- one: the nested values of key 1 have to be the SECOND set, not the first. +INSERT INTO pk_nested VALUES + ( + 1, + ARRAY[ARRAY[10, 20]], + ARRAY[CAST(ROW(11, 'xx') AS ROW)], + MAP['k9', ARRAY[10]], + MAP['k9', CAST(ROW(99, 'zz') AS ROW)], + CAST(ROW(2, ARRAY[70], MAP['mm', 30], ROW(50, 'deeper')) + AS ROW, r_map MAP, r_row ROW>) + ); + +-- --------------------------------------------------------------------------- +-- part_types: one partition column of every type fluss allows AND Doris can +-- read back. A partition's value is kept nowhere but in its name, so the type +-- decides whether it survives at all -- STRING is the only one the rest of these +-- fixtures use, and it is also the only one that could not tell a rendering bug +-- from a working one. +-- +-- Two rows, two partitions: one value of each column per partition, so a +-- predicate on any single column prunes to exactly one. +-- +-- p_bin is the type whose verdict depends on the catalog: fluss names its +-- partitions with the hex text of the bytes, which reads back as the text it is +-- unless 'enable.mapping.varbinary' asks for a VARBINARY column instead. +-- --------------------------------------------------------------------------- +CREATE TABLE part_types ( + id INT, + name STRING, + p_str STRING, + p_char CHAR(2), + p_bool BOOLEAN, + p_tiny TINYINT, + p_small SMALLINT, + p_int INT, + p_big BIGINT, + p_date DATE, + p_bin BINARY(2) +) PARTITIONED BY (p_str, p_char, p_bool, p_tiny, p_small, p_int, p_big, p_date, p_bin) +WITH ( + 'bucket.num' = '2' +); + +INSERT INTO part_types VALUES + (1, 'pt1', 'cn', CAST('c1' AS CHAR(2)), TRUE, CAST(1 AS TINYINT), CAST(10 AS SMALLINT), 100, + CAST(1000 AS BIGINT), DATE '2026-01-01', CAST(X'0102' AS BINARY(2))), + (2, 'pt2', 'us', CAST('c2' AS CHAR(2)), FALSE, CAST(2 AS TINYINT), CAST(20 AS SMALLINT), 200, + CAST(2000 AS BIGINT), DATE '2026-01-02', CAST(X'0304' AS BINARY(2))); + +-- --------------------------------------------------------------------------- +-- part_ts: a partition column whose value fluss cannot store verbatim. Its name +-- for 2026-01-01 01:02:03 is 2026-01-01-01-02-03_0 -- every character a +-- partition name may not hold rewritten, many-to-one, unrecoverable. Fluss +-- creates the table without complaint, which is why the refusal has to come from +-- the connector and has to name the column. +-- --------------------------------------------------------------------------- +CREATE TABLE part_ts ( + id INT, + name STRING, + p_ts TIMESTAMP(3) +) PARTITIONED BY (p_ts) +WITH ( + 'bucket.num' = '1' +); + +INSERT INTO part_ts VALUES + (1, 'ts1', TIMESTAMP '2026-01-01 01:02:03.000'); + +-- --------------------------------------------------------------------------- +-- log_time: a column of the one fluss type Doris has nowhere to put. The column +-- reads as UNSUPPORTED, so naming it -- or asking for * -- must fail, while the +-- rest of the table stays perfectly readable. A connector that mapped it to a +-- string or to elapsed millis instead would hand back a value meaning something +-- else, which no error would ever reveal. +-- --------------------------------------------------------------------------- +CREATE TABLE log_time ( + id INT, + name STRING, + f_time TIME(0) +) WITH ( + 'bucket.num' = '1' +); + +INSERT INTO log_time VALUES + (1, 'time1', TIME '01:02:03'), + (2, 'time2', TIME '04:05:06'); + +-- --------------------------------------------------------------------------- +-- pk_basic: primary-key table. Row 2 is updated and row 3 deleted, so a +-- correct read returns the merged view, not the raw change log. +-- --------------------------------------------------------------------------- +CREATE TABLE pk_basic ( + id INT NOT NULL, + name STRING, + score DOUBLE, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '3' +); + +INSERT INTO pk_basic VALUES + (1, 'k1', CAST(1.5 AS DOUBLE)), + (2, 'k2', CAST(2.5 AS DOUBLE)), + (3, 'k3', CAST(3.5 AS DOUBLE)), + (4, 'k4', CAST(4.5 AS DOUBLE)); + +INSERT INTO pk_basic VALUES + (2, 'k2-updated', CAST(22.5 AS DOUBLE)); + +SET 'execution.runtime-mode' = 'batch'; +DELETE FROM pk_basic WHERE id = 3; +SET 'execution.runtime-mode' = 'streaming'; + +-- --------------------------------------------------------------------------- +-- pk_types: same type coverage as log_types, but stored in the kv (compacted) +-- row format that primary-key tables use. +-- --------------------------------------------------------------------------- +CREATE TABLE pk_types ( + id INT NOT NULL, + f_boolean BOOLEAN, + f_tinyint TINYINT, + f_smallint SMALLINT, + f_int INT, + f_bigint BIGINT, + f_float FLOAT, + f_double DOUBLE, + f_decimal DECIMAL(20, 4), + f_char CHAR(5), + f_string STRING, + f_binary BINARY(3), + f_bytes BYTES, + f_date DATE, + f_timestamp TIMESTAMP(6), + f_timestamp_ltz TIMESTAMP_LTZ(3), + f_array ARRAY, + f_map MAP, + f_row ROW, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '1' +); + +INSERT INTO pk_types VALUES + ( + 1, + TRUE, + CAST(1 AS TINYINT), + CAST(2 AS SMALLINT), + 3, + CAST(4 AS BIGINT), + CAST(1.5 AS FLOAT), + CAST(2.5 AS DOUBLE), + CAST(123.4567 AS DECIMAL(20, 4)), + CAST('char1' AS CHAR(5)), + 'string1', + CAST(X'010203' AS BINARY(3)), + CAST(X'0a0b' AS BYTES), + DATE '2026-01-01', + TIMESTAMP '2026-01-01 01:02:03.456789', + CAST(TIMESTAMP '2026-01-01 01:02:03.456' AS TIMESTAMP_LTZ(3)), + ARRAY[1, 2, 3], + MAP['k1', 1, 'k2', 2], + CAST(ROW(1, 'nested1') AS ROW) + ), + ( + 2, + CAST(NULL AS BOOLEAN), + CAST(NULL AS TINYINT), + CAST(NULL AS SMALLINT), + CAST(NULL AS INT), + CAST(NULL AS BIGINT), + CAST(NULL AS FLOAT), + CAST(NULL AS DOUBLE), + CAST(NULL AS DECIMAL(20, 4)), + CAST(NULL AS CHAR(5)), + CAST(NULL AS STRING), + CAST(NULL AS BINARY(3)), + CAST(NULL AS BYTES), + CAST(NULL AS DATE), + CAST(NULL AS TIMESTAMP(6)), + CAST(NULL AS TIMESTAMP_LTZ(3)), + CAST(NULL AS ARRAY), + CAST(NULL AS MAP), + CAST(NULL AS ROW) + ); + +-- --------------------------------------------------------------------------- +-- pk_part: partitioned primary-key table. A partitioned primary-key table is +-- snapshotted per partition, so reading it wrong -- asking the table for its +-- snapshots instead of the partition -- resumes the change log at another +-- partition's offset. One row of one partition is updated and one deleted, so +-- the merge has to happen inside a partition and not across the table. +-- --------------------------------------------------------------------------- +CREATE TABLE pk_part ( + id INT NOT NULL, + name STRING, + dt STRING NOT NULL, + PRIMARY KEY (id, dt) NOT ENFORCED +) PARTITIONED BY (dt) +WITH ( + 'bucket.num' = '2' +); + +INSERT INTO pk_part VALUES + (1, 'q1a', '20260101'), + (2, 'q1b', '20260101'), + (3, 'q2a', '20260102'), + (4, 'q2b', '20260102'); + +INSERT INTO pk_part VALUES + (2, 'q1b-updated', '20260101'); + +SET 'execution.runtime-mode' = 'batch'; +DELETE FROM pk_part WHERE id = 4 AND dt = '20260102'; +SET 'execution.runtime-mode' = 'streaming'; + +-- --------------------------------------------------------------------------- +-- pk_empty: a primary-key table nothing was ever written to. It is not the same +-- shape as an empty log table: a primary-key read starts from a kv snapshot, +-- and there is none, so the path taken here is the one that has to notice the +-- table is empty rather than the one that reads no log records. +-- --------------------------------------------------------------------------- +CREATE TABLE pk_empty ( + id INT NOT NULL, + name STRING, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '2' +); + +-- =========================================================================== +-- Lake tables. 'table.datalake.enabled' makes the fluss coordinator create a +-- matching paimon table and lets the tiering service move data into it; the +-- lake settings themselves (warehouse, metastore) come from the cluster config +-- and are copied into each table's properties, which is where the Doris +-- connector reads them from. +-- +-- Freshness is the lag the tiering service is asked to keep. Three minutes by +-- default, which every environment start would then have to wait out. +-- =========================================================================== + +-- --------------------------------------------------------------------------- +-- lake_log: the ordinary union-read fixture. Three buckets and only two rows in +-- the tail, so at least one bucket is fully tiered and must contribute no log +-- range at all, while the others resume where the lake stops. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_log ( + id INT, + name STRING, + price DECIMAL(10, 2) +) COMMENT 'fluss log table tiered into paimon' +WITH ( + 'bucket.num' = '3', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_log VALUES + (1, 'lake1', CAST(1.10 AS DECIMAL(10, 2))), + (2, 'lake2', CAST(2.20 AS DECIMAL(10, 2))), + (3, 'lake3', CAST(3.30 AS DECIMAL(10, 2))), + (4, 'lake4', CAST(4.40 AS DECIMAL(10, 2))); + +-- --------------------------------------------------------------------------- +-- lake_cold: written once and never again, so after the tiering service has +-- caught up the lake holds the whole table and planning must emit no log range +-- whatsoever. One bucket, so "no log range" is an exact number and not a range. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_cold ( + id INT, + name STRING +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_cold VALUES + (1, 'cold1'), + (2, 'cold2'), + (3, 'cold3'); + +-- --------------------------------------------------------------------------- +-- lake_types: the type-parity fixture. The connector's fluss->Doris mapping is +-- required to equal fluss->paimon->Doris, so that the table and its $lake +-- sibling present one schema and not two; this is the only place that identity +-- is checked against the real paimon connector instead of by reading both +-- mappings side by side. One bucket keeps the split between the two halves +-- exact: everything here is tiered, the all-NULL row stays in the log. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_types ( + id INT, + f_boolean BOOLEAN, + f_tinyint TINYINT, + f_smallint SMALLINT, + f_int INT, + f_bigint BIGINT, + f_float FLOAT, + f_double DOUBLE, + f_decimal DECIMAL(20, 4), + f_char CHAR(5), + f_string STRING, + f_binary BINARY(3), + f_bytes BYTES, + f_date DATE, + f_timestamp TIMESTAMP(6), + f_timestamp_ltz TIMESTAMP_LTZ(3), + f_array ARRAY, + f_map MAP, + f_row ROW +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_types VALUES + ( + 1, + TRUE, + CAST(1 AS TINYINT), + CAST(2 AS SMALLINT), + 3, + CAST(4 AS BIGINT), + CAST(1.5 AS FLOAT), + CAST(2.5 AS DOUBLE), + CAST(123.4567 AS DECIMAL(20, 4)), + CAST('char1' AS CHAR(5)), + 'string1', + CAST(X'010203' AS BINARY(3)), + CAST(X'0a0b' AS BYTES), + DATE '2026-01-01', + TIMESTAMP '2026-01-01 01:02:03.456789', + CAST(TIMESTAMP '2026-01-01 01:02:03.456' AS TIMESTAMP_LTZ(3)), + ARRAY[1, 2, 3], + MAP['k1', 1, 'k2', 2], + CAST(ROW(1, 'nested1') AS ROW) + ); + +-- --------------------------------------------------------------------------- +-- lake_part: partitioned lake table. The tail goes into one partition only, so +-- the other one is served entirely from the lake -- the two halves have to be +-- stitched per (partition, bucket) and not per table. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_part ( + id INT, + name STRING, + dt STRING +) PARTITIONED BY (dt) +WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_part VALUES + (1, 'lp1a', '20260101'), + (2, 'lp1b', '20260101'), + (3, 'lp2a', '20260102'); + +-- --------------------------------------------------------------------------- +-- lake_pk: primary-key table tiered into paimon. Merging a lake with a change +-- log BY KEY is not implemented, so this table is read from fluss alone -- which +-- is the WHOLE table rather than a part of it, because fluss keeps a primary-key +-- table's state in full. That read is the baseline the future merge has to +-- reproduce: $lake shows what paimon holds, the front door shows the answer. +-- Row 2 is updated before tiering, so the lake already holds a merged view +-- rather than a raw change log. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_pk ( + id INT NOT NULL, + name STRING, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_pk VALUES + (1, 'lp1'), + (2, 'lp2'), + (3, 'lp3'); + +INSERT INTO lake_pk VALUES + (2, 'lp2-lake'); + +-- --------------------------------------------------------------------------- +-- lake_pk_multi: the same table over three buckets. A primary-key table is +-- merged with its log tail per BUCKET, and a single-bucket fixture cannot tell +-- that apart from merging per table: with one bucket the two are the same +-- arrangement. Here they are not -- the tail below touches some buckets and not +-- others, so a lake split may only be filtered by the tail of ITS OWN bucket. +-- Binding one bucket's tail to another's split suppresses nothing (a key lives +-- in exactly one bucket), and the rows that tail was meant to replace come back +-- as duplicates. +-- +-- Nine keys, because which bucket a key lands in is fluss's hash of it and not +-- ours to choose: enough of them to be spread over all three, few enough to +-- read. Row 5 is updated before tiering, so the lake half already holds a +-- merged view here too. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_pk_multi ( + id INT NOT NULL, + name STRING, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '3', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_pk_multi VALUES + (1, 'm1'), + (2, 'm2'), + (3, 'm3'), + (4, 'm4'), + (5, 'm5'), + (6, 'm6'), + (7, 'm7'), + (8, 'm8'), + (9, 'm9'); + +INSERT INTO lake_pk_multi VALUES + (5, 'm5-lake'); + +-- --------------------------------------------------------------------------- +-- lake_pk_part: partitioned primary-key table tiered into paimon. The tail is +-- written into one partition only and a third partition is created after +-- tiering has stopped, so one query reads all three ways a partition can stand +-- with respect to the lake: 20260101 is lake plus tail, 20260102 is lake alone, +-- and 20260103 exists in fluss only -- the lake has never heard of it, so its +-- buckets have to be read whole from fluss inside the very same scan. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_pk_part ( + id INT NOT NULL, + name STRING, + dt STRING NOT NULL, + PRIMARY KEY (id, dt) NOT ENFORCED +) PARTITIONED BY (dt) +WITH ( + 'bucket.num' = '2', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_pk_part VALUES + (1, 'pp1a', '20260101'), + (2, 'pp1b', '20260101'), + (3, 'pp2a', '20260102'), + (4, 'pp2b', '20260102'); + +-- --------------------------------------------------------------------------- +-- lake_pk_cold: a primary-key lake table with nothing left in the log. It gets +-- no tail at all, so planning must wrap no lake split and emit no tail range: +-- the merge has to cost nothing when there is nothing to merge, and a reader +-- that always builds a suppression set would still return the right rows here +-- while doing the work -- only the plan shows the difference. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_pk_cold ( + id INT NOT NULL, + name STRING, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '2', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_pk_cold VALUES + (1, 'c1'), + (2, 'c2'), + (3, 'c3'); + +-- --------------------------------------------------------------------------- +-- lake_nested: nested complex types that cross the seam. The type mapping this +-- connector applies has to equal fluss->paimon->Doris for these as well, and +-- nesting is where the two could most easily part ways: each level is converted +-- by its own rule on both sides. The populated row is tiered, the all-NULL row +-- stays in the log, so one union read decodes the same nested column through +-- paimon and through fluss. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_nested ( + id INT, + f_arr_arr ARRAY>, + f_arr_row ARRAY>, + f_map_arr MAP>, + f_map_row MAP>, + f_row_deep ROW, r_map MAP, r_row ROW> +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_nested VALUES + ( + 1, + ARRAY[ARRAY[1, 2], ARRAY[3]], + ARRAY[CAST(ROW(1, 'x') AS ROW)], + MAP['k1', ARRAY[1, 2]], + MAP['k1', CAST(ROW(9, 'z') AS ROW)], + CAST(ROW(1, ARRAY[7, 8], MAP['m', 3], ROW(5, 'deep')) + AS ROW, r_map MAP, r_row ROW>) + ); + +-- --------------------------------------------------------------------------- +-- lake_empty: tiering is on and has never committed anything, because nothing +-- was ever written. It is the state every lake table passes through, and the +-- one where "read the lake plus the log" has no lake to read: auto has to fall +-- back to the fluss-only read, required has to say why it will not. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_empty ( + id INT, + name STRING +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +-- --------------------------------------------------------------------------- +-- lake_part_int: a tiered log table partitioned by an INT. Concatenating a lake +-- with a log tail needs no partition value matched across the halves -- each +-- half prunes on its own -- so a non-STRING partition column has to work here, +-- and this table is what says the rule that stops the primary-key merge (below) +-- was not applied to everything partitioned. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_part_int ( + id INT, + name STRING, + p_int INT +) PARTITIONED BY (p_int) +WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_part_int VALUES + (1, 'li1a', 1), + (2, 'li1b', 1), + (3, 'li2a', 2); + +-- --------------------------------------------------------------------------- +-- lake_pk_part_int: a tiered PRIMARY-KEY table partitioned by an INT. Merging +-- its halves by key means matching a paimon split to a fluss partition by the +-- text each side renders that value as, and only STRING is guaranteed to render +-- alike -- so this table must NOT be merged. Under auto it falls back to the +-- fluss-only read, which returns every row anyway; under required it is an +-- error. Without this fixture the rule is only ever exercised on tables that had +-- no lake to merge in the first place. +-- --------------------------------------------------------------------------- +CREATE TABLE lake_pk_part_int ( + id INT NOT NULL, + name STRING, + p_int INT NOT NULL, + PRIMARY KEY (id, p_int) NOT ENFORCED +) PARTITIONED BY (p_int) +WITH ( + 'bucket.num' = '2', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO lake_pk_part_int VALUES + (1, 'pi1a', 1), + (2, 'pi1b', 1), + (3, 'pi2a', 2), + (4, 'pi2b', 2); + +-- =========================================================================== +-- The large fixtures. Everything above is a handful of rows chosen so that a +-- wrong answer is visible by eye; these two are the opposite question -- whether +-- the same machinery still returns the right answer when a scan is split across +-- many batches, a suppression set holds more keys than a debugger would print, +-- and a bucket's log runs to five figures. +-- +-- Both are lake tables with a tail, so ONE table answers three of the scenarios +-- at once: read through a union-read catalog it is lake plus log, read through a +-- disabled one it is the pure fluss read of the same rows -- a full log replay +-- for big_log, a kv snapshot plus its log for big_pk. Two catalogs over one +-- fixture also means the two paths can be compared to each other rather than to +-- a number someone wrote down. +-- +-- The values are derived from the sequence rather than randomly generated, so +-- every aggregate over them is a closed form: 1..100000 sums to 5000050000, and +-- a suite asserting that is checking arithmetic rather than repeating whatever +-- the fixture happened to produce. +-- =========================================================================== + +CREATE TEMPORARY TABLE big_seq (id INT) WITH ( + 'connector' = 'datagen', + 'fields.id.kind' = 'sequence', + 'fields.id.start' = '1', + 'fields.id.end' = '100000' +); + +-- --------------------------------------------------------------------------- +-- big_log: 100000 rows over three buckets, all of them tiered. The tail written +-- afterwards is small on purpose -- that is the shape of a real tiered table, +-- where the tail is only what the freshness window has not yet taken. +-- --------------------------------------------------------------------------- +CREATE TABLE big_log ( + id INT, + name STRING, + price DECIMAL(10, 2), + grp INT +) WITH ( + 'bucket.num' = '3', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO big_log +SELECT id, CONCAT('n', CAST(id AS STRING)), CAST(id AS DECIMAL(10, 2)), MOD(id, 7) FROM big_seq; + +-- --------------------------------------------------------------------------- +-- big_pk: 100000 keys over three buckets, all tiered. Its tail updates 500 of +-- them, deletes five and adds 500 more, so the suppression set that filters the +-- lake half holds a thousand-odd keys instead of the two or three every other +-- primary-key fixture here has. A merge that is right for three keys and wrong +-- for a thousand -- a set built per split, a cache keyed too coarsely -- has +-- nowhere to hide in the counts this produces. +-- --------------------------------------------------------------------------- +CREATE TABLE big_pk ( + id INT NOT NULL, + name STRING, + grp INT, + PRIMARY KEY (id) NOT ENFORCED +) WITH ( + 'bucket.num' = '3', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '30s' +); + +INSERT INTO big_pk +SELECT id, CONCAT('p', CAST(id AS STRING)), MOD(id, 7) FROM big_seq; + +-- No deletion-vector fixture, and the reason is worth recording where the next +-- person to want one will look. Fluss does forward a 'paimon.*' table property +-- into the paimon table it creates, so a table declared with +-- 'paimon.deletion-vectors.enabled' = 'true' really is created with deletion +-- vectors on -- that much was verified against this environment. What the +-- tiering service then writes into it is data files and no index: no deletion +-- vector is ever produced, and paimon's own reader answers COUNT(*) with 0 on a +-- table whose files are sitting right there. A fixture that reads as empty +-- proves nothing, so deletion vectors under the merge stay covered by the BE +-- unit tests until the tiering side writes them. diff --git a/docker/thirdparties/docker-compose/fluss/sql/lake-row-counts.sql b/docker/thirdparties/docker-compose/fluss/sql/lake-row-counts.sql new file mode 100644 index 00000000000000..53d8772c86ce6f --- /dev/null +++ b/docker/thirdparties/docker-compose/fluss/sql/lake-row-counts.sql @@ -0,0 +1,73 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Counts the rows the tiering service has committed to paimon, read straight +-- from the warehouse rather than through fluss. scripts/run-init-sql.sh runs +-- this in a loop and greps the markers until every table reports the count +-- init.sql wrote. +-- +-- Counting is what makes the environment deterministic. "A paimon snapshot +-- exists" would not: the service commits what it has consumed so far, so a +-- fixture could be frozen half-tiered, and then a table meant to be entirely in +-- the lake would still have a log tail -- for some runs and not others. +-- +-- __FLUSS_PAIMON_WAREHOUSE__ is substituted by scripts/run-init-sql.sh. + +SET 'execution.runtime-mode' = 'batch'; +SET 'sql-client.execution.result-mode' = 'tableau'; +SET 'parallelism.default' = '1'; + +CREATE CATALOG paimon_catalog WITH ( + 'type' = 'paimon', + 'warehouse' = '__FLUSS_PAIMON_WAREHOUSE__' +); + +USE CATALOG paimon_catalog; +USE fluss_test; + +-- One statement, so one Flink job per poll instead of one per table. The marker +-- is a single token with no spaces because it is matched with a literal grep. +SELECT CONCAT('LAKEROWS:lake_log=', CAST(COUNT(*) AS STRING)) AS marker FROM lake_log +UNION ALL +SELECT CONCAT('LAKEROWS:lake_cold=', CAST(COUNT(*) AS STRING)) FROM lake_cold +UNION ALL +SELECT CONCAT('LAKEROWS:lake_types=', CAST(COUNT(*) AS STRING)) FROM lake_types +UNION ALL +SELECT CONCAT('LAKEROWS:lake_part=', CAST(COUNT(*) AS STRING)) FROM lake_part +UNION ALL +SELECT CONCAT('LAKEROWS:lake_pk=', CAST(COUNT(*) AS STRING)) FROM lake_pk +UNION ALL +SELECT CONCAT('LAKEROWS:lake_pk_multi=', CAST(COUNT(*) AS STRING)) FROM lake_pk_multi +UNION ALL +SELECT CONCAT('LAKEROWS:lake_pk_part=', CAST(COUNT(*) AS STRING)) FROM lake_pk_part +UNION ALL +SELECT CONCAT('LAKEROWS:lake_pk_cold=', CAST(COUNT(*) AS STRING)) FROM lake_pk_cold +UNION ALL +SELECT CONCAT('LAKEROWS:lake_nested=', CAST(COUNT(*) AS STRING)) FROM lake_nested +UNION ALL +SELECT CONCAT('LAKEROWS:lake_part_int=', CAST(COUNT(*) AS STRING)) FROM lake_part_int +UNION ALL +SELECT CONCAT('LAKEROWS:lake_pk_part_int=', CAST(COUNT(*) AS STRING)) FROM lake_pk_part_int +UNION ALL +SELECT CONCAT('LAKEROWS:big_log=', CAST(COUNT(*) AS STRING)) FROM big_log +UNION ALL +SELECT CONCAT('LAKEROWS:big_pk=', CAST(COUNT(*) AS STRING)) FROM big_pk; + +-- lake_empty is absent on purpose: nothing is ever written to it, so there is +-- nothing for tiering to commit and no count to wait for. It is the fixture for +-- a lake table that has no snapshot at all, and waiting for one would hang the +-- environment on the very state it exists to provide. diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index 4ac5edb4a3322c..9929dffb14cc95 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -76,12 +76,12 @@ Usage: $0 --hive-modules comma separated hive modules to refresh All valid components: - mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,iceberg-rest,hudi,kafka,mariadb,db2,oceanbase,lakesoul,kerberos,ranger,polaris,minio + mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,iceberg-rest,hudi,kafka,mariadb,db2,oceanbase,lakesoul,kerberos,ranger,polaris,minio,fluss " exit 1 } DEFAULT_COMPONENTS="mysql,es,hive2,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,hudi,db2,oceanbase,kerberos,minio" -ALL_COMPONENTS="${DEFAULT_COMPONENTS},kafka,lakesoul,ranger,polaris" +ALL_COMPONENTS="${DEFAULT_COMPONENTS},kafka,lakesoul,ranger,polaris,fluss" COMPONENTS=$2 HELP=0 STOP=0 @@ -257,6 +257,7 @@ RUN_KERBEROS=0 RUN_MINIO=0 RUN_RANGER=0 RUN_POLARIS=0 +RUN_FLUSS=0 RESERVED_PORTS="65535" @@ -303,6 +304,8 @@ for element in "${COMPONENTS_ARR[@]}"; do RUN_RANGER=1 elif [[ "${element}"x == "polaris"x ]]; then RUN_POLARIS=1 + elif [[ "${element}"x == "fluss"x ]]; then + RUN_FLUSS=1 else echo "Invalid component: ${element}" usage @@ -1476,6 +1479,41 @@ start_mariadb() { "${ROOT}/docker-compose/mariadb/data" } +start_fluss() { + local fluss_dir="${ROOT}/docker-compose/fluss" + + # The compose file bind mounts remote.data.dir and the paimon warehouse at + # the same absolute paths it uses inside the containers, so Doris (running on + # the host) can read the kv snapshots and remote log segments the servers + # write, and the lake files the tiering service writes. + export FLUSS_COMPOSE_DIR="${fluss_dir}" + envsubst <"${fluss_dir}/fluss.env.tpl" >"${fluss_dir}/fluss.env" + set -a + # shellcheck source=/dev/null + . "${fluss_dir}/fluss.env" + set +a + + render_uid_template "${fluss_dir}/fluss.yaml.tpl" "${fluss_dir}/fluss.yaml" + register_stack_metadata "fluss" "${fluss_dir}/fluss.yaml" "${fluss_dir}/fluss.env" + compose_down_stack "${fluss_dir}/fluss.yaml" "${fluss_dir}/fluss.env" --remove-orphans + + if [[ "${STOP}" -eq 1 ]]; then + return 0 + fi + + # Fluss 1.0 is unreleased: both images are built from a local checkout. + FLUSS_DOCKER_REUSE_IMAGES="${FLUSS_DOCKER_REUSE_IMAGES:-1}" \ + bash "${fluss_dir}/build-images.sh" + + reset_data_dirs "${FLUSS_REMOTE_DATA_DIR}" "${FLUSS_PAIMON_WAREHOUSE_DIR}" + # The fluss and flink images run as uid 9999, the host directories are + # created by root. + sudo chmod 777 "${FLUSS_REMOTE_DATA_DIR}" "${FLUSS_PAIMON_WAREHOUSE_DIR}" + sudo chmod +x "${fluss_dir}/scripts/run-init-sql.sh" + + compose_up_stack "${fluss_dir}/fluss.yaml" "${fluss_dir}/fluss.env" -d --wait +} + start_lakesoul() { echo "RUN_LAKESOUL" cp "${ROOT}"/docker-compose/lakesoul/lakesoul.yaml.tpl "${ROOT}"/docker-compose/lakesoul/lakesoul.yaml @@ -1818,6 +1856,10 @@ if [[ "${RUN_LAKESOUL}" -eq 1 ]]; then launch_component "lakesoul" "${LOG_ROOT}/start_lakesoul.log" start_lakesoul fi +if [[ "${RUN_FLUSS}" -eq 1 ]]; then + launch_component "fluss" "${LOG_ROOT}/start_fluss.log" start_fluss +fi + if [[ "${RUN_MINIO}" -eq 1 ]]; then launch_component "minio" "${LOG_ROOT}/start_minio.log" start_minio fi diff --git a/fe/be-java-extensions/fluss-scanner/pom.xml b/fe/be-java-extensions/fluss-scanner/pom.xml new file mode 100644 index 00000000000000..3ce1b8b339001a --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/pom.xml @@ -0,0 +1,155 @@ + + + + + be-java-extensions + org.apache.doris + ${revision} + + 4.0.0 + + fluss-scanner + + + + 8 + 8 + + + + + org.apache.doris + java-common + ${project.version} + provided + + + + + org.apache.fluss + fluss-client + ${fluss.version} + + + + org.junit.jupiter + junit-jupiter + test + + + + + org.apache.fluss + fluss-server + ${fluss.version} + test + + + + + org.apache.fluss + fluss-server + ${fluss.version} + tests + test + + + + org.apache.fluss + fluss-test-utils + ${fluss.version} + test + + + + org.apache.curator + curator-test + 5.4.0 + test + + + + + fluss-scanner + ${project.basedir}/target/ + + + org.apache.maven.plugins + maven-surefire-plugin + + + --add-opens=java.base/java.nio=ALL-UNNAMED + + + + org.apache.maven.plugins + maven-assembly-plugin + + + src/main/resources/package.xml + + + + + + + + + + make-assembly + package + + single + + + + + + + diff --git a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogBatchScanner.java b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogBatchScanner.java new file mode 100644 index 00000000000000..aac1825b589343 --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogBatchScanner.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fluss; + +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.batch.BatchScanner; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.utils.CloseableIterator; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +/** + * One bucket of a fluss log table over {@code [logStartOffset, logStopOffset)}, as a bounded scanner. + * + *

Every record of a log table is a row of it, so this is {@link BoundedLogRecords} with the change + * type dropped — an append-only log has nothing to replay. + * + *

Shaped as a {@link BatchScanner} so that the log read and the primary-key reads + * ({@code KvSnapshotAndLogBatchScanner} and {@link PkTailBatchScanner}) present the same interface to + * {@link FlussJniScanner} — what separates them belongs here, not in the row loop. + */ +class BoundedLogBatchScanner implements BatchScanner { + + private final BoundedLogRecords records; + + /** + * @param projection table field indexes to read, in the order the caller wants them back; must + * not be empty, which fluss rejects outright + * @param logStartOffset a real offset, or fluss's {@code LogScanner.EARLIEST_OFFSET} sentinel + */ + BoundedLogBatchScanner(Table table, TableBucket tableBucket, int[] projection, + long logStartOffset, long logStopOffset) { + this.records = new BoundedLogRecords(table, tableBucket, projection, + logStartOffset, logStopOffset); + } + + @Override + public CloseableIterator pollBatch(Duration timeout) { + if (records.isFinished()) { + return null; + } + List batch = records.poll(timeout); + List rows = new ArrayList<>(batch.size()); + for (ScanRecord record : batch) { + rows.add(record.getRow()); + } + return CloseableIterator.wrap(rows.iterator()); + } + + @Override + public void close() throws IOException { + records.close(); + } +} diff --git a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogRecords.java b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogRecords.java new file mode 100644 index 00000000000000..9db6bb3f11e4b1 --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/BoundedLogRecords.java @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fluss; + +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.metadata.TableBucket; + +import java.io.Closeable; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +/** + * One bucket's log over {@code [logStartOffset, logStopOffset)}, polled in batches. + * + *

A fluss log scanner is a streaming reader with no end, so the bound has to be imposed here, and + * reaching it has to be detected three ways — all of them taken from fluss's own bounded reader, + * {@code KvSnapshotAndLogBatchScanner#pollLogRecords}: + *

+ * + *

This is the only place those three live. Both readers of a bounded log range consume it — + * {@link BoundedLogBatchScanner}, which wants the rows and throws the change type away, and + * {@link PkTailBatchScanner}, which replays the change types by key. A second copy of the stopping + * rules would be a second chance to get one of them wrong, and the one they exist for (the third) + * only shows itself as a query that never returns. + * + *

Records are handed out as fluss returned them, and the caller may hold them past the next poll: + * that is what fluss's own bounded primary-key reader does while it collects a whole range before + * merging it. + */ +class BoundedLogRecords implements Closeable { + + private final LogScanner logScanner; + private final TableBucket tableBucket; + private final long logStopOffset; + + private boolean finished; + + /** + * @param projection table field indexes to read, in the order the caller wants them back; must + * not be empty, which fluss rejects outright + * @param logStartOffset a real offset, or fluss's {@code LogScanner.EARLIEST_OFFSET} sentinel + */ + BoundedLogRecords(Table table, TableBucket tableBucket, int[] projection, + long logStartOffset, long logStopOffset) { + this.tableBucket = tableBucket; + this.logStopOffset = logStopOffset; + LogScanner scanner = table.newScan().project(projection).createLogScanner(); + try { + Long partitionId = tableBucket.getPartitionId(); + if (partitionId == null) { + scanner.subscribe(tableBucket.getBucket(), logStartOffset); + } else { + scanner.subscribe(partitionId, tableBucket.getBucket(), logStartOffset); + } + } catch (RuntimeException | Error e) { + // The scanner is already running its fetcher threads; leaving it unreferenced would keep + // them alive for the life of the BE process. + try { + scanner.close(); + } catch (Exception closeFailure) { + e.addSuppressed(closeFailure); + } + throw e; + } + this.logScanner = scanner; + } + + /** Whether the range has been read to its end; polling after that returns nothing more. */ + boolean isFinished() { + return finished; + } + + /** + * The next records of the range, in log order. May be empty while the fetch is still on its way, + * which says nothing about whether the range is done — {@link #isFinished()} does. + */ + List poll(Duration timeout) { + if (finished) { + return new ArrayList<>(); + } + ScanRecords scanRecords = logScanner.poll(timeout); + List records = new ArrayList<>(); + for (ScanRecord record : scanRecords.records(tableBucket)) { + long offset = record.logOffset(); + if (offset >= logStopOffset) { + // Past the end of this range: another query's rows, not ours. + finished = true; + break; + } + records.add(record); + if (offset >= logStopOffset - 1) { + // The last record of the range. Do not poll again: the record AT the stopping offset + // may not exist, and waiting for it never returns. + finished = true; + break; + } + } + Long consumedUpToOffset = scanRecords.consumedUpToOffset(tableBucket); + if (consumedUpToOffset != null && consumedUpToOffset >= logStopOffset) { + // The fetch reached the end of the range without necessarily yielding a record there — the + // tail can be control records, which take offsets but are never scanned. Without this the + // loop would poll for a row that is never coming. + finished = true; + } + return records; + } + + @Override + public void close() throws IOException { + try { + logScanner.close(); + } catch (Exception e) { + throw new IOException("Failed to close the fluss log scanner", e); + } + } +} diff --git a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussColumnValue.java b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussColumnValue.java new file mode 100644 index 00000000000000..98df50d05248b3 --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussColumnValue.java @@ -0,0 +1,334 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fluss; + +import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.common.jni.vec.ColumnValue; + +import org.apache.fluss.row.DataGetters; +import org.apache.fluss.row.InternalArray; +import org.apache.fluss.row.InternalMap; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.types.ArrayType; +import org.apache.fluss.types.BinaryType; +import org.apache.fluss.types.CharType; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.LocalZonedTimestampType; +import org.apache.fluss.types.MapType; +import org.apache.fluss.types.RowType; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * One column of one fluss row, read the way Doris's vector table wants it. + * + *

Two fluss-specific things this has to get right, both of which a paimon-shaped port would miss: + * + *

    + *
  • Fixed-width reads carry their width. {@code getChar} and {@code getBinary} take the declared + * length, so the fluss type — not just the Doris one — has to be around at read time.
    • + *
  • Timestamps come from two different accessors rather than one. A plain TIMESTAMP is + * {@code getTimestampNtz} and reads back as the wall clock that was written; a TIMESTAMP WITH + * LOCAL TIME ZONE is {@code getTimestampLtz}, an instant, which only becomes a wall clock once + * a zone is applied. Reading one through the other's accessor throws rather than silently + * shifting the value, which is why the fluss type is kept rather than inferred from the Doris + * side.
    • + *
+ * + *

Instances are reused across rows and, for nested values, across containers: the vector table + * consumes an unpacked value before the parent moves on, so a cache keyed by position is safe and + * keeps a wide nested column from allocating per row. + */ +public class FlussColumnValue implements ColumnValue { + + private static final Map DORIS_TIME_ZONE_ALIASES; + + static { + Map aliases = new HashMap<>(ZoneId.SHORT_IDS); + // The scanner cannot depend on FE's TimeUtils, so keep its accepted aliases and CST + // interpretation identical at this JNI boundary (same list as the paimon scanner's). + aliases.put("CST", "Asia/Shanghai"); + aliases.put("PRC", "Asia/Shanghai"); + aliases.put("UTC", "UTC"); + aliases.put("GMT", "UTC"); + DORIS_TIME_ZONE_ALIASES = Collections.unmodifiableMap(aliases); + } + + private DataGetters record; + private int idx; + private ColumnType dorisType; + private DataType flussType; + private ZoneId timeZone; + + // Lazy: a scalar column never pays for the nested-reuse bookkeeping. + private List arrayValues; + private List mapKeys; + private List mapValues; + private List structValues; + + public FlussColumnValue(String timeZone) { + this.timeZone = resolveTimeZone(timeZone); + } + + private FlussColumnValue(DataGetters record, int idx, ColumnType dorisType, DataType flussType, + ZoneId timeZone) { + this.record = record; + this.idx = idx; + this.dorisType = dorisType; + this.flussType = flussType; + this.timeZone = timeZone; + } + + /** Points this value at the row the scanner is currently emitting. */ + public void setRow(InternalRow record) { + this.record = record; + } + + /** Points this value at one column of that row. */ + public void setIdx(int idx, ColumnType dorisType, DataType flussType) { + this.idx = idx; + this.dorisType = dorisType; + this.flussType = flussType; + } + + @Override + public boolean canGetStringAsBytes() { + return true; + } + + @Override + public boolean isNull() { + boolean isNull = record.isNullAt(idx); + if (isNull) { + // A null container has no live descendants; drop wrappers its previous row left behind. + clearChildCaches(); + } + return isNull; + } + + @Override + public boolean getBoolean() { + return record.getBoolean(idx); + } + + @Override + public byte getByte() { + return record.getByte(idx); + } + + @Override + public short getShort() { + return record.getShort(idx); + } + + @Override + public int getInt() { + return record.getInt(idx); + } + + @Override + public long getLong() { + return record.getLong(idx); + } + + @Override + public float getFloat() { + return record.getFloat(idx); + } + + @Override + public double getDouble() { + return record.getDouble(idx); + } + + @Override + public BigInteger getBigInteger() { + return BigInteger.valueOf(record.getInt(idx)); + } + + @Override + public BigDecimal getDecimal() { + return record.getDecimal(idx, dorisType.getPrecision(), dorisType.getScale()).toBigDecimal(); + } + + @Override + public String getString() { + return new String(readBytes(), StandardCharsets.UTF_8); + } + + @Override + public byte[] getStringAsBytes() { + return readBytes(); + } + + @Override + public LocalDate getDate() { + return LocalDate.ofEpochDay(record.getInt(idx)); + } + + @Override + public LocalDateTime getDateTime() { + if (flussType instanceof LocalZonedTimestampType) { + // Stored as an instant, so it only becomes a wall clock in a zone — the session's. + return LocalDateTime.ofInstant( + record.getTimestampLtz(idx, precisionOf(flussType)).toInstant(), timeZone); + } + return record.getTimestampNtz(idx, precisionOf(flussType)).toLocalDateTime(); + } + + @Override + public LocalDateTime getTimeStampTz() { + // TIMESTAMPTZ keeps the instant itself, and Doris carries it as the UTC wall clock — NOT the + // session zone's, which is what getDateTime above applies. + return LocalDateTime.ofInstant( + record.getTimestampLtz(idx, precisionOf(flussType)).toInstant(), ZoneOffset.UTC); + } + + @Override + public byte[] getBytes() { + return readBytes(); + } + + @Override + public void unpackArray(List values) { + InternalArray array = record.getArray(idx); + if (arrayValues == null) { + arrayValues = new ArrayList<>(); + } + ColumnType elementDorisType = dorisType.getChildTypes().get(0); + DataType elementFlussType = ((ArrayType) flussType).getElementType(); + for (int i = 0; i < array.size(); i++) { + values.add(reuse(arrayValues, i, array, i, elementDorisType, elementFlussType)); + } + trim(arrayValues, array.size()); + } + + @Override + public void unpackMap(List keys, List values) { + InternalMap map = record.getMap(idx); + if (mapKeys == null) { + mapKeys = new ArrayList<>(); + mapValues = new ArrayList<>(); + } + InternalArray keyArray = map.keyArray(); + ColumnType keyDorisType = dorisType.getChildTypes().get(0); + DataType keyFlussType = ((MapType) flussType).getKeyType(); + for (int i = 0; i < keyArray.size(); i++) { + keys.add(reuse(mapKeys, i, keyArray, i, keyDorisType, keyFlussType)); + } + trim(mapKeys, keyArray.size()); + + InternalArray valueArray = map.valueArray(); + ColumnType valueDorisType = dorisType.getChildTypes().get(1); + DataType valueFlussType = ((MapType) flussType).getValueType(); + for (int i = 0; i < valueArray.size(); i++) { + values.add(reuse(mapValues, i, valueArray, i, valueDorisType, valueFlussType)); + } + trim(mapValues, valueArray.size()); + } + + @Override + public void unpackStruct(List structFieldIndex, List values) { + RowType rowType = (RowType) flussType; + // The indexes are into the FULL child list, so the nested row must keep its whole arity. + InternalRow row = record.getRow(idx, rowType.getFieldCount()); + if (structValues == null) { + structValues = new ArrayList<>(); + } + for (int i : structFieldIndex) { + values.add(reuse(structValues, i, row, i, dorisType.getChildTypes().get(i), + rowType.getTypeAt(i))); + } + } + + /** + * The raw bytes of a character or binary column, chosen by the FLUSS type rather than the Doris + * one. That distinction matters because the two do not line up one to one: fluss BYTES maps to + * Doris STRING unless the catalog turns on VARBINARY mapping, so the same fluss column can be + * asked for as a string here or as bytes there. Which accessor fluss needs is a property of how + * fluss stored the value, and CHAR and BINARY are stored fixed-width — reading them without their + * declared length gets nothing back. + */ + private byte[] readBytes() { + switch (flussType.getTypeRoot()) { + case CHAR: + return record.getChar(idx, ((CharType) flussType).getLength()).toBytes(); + case STRING: + return record.getString(idx).toBytes(); + case BINARY: + return record.getBinary(idx, ((BinaryType) flussType).getLength()); + case BYTES: + return record.getBytes(idx); + default: + throw new IllegalStateException( + "fluss type " + flussType + " has no byte representation to read"); + } + } + + private FlussColumnValue reuse(List cache, int cacheIndex, + DataGetters childRecord, int childIndex, ColumnType childDorisType, DataType childFlussType) { + while (cache.size() <= cacheIndex) { + cache.add(null); + } + FlussColumnValue value = cache.get(cacheIndex); + if (value == null) { + value = new FlussColumnValue(childRecord, childIndex, childDorisType, childFlussType, timeZone); + cache.set(cacheIndex, value); + } else { + value.record = childRecord; + value.idx = childIndex; + value.dorisType = childDorisType; + value.flussType = childFlussType; + value.timeZone = timeZone; + } + return value; + } + + private static int precisionOf(DataType type) { + return org.apache.fluss.types.DataTypeChecks.getPrecision(type); + } + + private static ZoneId resolveTimeZone(String timeZone) { + return ZoneId.of(timeZone, DORIS_TIME_ZONE_ALIASES); + } + + private static void trim(List cache, int liveSize) { + if (cache.size() > liveSize) { + // Keep only what the CURRENT container can address, not its historical maximum. + cache.subList(liveSize, cache.size()).clear(); + } + } + + private void clearChildCaches() { + arrayValues = null; + mapKeys = null; + mapValues = null; + structValues = null; + } +} diff --git a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java new file mode 100644 index 00000000000000..f4bf6d14931d11 --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/FlussJniScanner.java @@ -0,0 +1,392 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fluss; + +import org.apache.doris.common.jni.JniScanner; +import org.apache.doris.common.jni.vec.ColumnType; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.batch.BatchScanner; +import org.apache.fluss.client.table.scanner.batch.KvSnapshotAndLogBatchScanner; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.RowType; +import org.apache.fluss.utils.CloseableIterator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; + +/** + * Reads one fluss scan range — one bucket of one partition, bounded by log offsets. + * + *

The parameters are the two maps FE built, merged by BE: the scan-level one (connection, table) + * and the per-range one (which bucket, which offsets). Nothing between FE and here type-checks them, + * so every one this class needs is read through {@link #required}, which names the missing key rather + * than letting a null reach fluss. + * + *

Three ways to read, one loop. A log table's range is its bucket's records over + * {@code [start, stop)}, in log order. A primary-key table's range cannot be read that way — its log + * is a change log, so replaying it verbatim returns superseded and deleted rows — and is read instead + * as a kv snapshot merged with the change log that followed it, by fluss's own + * {@code KvSnapshotAndLogBatchScanner}. A primary-key table whose rows are mostly in its lake needs + * only the tail of that change log, replayed by key into the state it ended in + * ({@link PkTailBatchScanner}); the lake half of that read never comes through here. All three are + * fluss {@code BatchScanner}s, so the row loop below does not know which one it is draining. + * + *

Partition columns are not read here. FE declares them to the engine, which leaves them out of + * {@code required_fields} and fills them from the range itself, so the projection this builds covers + * only the data columns — the same division the paimon scanner works under, and the one a union read + * needs both halves to agree on. + */ +public class FlussJniScanner extends JniScanner { + + private static final Logger LOG = LoggerFactory.getLogger(FlussJniScanner.class); + + /** Prefix for the fluss client configuration; the rest of the key is fluss's own option name. */ + private static final String CLIENT_PREFIX = "fluss.client."; + private static final String DB_NAME = "fluss.db_name"; + private static final String TABLE_NAME = "fluss.table_name"; + private static final String RANGE_TYPE = "fluss.range_type"; + private static final String PARTITION_ID = "fluss.partition_id"; + private static final String BUCKET_ID = "fluss.bucket_id"; + private static final String LOG_START_OFFSET = "fluss.log_start_offset"; + private static final String LOG_STOP_OFFSET = "fluss.log_stop_offset"; + private static final String KV_SNAPSHOT_ID = "fluss.kv_snapshot_id"; + /** Scan-level, and only when a primary-key table is read as its lake plus this tail. */ + private static final String MAX_TAIL_ROWS = "fluss.union.max_tail_rows"; + + private static final String RANGE_TYPE_LOG = "LOG"; + private static final String RANGE_TYPE_PK_FULL = "PK_FULL"; + private static final String RANGE_TYPE_PK_TAIL = "PK_TAIL"; + + /** + * How long one poll waits for data. Only affects how often the loop spins, never correctness: the + * loop keeps polling until the scanner reports it has reached the end of the range. + */ + private static final Duration POLL_TIMEOUT = Duration.ofSeconds(1); + + private final Map params; + private final ClassLoader classLoader; + private final FlussColumnValue columnValue; + + private final String rangeType; + private final long logStopOffset; + private final long logStartOffset; + /** Kv snapshot to start a primary-key read from; {@code -1} when the bucket has never had one. */ + private final long kvSnapshotId; + /** How many change log records a {@code PK_TAIL} range may hold while replaying it. */ + private final long maxTailRows; + private final int bucketId; + /** {@code null} on an unpartitioned table, which fluss subscribes to by bucket alone. */ + private final Long partitionId; + + private Connection connection; + private Table table; + private BatchScanner scanner; + /** The same object as {@link #scanner} on a {@code PK_TAIL} range, for what it counted. */ + private PkTailBatchScanner tailScanner; + + /** Fluss types of the projected columns, positionally aligned with {@link #fields}. */ + private List projectedTypes; + + private CloseableIterator currentBatch; + private boolean finished; + private long rowsRead; + + public FlussJniScanner(int batchSize, Map params) { + this.params = params; + this.classLoader = this.getClass().getClassLoader(); + + this.rangeType = required(RANGE_TYPE); + if (!RANGE_TYPE_LOG.equals(rangeType) && !RANGE_TYPE_PK_FULL.equals(rangeType) + && !RANGE_TYPE_PK_TAIL.equals(rangeType)) { + // The suppression descriptor a union read attaches to a lake split also travels as a range + // type, and is read by BE's C++ side alone; one reaching here would mean a lake split was + // handed to the fluss scanner, and reading it as a fluss range would return nothing at all. + throw new IllegalArgumentException( + "fluss scan range type '" + rangeType + "' is not one this scanner reads; expected " + + RANGE_TYPE_LOG + ", " + RANGE_TYPE_PK_FULL + " or " + RANGE_TYPE_PK_TAIL); + } + // Every range field is parsed here, before open() creates a connection: a range that cannot be + // read should say so without having contacted the cluster first. + this.logStopOffset = Long.parseLong(required(LOG_STOP_OFFSET)); + this.logStartOffset = Long.parseLong(required(LOG_START_OFFSET)); + this.kvSnapshotId = RANGE_TYPE_PK_FULL.equals(rangeType) + ? Long.parseLong(required(KV_SNAPSHOT_ID)) : -1L; + this.maxTailRows = RANGE_TYPE_PK_TAIL.equals(rangeType) ? maxTailRows() : 0L; + if (RANGE_TYPE_PK_TAIL.equals(rangeType) && logStartOffset >= logStopOffset) { + // A bucket whose lake has caught up with its log contributes nothing, and planning says so + // by not planning a range at all. One arriving here means the lake half and the fluss half + // were bounded by different offsets, which is a duplicate or a missing row either way. + throw new IllegalArgumentException("a fluss log tail must read something, but bucket " + + required(BUCKET_ID) + " was given [" + logStartOffset + ", " + logStopOffset + + ")"); + } + this.bucketId = Integer.parseInt(required(BUCKET_ID)); + String partition = params.get(PARTITION_ID); + this.partitionId = partition == null ? null : Long.parseLong(partition); + + String[] requiredFields = splitOn(params.get("required_fields"), ","); + String[] requiredTypes = splitOn(params.get("columns_types"), "#"); + if (requiredFields.length != requiredTypes.length) { + throw new IllegalArgumentException("required_fields size " + requiredFields.length + + " does not match columns_types size " + requiredTypes.length); + } + ColumnType[] columnTypes = new ColumnType[requiredTypes.length]; + for (int i = 0; i < requiredTypes.length; i++) { + columnTypes[i] = ColumnType.parseType(requiredFields[i], requiredTypes[i]); + } + initTableInfo(columnTypes, requiredFields, batchSize); + this.columnValue = new FlussColumnValue( + params.getOrDefault("time_zone", TimeZone.getDefault().getID())); + } + + @Override + public void open() throws IOException { + ClassLoader callerLoader = Thread.currentThread().getContextClassLoader(); + // The fluss client spawns its own threads (netty IO, metadata updater) while connecting, and a + // thread inherits the context classloader of whoever created it. Started under BE's loader they + // would not see fluss at all. + Thread.currentThread().setContextClassLoader(classLoader); + try { + connection = ConnectionFactory.createConnection(clientConfig()); + table = connection.getTable(TablePath.of(required(DB_NAME), required(TABLE_NAME))); + + RowType rowType = table.getTableInfo().getRowType(); + int[] projection = projectionOf(rowType); + projectedTypes = new ArrayList<>(projection.length); + for (int index : projection) { + projectedTypes.add(rowType.getTypeAt(index)); + } + + long tableId = table.getTableInfo().getTableId(); + TableBucket tableBucket = partitionId == null + ? new TableBucket(tableId, bucketId) + : new TableBucket(tableId, partitionId, bucketId); + scanner = createScanner(tableBucket, projection); + } catch (Throwable e) { + try { + close(); + } catch (IOException closeFailure) { + e.addSuppressed(closeFailure); + } + throw new IOException("Failed to open the fluss scanner for " + + params.get(DB_NAME) + "." + params.get(TABLE_NAME) + + " bucket " + params.get(BUCKET_ID), e); + } finally { + Thread.currentThread().setContextClassLoader(callerLoader); + } + } + + /** + * The reader for this kind of range. All three are fluss {@code BatchScanner}s, which is what lets + * {@link #getNext} drain any of them without knowing which it has. + */ + private BatchScanner createScanner(TableBucket tableBucket, int[] projection) { + switch (rangeType) { + case RANGE_TYPE_PK_FULL: + return primaryKeyScanner(tableBucket, projection); + case RANGE_TYPE_PK_TAIL: + tailScanner = new PkTailBatchScanner(table, tableBucket, projection, + logStartOffset, logStopOffset, maxTailRows); + return tailScanner; + default: + return new BoundedLogBatchScanner(table, tableBucket, scanProjection(projection), + logStartOffset, logStopOffset); + } + } + + /** + * The kv snapshot of this bucket merged with the change log that followed it, by fluss's own + * bounded primary-key reader. That class is {@code @Internal} — this connector is pinned to the + * fluss version it ships with, and an upgrade has to re-run these tests. + * + *

The projection goes in as-is, including when it is empty, unlike the log path: this reader + * appends the primary-key columns to what it asks fluss for (it needs them to merge on) and + * projects the merged row back down to exactly what was requested, so it never asks fluss for the + * empty projection fluss rejects. + * + *

It buffers the whole bounded change-log range in memory before merging, which is fluss's own + * design for a batch read; the log tail after a snapshot is what bounds that, so a bucket whose + * snapshot is far behind costs the most. + */ + private BatchScanner primaryKeyScanner(TableBucket tableBucket, int[] projection) { + return new KvSnapshotAndLogBatchScanner( + table, tableBucket, kvSnapshotId, logStartOffset, logStopOffset, projection); + } + + /** + * Where each required column sits in the table's row type. Resolved by NAME at open time rather + * than shipped as indexes from FE: the two are separated by planning, and a column added in + * between would shift every index after it. + */ + private int[] projectionOf(RowType rowType) { + List fieldNames = rowType.getFieldNames(); + int[] projection = new int[fields.length]; + for (int i = 0; i < fields.length; i++) { + int index = fieldNames.indexOf(fields[i]); + if (index < 0) { + throw new IllegalStateException("Column '" + fields[i] + + "' is not in the fluss table, which has " + fieldNames + + ". The table's schema changed after the query was planned."); + } + projection[i] = index; + } + return projection; + } + + /** + * What to actually ask fluss for. Normally the projected columns, but a query can need none of + * them — selecting only partition columns leaves this scanner with an empty projection, because + * BE materializes those from the range instead. Fluss rejects an empty projection outright + * ({@code Projection.of}), so the narrowest legal request stands in for it. Nothing reads the + * column that comes back: {@link #getNext} iterates over {@link #fields}, which is empty, and + * reports the row count alone. A fluss table always has a first column. + */ + private static int[] scanProjection(int[] projection) { + return projection.length == 0 ? new int[] {0} : projection; + } + + private Configuration clientConfig() { + Configuration config = new Configuration(); + for (Map.Entry entry : params.entrySet()) { + if (entry.getKey().startsWith(CLIENT_PREFIX)) { + config.setString(entry.getKey().substring(CLIENT_PREFIX.length()), entry.getValue()); + } + } + return config; + } + + @Override + protected int getNext() throws IOException { + int rows = 0; + while (rows < getBatchSize()) { + if (currentBatch == null || !currentBatch.hasNext()) { + if (finished) { + break; + } + // A batch scanner returns null only at the end of the range; an empty batch means it + // has more to do first (the primary-key reader drains the change log that way). + currentBatch = scanner.pollBatch(POLL_TIMEOUT); + finished = currentBatch == null; + continue; + } + columnValue.setRow(currentBatch.next()); + for (int i = 0; i < fields.length; i++) { + columnValue.setIdx(i, types[i], projectedTypes.get(i)); + appendData(i, columnValue); + } + rows++; + } + if (fields.length == 0 && rows > 0) { + // A count-shaped read projects nothing; the vector table still needs the row count. + vectorTable.appendVirtualData(rows); + } + rowsRead += rows; + return rows; + } + + @Override + public void close() throws IOException { + IOException failure = null; + // Close everything even if an earlier close throws: a leaked fluss connection keeps its netty + // and metadata-updater threads alive for the life of the BE process. + failure = closeQuietly(scanner, "scanner", failure); + scanner = null; + failure = closeQuietly(table, "table", failure); + table = null; + failure = closeQuietly(connection, "connection", failure); + connection = null; + currentBatch = null; + if (failure != null) { + throw failure; + } + } + + private IOException closeQuietly(AutoCloseable closeable, String what, IOException previous) { + if (closeable == null) { + return previous; + } + try { + closeable.close(); + return previous; + } catch (Exception e) { + LOG.warn("Failed to close the fluss {}", what, e); + IOException failure = new IOException("Failed to close the fluss " + what, e); + if (previous == null) { + return failure; + } + previous.addSuppressed(failure); + return previous; + } + } + + @Override + public Map getStatistics() { + Map statistics = new HashMap<>(); + statistics.put("counter:FlussJniRowsRead", String.valueOf(rowsRead)); + statistics.put("gauge:FlussJniRequiredFieldCount", String.valueOf(fields.length)); + if (tailScanner != null) { + // What the tail cost and what it hid: the records replayed, and the keys it ended deleted — + // those are lake rows that disappear with nothing returned in their place. + statistics.put("counter:FlussJniTailRecordsRead", + String.valueOf(tailScanner.getRecordsRead())); + statistics.put("counter:FlussJniTailTombstoneKeys", + String.valueOf(tailScanner.getTombstoneKeys())); + } + return statistics; + } + + /** The tail ceiling, which planning sends with every {@code PK_TAIL} range of a union read. */ + private long maxTailRows() { + long rows = Long.parseLong(required(MAX_TAIL_ROWS)); + if (rows <= 0) { + throw new IllegalArgumentException("fluss scanner parameter '" + MAX_TAIL_ROWS + + "' is " + rows + "; expected a positive number of rows"); + } + return rows; + } + + private String required(String key) { + String value = params.get(key); + if (value == null) { + throw new IllegalArgumentException( + "fluss scanner parameter '" + key + "' is missing; got keys " + params.keySet()); + } + return value; + } + + private static String[] splitOn(String value, String separator) { + if (value == null || value.isEmpty()) { + return new String[0]; + } + return value.split(separator); + } +} diff --git a/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/PkTailBatchScanner.java b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/PkTailBatchScanner.java new file mode 100644 index 00000000000000..7bdd80d7590e56 --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/main/java/org/apache/doris/fluss/PkTailBatchScanner.java @@ -0,0 +1,238 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fluss; + +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.batch.BatchScanner; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.row.ProjectedRow; +import org.apache.fluss.row.encode.KeyEncoder; +import org.apache.fluss.types.RowType; +import org.apache.fluss.utils.CloseableIterator; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The log tail of one bucket of a primary-key table, returned as the state that tail ended in. + * + *

This is the fluss half of a union read: the lake holds the bucket up to the offset it was tiered + * at, and this contributes what has happened to that bucket since. The change log cannot be returned + * as it stands — it records every intermediate state, so a key written three times appears three + * times — so the range is replayed by key first, exactly the way fluss's own bounded primary-key + * reader reduces the log half of a snapshot read. + * + *

A delete leaves a tombstone rather than removing the key. The two look interchangeable + * from here, and are not: the lake rows this tail supersedes are hidden by a key set that BE's C++ + * side builds from the same offsets, and that set covers every key the tail touched — deleted ones + * included. So a key deleted in the tail is already gone from the lake half, and what this reader owes + * is only the keys that survived. Keeping the tombstone keeps that fact countable + * ({@link #getTombstoneKeys()}) instead of indistinguishable from a key that was never written. + * + *

The key is the physical primary key — the primary key minus the partition columns. Rows + * are only ever compared within one bucket of one partition, where the partition columns are equal by + * construction, and both halves of the union agree on that: fluss's own kv storage keys the same way. + * + *

The whole range is collected before anything is returned, which is what a merge of any shape + * costs; {@code fluss.union.max_tail_rows} is what stops a tail that grew unbounded (tiering stopped, + * say) from being read into a shared BE process without a limit. + * + *

Rows are held across polls, as fluss's own bounded reader does with the same records. + */ +class PkTailBatchScanner implements BatchScanner { + + private final BoundedLogRecords records; + private final TableBucket tableBucket; + private final long maxTailRows; + + /** Reads the key columns out of a scan row; reused, because each key is encoded on the spot. */ + private final ProjectedRow keyRow; + private final KeyEncoder keyEncoder; + + /** + * The replayed state: the last record of each key, in the order the key was first seen. A + * {@code null} value is a tombstone — the key ended the range deleted. + */ + private final Map state = new LinkedHashMap<>(); + + private long recordsRead; + private long tombstoneKeys; + private boolean emitted; + + /** + * @param projection table field indexes the query asked for, in the order it wants them back; + * may be empty, which a count-shaped query produces + * @param logStartOffset where the lake snapshot this tail follows left off + * @param maxTailRows how many change log records this range may hold before it is refused + */ + PkTailBatchScanner(Table table, TableBucket tableBucket, int[] projection, + long logStartOffset, long logStopOffset, long maxTailRows) { + TableInfo tableInfo = table.getTableInfo(); + if (!tableInfo.hasPrimaryKey()) { + // Replaying by key needs a key. A log table's records are rows in their own right and are + // read as a LOG range; reaching here with one means the range was planned as the wrong kind. + throw new IllegalArgumentException("a primary-key log tail was planned for " + + tableInfo.getTablePath() + ", which has no primary key"); + } + this.tableBucket = tableBucket; + this.maxTailRows = maxTailRows; + + List primaryKeys = tableInfo.getPhysicalPrimaryKeys(); + RowType rowType = tableInfo.getRowType(); + // What to ask fluss for: the requested columns, plus any key column not among them. The key is + // needed to replay whether or not the query selects it, and appending keeps the requested + // columns at the front — so a returned row's first fields are the requested ones, in order, and + // the row loop reads them positionally without knowing this reader added anything. + List scanProjection = new ArrayList<>(projection.length + primaryKeys.size()); + for (int index : projection) { + scanProjection.add(index); + } + int[] keyIndexesInTable = new int[primaryKeys.size()]; + int[] keyIndexesInScanRow = new int[primaryKeys.size()]; + for (int i = 0; i < primaryKeys.size(); i++) { + int indexInTable = rowType.getFieldIndex(primaryKeys.get(i)); + keyIndexesInTable[i] = indexInTable; + int indexInScanRow = scanProjection.indexOf(indexInTable); + if (indexInScanRow < 0) { + scanProjection.add(indexInTable); + indexInScanRow = scanProjection.size() - 1; + } + keyIndexesInScanRow[i] = indexInScanRow; + } + this.keyRow = ProjectedRow.from(keyIndexesInScanRow); + // The table's own key encoder, chosen the way fluss chooses it for this table. Nothing outside + // this JVM sees these bytes — they only have to tell two different keys apart, which is the one + // thing every encoder this factory returns guarantees. + this.keyEncoder = KeyEncoder.ofPrimaryKeyEncoder( + Schema.getKeyRowType(tableInfo.getSchema(), keyIndexesInTable), + primaryKeys, tableInfo.getTableConfig(), tableInfo.isDefaultBucketKey()); + + int[] fields = new int[scanProjection.size()]; + for (int i = 0; i < fields.length; i++) { + fields[i] = scanProjection.get(i); + } + this.records = new BoundedLogRecords(table, tableBucket, fields, + logStartOffset, logStopOffset); + } + + @Override + public CloseableIterator pollBatch(Duration timeout) throws IOException { + if (emitted) { + return null; + } + if (!records.isFinished()) { + for (ScanRecord record : records.poll(timeout)) { + replay(record); + } + // An empty batch, not the end: the caller polls again. Only when the range has been read to + // its end is there an answer to give. + return CloseableIterator.emptyIterator(); + } + emitted = true; + List surviving = new ArrayList<>(); + for (InternalRow row : state.values()) { + if (row != null) { + surviving.add(row); + } + } + tombstoneKeys = state.size() - surviving.size(); + // The surviving rows are held by the list now; the keys that indexed them are the tail's whole + // memory cost and nothing reads them again. + state.clear(); + return CloseableIterator.wrap(surviving.iterator()); + } + + private void replay(ScanRecord record) { + recordsRead++; + if (recordsRead > maxTailRows) { + throw new IllegalStateException("the log tail of bucket " + tableBucket.getBucket() + + " holds more than " + maxTailRows + " change log records, the limit set by '" + + "fluss.union_read.max_tail_rows'. Read the table as pure fluss with" + + " 'fluss.union_read.mode=disabled', wait for tiering to move the tail into the" + + " lake, or raise the limit"); + } + InternalRow row = record.getRow(); + TailKey key = new TailKey(keyEncoder.encodeKey(keyRow.replaceRow(row))); + switch (record.getChangeType()) { + case INSERT: + case UPDATE_AFTER: + state.put(key, row); + break; + case DELETE: + case UPDATE_BEFORE: + state.put(key, null); + break; + default: + // A primary-key table's change log holds no other kind. One appearing means fluss now + // describes a change this reader has never replayed, and guessing at it would return a + // row nobody can check. + throw new IllegalStateException("the change log of a primary-key table produced a '" + + record.getChangeType() + "' record, which this reader does not know how to" + + " replay"); + } + } + + /** How many change log records the tail held. */ + long getRecordsRead() { + return recordsRead; + } + + /** + * How many keys the tail ended deleted, known once the range has been replayed. Those are the keys + * whose lake rows disappear without anything being returned in their place. + */ + long getTombstoneKeys() { + return tombstoneKeys; + } + + @Override + public void close() throws IOException { + records.close(); + } + + /** An encoded primary key, by value: {@code byte[]} is compared by identity on its own. */ + private static final class TailKey { + + private final byte[] encoded; + private final int hash; + + TailKey(byte[] encoded) { + this.encoded = encoded; + this.hash = Arrays.hashCode(encoded); + } + + @Override + public boolean equals(Object other) { + return other instanceof TailKey && Arrays.equals(encoded, ((TailKey) other).encoded); + } + + @Override + public int hashCode() { + return hash; + } + } +} diff --git a/fe/be-java-extensions/fluss-scanner/src/main/resources/package.xml b/fe/be-java-extensions/fluss-scanner/src/main/resources/package.xml new file mode 100644 index 00000000000000..6566bb46f110d4 --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/main/resources/package.xml @@ -0,0 +1,47 @@ + + + + jar-with-dependencies + + jar + + false + + + / + true + true + runtime + + + **/Log4j2Plugins.dat + + + + + + + + metaInf-services + + + diff --git a/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java new file mode 100644 index 00000000000000..f90f2a88cacf02 --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerLogTest.java @@ -0,0 +1,582 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fluss; + +import org.apache.doris.common.jni.utils.OffHeap; +import org.apache.doris.common.jni.vec.VectorTable; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.writer.AppendWriter; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The scanner against a real fluss cluster started in this JVM. + * + *

Everything here is end to end on purpose. The whole job of this class is to turn what a fluss + * cluster actually stores into what Doris's vector table expects, and both ends of that are other + * people's formats: a test that fed it hand-built rows would only check the conversion against our own + * idea of fluss's encoding. So the fixtures are written through the fluss client — which puts them in + * the real arrow log format — and read back through the scanner into a real off-heap vector table. + * + *

Named {@code ...Test} rather than {@code ...ITCase}: surefire's default includes do not match + * {@code *ITCase}, so that name would leave the class silently unexecuted with a green build. + */ +public class FlussJniScannerLogTest { + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER = FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .build(); + + /** One bucket, so a fixture's rows land in a known place and the offsets are the row numbers. */ + private static final int BUCKETS = 1; + + private static int databaseCounter; + + private static Connection connection; + private static Admin admin; + private static String bootstrapServers; + + private String db; + + @BeforeAll + public static void connectToCluster() { + OffHeap.setTesting(); + bootstrapServers = FLUSS_CLUSTER.getBootstrapServers(); + connection = ConnectionFactory.createConnection(FLUSS_CLUSTER.getClientConfig()); + admin = connection.getAdmin(); + } + + @AfterAll + public static void disconnect() throws Exception { + if (connection != null) { + connection.close(); + } + } + + @BeforeEach + public void createDatabase() throws Exception { + // The cluster extension drops every non-built-in database after each test. + db = "doris_fluss_scanner_test_" + (++databaseCounter); + admin.createDatabase(db, DatabaseDescriptor.EMPTY, true).get(); + } + + // ---------------------------------------------------------------- values + + /** + * Every type the connector maps, written by fluss and read back through the scanner, plus an + * all-null row. The null row is not padding: a fluss column read through the wrong accessor often + * still produces a plausible value for real data and only misbehaves on null, and a value that is + * null on the way in but not on the way out is the failure this catches. + */ + @Test + public void everyMappedTypeSurvivesTheRoundTrip() throws Exception { + TablePath tablePath = TablePath.of(db, "all_types"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("c_boolean", DataTypes.BOOLEAN()) + .column("c_tinyint", DataTypes.TINYINT()) + .column("c_smallint", DataTypes.SMALLINT()) + .column("c_int", DataTypes.INT()) + .column("c_bigint", DataTypes.BIGINT()) + .column("c_float", DataTypes.FLOAT()) + .column("c_double", DataTypes.DOUBLE()) + .column("c_char", DataTypes.CHAR(5)) + .column("c_string", DataTypes.STRING()) + .column("c_decimal", DataTypes.DECIMAL(20, 4)) + .column("c_date", DataTypes.DATE()) + .column("c_timestamp", DataTypes.TIMESTAMP(6)) + .column("c_timestamp_ltz", DataTypes.TIMESTAMP_LTZ(6)) + .column("c_binary", DataTypes.BINARY(4)) + .column("c_bytes", DataTypes.BYTES()) + .build()) + .distributedBy(BUCKETS) + .build(), true).get(); + + LocalDateTime timestamp = LocalDateTime.of(2026, 8, 2, 12, 34, 56, 123456000); + Instant instant = Instant.parse("2026-08-02T04:34:56.123456Z"); + appendRows(tablePath, + GenericRow.of(true, (byte) 1, (short) 2, 3, 4L, 5.5f, 6.5d, + BinaryString.fromString("abcde"), BinaryString.fromString("hello"), + Decimal.fromBigDecimal(new BigDecimal("12345.6789"), 20, 4), + (int) LocalDate.of(2026, 8, 2).toEpochDay(), + TimestampNtz.fromLocalDateTime(timestamp), + TimestampLtz.fromInstant(instant), + new byte[] {1, 2, 3, 4}, new byte[] {9, 8}), + GenericRow.of(null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null)); + + Object[][] rows = scanAll(tablePath, columns( + "c_boolean", "boolean", + "c_tinyint", "tinyint", + "c_smallint", "smallint", + "c_int", "int", + "c_bigint", "bigint", + "c_float", "float", + "c_double", "double", + "c_char", "char(5)", + "c_string", "string", + "c_decimal", "decimal(20,4)", + "c_date", "datev2", + "c_timestamp", "datetimev2(6)", + "c_timestamp_ltz", "timestamptz(6)", + "c_binary", "varbinary", + "c_bytes", "varbinary"), 0, 2, 1024); + + Assertions.assertEquals(2, rows.length); + Object[] row = rows[0]; + Assertions.assertEquals(true, row[0]); + Assertions.assertEquals((byte) 1, row[1]); + Assertions.assertEquals((short) 2, row[2]); + Assertions.assertEquals(3, row[3]); + Assertions.assertEquals(4L, row[4]); + Assertions.assertEquals(5.5f, row[5]); + Assertions.assertEquals(6.5d, row[6]); + Assertions.assertEquals("abcde", row[7]); + Assertions.assertEquals("hello", row[8]); + Assertions.assertEquals(new BigDecimal("12345.6789"), row[9]); + Assertions.assertEquals(LocalDate.of(2026, 8, 2), row[10]); + Assertions.assertEquals(timestamp, row[11]); + // TIMESTAMPTZ is carried as the UTC wall clock of the instant, not the session zone's. + Assertions.assertEquals(LocalDateTime.ofInstant(instant, java.time.ZoneOffset.UTC), row[12]); + // Content of the two binary columns is asserted separately, through the STRING mapping: the + // VARBINARY read-BACK in java-common (VectorColumn.getBytesVarbinary) decodes a StringView + // layout that nothing writes -- appendVarbinary and BE's _fill_varbinary_column both use + // [len:long][addr:long] -- so it cannot be used to check a value. What it does show here is + // that the column arrived non-null, and the all-null row below that it can be null. + Assertions.assertNotNull(row[13]); + Assertions.assertNotNull(row[14]); + + for (int i = 0; i < row.length; i++) { + Assertions.assertNull(rows[1][i], "column " + i + " of the all-null row came back non-null"); + } + } + + /** + * Fluss BYTES maps to Doris STRING unless the catalog opts into VARBINARY, and BINARY is + * fixed-width where BYTES is not — so which fluss accessor a column needs is a property of the + * fluss type, not of the Doris one it was mapped to. Both are read here through the STRING + * mapping, which is also the only read-back path that can check a byte value at all (see the + * java-common note in the round-trip test above). + */ + @Test + public void binaryAndBytesReadAsStringsGiveTheirBytes() throws Exception { + TablePath tablePath = TablePath.of(db, "bytes_as_string"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("c_bytes", DataTypes.BYTES()) + .column("c_binary", DataTypes.BINARY(2)) + .column("c_char", DataTypes.CHAR(2)) + .build()) + .distributedBy(BUCKETS) + .build(), true).get(); + appendRows(tablePath, GenericRow.of(new byte[] {(byte) 'h', (byte) 'i'}, + new byte[] {(byte) 'o', (byte) 'k'}, BinaryString.fromString("ab"))); + + Object[][] rows = scanAll(tablePath, + columns("c_bytes", "string", "c_binary", "string", "c_char", "string"), 0, 1, 1024); + + Assertions.assertArrayEquals(new Object[] {"hi", "ok", "ab"}, rows[0]); + } + + @Test + public void nestedTypesComeBackWithTheirStructure() throws Exception { + TablePath tablePath = TablePath.of(db, "nested_types"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("c_array", DataTypes.ARRAY(DataTypes.INT())) + .column("c_map", DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())) + .column("c_struct", DataTypes.ROW( + DataTypes.FIELD("a", DataTypes.INT()), + DataTypes.FIELD("b", DataTypes.STRING()))) + .build()) + .distributedBy(BUCKETS) + .build(), true).get(); + + Map map = new LinkedHashMap<>(); + map.put(BinaryString.fromString("k1"), 1); + map.put(BinaryString.fromString("k2"), 2); + appendRows(tablePath, + GenericRow.of(new GenericArray(new int[] {10, 20, 30}), new GenericMap(map), + GenericRow.of(7, BinaryString.fromString("inner"))), + // A shorter array right after a longer one: the value wrappers are cached by position + // and reused across rows, so a stale wrapper would surface as a phantom fourth element. + GenericRow.of(new GenericArray(new int[] {40}), + new GenericMap(Collections.singletonMap(BinaryString.fromString("k3"), 3)), + GenericRow.of(8, BinaryString.fromString("other")))); + + Object[][] rows = scanAll(tablePath, columns( + "c_array", "array", + "c_map", "map", + "c_struct", "struct"), 0, 2, 1024); + + Assertions.assertEquals(2, rows.length); + Assertions.assertEquals("[10, 20, 30]", String.valueOf(rows[0][0])); + Assertions.assertEquals("{k1=1, k2=2}", String.valueOf(rows[0][1])); + Assertions.assertEquals("{a=7, b=inner}", String.valueOf(rows[0][2])); + Assertions.assertEquals("[40]", String.valueOf(rows[1][0])); + Assertions.assertEquals("{k3=3}", String.valueOf(rows[1][1])); + Assertions.assertEquals("{a=8, b=other}", String.valueOf(rows[1][2])); + } + + // ---------------------------------------------------------------- projection and bounds + + /** + * The projection is by name, in the order Doris asked for — which is not the table's order. Passing + * the table's own order through instead would silently transpose columns of the same type. + */ + @Test + public void projectionFollowsTheRequestedNamesAndOrder() throws Exception { + TablePath tablePath = TablePath.of(db, "projection"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("a", DataTypes.INT()) + .column("b", DataTypes.INT()) + .column("c", DataTypes.INT()) + .build()) + .distributedBy(BUCKETS) + .build(), true).get(); + appendRows(tablePath, GenericRow.of(1, 2, 3)); + + Object[][] rows = scanAll(tablePath, columns("c", "int", "a", "int"), 0, 1, 1024); + + Assertions.assertEquals(1, rows.length); + Assertions.assertArrayEquals(new Object[] {3, 1}, rows[0]); + } + + /** + * A query can ask for no column of this scanner at all: {@code select dt, count(*) ... group by dt} + * over a partitioned table needs only the partition column, and BE materializes that one from the + * range. Fluss refuses an empty projection, so the scanner has to stand something in for it and + * still report how many rows the range held — answering 0 rows would turn a populated partition + * into an empty one. + */ + @Test + public void projectingNoColumnStillCountsTheRows() throws Exception { + TablePath tablePath = TablePath.of(db, "no_projection"); + createIntTable(tablePath); + appendInts(tablePath, 0, 7); + + Object[][] rows = scanAll(tablePath, columns(), 0, 7, 1024); + + Assertions.assertEquals(7, rows.length); + Assertions.assertEquals(0, rows[0].length, "a column was returned for an empty projection"); + } + + /** The range is half-open: rows at or past the stopping offset belong to nobody's scan yet. */ + @Test + public void theStoppingOffsetBoundsTheRead() throws Exception { + TablePath tablePath = TablePath.of(db, "bounded"); + createIntTable(tablePath); + appendInts(tablePath, 0, 10); + + Object[][] rows = scanAll(tablePath, columns("id", "int"), 0, 4, 1024); + + Assertions.assertEquals(4, rows.length); + Assertions.assertArrayEquals(new Object[] {0}, rows[0]); + Assertions.assertArrayEquals(new Object[] {3}, rows[3]); + } + + @Test + public void theStartingOffsetSkipsWhatCameBefore() throws Exception { + TablePath tablePath = TablePath.of(db, "start_offset"); + createIntTable(tablePath); + appendInts(tablePath, 0, 10); + + Object[][] rows = scanAll(tablePath, columns("id", "int"), 6, 10, 1024); + + Assertions.assertEquals(4, rows.length); + Assertions.assertArrayEquals(new Object[] {6}, rows[0]); + Assertions.assertArrayEquals(new Object[] {9}, rows[3]); + } + + /** Rows written after planning are past the stopping offset and must not leak into the scan. */ + @Test + public void rowsWrittenAfterPlanningAreNotRead() throws Exception { + TablePath tablePath = TablePath.of(db, "late_rows"); + createIntTable(tablePath); + appendInts(tablePath, 0, 5); + // Planning would have stopped at 5; these arrive afterwards, inside the same fetched batch range. + appendInts(tablePath, 100, 105); + + Object[][] rows = scanAll(tablePath, columns("id", "int"), 0, 5, 1024); + + Assertions.assertEquals(5, rows.length); + Assertions.assertArrayEquals(new Object[] {4}, rows[4]); + } + + /** Doris pulls fixed-size batches; the range's rows must arrive whole across several of them. */ + @Test + public void rangeLargerThanOneBatchIsReadAcrossBatches() throws Exception { + TablePath tablePath = TablePath.of(db, "batched"); + createIntTable(tablePath); + appendInts(tablePath, 0, 25); + + Object[][] rows = scanAll(tablePath, columns("id", "int"), 0, 25, 10); + + Assertions.assertEquals(25, rows.length); + for (int i = 0; i < 25; i++) { + Assertions.assertArrayEquals(new Object[] {i}, rows[i], "row " + i); + } + } + + /** + * A partitioned table is addressed by partition id plus bucket. Subscribing without the partition + * would read the wrong bucket entirely. + */ + @Test + public void partitionedTableIsReadThroughItsPartitionId() throws Exception { + TablePath tablePath = TablePath.of(db, "partitioned"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .build()) + .partitionedBy("dt") + .distributedBy(BUCKETS) + .build(), true).get(); + admin.createPartition(tablePath, + new PartitionSpec(Collections.singletonMap("dt", "2026_08_02")), true).get(); + admin.createPartition(tablePath, + new PartitionSpec(Collections.singletonMap("dt", "2026_08_03")), true).get(); + try (Table table = connection.getTable(tablePath)) { + AppendWriter writer = table.newAppend().createWriter(); + writer.append(GenericRow.of(1, BinaryString.fromString("2026_08_02"))); + writer.append(GenericRow.of(2, BinaryString.fromString("2026_08_03"))); + writer.flush(); + } + long partitionId = admin.listPartitionInfos(tablePath).get().stream() + .filter(p -> p.getPartitionName().equals("2026_08_03")) + .findFirst().orElseThrow(AssertionError::new) + .getPartitionId(); + + Map params = params(tablePath, columns("id", "int"), 0, 1); + params.put("fluss.partition_id", String.valueOf(partitionId)); + Object[][] rows = runScanner(params, 1024); + + Assertions.assertEquals(1, rows.length); + Assertions.assertArrayEquals(new Object[] {2}, rows[0]); + } + + // ---------------------------------------------------------------- lifecycle and fail-loud + + /** + * A fluss connection owns netty and metadata-updater threads that outlive the query if the scanner + * leaks it — and a BE runs scanners for the life of the process. + */ + @Test + public void closingTheScannerReleasesItsClientThreads() throws Exception { + TablePath tablePath = TablePath.of(db, "threads"); + createIntTable(tablePath); + appendInts(tablePath, 0, 3); + int before = countClientThreads(); + + for (int i = 0; i < 3; i++) { + scanAll(tablePath, columns("id", "int"), 0, 3, 1024); + } + + long deadline = System.currentTimeMillis() + 30_000; + while (countClientThreads() > before && System.currentTimeMillis() < deadline) { + Thread.sleep(100); + } + Assertions.assertTrue(countClientThreads() <= before, + "fluss client threads leaked: " + before + " before, " + countClientThreads() + " after"); + } + + /** An unreadable range must name what is wrong, not hand fluss a null and fail somewhere inside. */ + @Test + public void missingParameterIsNamed() { + Map params = params(TablePath.of(db, "x"), columns("id", "int"), 0, 1); + params.remove("fluss.bucket_id"); + + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> new FlussJniScanner(1024, params).open()); + Assertions.assertTrue(e.getMessage().contains("fluss.bucket_id"), e.getMessage()); + } + + /** + * A union read of a primary-key table puts a fourth range type on the wire — the suppression + * descriptor that rides along with a lake split — and BE's C++ side is the only reader of it. One + * arriving here means a lake split was handed to this scanner, which would return no rows at all + * for a split that holds many. + */ + @Test + public void rangeTypeThisScannerDoesNotReadIsRefused() { + Map params = params(TablePath.of(db, "x"), columns("id", "int"), 0, 1); + params.put("fluss.range_type", "LAKE_SUPPRESS"); + + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> new FlussJniScanner(1024, params)); + Assertions.assertTrue(e.getMessage().contains("LAKE_SUPPRESS"), e.getMessage()); + } + + /** A column dropped between planning and reading must say so, not read a neighbouring column. */ + @Test + public void columnThatIsNoLongerThereIsNamed() throws Exception { + TablePath tablePath = TablePath.of(db, "missing_column"); + createIntTable(tablePath); + appendInts(tablePath, 0, 1); + + Exception e = Assertions.assertThrows(Exception.class, + () -> runScanner(params(tablePath, columns("nope", "int"), 0, 1), 1024)); + Assertions.assertTrue(rootMessage(e).contains("nope"), rootMessage(e)); + } + + // ---------------------------------------------------------------- helpers + + private static String rootMessage(Throwable e) { + StringBuilder message = new StringBuilder(); + for (Throwable t = e; t != null; t = t.getCause()) { + message.append(t.getMessage()).append(" | "); + } + return message.toString(); + } + + private static int countClientThreads() { + int count = 0; + for (Thread thread : Thread.getAllStackTraces().keySet()) { + // Client-side only: the embedded cluster's own threads are fluss-netty-server-*. + if (thread.getName().startsWith("fluss-netty-client")) { + count++; + } + } + return count; + } + + private void createIntTable(TablePath tablePath) throws Exception { + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder().column("id", DataTypes.INT()).build()) + .distributedBy(BUCKETS) + .build(), true).get(); + } + + private void appendInts(TablePath tablePath, int fromInclusive, int toExclusive) throws Exception { + GenericRow[] rows = new GenericRow[toExclusive - fromInclusive]; + for (int i = fromInclusive; i < toExclusive; i++) { + rows[i - fromInclusive] = GenericRow.of(i); + } + appendRows(tablePath, rows); + } + + private static void appendRows(TablePath tablePath, GenericRow... rows) throws Exception { + try (Table table = connection.getTable(tablePath)) { + AppendWriter writer = table.newAppend().createWriter(); + for (GenericRow row : rows) { + writer.append(row); + } + writer.flush(); + } + } + + /** {@code name, dorisType, name, dorisType, ...} as the two params BE sends. */ + private static Map columns(String... nameThenType) { + List names = new ArrayList<>(); + List types = new ArrayList<>(); + for (int i = 0; i < nameThenType.length; i += 2) { + names.add(nameThenType[i]); + types.add(nameThenType[i + 1]); + } + Map columns = new LinkedHashMap<>(); + columns.put("required_fields", String.join(",", names)); + columns.put("columns_types", String.join("#", types)); + return columns; + } + + /** The merged map BE hands the scanner: the scan-level properties plus this range's. */ + private static Map params(TablePath tablePath, Map columns, + long startOffset, long stopOffset) { + Map params = new HashMap<>(columns); + params.put("fluss.client.bootstrap.servers", bootstrapServers); + params.put("fluss.db_name", tablePath.getDatabaseName()); + params.put("fluss.table_name", tablePath.getTableName()); + params.put("fluss.range_type", "LOG"); + params.put("fluss.bucket_id", "0"); + params.put("fluss.log_start_offset", String.valueOf(startOffset)); + params.put("fluss.log_stop_offset", String.valueOf(stopOffset)); + params.put("time_zone", "UTC"); + return params; + } + + private static Object[][] scanAll(TablePath tablePath, Map columns, + long startOffset, long stopOffset, int batchSize) throws Exception { + return runScanner(params(tablePath, columns, startOffset, stopOffset), batchSize); + } + + /** + * Drives the scanner the way BE does — batch by batch until it reports none left — and returns the + * rows it produced. {@code getMaterializedData} hands back COLUMN-major arrays, so this transposes; + * reading it as rows would silently compare a column against a row and mostly pass on square data. + */ + private static Object[][] runScanner(Map params, int batchSize) throws Exception { + List allRows = new ArrayList<>(); + FlussJniScanner scanner = new FlussJniScanner(batchSize, params); + try { + scanner.open(); + while (scanner.getNextBatchMeta() != 0) { + VectorTable table = scanner.getTable(); + Object[][] byColumn = table.getMaterializedData(); + int rows = table.getNumRows(); + for (int row = 0; row < rows; row++) { + Object[] values = new Object[byColumn.length]; + for (int column = 0; column < byColumn.length; column++) { + values[column] = byColumn[column][row]; + } + allRows.add(values); + } + scanner.resetTable(); + } + } finally { + scanner.releaseTable(); + scanner.close(); + } + return allRows.toArray(new Object[0][]); + } +} diff --git a/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerPkTailTest.java b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerPkTailTest.java new file mode 100644 index 00000000000000..7b5c5e07256c00 --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerPkTailTest.java @@ -0,0 +1,559 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fluss; + +import org.apache.doris.common.jni.utils.OffHeap; +import org.apache.doris.common.jni.vec.VectorTable; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.writer.AppendWriter; +import org.apache.fluss.client.table.writer.UpsertWriter; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Reading the log tail of a primary-key table — the fluss half of a union read — against a real + * cluster in this JVM. + * + *

What this range means is narrower than the whole-bucket read next door, and the difference is the + * point of every test here: the lake holds the bucket as it stood at the offset it was tiered at, so + * this range owes exactly what has happened since, replayed into the state it ended in. A key nobody + * touched in the tail must NOT come back — the lake already has it, and returning it again is a + * duplicate row. A key updated in the tail must come back once, with the final value. A key deleted in + * the tail must come back not at all, and that is not the same as never having been written: the lake + * row it supersedes is hidden by a key set BE's C++ side builds from these same offsets, so the row + * disappearing is the whole intent. + * + *

Named {@code ...Test}, not {@code ...ITCase}: surefire's default includes do not match + * {@code *ITCase}, so that name would leave the class silently unexecuted with a green build. + */ +public class FlussJniScannerPkTailTest { + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER = FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .build(); + + /** One bucket, so every fixture row lands in the bucket the test reads the tail of. */ + private static final int BUCKETS = 1; + + /** A ceiling no fixture here comes near; the one test about the ceiling sets its own. */ + private static final long NO_CEILING_IN_PRACTICE = 1_000_000L; + + private static int databaseCounter; + + private static Connection connection; + private static Admin admin; + private static String bootstrapServers; + + private String db; + + @BeforeAll + public static void connectToCluster() { + OffHeap.setTesting(); + bootstrapServers = FLUSS_CLUSTER.getBootstrapServers(); + connection = ConnectionFactory.createConnection(FLUSS_CLUSTER.getClientConfig()); + admin = connection.getAdmin(); + } + + @AfterAll + public static void disconnect() throws Exception { + if (connection != null) { + connection.close(); + } + } + + @BeforeEach + public void createDatabase() throws Exception { + // The cluster extension drops every non-built-in database after each test. + db = "doris_fluss_tail_test_" + (++databaseCounter); + admin.createDatabase(db, DatabaseDescriptor.EMPTY, true).get(); + } + + // ---------------------------------------------------------------- what the tail owes + + /** + * The three things at once, on one tail: a key the tail updated comes back once with the later + * value, a key the tail deleted does not come back, a key the tail inserted comes back — and the + * key nobody touched does not come back at all, because it is the lake's to return. + * + *

Returning the change log as it stands would give five rows for these three keys. + */ + @Test + public void tailComesBackAsTheStateItEndedIn() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_mixed"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two"), row(3, "three")); + long lakeEnd = latestOffset(tablePath); + upsert(tablePath, row(1, "one-updated")); + delete(tablePath, row(2, "two")); + upsert(tablePath, row(4, "four")); + + Object[][] rows = read(tailParams(tablePath, columns("id", "int", "name", "string"), + lakeEnd, latestOffset(tablePath), NO_CEILING_IN_PRACTICE), 1024); + + Assertions.assertArrayEquals( + new Object[][] {{1, "one-updated"}, {4, "four"}}, sortById(rows)); + } + + /** + * A key written only before the tail belongs to the lake half. Reading from the beginning of the + * log instead of from where the lake left off returns it here too, and the query then has it twice. + */ + @Test + public void rowsBeforeTheTailAreLeftToTheLake() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_start"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two")); + long lakeEnd = latestOffset(tablePath); + upsert(tablePath, row(3, "three")); + + Object[][] rows = read(tailParams(tablePath, columns("id", "int", "name", "string"), + lakeEnd, latestOffset(tablePath), NO_CEILING_IN_PRACTICE), 1024); + + Assertions.assertArrayEquals(new Object[][] {{3, "three"}}, sortById(rows)); + } + + /** + * The tail ends where planning said it ends. A write that lands after that belongs to the next + * query — and here it would also disagree with the key set the lake half was filtered by, which was + * built from the same two offsets. + */ + @Test + public void writesAfterTheStoppingOffsetAreNotRead() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_late_write"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one")); + long lakeEnd = latestOffset(tablePath); + upsert(tablePath, row(2, "two")); + long stop = latestOffset(tablePath); + upsert(tablePath, row(3, "three")); + + Object[][] rows = read(tailParams(tablePath, columns("id", "int", "name", "string"), + lakeEnd, stop, NO_CEILING_IN_PRACTICE), 1024); + + Assertions.assertArrayEquals(new Object[][] {{2, "two"}}, sortById(rows)); + } + + /** + * A tail that only deletes returns nothing, and says so: the deleted key is counted, because the + * lake row it hides is dropped on the strength of this range having seen the key. A tail that + * quietly returned no rows and counted nothing would look exactly like a tail that read nothing at + * all. + * + *

The record count is asserted too — an update of an existing key is logged as a pair, a delete + * as one record. A fluss upgrade that changes the shape of the change log should land here. + */ + @Test + public void keysDeletedInTheTailAreCountedRatherThanReturned() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_delete"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two")); + long lakeEnd = latestOffset(tablePath); + upsert(tablePath, row(1, "one-updated")); + delete(tablePath, row(2, "two")); + + Map params = tailParams(tablePath, columns("id", "int", "name", "string"), + lakeEnd, latestOffset(tablePath), NO_CEILING_IN_PRACTICE); + Map statistics = new HashMap<>(); + Object[][] rows = read(params, 1024, statistics); + + Assertions.assertArrayEquals(new Object[][] {{1, "one-updated"}}, sortById(rows)); + Assertions.assertEquals("1", statistics.get("counter:FlussJniTailTombstoneKeys")); + Assertions.assertEquals("3", statistics.get("counter:FlussJniTailRecordsRead"), + "an update is two change log records and a delete is one"); + } + + // ---------------------------------------------------------------- projection and shape + + /** + * Replaying needs the primary key whether or not the query selects it, so the reader asks fluss for + * it and does not return it. Getting that wrong shows up as an extra column — or, if the key were + * simply not fetched, as two rows for one key. + */ + @Test + public void projectionExcludingThePrimaryKeyKeepsTheRequestedShape() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_projection"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("a", DataTypes.INT()) + .column("b", DataTypes.STRING()) + .primaryKey("id") + .build()) + .distributedBy(BUCKETS, "id") + .build(), true).get(); + upsert(tablePath, GenericRow.of(1, 10, BinaryString.fromString("x"))); + long lakeEnd = latestOffset(tablePath); + upsert(tablePath, GenericRow.of(1, 11, BinaryString.fromString("y"))); + upsert(tablePath, GenericRow.of(1, 12, BinaryString.fromString("z"))); + + Object[][] rows = read(tailParams(tablePath, columns("b", "string", "a", "int"), + lakeEnd, latestOffset(tablePath), NO_CEILING_IN_PRACTICE), 1024); + + Assertions.assertEquals(1, rows.length); + Assertions.assertArrayEquals(new Object[] {"z", 12}, rows[0]); + } + + /** + * A query can need no column of this scanner at all — {@code select dt, count(*) ... group by dt} + * over a partitioned table — and the count still has to be of the replayed rows. Counting change + * log records instead reports four where the tail contributes two. + */ + @Test + public void projectingNoColumnStillCountsTheReplayedRows() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_count_only"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two")); + long lakeEnd = latestOffset(tablePath); + upsert(tablePath, row(1, "one-updated")); + delete(tablePath, row(2, "two")); + upsert(tablePath, row(3, "three")); + + Map params = tailParams(tablePath, columns(), lakeEnd, + latestOffset(tablePath), NO_CEILING_IN_PRACTICE); + int rows = 0; + FlussJniScanner scanner = new FlussJniScanner(1024, params); + try { + scanner.open(); + while (scanner.getNextBatchMeta() != 0) { + rows += scanner.getTable().getNumRows(); + scanner.resetTable(); + } + } finally { + scanner.releaseTable(); + scanner.close(); + } + + Assertions.assertEquals(2, rows); + } + + /** More surviving rows than fit in one batch: the reader has to resume where it stopped. */ + @Test + public void replayedRowsSpanSeveralBatches() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_batches"); + createPkTable(tablePath); + upsert(tablePath, row(0, "v0")); + long lakeEnd = latestOffset(tablePath); + GenericRow[] tail = new GenericRow[50]; + for (int i = 1; i <= 50; i++) { + tail[i - 1] = row(i, "v" + i); + } + upsert(tablePath, tail); + + Object[][] rows = read(tailParams(tablePath, columns("id", "int", "name", "string"), + lakeEnd, latestOffset(tablePath), NO_CEILING_IN_PRACTICE), 7); + + Object[][] sorted = sortById(rows); + Assertions.assertEquals(50, sorted.length); + for (int i = 1; i <= 50; i++) { + Assertions.assertArrayEquals(new Object[] {i, "v" + i}, sorted[i - 1], "row " + i); + } + } + + /** + * A partitioned table's tail is a tail of one partition's bucket. Subscribing without the partition + * reads a different tablet altogether, and the lake half it is joined to is this partition's. + */ + @Test + public void partitionedTailIsReadThroughItsPartitionBucket() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_part"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("dt", DataTypes.STRING().copy(false)) + .column("name", DataTypes.STRING()) + .primaryKey("id", "dt") + .build()) + .partitionedBy("dt") + .distributedBy(BUCKETS, "id") + .build(), true).get(); + admin.createPartition(tablePath, + new PartitionSpec(Collections.singletonMap("dt", "2026_08_03")), true).get(); + admin.createPartition(tablePath, + new PartitionSpec(Collections.singletonMap("dt", "2026_08_04")), true).get(); + long partitionId = partitionId(tablePath, "2026_08_03"); + upsert(tablePath, + GenericRow.of(1, BinaryString.fromString("2026_08_03"), BinaryString.fromString("a")), + GenericRow.of(2, BinaryString.fromString("2026_08_04"), BinaryString.fromString("b"))); + long lakeEnd = partitionOffset(tablePath, "2026_08_03"); + upsert(tablePath, + GenericRow.of(1, BinaryString.fromString("2026_08_03"), + BinaryString.fromString("a-updated")), + GenericRow.of(3, BinaryString.fromString("2026_08_04"), + BinaryString.fromString("c"))); + + // The partition column is not read here: FE declares it and BE materializes it from the range. + Map params = tailParams(tablePath, columns("id", "int", "name", "string"), + lakeEnd, partitionOffset(tablePath, "2026_08_03"), NO_CEILING_IN_PRACTICE); + params.put("fluss.partition_id", String.valueOf(partitionId)); + + Object[][] rows = read(params, 1024); + + Assertions.assertArrayEquals(new Object[][] {{1, "a-updated"}}, sortById(rows)); + } + + // ---------------------------------------------------------------- what must fail loudly + + /** + * The tail is held in memory to be replayed, in a process shared by every query on this BE. A tail + * that outgrew its ceiling — tiering stopped, say — has to say so rather than read on. + */ + @Test + public void tailBiggerThanItsCeilingIsRefused() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_ceiling"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one")); + long lakeEnd = latestOffset(tablePath); + upsert(tablePath, row(2, "two"), row(3, "three"), row(4, "four")); + + Map params = tailParams(tablePath, columns("id", "int", "name", "string"), + lakeEnd, latestOffset(tablePath), 2L); + + Exception failure = Assertions.assertThrows(Exception.class, () -> read(params, 1024)); + Assertions.assertTrue(messageChain(failure).contains("fluss.union_read.max_tail_rows"), + "the ceiling that was hit should be named, but the failure was: " + + messageChain(failure)); + } + + /** + * An empty tail is not something planning produces — a bucket whose lake has caught up contributes + * no range at all. One arriving means the two halves were bounded by different offsets, which is a + * duplicated or a missing row either way, and reading nothing would hide it. + */ + @Test + public void emptyTailRangeIsRefused() { + Map params = tailParams(TablePath.of(db, "tail_empty"), + columns("id", "int"), 7L, 7L, NO_CEILING_IN_PRACTICE); + + Exception failure = Assertions.assertThrows(IllegalArgumentException.class, + () -> new FlussJniScanner(1024, params)); + Assertions.assertTrue(failure.getMessage().contains("must read something"), failure.getMessage()); + } + + /** + * A log table has no key to replay by, and its records are rows in their own right. A tail range + * pointed at one means planning produced the wrong kind of range; replaying it by "the key" would + * mean inventing one. + */ + @Test + public void tailOfALogTableIsRefused() throws Exception { + TablePath tablePath = TablePath.of(db, "tail_log_table"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("name", DataTypes.STRING()) + .build()) + .distributedBy(BUCKETS) + .build(), true).get(); + try (Table table = connection.getTable(tablePath)) { + AppendWriter writer = table.newAppend().createWriter(); + writer.append(row(1, "one")); + writer.flush(); + } + + Map params = tailParams(tablePath, columns("id", "int", "name", "string"), + 0L, latestOffset(tablePath), NO_CEILING_IN_PRACTICE); + + Exception failure = Assertions.assertThrows(Exception.class, () -> read(params, 1024)); + Assertions.assertTrue(messageChain(failure).contains("no primary key"), + "the failure should name what is missing, but it was: " + messageChain(failure)); + } + + /** A ceiling that cannot bound anything is a configuration mistake, not a tail of size zero. */ + @Test + public void nonPositiveCeilingIsRefused() { + Map params = tailParams(TablePath.of(db, "tail_bad_ceiling"), + columns("id", "int"), 0L, 7L, 0L); + + Exception failure = Assertions.assertThrows(IllegalArgumentException.class, + () -> new FlussJniScanner(1024, params)); + Assertions.assertTrue(failure.getMessage().contains("fluss.union.max_tail_rows"), + failure.getMessage()); + } + + // ---------------------------------------------------------------- helpers + + private void createPkTable(TablePath tablePath) throws Exception { + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("name", DataTypes.STRING()) + .primaryKey("id") + .build()) + .distributedBy(BUCKETS, "id") + .build(), true).get(); + } + + private static GenericRow row(int id, String name) { + return GenericRow.of(id, BinaryString.fromString(name)); + } + + private static void upsert(TablePath tablePath, GenericRow... rows) throws Exception { + try (Table table = connection.getTable(tablePath)) { + UpsertWriter writer = table.newUpsert().createWriter(); + for (GenericRow row : rows) { + writer.upsert(row); + } + writer.flush(); + } + } + + private static void delete(TablePath tablePath, GenericRow row) throws Exception { + try (Table table = connection.getTable(tablePath)) { + UpsertWriter writer = table.newUpsert().createWriter(); + writer.delete(row); + writer.flush(); + } + } + + /** Where bucket 0's log has got to — what planning uses to bound a tail. */ + private static long latestOffset(TablePath tablePath) throws Exception { + return admin.listOffsets(tablePath, Collections.singletonList(0), new OffsetSpec.LatestSpec()) + .all().get().get(0); + } + + private static long partitionOffset(TablePath tablePath, String partitionName) throws Exception { + return admin.listOffsets(tablePath, partitionName, Collections.singletonList(0), + new OffsetSpec.LatestSpec()).all().get().get(0); + } + + private static long partitionId(TablePath tablePath, String partitionName) throws Exception { + return admin.listPartitionInfos(tablePath).get().stream() + .filter(p -> p.getPartitionName().equals(partitionName)) + .findFirst().orElseThrow(AssertionError::new) + .getPartitionId(); + } + + /** {@code name, dorisType, name, dorisType, ...} as the two params BE sends. */ + private static Map columns(String... nameThenType) { + List names = new ArrayList<>(); + List types = new ArrayList<>(); + for (int i = 0; i < nameThenType.length; i += 2) { + names.add(nameThenType[i]); + types.add(nameThenType[i + 1]); + } + Map columns = new LinkedHashMap<>(); + columns.put("required_fields", String.join(",", names)); + columns.put("columns_types", String.join("#", types)); + return columns; + } + + /** + * The merged map BE hands the scanner for a tail range: the range's own keys, plus the scan-level + * ceiling that only a union read of a primary-key table carries. No kv snapshot — the state before + * the tail is the lake's, not a snapshot's. + */ + private static Map tailParams(TablePath tablePath, Map columns, + long logStartOffset, long logStopOffset, long maxTailRows) { + Map params = new HashMap<>(columns); + params.put("fluss.client.bootstrap.servers", bootstrapServers); + params.put("fluss.db_name", tablePath.getDatabaseName()); + params.put("fluss.table_name", tablePath.getTableName()); + params.put("fluss.range_type", "PK_TAIL"); + params.put("fluss.bucket_id", "0"); + params.put("fluss.log_start_offset", String.valueOf(logStartOffset)); + params.put("fluss.log_stop_offset", String.valueOf(logStopOffset)); + params.put("fluss.union.max_tail_rows", String.valueOf(maxTailRows)); + params.put("time_zone", "UTC"); + return params; + } + + private static Object[][] read(Map params, int batchSize) throws Exception { + return read(params, batchSize, new HashMap<>()); + } + + /** + * Drives the scanner the way BE does — batch by batch until it reports none left — and returns the + * rows it produced, filling {@code statistics} with what the scanner counted. + * {@code getMaterializedData} hands back COLUMN-major arrays, so this transposes; reading it as + * rows would silently compare a column against a row. + */ + private static Object[][] read(Map params, int batchSize, + Map statistics) throws Exception { + List allRows = new ArrayList<>(); + FlussJniScanner scanner = new FlussJniScanner(batchSize, params); + try { + scanner.open(); + while (scanner.getNextBatchMeta() != 0) { + VectorTable table = scanner.getTable(); + Object[][] byColumn = table.getMaterializedData(); + int rows = table.getNumRows(); + for (int row = 0; row < rows; row++) { + Object[] values = new Object[byColumn.length]; + for (int column = 0; column < byColumn.length; column++) { + values[column] = byColumn[column][row]; + } + allRows.add(values); + } + scanner.resetTable(); + } + statistics.putAll(scanner.getStatistics()); + } finally { + scanner.releaseTable(); + scanner.close(); + } + return allRows.toArray(new Object[0][]); + } + + /** Every message down the cause chain, because open() wraps what the reader threw. */ + private static String messageChain(Throwable failure) { + StringBuilder messages = new StringBuilder(); + for (Throwable cause = failure; cause != null; cause = cause.getCause()) { + messages.append(cause.getMessage()).append(" | "); + } + return messages.toString(); + } + + /** + * Sorted by the first column, which every fixture here makes the primary key. Replay returns rows + * in the order their keys were first seen in the tail, which is fluss's write order, not an order + * this reader promises. + */ + private static Object[][] sortById(Object[][] rows) { + Object[][] sorted = Arrays.copyOf(rows, rows.length); + Arrays.sort(sorted, Comparator.comparingInt(row -> (Integer) row[0])); + return sorted; + } +} diff --git a/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerPkTest.java b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerPkTest.java new file mode 100644 index 00000000000000..de91df8ed3b7cf --- /dev/null +++ b/fe/be-java-extensions/fluss-scanner/src/test/java/org/apache/doris/fluss/FlussJniScannerPkTest.java @@ -0,0 +1,594 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fluss; + +import org.apache.doris.common.jni.utils.OffHeap; +import org.apache.doris.common.jni.vec.VectorTable; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.client.metadata.KvSnapshots; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.writer.UpsertWriter; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.Decimal; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericMap; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.TimestampNtz; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Reading a fluss primary-key table through the scanner, against a real cluster in this JVM. + * + *

What makes this different from the log test is that the answer is not what was written — it is + * what the writes add up to. A key upserted twice must come back once, with the later value; a deleted + * key must not come back at all; and none of that is visible in the change log the table stores, which + * holds every intermediate state. This is the first line of defence for that merge: a scanner that got + * it wrong would return extra rows that look entirely plausible one at a time. + * + *

Snapshots are triggered, not waited for. The cluster's periodic interval is ten minutes, + * so a bucket has a snapshot only where a test asked for one, which is what makes "these rows came out + * of the snapshot and those out of the log after it" a fact rather than a race. + * + *

Named {@code ...Test}, not {@code ...ITCase}: surefire's default includes do not match + * {@code *ITCase}, so that name would leave the class silently unexecuted with a green build. + */ +public class FlussJniScannerPkTest { + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER = FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .build(); + + /** One bucket, so every fixture row lands where the test triggers its snapshot. */ + private static final int BUCKETS = 1; + + /** {@code fluss.kv_snapshot_id} for a bucket fluss has never snapshotted. */ + private static final long NO_SNAPSHOT = -1L; + + /** Fluss's "start at the beginning of the log" sentinel. */ + private static final long EARLIEST = -2L; + + private static int databaseCounter; + + private static Connection connection; + private static Admin admin; + private static String bootstrapServers; + + private String db; + + @BeforeAll + public static void connectToCluster() { + OffHeap.setTesting(); + bootstrapServers = FLUSS_CLUSTER.getBootstrapServers(); + connection = ConnectionFactory.createConnection(FLUSS_CLUSTER.getClientConfig()); + admin = connection.getAdmin(); + } + + @AfterAll + public static void disconnect() throws Exception { + if (connection != null) { + connection.close(); + } + } + + @BeforeEach + public void createDatabase() throws Exception { + // The cluster extension drops every non-built-in database after each test. + db = "doris_fluss_pk_test_" + (++databaseCounter); + admin.createDatabase(db, DatabaseDescriptor.EMPTY, true).get(); + } + + // ---------------------------------------------------------------- the merged view + + /** + * With no snapshot the whole state has to be rebuilt from the change log, and the change log is + * exactly what must not be returned as-is: these five writes produce an insert, an update (which + * fluss logs as two records) and a delete, and the answer is two rows. + */ + @Test + public void changeLogAloneStillYieldsTheMergedView() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_log_only"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two"), row(3, "three")); + upsert(tablePath, row(2, "two-updated")); + delete(tablePath, row(3, "three")); + + Object[][] rows = readMerged(tablePath, NO_SNAPSHOT, EARLIEST, + columns("id", "int", "name", "string")); + + Assertions.assertArrayEquals( + new Object[][] {{1, "one"}, {2, "two-updated"}}, sortById(rows)); + } + + /** + * The two halves have to be merged, not concatenated: a key the snapshot holds and the log updates + * must come back once with the log's value, and a key the log deletes must not come back at all + * even though the snapshot still has it. + */ + @Test + public void logAfterASnapshotOverridesAndDeletesWhatTheSnapshotHolds() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_snapshot_and_log"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two"), row(3, "three")); + long snapshotId = snapshot(tablePath); + upsert(tablePath, row(2, "two-after-snapshot"), row(4, "four")); + delete(tablePath, row(3, "three")); + + Object[][] rows = readMerged(tablePath, snapshotId, snapshotLogOffset(tablePath), + columns("id", "int", "name", "string")); + + Assertions.assertArrayEquals(new Object[][] { + {1, "one"}, {2, "two-after-snapshot"}, {4, "four"}}, sortById(rows)); + } + + /** Nothing written since the snapshot: the snapshot alone is the whole answer. */ + @Test + public void snapshotWithNoLogTailIsReadOnItsOwn() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_snapshot_only"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two")); + long snapshotId = snapshot(tablePath); + long logOffset = snapshotLogOffset(tablePath); + + // Start and stop meet, which is what planning produces for a bucket whose log has caught up. + Object[][] rows = read(pkParams(tablePath, columns("id", "int", "name", "string"), + snapshotId, logOffset, logOffset), 1024); + + Assertions.assertArrayEquals(new Object[][] {{1, "one"}, {2, "two"}}, sortById(rows)); + } + + /** + * Rows are compared to the state at the stopping offset, not to the state now. A write that lands + * after planning belongs to the next query; letting it through would make two scans of the same + * plan disagree. + */ + @Test + public void writesAfterTheStoppingOffsetAreNotRead() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_late_write"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two")); + long stopOffset = latestOffset(tablePath); + upsert(tablePath, row(2, "two-later"), row(3, "three")); + + Object[][] rows = read(pkParams(tablePath, columns("id", "int", "name", "string"), + NO_SNAPSHOT, EARLIEST, stopOffset), 1024); + + Assertions.assertArrayEquals(new Object[][] {{1, "one"}, {2, "two"}}, sortById(rows)); + } + + // ---------------------------------------------------------------- values + + /** + * Every mapped type, half of it read out of a kv snapshot and half out of the change log after it. + * The two are stored in different row formats, and a value wrapper that only handles one of them + * would still pass a test that read a single row from a single place. + */ + @Test + public void everyMappedTypeSurvivesBothHalvesOfTheMerge() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_all_types"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("c_boolean", DataTypes.BOOLEAN()) + .column("c_tinyint", DataTypes.TINYINT()) + .column("c_smallint", DataTypes.SMALLINT()) + .column("c_bigint", DataTypes.BIGINT()) + .column("c_float", DataTypes.FLOAT()) + .column("c_double", DataTypes.DOUBLE()) + .column("c_char", DataTypes.CHAR(5)) + .column("c_string", DataTypes.STRING()) + .column("c_decimal", DataTypes.DECIMAL(20, 4)) + .column("c_date", DataTypes.DATE()) + .column("c_timestamp", DataTypes.TIMESTAMP(6)) + .column("c_timestamp_ltz", DataTypes.TIMESTAMP_LTZ(6)) + .column("c_array", DataTypes.ARRAY(DataTypes.INT())) + .column("c_map", DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())) + .column("c_struct", DataTypes.ROW( + DataTypes.FIELD("a", DataTypes.INT()), + DataTypes.FIELD("b", DataTypes.STRING()))) + .primaryKey("id") + .build()) + .distributedBy(BUCKETS, "id") + .build(), true).get(); + + LocalDateTime timestamp = LocalDateTime.of(2026, 8, 3, 12, 34, 56, 123456000); + Instant instant = Instant.parse("2026-08-03T04:34:56.123456Z"); + Map map = new LinkedHashMap<>(); + map.put(BinaryString.fromString("k1"), 1); + upsert(tablePath, GenericRow.of(1, true, (byte) 1, (short) 2, 4L, 5.5f, 6.5d, + BinaryString.fromString("abcde"), BinaryString.fromString("hello"), + Decimal.fromBigDecimal(new BigDecimal("12345.6789"), 20, 4), + (int) LocalDate.of(2026, 8, 3).toEpochDay(), + TimestampNtz.fromLocalDateTime(timestamp), + TimestampLtz.fromInstant(instant), + new GenericArray(new int[] {10, 20}), new GenericMap(map), + GenericRow.of(7, BinaryString.fromString("inner")))); + long snapshotId = snapshot(tablePath); + // Row 2 goes only into the log after the snapshot, and is all-null besides its key: a column + // read through the wrong accessor often produces a plausible value for real data and only + // misbehaves on null. + upsert(tablePath, GenericRow.of(2, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null)); + + Object[][] rows = readMerged(tablePath, snapshotId, snapshotLogOffset(tablePath), columns( + "id", "int", + "c_boolean", "boolean", + "c_tinyint", "tinyint", + "c_smallint", "smallint", + "c_bigint", "bigint", + "c_float", "float", + "c_double", "double", + "c_char", "char(5)", + "c_string", "string", + "c_decimal", "decimal(20,4)", + "c_date", "datev2", + "c_timestamp", "datetimev2(6)", + "c_timestamp_ltz", "timestamptz(6)", + "c_array", "array", + "c_map", "map", + "c_struct", "struct")); + + Object[][] sorted = sortById(rows); + Assertions.assertEquals(2, sorted.length); + Object[] fromSnapshot = sorted[0]; + Assertions.assertEquals(1, fromSnapshot[0]); + Assertions.assertEquals(true, fromSnapshot[1]); + Assertions.assertEquals((byte) 1, fromSnapshot[2]); + Assertions.assertEquals((short) 2, fromSnapshot[3]); + Assertions.assertEquals(4L, fromSnapshot[4]); + Assertions.assertEquals(5.5f, fromSnapshot[5]); + Assertions.assertEquals(6.5d, fromSnapshot[6]); + Assertions.assertEquals("abcde", fromSnapshot[7]); + Assertions.assertEquals("hello", fromSnapshot[8]); + Assertions.assertEquals(new BigDecimal("12345.6789"), fromSnapshot[9]); + Assertions.assertEquals(LocalDate.of(2026, 8, 3), fromSnapshot[10]); + Assertions.assertEquals(timestamp, fromSnapshot[11]); + // TIMESTAMPTZ is carried as the UTC wall clock of the instant, not the session zone's. + Assertions.assertEquals(LocalDateTime.ofInstant(instant, java.time.ZoneOffset.UTC), + fromSnapshot[12]); + Assertions.assertEquals("[10, 20]", String.valueOf(fromSnapshot[13])); + Assertions.assertEquals("{k1=1}", String.valueOf(fromSnapshot[14])); + Assertions.assertEquals("{a=7, b=inner}", String.valueOf(fromSnapshot[15])); + + Object[] fromLog = sorted[1]; + Assertions.assertEquals(2, fromLog[0]); + for (int i = 1; i < fromLog.length; i++) { + Assertions.assertNull(fromLog[i], "column " + i + " of the all-null row came back non-null"); + } + } + + // ---------------------------------------------------------------- projection and shape + + /** + * The merge needs the primary key even when the query does not ask for it, so the reader adds it + * to what it fetches and projects it back out. Getting that wrong shows up as an extra column, or + * as columns in the fetch order rather than the requested one. + */ + @Test + public void projectionExcludingThePrimaryKeyKeepsTheRequestedShape() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_projection"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("a", DataTypes.INT()) + .column("b", DataTypes.STRING()) + .primaryKey("id") + .build()) + .distributedBy(BUCKETS, "id") + .build(), true).get(); + upsert(tablePath, GenericRow.of(1, 10, BinaryString.fromString("x"))); + long snapshotId = snapshot(tablePath); + upsert(tablePath, GenericRow.of(1, 11, BinaryString.fromString("y"))); + + Object[][] rows = readMerged(tablePath, snapshotId, snapshotLogOffset(tablePath), + columns("b", "string", "a", "int")); + + Assertions.assertEquals(1, rows.length); + Assertions.assertArrayEquals(new Object[] {"y", 11}, rows[0]); + } + + /** + * A query can need no column of this scanner at all — {@code select dt, count(*) ... group by dt} + * over a partitioned table — and the row count still has to be the merged one. Counting change log + * records instead would report four rows where the table holds two. + */ + @Test + public void projectingNoColumnStillCountsTheMergedRows() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_count_only"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one"), row(2, "two")); + upsert(tablePath, row(1, "one-updated")); + delete(tablePath, row(2, "two")); + upsert(tablePath, row(3, "three")); + + Map params = pkParams(tablePath, columns(), NO_SNAPSHOT, EARLIEST, + latestOffset(tablePath)); + int rows = 0; + FlussJniScanner scanner = new FlussJniScanner(1024, params); + try { + scanner.open(); + while (scanner.getNextBatchMeta() != 0) { + rows += scanner.getTable().getNumRows(); + scanner.resetTable(); + } + } finally { + scanner.releaseTable(); + scanner.close(); + } + + Assertions.assertEquals(2, rows); + } + + /** More merged rows than fit in one batch: the reader has to resume where it stopped. */ + @Test + public void mergedRowsSpanSeveralBatches() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_batches"); + createPkTable(tablePath); + GenericRow[] first = new GenericRow[50]; + for (int i = 0; i < 50; i++) { + first[i] = row(i, "v" + i); + } + upsert(tablePath, first); + long snapshotId = snapshot(tablePath); + GenericRow[] second = new GenericRow[50]; + for (int i = 50; i < 100; i++) { + second[i - 50] = row(i, "v" + i); + } + upsert(tablePath, second); + + Object[][] rows = read(pkParams(tablePath, columns("id", "int", "name", "string"), + snapshotId, snapshotLogOffset(tablePath), latestOffset(tablePath)), 7); + + Object[][] sorted = sortById(rows); + Assertions.assertEquals(100, sorted.length); + for (int i = 0; i < 100; i++) { + Assertions.assertArrayEquals(new Object[] {i, "v" + i}, sorted[i], "row " + i); + } + } + + /** + * A partitioned primary-key table is snapshotted per partition, and the scanner has to subscribe to + * the partition's bucket rather than the table's. Subscribing without the partition reads a + * different tablet altogether. + */ + @Test + public void partitionedTableIsReadThroughItsPartitionBucket() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_part"); + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("dt", DataTypes.STRING().copy(false)) + .column("name", DataTypes.STRING()) + .primaryKey("id", "dt") + .build()) + .partitionedBy("dt") + .distributedBy(BUCKETS, "id") + .build(), true).get(); + admin.createPartition(tablePath, + new PartitionSpec(Collections.singletonMap("dt", "2026_08_03")), true).get(); + admin.createPartition(tablePath, + new PartitionSpec(Collections.singletonMap("dt", "2026_08_04")), true).get(); + long partitionId = partitionId(tablePath, "2026_08_03"); + upsert(tablePath, + GenericRow.of(1, BinaryString.fromString("2026_08_03"), BinaryString.fromString("a")), + GenericRow.of(2, BinaryString.fromString("2026_08_04"), BinaryString.fromString("b"))); + FLUSS_CLUSTER.triggerAndWaitSnapshot(new TableBucket(tableId(tablePath), partitionId, 0)); + upsert(tablePath, + GenericRow.of(1, BinaryString.fromString("2026_08_03"), + BinaryString.fromString("a-updated"))); + + KvSnapshots snapshots = admin.getLatestKvSnapshots(tablePath, "2026_08_03").get(); + long stop = admin.listOffsets(tablePath, "2026_08_03", Collections.singletonList(0), + new OffsetSpec.LatestSpec()).all().get().get(0); + // The partition column is not read here: FE declares it and BE materializes it from the range. + Map params = pkParams(tablePath, columns("id", "int", "name", "string"), + snapshots.getSnapshotId(0).orElse(NO_SNAPSHOT), + snapshots.getLogOffset(0).orElse(EARLIEST), stop); + params.put("fluss.partition_id", String.valueOf(partitionId)); + + Object[][] rows = read(params, 1024); + + // Only this partition's row, and with the update applied. + Assertions.assertArrayEquals(new Object[][] {{1, "a-updated"}}, sortById(rows)); + } + + /** A snapshot id pointing at a snapshot that does not exist must fail, not read an empty table. */ + @Test + public void unknownSnapshotIdFailsRatherThanReadingNothing() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_bad_snapshot"); + createPkTable(tablePath); + upsert(tablePath, row(1, "one")); + + Map params = pkParams(tablePath, columns("id", "int", "name", "string"), + 999L, 0L, latestOffset(tablePath)); + + Assertions.assertThrows(Exception.class, () -> read(params, 1024)); + } + + // ---------------------------------------------------------------- helpers + + private void createPkTable(TablePath tablePath) throws Exception { + admin.createTable(tablePath, TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .column("name", DataTypes.STRING()) + .primaryKey("id") + .build()) + .distributedBy(BUCKETS, "id") + .build(), true).get(); + } + + private static GenericRow row(int id, String name) { + return GenericRow.of(id, BinaryString.fromString(name)); + } + + private static void upsert(TablePath tablePath, GenericRow... rows) throws Exception { + try (Table table = connection.getTable(tablePath)) { + UpsertWriter writer = table.newUpsert().createWriter(); + for (GenericRow row : rows) { + writer.upsert(row); + } + writer.flush(); + } + } + + private static void delete(TablePath tablePath, GenericRow row) throws Exception { + try (Table table = connection.getTable(tablePath)) { + UpsertWriter writer = table.newUpsert().createWriter(); + writer.delete(row); + writer.flush(); + } + } + + /** Takes a kv snapshot of bucket 0 and returns its id. */ + private static long snapshot(TablePath tablePath) throws Exception { + FLUSS_CLUSTER.triggerAndWaitSnapshot(new TableBucket(tableId(tablePath), 0)); + return admin.getLatestKvSnapshots(tablePath).get().getSnapshotId(0) + .orElseThrow(() -> new AssertionError("the trigger produced no snapshot")); + } + + private static long snapshotLogOffset(TablePath tablePath) throws Exception { + return admin.getLatestKvSnapshots(tablePath).get().getLogOffset(0) + .orElseThrow(() -> new AssertionError("the bucket has no snapshot")); + } + + private static long latestOffset(TablePath tablePath) throws Exception { + return admin.listOffsets(tablePath, Collections.singletonList(0), new OffsetSpec.LatestSpec()) + .all().get().get(0); + } + + private static long tableId(TablePath tablePath) throws Exception { + return admin.getTableInfo(tablePath).get().getTableId(); + } + + private static long partitionId(TablePath tablePath, String partitionName) throws Exception { + return admin.listPartitionInfos(tablePath).get().stream() + .filter(p -> p.getPartitionName().equals(partitionName)) + .findFirst().orElseThrow(AssertionError::new) + .getPartitionId(); + } + + /** Reads the whole bucket up to where its log has got to, the way a planned range would. */ + private static Object[][] readMerged(TablePath tablePath, long snapshotId, long logStartOffset, + Map columns) throws Exception { + return read(pkParams(tablePath, columns, snapshotId, logStartOffset, latestOffset(tablePath)), + 1024); + } + + /** {@code name, dorisType, name, dorisType, ...} as the two params BE sends. */ + private static Map columns(String... nameThenType) { + List names = new ArrayList<>(); + List types = new ArrayList<>(); + for (int i = 0; i < nameThenType.length; i += 2) { + names.add(nameThenType[i]); + types.add(nameThenType[i + 1]); + } + Map columns = new LinkedHashMap<>(); + columns.put("required_fields", String.join(",", names)); + columns.put("columns_types", String.join("#", types)); + return columns; + } + + /** The merged map BE hands the scanner for a primary-key range. */ + private static Map pkParams(TablePath tablePath, Map columns, + long snapshotId, long logStartOffset, long logStopOffset) { + Map params = new HashMap<>(columns); + params.put("fluss.client.bootstrap.servers", bootstrapServers); + params.put("fluss.db_name", tablePath.getDatabaseName()); + params.put("fluss.table_name", tablePath.getTableName()); + params.put("fluss.range_type", "PK_FULL"); + params.put("fluss.bucket_id", "0"); + params.put("fluss.kv_snapshot_id", String.valueOf(snapshotId)); + params.put("fluss.log_start_offset", String.valueOf(logStartOffset)); + params.put("fluss.log_stop_offset", String.valueOf(logStopOffset)); + params.put("time_zone", "UTC"); + return params; + } + + /** + * Drives the scanner the way BE does — batch by batch until it reports none left — and returns the + * rows it produced. {@code getMaterializedData} hands back COLUMN-major arrays, so this transposes; + * reading it as rows would silently compare a column against a row. + */ + private static Object[][] read(Map params, int batchSize) throws Exception { + List allRows = new ArrayList<>(); + FlussJniScanner scanner = new FlussJniScanner(batchSize, params); + try { + scanner.open(); + while (scanner.getNextBatchMeta() != 0) { + VectorTable table = scanner.getTable(); + Object[][] byColumn = table.getMaterializedData(); + int rows = table.getNumRows(); + for (int row = 0; row < rows; row++) { + Object[] values = new Object[byColumn.length]; + for (int column = 0; column < byColumn.length; column++) { + values[column] = byColumn[column][row]; + } + allRows.add(values); + } + scanner.resetTable(); + } + } finally { + scanner.releaseTable(); + scanner.close(); + } + return allRows.toArray(new Object[0][]); + } + + /** + * Sorted by the first column, which every fixture here makes the primary key. The merge returns + * rows in primary-key encoding order, which is not the natural order of the key's Java type, so + * asserting the reader's order would be asserting fluss's key encoding. + */ + private static Object[][] sortById(Object[][] rows) { + Object[][] sorted = Arrays.copyOf(rows, rows.length); + Arrays.sort(sorted, Comparator.comparingInt(row -> (Integer) row[0])); + return sorted; + } +} diff --git a/fe/be-java-extensions/pom.xml b/fe/be-java-extensions/pom.xml index 075b59cc8d6c23..a76f34c71493e1 100644 --- a/fe/be-java-extensions/pom.xml +++ b/fe/be-java-extensions/pom.xml @@ -28,6 +28,7 @@ under the License. java-udf jdbc-scanner paimon-scanner + fluss-scanner max-compute-connector diff --git a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProvider.java b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProvider.java index acd44d5dfc9136..a13af026a759b0 100644 --- a/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-api/src/main/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProvider.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalLong; +import java.util.Set; /** * Plans the set of scan ranges (splits) needed to read a connector table. @@ -148,6 +149,42 @@ default TFileCompressType adjustFileCompressType(TFileCompressType inferred) { return inferred; } + /** + * The columns BE must READ for this scan even when the query references none of them, by Doris-side + * column name. The engine keeps their slots in the scan's tuple instead of pruning them away; the + * projection above the scan still removes them from the query's output, so the answer changes what is + * read, never what is returned. + * + *

This exists for a connector whose BE-side reader needs a column to produce CORRECT ROWS rather than + * to answer the query — a merge key, a suppression key, a row identity. Doris does the same thing for its + * own aggregate / merge-on-read unique-key tables ({@code PhysicalPlanTranslator.preserveExtraStorageKeySlots}): + * BE merges by key whether or not the user selected the key. Trino has no counterpart because its + * connectors own the page source and can add such columns privately; here the reader is BE, so the columns + * have to reach it through the plan.

+ * + *

Answer per SCAN, not per table: a connector that only sometimes needs the column (e.g. only when it + * decides to combine two sources) must return it only for those scans, and must reach the SAME decision + * when it later plans the splits — the engine asks this during plan translation, strictly before + * {@link #planScan}. Memoize that decision on the provider instance (the engine keeps one per scan node) + * rather than deciding twice: two independent decisions can disagree, and then BE is asked to read a + * column the tuple does not carry.

+ * + *

Every name returned must be a column of the scanned table, spelled as Doris knows it (the same + * identifier-mapped name {@link #classifyColumn} receives). A name that matches no slot in the scan's + * tuple fails the query loud: it means the connector and the engine disagree about the table, and reading + * on would silently produce whatever the connector's reader does without that column.

+ * + *

The default returns an empty set — every connector whose reader needs nothing beyond the projection + * is untouched, and its scans prune exactly as before.

+ * + * @param session the current session + * @param handle the table handle being scanned + * @return Doris-side names of the columns to read regardless of the projection (default: empty) + */ + default Set getMustReadColumns(ConnectorSession session, ConnectorTableHandle handle) { + return Collections.emptySet(); + } + /** * Plans the scan described by {@code request}, returning the ranges that cover the requested data. * diff --git a/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProviderMustReadColumnsTest.java b/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProviderMustReadColumnsTest.java new file mode 100644 index 00000000000000..f5e6cdb1d8d4dc --- /dev/null +++ b/fe/fe-connector/fe-connector-api/src/test/java/org/apache/doris/connector/api/scan/ConnectorScanPlanProviderMustReadColumnsTest.java @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.api.scan; + +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Set; + +/** + * Guards the additive {@code getMustReadColumns} SPI default on {@link ConnectorScanPlanProvider}. + * + *

WHY: the engine consults this on EVERY plugin-table scan that has a projection above it, and widens the + * scan's tuple by whatever comes back. The default must therefore be empty, or every connector that never + * asked for anything would start reading extra columns — and, worse, would fail the query loud when a name + * it never returned matches no slot. This is the zero-break guard for es/jdbc/paimon/iceberg/hive/maxcompute, + * none of which override it.

+ */ +public class ConnectorScanPlanProviderMustReadColumnsTest { + + /** Bare provider: only the abstract planScan implemented; everything else inherits SPI defaults. */ + private static final class BareProvider implements ConnectorScanPlanProvider { + @Override + public List planScan(ConnectorSession session, ConnectorScanRequest request) { + return Collections.emptyList(); + } + } + + /** A connector whose BE-side reader needs a merge key the query may not have selected. */ + private static final class KeyReadingProvider implements ConnectorScanPlanProvider { + @Override + public List planScan(ConnectorSession session, ConnectorScanRequest request) { + return Collections.emptyList(); + } + + @Override + public Set getMustReadColumns(ConnectorSession session, ConnectorTableHandle handle) { + return Collections.singleton("id"); + } + } + + @Test + public void defaultAsksForNoExtraColumns() { + ConnectorScanPlanProvider provider = new BareProvider(); + + // MUTATION: a default returning anything non-empty would widen every connector's scans and fail + // loud on the first name that matches no slot -> red here first. + Assertions.assertEquals(Collections.emptySet(), provider.getMustReadColumns(null, null), + "a connector that never opted in must ask for no extra columns"); + } + + @Test + public void connectorThatOptsInIsObeyed() { + ConnectorScanPlanProvider provider = new KeyReadingProvider(); + + Assertions.assertEquals(Collections.singleton("id"), provider.getMustReadColumns(null, null), + "the engine must read back exactly what the connector asked for"); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/pom.xml b/fe/fe-connector/fe-connector-fluss/pom.xml new file mode 100644 index 00000000000000..0dc9edadab7812 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/pom.xml @@ -0,0 +1,188 @@ + + + + 4.0.0 + + + org.apache.doris + fe-connector + ${revision} + ../pom.xml + + + fe-connector-fluss + jar + Doris FE Connector - Fluss + + Fluss connector plugin for Doris FE. + Provides metadata operations (list databases, list tables, get schema) and read-side scan + planning for Fluss clusters via the connector SPI. + + + + + ${project.groupId} + fe-connector-spi + ${project.version} + + + + + ${project.groupId} + fe-connector-api + ${project.version} + + + + + org.apache.fluss + fluss-client + ${fluss.version} + + + + + ${project.groupId} + fe-thrift + ${project.version} + provided + + + + org.apache.logging.log4j + log4j-api + + + + org.junit.jupiter + junit-jupiter + test + + + + + org.apache.fluss + fluss-server + ${fluss.version} + test + + + + + org.apache.fluss + fluss-server + ${fluss.version} + tests + test + + + + + org.apache.fluss + fluss-test-utils + ${fluss.version} + test + + + + + org.apache.curator + curator-test + 5.4.0 + test + + + + + doris-fe-connector-fluss + + + org.apache.maven.plugins + maven-surefire-plugin + + + --add-opens=java.base/java.nio=ALL-UNNAMED + + + + maven-assembly-plugin + + false + + src/main/assembly/plugin-zip.xml + + + + + make-assembly + package + + single + + + + + + + diff --git a/fe/fe-connector/fe-connector-fluss/src/main/assembly/plugin-zip.xml b/fe/fe-connector/fe-connector-fluss/src/main/assembly/plugin-zip.xml new file mode 100644 index 00000000000000..f10c88f571a01a --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/assembly/plugin-zip.xml @@ -0,0 +1,65 @@ + + + + + plugin + + zip + + false + + + + ${project.build.directory}/${project.build.finalName}.jar + / + + + + + + /lib + false + runtime + + org.apache.doris:fe-connector-api + org.apache.doris:fe-connector-spi + org.apache.doris:fe-extension-spi + org.apache.doris:fe-filesystem-api + org.apache.doris:fe-thrift + org.apache.thrift:libthrift + + org.apache.logging.log4j:* + org.slf4j:* + + + + diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/ConnectionBackedFlussAdminOps.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/ConnectionBackedFlussAdminOps.java new file mode 100644 index 00000000000000..4dc6d7b80cf96a --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/ConnectionBackedFlussAdminOps.java @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.DorisConnectorException; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.client.metadata.KvSnapshots; +import org.apache.fluss.client.metadata.LakeSnapshot; +import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.metadata.TableStats; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * {@link FlussAdminOps} over a live fluss {@link Connection}: the one place in this connector that + * deals with futures, timeouts and error wrapping. + * + *

The connection is owned by {@link FlussConnector} and outlives this object, which is created per + * statement. {@code Connection#getAdmin()} is a memoized, connection-scoped singleton, so it is fetched + * per call and never closed here — closing it would pull the admin out from under every other statement + * on the same catalog. + */ +public class ConnectionBackedFlussAdminOps implements FlussAdminOps { + + /** + * How long any single metadata call may take before it is abandoned. These are small RPCs against + * the coordinator, so a wait this long means the cluster is unreachable rather than busy, and the + * caller is a query that should say so instead of hanging. + */ + private static final long TIMEOUT_MS = 60_000L; + + private final Connection connection; + private final String catalogName; + private final String bootstrapServers; + + public ConnectionBackedFlussAdminOps(Connection connection, String catalogName, String bootstrapServers) { + this.connection = connection; + this.catalogName = catalogName; + this.bootstrapServers = bootstrapServers; + } + + @Override + public List listDatabases() { + return await(admin().listDatabases(), "listDatabases"); + } + + @Override + public boolean databaseExists(String databaseName) { + return await(admin().databaseExists(databaseName), "databaseExists(" + databaseName + ")"); + } + + @Override + public List listTables(String databaseName) { + return await(admin().listTables(databaseName), "listTables(" + databaseName + ")"); + } + + @Override + public boolean tableExists(TablePath tablePath) { + return await(admin().tableExists(tablePath), "tableExists(" + tablePath + ")"); + } + + @Override + public TableInfo getTableInfo(TablePath tablePath) { + return await(admin().getTableInfo(tablePath), "getTableInfo(" + tablePath + ")"); + } + + @Override + public List listPartitionInfos(TablePath tablePath) { + return await(admin().listPartitionInfos(tablePath), "listPartitionInfos(" + tablePath + ")"); + } + + @Override + public List listPartitionInfos(TablePath tablePath, PartitionSpec partialPartitionSpec) { + return await(admin().listPartitionInfos(tablePath, partialPartitionSpec), + "listPartitionInfos(" + tablePath + ", " + partialPartitionSpec + ")"); + } + + @Override + public TableStats getTableStats(TablePath tablePath) { + return await(admin().getTableStats(tablePath), "getTableStats(" + tablePath + ")"); + } + + @Override + public KvSnapshots getLatestKvSnapshots(TablePath tablePath) { + return await(admin().getLatestKvSnapshots(tablePath), "getLatestKvSnapshots(" + tablePath + ")"); + } + + @Override + public KvSnapshots getLatestKvSnapshots(TablePath tablePath, String partitionName) { + return await(admin().getLatestKvSnapshots(tablePath, partitionName), + "getLatestKvSnapshots(" + tablePath + ", " + partitionName + ")"); + } + + @Override + public LakeSnapshot getReadableLakeSnapshot(TablePath tablePath) { + return await(admin().getReadableLakeSnapshot(tablePath), "getReadableLakeSnapshot(" + tablePath + ")"); + } + + @Override + public Map listOffsets(TablePath tablePath, Collection buckets, OffsetSpec offsetSpec) { + return await(admin().listOffsets(tablePath, buckets, offsetSpec).all(), + "listOffsets(" + tablePath + ", " + buckets + ")"); + } + + @Override + public Map listOffsets(TablePath tablePath, String partitionName, + Collection buckets, OffsetSpec offsetSpec) { + return await(admin().listOffsets(tablePath, partitionName, buckets, offsetSpec).all(), + "listOffsets(" + tablePath + ", " + partitionName + ", " + buckets + ")"); + } + + private Admin admin() { + return connection.getAdmin(); + } + + private T await(CompletableFuture future, String operation) { + try { + return future.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new DorisConnectorException(describe(operation, "was interrupted"), e); + } catch (TimeoutException e) { + future.cancel(true); + throw new DorisConnectorException(describe(operation, "timed out after " + TIMEOUT_MS + " ms"), e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException) { + // Fluss's own errors (ApiException subclasses) are RuntimeExceptions, and callers + // discriminate on their type — LakeTableSnapshotNotExistException is what tells scan + // planning that this table has nothing readable in the lake yet. Wrapping them here + // would turn that decision into string matching, so they travel unchanged. + throw (RuntimeException) cause; + } + throw new DorisConnectorException(describe(operation, "failed"), cause == null ? e : cause); + } + } + + private String describe(String operation, String outcome) { + return "fluss catalog '" + catalogName + "': " + operation + " " + outcome + + " (" + FlussConnectorProperties.BOOTSTRAP_SERVERS + "=" + bootstrapServers + ")"; + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussAdminOps.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussAdminOps.java new file mode 100644 index 00000000000000..b544331ea04f39 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussAdminOps.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.client.metadata.KvSnapshots; +import org.apache.fluss.client.metadata.LakeSnapshot; +import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.metadata.TableStats; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * Every remote call this connector makes to a fluss cluster, as one synchronous interface. + * + *

It exists to make the connector testable without a mocking framework. Metadata mapping and, above + * all, split generation are pure functions of what these methods return — offsets, snapshot ids, + * partition lists — so a recording implementation lets a unit test drive the interesting boundaries + * (an empty bucket, a lake offset that has caught up with the log, a partition that exists only in the + * lake) directly, instead of trying to coax a live cluster into each of those states. The + * connection-backed implementation is then the only place that has to worry about futures, timeouts + * and error wrapping, and integration tests against a real cluster cover it. + * + *

Signatures mirror {@code org.apache.fluss.client.admin.Admin} one for one, minus the asynchrony: + * every method blocks with a timeout and returns the value, so nothing downstream handles a + * {@link java.util.concurrent.CompletableFuture}. Fluss's own exceptions (subclasses of + * {@code org.apache.fluss.exception.ApiException}) propagate unwrapped — callers discriminate on them, + * notably {@code LakeTableSnapshotNotExistException} when deciding whether a union read is possible. + * + *

Implementations are NOT thread-safe and are scoped to one statement. + */ +public interface FlussAdminOps { + + List listDatabases(); + + boolean databaseExists(String databaseName); + + List listTables(String databaseName); + + boolean tableExists(TablePath tablePath); + + TableInfo getTableInfo(TablePath tablePath); + + List listPartitionInfos(TablePath tablePath); + + /** Server-side partial partition pruning: only partitions matching {@code partialPartitionSpec}. */ + List listPartitionInfos(TablePath tablePath, PartitionSpec partialPartitionSpec); + + TableStats getTableStats(TablePath tablePath); + + /** Latest kv snapshot per bucket of a primary-key table, with the log offset each snapshot ends at. */ + KvSnapshots getLatestKvSnapshots(TablePath tablePath); + + /** As above, for one partition of a partitioned primary-key table. */ + KvSnapshots getLatestKvSnapshots(TablePath tablePath, String partitionName); + + /** + * The lake snapshot a reader may actually read, together with the per-bucket log offset the lake + * ends at. + * + *

Deliberately not the latest lake snapshot: data that tiering has just committed can + * still be unreadable in the lake (a paimon deletion-vector table keeps it in L0 for a while), and + * a reader that took the latest snapshot id would skip the log covering it and lose rows. Both + * halves — snapshot id and offsets — come from this one call so they cannot disagree. + * + * @throws org.apache.fluss.exception.LakeTableSnapshotNotExistException if nothing is readable in + * the lake yet, which is the normal state of a table whose tiering has not committed once + */ + LakeSnapshot getReadableLakeSnapshot(TablePath tablePath); + + /** Offsets for {@code buckets} of a non-partitioned table, keyed by bucket id. */ + Map listOffsets(TablePath tablePath, Collection buckets, OffsetSpec offsetSpec); + + /** Offsets for {@code buckets} of one partition, keyed by bucket id. */ + Map listOffsets(TablePath tablePath, String partitionName, + Collection buckets, OffsetSpec offsetSpec); +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussColumnHandle.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussColumnHandle.java new file mode 100644 index 00000000000000..d054eed5ad948f --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussColumnHandle.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; + +import java.util.Objects; + +/** + * A fluss column: its name and its position in the table's row type. + * + *

The position is what the scan side needs — fluss projects by field index + * ({@code TableScan.project(int[])}) and its readers hand back positional rows — so carrying it here + * keeps the projection from having to re-derive it from the schema. + */ +public class FlussColumnHandle implements ConnectorColumnHandle { + + private static final long serialVersionUID = 1L; + + private final String name; + private final int fieldIndex; + + public FlussColumnHandle(String name, int fieldIndex) { + this.name = Objects.requireNonNull(name, "name"); + this.fieldIndex = fieldIndex; + } + + public String getName() { + return name; + } + + public int getFieldIndex() { + return fieldIndex; + } + + /** Identity is the name and the position: the same name at a different index is a different column. */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof FlussColumnHandle)) { + return false; + } + FlussColumnHandle that = (FlussColumnHandle) o; + return fieldIndex == that.fieldIndex && name.equals(that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name, fieldIndex); + } + + @Override + public String toString() { + return name + "[" + fieldIndex + "]"; + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java new file mode 100644 index 00000000000000..3fa746617ca943 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java @@ -0,0 +1,259 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.ConnectorMetadata; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorTestResult; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; +import org.apache.doris.connector.spi.ConnectorContext; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.config.Configuration; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Fluss connector: one instance per catalog, owning that catalog's single fluss {@link Connection}. + * + *

The connection is the expensive, thread-safe, long-lived object (it carries the RPC client, + * metadata updater and their threads); {@code Admin} handles taken from it are cheap and memoized by + * the connection itself. So the connection is created once, lazily, and closed when the catalog goes + * away, while each statement gets its own {@link ConnectorMetadata} over a fresh + * {@link ConnectionBackedFlussAdminOps} view of it. + */ +public class FlussConnector implements Connector { + + /** + * The sibling connector type a fluss lake table ({@code tbl$lake}) is delegated to. A string literal, + * not the paimon plugin's own constant: that plugin loads child-first and none of its classes are + * visible from here. Matches the type {@code PaimonConnectorProvider} registers. + */ + private static final String PAIMON_CONNECTOR_TYPE = "paimon"; + + private final Map properties; + private final String catalogName; + private final ConnectorContext context; + + private volatile Connection connection; + + // The embedded paimon SIBLING connector this catalog delegates its lake tables to. Built lazily in the + // PAIMON plugin's OWN child-first classloader via context.createSiblingConnector, never co-packaged + // into the fluss zip (a second copy of paimon in one JVM). Held ONLY as the parent-first Connector + // interface and NEVER cast: the concrete type is invisible to the fluss loader, so a cast would CCE + // across the loader split. + // + // Exactly ONE, and that is a correctness constraint rather than thrift. Handle routing asks each + // sibling "is this handle yours?", and two paimon siblings answer that question with the SAME class + // test — so a second one could never be told apart from the first and a table would silently read the + // wrong warehouse. One fluss cluster injects one lake configuration into all of its datalake tables + // (LakeCatalogDynamicLoader), so one is also all a healthy catalog ever needs; a cluster reconfigured + // under a live catalog gets a loud error instead (see getOrCreateLakeSibling). + private volatile Connector lakeSibling; + + // The configuration lakeSibling was built from, for the "reconfigured under a live catalog" check. + // Written BEFORE the volatile lakeSibling publishes it, so a reader that sees the sibling sees this. + private Map lakeSiblingProperties; + + public FlussConnector(Map properties, ConnectorContext context) { + FlussConnectorProperties.validate(properties); + this.properties = Collections.unmodifiableMap(new HashMap<>(properties)); + this.catalogName = context.getCatalogName(); + this.context = context; + } + + @Override + public ConnectorMetadata getMetadata(ConnectorSession session) { + return new FlussConnectorMetadata(adminOps(), + FlussConnectorProperties.typeMappingOptions(properties), + this::getOrCreateLakeSibling, this::lakeSiblingOwning); + } + + /** + * A fresh provider per call, which is what the engine wants: it memoizes one instance per scan node + * and that instance keeps the just-planned range counts for the node's EXPLAIN line. + */ + @Override + public ConnectorScanPlanProvider getScanPlanProvider() { + return new FlussScanPlanProvider(adminOps(), properties, this::getOrCreateLakeSibling); + } + + /** + * Routes scan planning by handle, so a lake table's scan is planned by the paimon sibling that owns + * its handle and reads as a plain paimon table (native readers, its own statistics and time travel). + * A fluss handle keeps this connector's own provider. + */ + @Override + public ConnectorScanPlanProvider getScanPlanProvider(ConnectorTableHandle handle) { + Connector owner = lakeSiblingOwning(handle); + return owner == null ? getScanPlanProvider() : owner.getScanPlanProvider(handle); + } + + /** + * Only fluss's own handles. The engine asks this to route a handle back to the connector that made it, + * and a lake table's handle was made by the paimon sibling, which answers for itself. + */ + @Override + public boolean ownsHandle(ConnectorTableHandle handle) { + return handle instanceof FlussTableHandle; + } + + /** + * The already-built lake sibling when it owns {@code handle}, else null (i.e. a fluss handle). Asks the + * sibling's {@code ownsHandle} because the sibling tests its OWN in-loader handle type, which this side + * cannot {@code instanceof} across the plugin split. + * + *

Deliberately a peek, never a build: a query that never touches a lake table must not construct a + * paimon catalog (nor fail when the paimon plugin is absent). + */ + private Connector lakeSiblingOwning(ConnectorTableHandle handle) { + Connector sibling = lakeSibling; + return sibling != null && sibling.ownsHandle(handle) ? sibling : null; + } + + /** + * Builds (once) and returns this catalog's paimon sibling. Fails loud when no paimon provider is + * available, e.g. the plugin is not installed; that failure is NOT memoized (the field stays unset), so + * a later-installed plugin recovers on the next access. + * + *

Also fails loud when a second, DIFFERENT lake configuration shows up — see the field comment: a + * second paimon sibling could not be routed apart from the first, so serving both would mean reading + * one warehouse under the other's name. The message asks for a catalog refresh, which rebuilds this + * connector and picks up the new configuration. It deliberately does not print either configuration: + * they carry storage credentials. + * + *

Package-private (not private) so a unit test can drive the sibling wiring without + * {@link #getMetadata} first opening a real fluss connection. + */ + Connector getOrCreateLakeSibling(Map siblingProperties) { + if (lakeSibling == null) { + synchronized (this) { + if (lakeSibling == null) { + Connector sibling = + context.createSiblingConnector(PAIMON_CONNECTOR_TYPE, siblingProperties); + if (sibling == null) { + throw new DorisConnectorException( + "Cannot read the lake table of fluss catalog '" + catalogName + + "': the paimon connector plugin is not available"); + } + lakeSiblingProperties = Collections.unmodifiableMap(new HashMap<>(siblingProperties)); + lakeSibling = sibling; + } + } + } + if (!lakeSiblingProperties.equals(siblingProperties)) { + throw new DorisConnectorException( + "Fluss catalog '" + catalogName + "' is already serving lake tables with a different" + + " paimon configuration than this table's. Its fluss cluster was reconfigured;" + + " refresh the catalog to pick up the new lake configuration"); + } + return lakeSibling; + } + + @Override + public ConnectorTestResult testConnection(ConnectorSession session) { + try { + adminOps().listDatabases(); + return ConnectorTestResult.success(); + } catch (Exception e) { + return ConnectorTestResult.failure("Fluss connectivity test failed: " + e.getMessage()); + } + } + + /** + * Closes this catalog's fluss connection and its lake sibling. The engine closes only a catalog's + * PRIMARY connector, so the sibling's lifecycle is this connector's to own — an unclosed one leaks its + * paimon catalog (file handles, client pools) for the life of the FE. The sibling is closed first and + * its failure is held back, so it cannot leak the fluss connection. + */ + @Override + public void close() throws IOException { + IOException siblingFailure = null; + Connector sibling = lakeSibling; + lakeSibling = null; + lakeSiblingProperties = null; + if (sibling != null) { + try { + sibling.close(); + } catch (Exception e) { + siblingFailure = new IOException( + "Failed to close the paimon lake sibling of catalog '" + catalogName + "'", e); + } + } + + Connection toClose = connection; + connection = null; + if (toClose != null) { + try { + toClose.close(); + } catch (Exception e) { + throw new IOException( + "Failed to close the fluss connection of catalog '" + catalogName + "'", e); + } + } + if (siblingFailure != null) { + throw siblingFailure; + } + } + + private FlussAdminOps adminOps() { + return new ConnectionBackedFlussAdminOps(getOrCreateConnection(), catalogName, + FlussConnectorProperties.bootstrapServers(properties)); + } + + private Connection getOrCreateConnection() { + if (connection == null) { + synchronized (this) { + if (connection == null) { + connection = createConnection(); + } + } + } + return connection; + } + + private Connection createConnection() { + Configuration config = new Configuration(); + FlussConnectorProperties.toFlussClientConfig(properties).forEach(config::setString); + + // TCCL pin, and this is the locus that matters: creating the connection is what spawns the + // fluss client's own threads (netty IO, metadata updater), and a thread inherits the context + // classloader of whoever created it. Started under the engine's loader they would resolve + // fluss classes against fe-core instead of this child-first plugin, where fluss does not exist + // at all. Pinning here fixes every thread the connection owns for its whole life, which is why + // the per-call admin path needs no pin of its own. + ClassLoader callerLoader = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); + try { + return ConnectionFactory.createConnection(config); + } catch (RuntimeException e) { + throw new DorisConnectorException("Failed to connect fluss catalog '" + catalogName + "' (" + + FlussConnectorProperties.BOOTSTRAP_SERVERS + "=" + + FlussConnectorProperties.bootstrapServers(properties) + ")", e); + } finally { + Thread.currentThread().setContextClassLoader(callerLoader); + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java new file mode 100644 index 00000000000000..b4c25e6f40632c --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorMetadata.java @@ -0,0 +1,472 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.ConnectorColumn; +import org.apache.doris.connector.api.ConnectorMetadata; +import org.apache.doris.connector.api.ConnectorPartitionInfo; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorTableSchema; +import org.apache.doris.connector.api.ConnectorTableStatistics; +import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.pushdown.ConnectorExpression; +import org.apache.doris.thrift.THiveTable; +import org.apache.doris.thrift.TTableDescriptor; +import org.apache.doris.thrift.TTableType; + +import org.apache.fluss.exception.DatabaseNotExistException; +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypeRoot; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +/** + * Fluss metadata for one statement: a thin mapping from the connector SPI onto {@link FlussAdminOps}. + * + *

Fluss has a real two-level namespace (database, table), so the listing calls are direct + * pass-throughs and carry no Doris-side naming convention. + * + *

Every method that needs the table's schema goes through {@link #tableInfo}, which memoizes the + * fetch for the statement — the handle, the schema, the column handles and (later) split planning + * therefore see one coherent version of the table for one round trip. + * + *

It is also the gateway for lake tables: a datalake-enabled table exposes a {@code lake} system + * table, so {@code tbl$lake} resolves to a handle made by the embedded paimon sibling and every later + * per-handle call is forwarded to that sibling's metadata. Those forwards are the guarded methods below; + * each one first asks {@link #siblingOwner} whether the handle is foreign, and only then falls through to + * fluss's own implementation. A guard that is missing would not degrade gracefully — fluss's body casts to + * {@link FlussTableHandle} and would throw {@link ClassCastException} on a paimon handle. Every method + * that performs that cast is guarded; a new one must be too. + * + *

The methods this connector does NOT implement are deliberately left unguarded: they inherit the SPI's + * neutral defaults, so a lake handle reaching them is answered "not supported" rather than crashing. That + * costs the lake table paimon's MVCC pin and time travel, and the reason is structural rather than an + * oversight: fe-core picks a table's MVCC-capable class from the FRONT DOOR connector's capabilities + * ({@code SUPPORTS_MVCC_SNAPSHOT}), which fluss does not declare, so the engine never asks for a pin on a + * fluss catalog's table — forwarding those calls today would be unreachable code. Schema and data stay + * consistent (both read latest). Should fluss ever declare that capability, add forwards for + * {@code beginQuerySnapshot} / {@code resolveTimeTravel} / {@code applySnapshot} and the snapshot overloads + * of {@code getTableSchema} / {@code getTableStatistics} in the same breath. + */ +public class FlussConnectorMetadata implements ConnectorMetadata { + + private static final Logger LOG = LogManager.getLogger(FlussConnectorMetadata.class); + + /** What {@code getTableSchema} reports as the table's format; surfaces in DESCRIBE / EXPLAIN. */ + private static final String TABLE_FORMAT_TYPE = "FLUSS"; + + /** + * The system-table name that reads a datalake table's lake side, i.e. {@code tbl$lake}. Matches the + * suffix fluss's own Flink catalog uses ({@code FlinkCatalog.LAKE_TABLE_SPLITTER = "$lake"}); the + * engine supplies the {@code $}. + */ + private static final String LAKE_SYS_TABLE = "lake"; + + /** The only lake format that can be delegated today; fluss also defines iceberg / lance / hudi. */ + private static final String PAIMON_LAKE_FORMAT = "paimon"; + + private final FlussAdminOps adminOps; + private final FlussTypeMapping.Options typeMappingOptions; + private final Function, Connector> lakeSiblingFactory; + private final Function siblingOwner; + + public FlussConnectorMetadata(FlussAdminOps adminOps, FlussTypeMapping.Options typeMappingOptions, + Function, Connector> lakeSiblingFactory, + Function siblingOwner) { + this.adminOps = adminOps; + this.typeMappingOptions = typeMappingOptions; + this.lakeSiblingFactory = lakeSiblingFactory; + this.siblingOwner = siblingOwner; + } + + /** + * Forwards one call to the lake sibling's metadata. Only SPI types are touched, and neither the metadata + * nor any handle it produces may be cast (cross-loader {@code CCE}). {@link LakeSibling#forward} owns the + * classloader pin and the per-statement instance, so both are shared with the scan planner. + */ + private T forward(ConnectorSession session, Connector sibling, + Function call) { + return LakeSibling.forward(session, sibling, call); + } + + @Override + public List listDatabaseNames(ConnectorSession session) { + return adminOps.listDatabases(); + } + + @Override + public boolean databaseExists(ConnectorSession session, String dbName) { + return adminOps.databaseExists(dbName); + } + + @Override + public List listTableNames(ConnectorSession session, String dbName) { + return adminOps.listTables(dbName); + } + + /** + * Resolves a table, or reports that it does not exist. + * + *

Only fluss's own "not there" exceptions become an empty handle. Anything else — an + * unreachable coordinator, a timeout, an auth failure — propagates, because answering "no such + * table" to those makes a broken catalog look like an empty one: {@code SELECT} then fails with + * "table not found", a {@code CREATE TABLE IF NOT EXISTS} would look free to proceed, and the real + * error never reaches the user. + */ + @Override + public Optional getTableHandle( + ConnectorSession session, String dbName, String tableName) { + try { + return Optional.of(FlussTableHandle.of(tableInfo(session, TablePath.of(dbName, tableName)))); + } catch (TableNotExistException | DatabaseNotExistException e) { + return Optional.empty(); + } + } + + /** + * Reports the {@code lake} system table for a table that has a lake side, so {@code tbl$lake} resolves. + * + *

Gated on {@code table.datalake.enabled} alone, not on the lake FORMAT: a table with an unsupported + * lake format still advertises {@code $lake} so that reading it produces + * {@link #getSysTableHandle}'s precise "this format is not supported" error rather than fe-core's + * generic "no such table". A table with no lake at all advertises nothing — announcing a sub-table that + * can only fail would be worse than not offering it. + */ + @Override + public List listSupportedSysTables(ConnectorSession session, + ConnectorTableHandle baseTableHandle) { + Connector owner = siblingOwner.apply(baseTableHandle); + if (owner != null) { + return forward(session, owner, m -> m.listSupportedSysTables(session, baseTableHandle)); + } + FlussTableHandle flussHandle = (FlussTableHandle) baseTableHandle; + return flussHandle.isDataLakeEnabled() + ? Collections.singletonList(LAKE_SYS_TABLE) + : Collections.emptyList(); + } + + /** + * Resolves {@code tbl$lake} to the paimon sibling's handle for the same {@code db.table} name — which is + * the name fluss's tiering service writes the lake table under. From here on the table IS a paimon + * table: the engine routes its scan by handle to the sibling's plan provider, and this metadata's + * guarded methods forward the rest. + * + *

The sibling is configured from THIS table's properties, where the fluss coordinator puts the + * cluster's lake settings; see {@link PaimonSiblingProperties}. + */ + @Override + public Optional getSysTableHandle(ConnectorSession session, + ConnectorTableHandle baseTableHandle, String sysName) { + Connector owner = siblingOwner.apply(baseTableHandle); + if (owner != null) { + return forward(session, owner, m -> m.getSysTableHandle(session, baseTableHandle, sysName)); + } + if (!LAKE_SYS_TABLE.equals(sysName)) { + return Optional.empty(); + } + + FlussTableHandle flussHandle = (FlussTableHandle) baseTableHandle; + // Re-checked rather than assumed from listSupportedSysTables: discovery and resolution are two + // round trips, and the table could have had its lake turned off in between. + if (!flussHandle.isDataLakeEnabled()) { + throw new DorisConnectorException("Table '" + flussHandle.getDatabaseName() + "." + + flussHandle.getTableName() + "' has no lake table: it is not created with" + + " table.datalake.enabled = true"); + } + String lakeFormat = flussHandle.getDataLakeFormat(); + if (lakeFormat == null || !PAIMON_LAKE_FORMAT.equalsIgnoreCase(lakeFormat)) { + throw new DorisConnectorException("Cannot read the lake table of '" + + flussHandle.getDatabaseName() + "." + flussHandle.getTableName() + + "': its table.datalake.format is '" + lakeFormat + + "', and the fluss connector currently supports only '" + PAIMON_LAKE_FORMAT + "'"); + } + + Connector sibling = lakeSiblingFactory.apply( + PaimonSiblingProperties.synthesize(flussHandle.getProperties())); + Optional lakeHandle = forward(session, sibling, m -> m.getTableHandle( + session, flussHandle.getDatabaseName(), flussHandle.getTableName())); + if (!lakeHandle.isPresent()) { + // The lake table is created by the tiering service on its first commit, so "not there" means + // nothing has been tiered yet — a state that resolves itself and is worth saying out loud. + // Returning empty here would instead surface as fe-core's generic "no such table", pointing the + // user at a name that IS correct. + throw new DorisConnectorException("The lake table of '" + flussHandle.getDatabaseName() + "." + + flussHandle.getTableName() + "' does not exist yet: nothing has been tiered to the" + + " lake. Start (or wait for) the fluss tiering service for this table"); + } + // From here on this handle travels back through the engine and returns to the guards below, which + // route it by asking the sibling whether it is its own. Checked once, here, where a failure still + // has a cause attached to it. + return Optional.of(LakeSibling.requireOwned(sibling, lakeHandle.get())); + } + + @Override + public boolean isPartitionValuesSysTable(ConnectorSession session, + ConnectorTableHandle baseTableHandle, String sysName) { + Connector owner = siblingOwner.apply(baseTableHandle); + if (owner != null) { + return forward(session, owner, + m -> m.isPartitionValuesSysTable(session, baseTableHandle, sysName)); + } + // The lake table is a real data table served by the paimon sibling, not the generic + // partition_values function. + return false; + } + + @Override + public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle handle) { + Connector owner = siblingOwner.apply(handle); + if (owner != null) { + return forward(session, owner, m -> m.getTableSchema(session, handle)); + } + FlussTableHandle flussHandle = (FlussTableHandle) handle; + TableInfo info = tableInfo(session, flussHandle.toTablePath()); + + List schemaColumns = info.getSchema().getColumns(); + List columns = new ArrayList<>(schemaColumns.size()); + for (Schema.Column column : schemaColumns) { + columns.add(toConnectorColumn(column)); + } + + // LinkedHashMap: SHOW CREATE TABLE renders PROPERTIES from this map, and a stable order keeps + // the rendered DDL from churning between runs. + Map properties = new LinkedHashMap<>(info.getProperties().toMap()); + if (flussHandle.isPartitioned()) { + // "partition_columns" is the key the generic fe-core consumer reads; without it the table is + // treated as unpartitioned and partition pruning is silently lost. Names are case-preserved + // to stay matchable against the column names emitted above. + properties.put(ConnectorTableSchema.PARTITION_COLUMNS_KEY, + String.join(",", flussHandle.getPartitionKeys())); + } + return new ConnectorTableSchema( + flussHandle.getTableName(), columns, TABLE_FORMAT_TYPE, properties); + } + + @Override + public Map getColumnHandles( + ConnectorSession session, ConnectorTableHandle handle) { + Connector owner = siblingOwner.apply(handle); + if (owner != null) { + return forward(session, owner, m -> m.getColumnHandles(session, handle)); + } + FlussTableHandle flussHandle = (FlussTableHandle) handle; + List columns = tableInfo(session, flussHandle.toTablePath()).getSchema().getColumns(); + Map handles = new LinkedHashMap<>(); + for (int i = 0; i < columns.size(); i++) { + String name = columns.get(i).getName(); + handles.put(name, new FlussColumnHandle(name, i)); + } + return handles; + } + + @Override + public String getTableComment(ConnectorSession session, String dbName, String tableName) { + return tableInfo(session, TablePath.of(dbName, tableName)).getComment().orElse(""); + } + + @Override + public List listPartitionNames(ConnectorSession session, ConnectorTableHandle handle) { + Connector owner = siblingOwner.apply(handle); + if (owner != null) { + return forward(session, owner, m -> m.listPartitionNames(session, handle)); + } + List partitions = listPartitions(session, handle, Optional.empty()); + List names = new ArrayList<>(partitions.size()); + for (ConnectorPartitionInfo partition : partitions) { + names.add(partition.getPartitionName()); + } + return names; + } + + /** + * Lists the table's partitions in the Hive-style {@code k1=v1/k2=v2} naming every Doris catalog + * uses, which is not fluss's own {@code v1$v2} spelling — fe-core parses the segments back out of + * the name for {@code SHOW PARTITIONS} and the {@code partition_values} function. The rendering is + * {@link FlussPartitions}', shared with split planning: the pruned names the engine derives from + * this listing are what planning matches its partitions against, so one renderer or none. + * + *

{@code filter} is ignored: server-side partition pruning exists in fluss (a partial + * {@code PartitionSpec}) but it takes a spec, not a predicate, and the predicate-to-spec reduction + * belongs with split planning, which is the caller that has a predicate worth pushing. Listing + * returns everything, as the paimon and maxcompute connectors do. + */ + @Override + public List listPartitions(ConnectorSession session, + ConnectorTableHandle handle, Optional filter) { + Connector owner = siblingOwner.apply(handle); + if (owner != null) { + return forward(session, owner, m -> m.listPartitions(session, handle, filter)); + } + FlussTableHandle flussHandle = (FlussTableHandle) handle; + List partitionKeys = flussHandle.getPartitionKeys(); + if (partitionKeys.isEmpty()) { + // Guard before the remote call: asking fluss for the partitions of an unpartitioned table is + // an error there, and "this table has no partitions" is already known from the handle. + return Collections.emptyList(); + } + // Before the remote call and before any name is rendered, because the answer depends on the + // table's schema alone: a partition column whose value fluss cannot store verbatim in a partition + // name makes every partition of this table unreadable, whether it has any yet or not. + rejectUnreadablePartitionColumns(flussHandle, partitionKeys); + + List flussPartitions = adminOps.listPartitionInfos(flussHandle.toTablePath()); + List result = new ArrayList<>(flussPartitions.size()); + for (PartitionInfo partition : flussPartitions) { + FlussScanRange.Partition resolved = FlussPartitions.toScanPartition(partition, partitionKeys); + // The values already follow partition-COLUMN order (fe-core zips them positionally against + // the partition columns); the null-flag list stays empty because fluss allows no null value. + result.add(new ConnectorPartitionInfo( + resolved.getName(), resolved.getValues(), Collections.emptyMap(), + new ArrayList<>(resolved.getValues().values()), Collections.emptyList())); + } + return result; + } + + /** + * Refuses a table whose partition values cannot be read back out of the names fluss stores them in. + * + *

This is the only place it can be said well. Further down, the name is all there is: fe-core's + * parser sees {@code 1_5}, has a FLOAT column to put it in, and reports that it failed to convert a + * partition — naming neither the column nor fluss nor the property that would help. Here the column + * and its fluss type are both still in hand. + * + *

Refusing is the answer rather than guessing because the rewriting is many-to-one: {@code 1_5} + * was {@code 1.5}, and {@code 01-02-03} was {@code 01:02:03}, and neither the connector nor fluss + * itself can say which character came back out. + */ + private void rejectUnreadablePartitionColumns(FlussTableHandle handle, List partitionKeys) { + Map keyColumnTypes = handle.getKeyColumnTypes(); + for (String partitionKey : partitionKeys) { + DataType type = keyColumnTypes.get(partitionKey); + if (type == null) { + // The handle records a type for every partition column of the table it was built from, so + // a missing one means this handle and that schema have come apart. Guessing "readable" + // here would put the mangled name back on the path this method exists to close. + throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." + + handle.getTableName() + "' lists '" + partitionKey + "' as a partition column" + + " but carries no type for it; refresh the catalog and try again."); + } + String rejection = FlussPartitionColumnTypes.rejection(type, typeMappingOptions); + if (rejection != null) { + throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." + + handle.getTableName() + "' cannot be read: its partition column '" + partitionKey + + "' has fluss type " + type + ", and " + rejection + ". Partition columns of type " + + FlussPartitionColumnTypes.READABLE_TYPES + " are stored as written."); + } + } + } + + /** + * The table's row count, when fluss has one. + * + *

Fluss reports row count only (no data size, no per-column statistics), and only for a table + * whose statistics are enabled — otherwise the count comes back as zero, which is reported as + * unknown rather than as "the table is empty". Statistics are best effort by contract: a failure + * degrades to unknown instead of failing the statement, because this runs in background analysis + * and in SHOW, where a transient coordinator error must not surface as a query error. + */ + @Override + public Optional getTableStatistics( + ConnectorSession session, ConnectorTableHandle handle) { + Connector owner = siblingOwner.apply(handle); + if (owner != null) { + return forward(session, owner, m -> m.getTableStatistics(session, handle)); + } + FlussTableHandle flussHandle = (FlussTableHandle) handle; + long rowCount; + try { + rowCount = adminOps.getTableStats(flussHandle.toTablePath()).getRowCount(); + } catch (Exception e) { + LOG.warn("Failed to read fluss table statistics for {}", flussHandle, e); + return Optional.empty(); + } + if (rowCount <= 0) { + return Optional.empty(); + } + // -1 = unknown data size. Zero would tell the optimizer the table costs nothing to scan. + return Optional.of(new ConnectorTableStatistics(rowCount, -1)); + } + + /** + * The Thrift table descriptor the BE receives. A fluss scan reaches the BE through the same file + * scan node the lake connectors use (the fluss ranges ride in its format-specific descriptor), so + * the table descriptor is the generic hive-shaped one those connectors send, exactly as paimon and + * hudi do; a fluss-specific Thrift table type would buy nothing the scan path reads. + * + *

Not guarded for a lake handle, and it could not be: the signature carries names, not a handle. It + * needs no guard because the paimon connector builds the byte-identical descriptor + * ({@code PaimonConnectorMetadata.buildTableDescriptor}) — verified, not assumed. If either side's + * descriptor ever diverges, this becomes a real gap that a handle-less signature cannot close here. + */ + @Override + public TTableDescriptor buildTableDescriptor(ConnectorSession session, + long tableId, String tableName, String dbName, + String remoteName, int numCols, long catalogId) { + TTableDescriptor descriptor = new TTableDescriptor( + tableId, TTableType.HIVE_TABLE, numCols, 0, tableName, dbName); + descriptor.setHiveTable(new THiveTable(dbName, tableName, new LinkedHashMap<>())); + return descriptor; + } + + private ConnectorColumn toConnectorColumn(Schema.Column column) { + ConnectorType type = FlussTypeMapping.toConnectorType(column.getDataType(), typeMappingOptions); + // isKey=true for every column and nullable=true for every column: this is what every Doris + // external catalog reports. The nullability one matters beyond convention — fluss marks its + // primary-key columns NOT NULL, and propagating that would let the planner fold null-rejecting + // predicates, while the same table read through its lake sibling (the paimon connector, which + // reports every column nullable) would keep them. One table must not get two different plans + // depending on which door it was read through. + ConnectorColumn connectorColumn = new ConnectorColumn( + column.getName(), + type, + column.getComment().orElse(""), + true, + null, + true); + // A "with local time zone" timestamp carries the WITH_TIMEZONE marker DESCRIBE shows in Extra. + // Keyed on the SOURCE fluss type, so it survives whether enable.mapping.timestamp_tz mapped the + // column to TIMESTAMPTZ or to plain DATETIME. + if (column.getDataType().getTypeRoot() == DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) { + connectorColumn = connectorColumn.withTimeZone(); + } + return connectorColumn; + } + + /** The table's metadata, fetched once per statement (see {@link FlussStatementScope}). */ + private TableInfo tableInfo(ConnectorSession session, TablePath tablePath) { + return FlussStatementScope.sharedTableInfo(session, tablePath, + () -> adminOps.getTableInfo(tablePath)); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProperties.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProperties.java new file mode 100644 index 00000000000000..954903ac476443 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProperties.java @@ -0,0 +1,229 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import java.util.Locale; +import java.util.Map; +import java.util.TreeMap; + +/** + * The catalog properties a fluss catalog accepts, and their translation into a fluss client + * configuration. + * + *

Naming rule. Everything a user may write is prefixed {@code fluss.}, and every such key + * except the Doris-only ones listed in {@link #isDorisOnly(String)} is handed to the fluss client + * verbatim with that prefix stripped. So {@code fluss.bootstrap.servers} becomes fluss's + * {@code bootstrap.servers} and {@code fluss.client.security.protocol} becomes + * {@code client.security.protocol}. Nothing here enumerates the fluss client's own option set: fluss + * owns those names and adds to them between releases, and a Doris-side allowlist would silently + * reject options that fluss understands perfectly well. + * + *

No lake/paimon connection property belongs here. The tiering service writes the lake catalog + * configuration into each datalake-enabled table's own properties ({@code table.datalake.paimon.*}), + * so union read reads it from the table, never from the catalog. + */ +public final class FlussConnectorProperties { + + /** Prefix every fluss catalog property carries; also what is stripped on the way to the client. */ + public static final String PROPERTY_PREFIX = "fluss."; + + /** Required. Comma-separated {@code host:port} list used to bootstrap the fluss cluster. */ + public static final String BOOTSTRAP_SERVERS = "fluss.bootstrap.servers"; + + /** + * Optional. How a scan of a datalake-enabled table combines the paimon lake with the fluss log. + * + *

Doris-only: it selects a planning strategy and is never passed to the fluss client. The + * three values exist because "did this query actually read the lake?" is otherwise unobservable — + * {@code auto} silently falls back to a fluss-only read when the lake has no readable snapshot + * yet, which makes a union-read regression test pass for the wrong reason. {@code required} turns + * that fallback into an error and {@code disabled} forces the fluss-only path, so a test can pin + * both sides of the comparison. + */ + public static final String UNION_READ_MODE = "fluss.union_read.mode"; + + /** + * Optional. How many log-tail rows one bucket of a primary-key table may contribute to a union read. + * + *

Doris-only. Reading a primary-key table's lake together with its log tail holds that tail in BE + * memory — once as the set of keys that suppress lake rows, once as the rows the tail itself + * contributes — so an unbounded tail is an unbounded allocation inside a process that serves every + * other query too. Fluss's own engines have no such limit (Flink's tiering reader accumulates the + * tail on the heap and lets the task manager die and restart if it does not fit), which is exactly + * the difference: BE has no restart to fall back on. A tail is by design the data written within one + * {@code table.datalake.freshness} period, so the default is far above any healthy table; hitting it + * means tiering has stalled, and the error says so. + */ + public static final String UNION_READ_MAX_TAIL_ROWS = "fluss.union_read.max_tail_rows"; + + /** Default {@link #UNION_READ_MAX_TAIL_ROWS}: two million rows per bucket. */ + public static final long DEFAULT_MAX_TAIL_ROWS = 2_000_000L; + + /** + * Optional. Whether a fluss BINARY/BYTES column reads as Doris VARBINARY instead of STRING. + * + *

Unprefixed on purpose: this is the engine-wide catalog property + * ({@code CatalogProperty.ENABLE_MAPPING_VARBINARY}) that the hive, paimon and iceberg catalogs + * already answer to, and a user should not have to learn a fluss-specific spelling for it. Being + * unprefixed also keeps it out of {@link #toFlussClientConfig} for free. + */ + public static final String ENABLE_MAPPING_VARBINARY = "enable.mapping.varbinary"; + + /** + * Optional. Whether a fluss TIMESTAMP_LTZ column reads as Doris TIMESTAMPTZ instead of DATETIMEV2. + * Engine-wide catalog property, same reasoning as {@link #ENABLE_MAPPING_VARBINARY}. + */ + public static final String ENABLE_MAPPING_TIMESTAMP_TZ = "enable.mapping.timestamp_tz"; + + /** Value set of {@link #UNION_READ_MODE}. */ + public enum UnionReadMode { + /** Union read when the lake has a readable snapshot, fluss-only when it does not. */ + AUTO, + /** Union read, or fail: a datalake table with no readable lake snapshot is an error. */ + REQUIRED, + /** Never union read: scan fluss only, even for a datalake table with a lake snapshot. */ + DISABLED; + + static UnionReadMode parse(String raw) { + for (UnionReadMode mode : values()) { + if (mode.name().equalsIgnoreCase(raw.trim())) { + return mode; + } + } + throw new IllegalArgumentException("Invalid value '" + raw + "' for property '" + + UNION_READ_MODE + "'; expected one of auto, required, disabled"); + } + + /** The lower-case spelling users write, and what {@code appendExplainInfo} prints. */ + public String propertyValue() { + return name().toLowerCase(Locale.ROOT); + } + } + + private FlussConnectorProperties() { + } + + /** + * Fails on a catalog that cannot work, at {@code CREATE CATALOG} time rather than at first query. + * Only checks what is decidable without touching the cluster; reachability is + * {@code testConnection}'s job. + */ + public static void validate(Map properties) { + validateBootstrapServers(bootstrapServers(properties)); + unionReadMode(properties); + maxTailRows(properties); + } + + /** The declared bootstrap servers, or the empty string when the property is absent. */ + public static String bootstrapServers(Map properties) { + String value = properties.get(BOOTSTRAP_SERVERS); + return value == null ? "" : value.trim(); + } + + /** The declared union-read mode, {@link UnionReadMode#AUTO} when the property is absent. */ + public static UnionReadMode unionReadMode(Map properties) { + String value = properties.get(UNION_READ_MODE); + return value == null ? UnionReadMode.AUTO : UnionReadMode.parse(value); + } + + /** + * The declared per-bucket tail ceiling, {@link #DEFAULT_MAX_TAIL_ROWS} when the property is absent. + * Zero and negative values are rejected rather than read as "no limit": a user who wants no ceiling + * has no way to say so, which is the point. + */ + public static long maxTailRows(Map properties) { + String value = properties.get(UNION_READ_MAX_TAIL_ROWS); + if (value == null) { + return DEFAULT_MAX_TAIL_ROWS; + } + long rows; + try { + rows = Long.parseLong(value.trim()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid value '" + value + "' for property '" + + UNION_READ_MAX_TAIL_ROWS + "'; expected a positive number of rows"); + } + if (rows <= 0) { + throw new IllegalArgumentException("Invalid value '" + value + "' for property '" + + UNION_READ_MAX_TAIL_ROWS + "'; expected a positive number of rows"); + } + return rows; + } + + /** The type-mapping switches this catalog declares; both default to off. */ + public static FlussTypeMapping.Options typeMappingOptions(Map properties) { + return new FlussTypeMapping.Options( + booleanValue(properties, ENABLE_MAPPING_VARBINARY), + booleanValue(properties, ENABLE_MAPPING_TIMESTAMP_TZ)); + } + + private static boolean booleanValue(Map properties, String key) { + return Boolean.parseBoolean(properties.getOrDefault(key, "false")); + } + + /** + * The fluss client configuration this catalog describes: every {@code fluss.}-prefixed property + * that is not Doris-only, with the prefix stripped. + * + *

Returned sorted so that a configuration is printable and comparable in a test without the map + * iteration order leaking in. + */ + public static Map toFlussClientConfig(Map properties) { + Map config = new TreeMap<>(); + for (Map.Entry entry : properties.entrySet()) { + String key = entry.getKey(); + if (!key.startsWith(PROPERTY_PREFIX) || isDorisOnly(key)) { + continue; + } + config.put(key.substring(PROPERTY_PREFIX.length()), entry.getValue()); + } + return config; + } + + /** Whether {@code key} configures Doris's own behaviour and must not reach the fluss client. */ + private static boolean isDorisOnly(String key) { + return UNION_READ_MODE.equals(key) || UNION_READ_MAX_TAIL_ROWS.equals(key); + } + + private static void validateBootstrapServers(String value) { + if (value.isEmpty()) { + throw new IllegalArgumentException( + "Required property '" + BOOTSTRAP_SERVERS + "' is missing"); + } + for (String entry : value.split(",", -1)) { + String server = entry.trim(); + // lastIndexOf, not indexOf: a bracketed IPv6 literal ("[::1]:9123") contains colons of its own. + int colon = server.lastIndexOf(':'); + if (colon <= 0 || colon == server.length() - 1) { + throw new IllegalArgumentException("Invalid value '" + value + "' for property '" + + BOOTSTRAP_SERVERS + "'; expected a comma-separated host:port list"); + } + int port; + try { + port = Integer.parseInt(server.substring(colon + 1)); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid port in '" + server + "' for property '" + + BOOTSTRAP_SERVERS + "'; expected a number between 1 and 65535"); + } + if (port < 1 || port > 65535) { + throw new IllegalArgumentException("Invalid port in '" + server + "' for property '" + + BOOTSTRAP_SERVERS + "'; expected a number between 1 and 65535"); + } + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProvider.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProvider.java new file mode 100644 index 00000000000000..8b5233be836ad8 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnectorProvider.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.spi.ConnectorContext; +import org.apache.doris.connector.spi.ConnectorProvider; + +import java.util.Map; + +/** + * SPI entry point for the fluss connector. + * Discovered via META-INF/services/org.apache.doris.connector.spi.ConnectorProvider. + */ +public class FlussConnectorProvider implements ConnectorProvider { + + @Override + public String getType() { + return "fluss"; + } + + @Override + public void validateProperties(Map properties) { + FlussConnectorProperties.validate(properties); + } + + @Override + public Connector create(Map properties, ConnectorContext context) { + return new FlussConnector(properties, context); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussPartitionColumnTypes.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussPartitionColumnTypes.java new file mode 100644 index 00000000000000..253bfb9720fa79 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussPartitionColumnTypes.java @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.fluss.types.DataType; + +/** + * Which column types a fluss table can be PARTITIONED by and still be readable here. + * + *

A fluss partition carries its value nowhere but in its own name, and fluss restricts what that name + * may contain to ASCII letters, digits, {@code _} and {@code -} — so a value holding anything else is + * rewritten on the way in. A FLOAT {@code 1.5} is named {@code 1_5}; a TIMESTAMP + * {@code 2026-01-01 01:02:03.0} is named {@code 2026-01-01-01-02-03_0}. Nothing reads those back: the + * substitution is many-to-one (a {@code _} was a {@code .} or a {@code :}), and Doris is handed the name, + * not the value. + * + *

Which is why this is a refusal and not a conversion. Left alone, such a table reaches fe-core's + * partition parser and dies there with {@code failed to convert partition [1_5] to list partition} — a + * message that names neither the column nor fluss nor what to do about it, on a table whose {@code DESC} + * looks perfectly ordinary. The verdict below is asked before the partitions are listed, so the answer + * depends on the table's schema alone and is the same whether the table has partitions yet or not. + * + *

Not to be confused with {@link FlussUnionKeyTypes#partitionColumnRejection}, which asks a stricter + * and later question: whether a LAKE split can be matched to a fluss partition by comparing two engines' + * renderings of the same value. That one allows STRING only. This one asks whether Doris can read the + * partition at all, and its answer is a precondition for the other's. + */ +final class FlussPartitionColumnTypes { + + /** The types whose value survives fluss's partition naming, for the error message. */ + static final String READABLE_TYPES = "CHAR, STRING, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT and DATE"; + + private FlussPartitionColumnTypes() { + } + + /** + * Why {@code type} cannot be a partition column of a readable table, or null when it can. + * + *

The switch is exhaustive over fluss's type roots on purpose: a type a future fluss release adds + * to its own partition-key whitelist lands in the default branch and is refused with a message, rather + * than being waved through into fe-core's parser by a rule written before it existed. + * {@link FlussPartitionColumnTypesTest} fails the build when that happens. + * + *

BINARY and BYTES are the one verdict that depends on the catalog: fluss names their partitions + * with the hex text of the bytes, which is a perfectly good STRING and not a VARBINARY at all. + */ + static String rejection(DataType type, FlussTypeMapping.Options options) { + switch (type.getTypeRoot()) { + case CHAR: + case STRING: + case BOOLEAN: + case TINYINT: + case SMALLINT: + case INTEGER: + case BIGINT: + case DATE: + return null; + case BINARY: + case BYTES: + return options.isMapBinaryToVarbinary() + ? "fluss names such a partition with the hex text of the bytes, which this catalog" + + " cannot read back as the VARBINARY column that '" + + FlussConnectorProperties.ENABLE_MAPPING_VARBINARY + "=true' asks for;" + + " turning that property off reads the column, and the partition, as text" + : null; + case FLOAT: + case DOUBLE: + case TIME_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return "fluss keeps a partition's value only in its name, where it rewrites every character" + + " a name may not contain — 1.5 is named 1_5 and 2026-01-01 01:02:03 is named" + + " 2026-01-01-01-02-03 — so the value that was written cannot be read back"; + default: + // ARRAY, MAP and ROW among them: fluss refuses these as partition keys itself, and this + // branch is what keeps that from being the only thing standing in the way. + return "the fluss connector does not know how a partition of this type is named"; + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussPartitions.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussPartitions.java new file mode 100644 index 00000000000000..ab7b5d5854a523 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussPartitions.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.fluss.metadata.PartitionInfo; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * How a fluss partition is named on the Doris side — in one place, because two callers must agree on it. + * + *

Fluss names a partition by joining its values ({@code 20260101$cn}); Doris names it the Hive way + * ({@code dt=20260101/region=cn}), which is what fe-core parses back out for {@code SHOW PARTITIONS} + * and what it hands back as the pruned partition set. So the metadata listing and split planning both + * render this name, and if the two renderings ever drifted apart, planning would match none of the + * pruned names and silently scan nothing. + * + *

No escaping is needed: fluss rejects a partition value that is not ASCII alphanumerics, {@code _} + * or {@code -} (TablePath#detectInvalidName via PartitionUtils#validatePartitionValues), so a value can + * contain neither {@code =} nor {@code /}, and cannot be null. + */ +final class FlussPartitions { + + private FlussPartitions() { + } + + /** + * The scan-side view of {@code partition}: fluss's partition id, the Doris partition name and the + * per-column values, in partition-key order (fe-core zips values against the partition columns + * positionally). + */ + static FlussScanRange.Partition toScanPartition(PartitionInfo partition, List partitionKeys) { + Map spec = partition.getPartitionSpec().getSpecMap(); + Map values = new LinkedHashMap<>(); + StringBuilder name = new StringBuilder(); + for (String partitionKey : partitionKeys) { + String value = spec.get(partitionKey); + values.put(partitionKey, value); + if (name.length() > 0) { + name.append('/'); + } + name.append(partitionKey).append('=').append(value); + } + return FlussScanRange.Partition.of(partition.getPartitionId(), name.toString(), values); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java new file mode 100644 index 00000000000000..e343547ce11b58 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanPlanProvider.java @@ -0,0 +1,1051 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.mvcc.ConnectorMvccSnapshot; +import org.apache.doris.connector.api.pushdown.ConnectorExpression; +import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.connector.api.scan.ConnectorScanRequest; +import org.apache.doris.connector.api.scan.ScanNodePropertyKeys; +import org.apache.doris.thrift.TFileScanRangeParams; +import org.apache.doris.thrift.TTableFormatFileDesc; + +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.client.metadata.KvSnapshots; +import org.apache.fluss.client.metadata.LakeSnapshot; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.exception.LakeTableSnapshotNotExistException; +import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +/** + * Turns a fluss table into the scan ranges that cover it. + * + *

A log table is read one bucket at a time, from the earliest offset fluss still holds up to the + * offset the log had reached when planning ran. That stopping offset is taken once, for all of a + * partition's buckets, so every bucket of a partition stops at the same view of the table; without it + * a bucket planned later would read rows written after the query started while an earlier bucket did + * not. A bucket that has never been written to (stopping offset 0) yields no range at all. + * + *

A primary-key table cannot be read that way: its log is a change log, so replaying it verbatim + * returns superseded and deleted rows. Each bucket is read instead as its latest kv snapshot plus the + * change log that followed, which the scanner merges by key. A bucket fluss has never snapshotted is + * rebuilt by replaying its whole change log, which is equally correct and only slower. + * + *

Planning only reads metadata — partition lists, offsets and snapshot ids — so it is safe to run + * for an {@code EXPLAIN}, which does reach {@code planScan}. (There is no explain-only signal on the + * SPI in this branch; the point is that fluss does not need one. A future change that takes a snapshot + * lease during planning would.) + * + *

A table that is tiered into a lake is read as the union of the two: the lake at the snapshot fluss + * says is readable, plus each bucket's log from where that snapshot ended. The lake half is not planned + * here — it is planned by the paimon sibling connector, pinned to that snapshot, and its ranges are mixed + * into the same scan node (BE builds a reader per range, so the two kinds coexist). That keeps this plugin + * free of any paimon dependency and gives the lake half paimon's native readers, deletion vectors and file + * cache for free. Refusing to serve such a table at all is the alternative to avoid: a datalake table read + * as fluss-only silently returns just the rows tiering has not moved yet, which looks like a working + * query. {@code fluss.union_read.mode=disabled} is how a user asks for that fluss-only read on purpose. + * + *

A tiered PRIMARY-KEY table is read as a union too, but its halves overlap by KEY rather than meeting + * at an offset: the log tail carries updates and deletes of rows the lake already holds. It is split into + * three parts, per bucket. The lake splits of a bucket whose log has moved on are wrapped + * ({@link FlussSuppressedLakeRange}) with the offsets of that tail, and BE drops the lake rows whose keys + * the tail names. The surviving state of the tail itself is contributed once, by a + * {@code PK_TAIL} range. A bucket the lake has never seen is read whole from fluss, as {@code PK_FULL}, + * exactly as it would be without a lake. Every row is therefore produced exactly once, and the read + * matches what fluss alone would return — which is what makes the fluss-only read the reference the + * regression tests compare against. + * + *

Falling back to that fluss-only read is always safe for a primary-key table, in a way it is not for + * a log table: fluss keeps such a table's state in full, so reading it alone returns the whole table, + * only slower than reading the lake's columnar files would be. That is why {@code auto} answers every + * question it cannot answer well — a key column it cannot compare exactly, a tail the log no longer + * holds — by reading fluss alone rather than by failing, and why {@code required} answers the same + * questions with an error: that mode exists to make "did this actually read the lake?" answerable in a + * test. + */ +public class FlussScanPlanProvider implements ConnectorScanPlanProvider { + + /** + * Prefix marking a node property that belongs to BE rather than to the engine. Everything under it + * is copied verbatim into {@code TFileScanRangeParams.fluss_properties}; everything else (the + * engine's own keys, e.g. the partition keys) is not. + */ + static final String BE_PROPERTY_PREFIX = "fluss."; + + /** + * Prefix for the fluss client configuration. The scanner strips it and hands the rest to fluss's + * {@code Configuration}, so the connector never has to enumerate fluss's own option names. + */ + static final String PROP_CLIENT_PREFIX = "fluss.client."; + + static final String PROP_DB_NAME = "fluss.db_name"; + static final String PROP_TABLE_NAME = "fluss.table_name"; + + /** + * Node property naming the key columns BE compares when it suppresses lake rows: the primary key + * minus the partition columns, in Doris's own column names. Only the names — the types travel the + * ordinary way, as the slot descriptors of columns that {@link #getMustReadColumns} kept in the + * scan's tuple. + */ + static final String PROP_UNION_PK_NAMES = "fluss.union.pk_names"; + + /** Node property carrying {@link FlussConnectorProperties#UNION_READ_MAX_TAIL_ROWS} to both readers. */ + static final String PROP_UNION_MAX_TAIL_ROWS = "fluss.union.max_tail_rows"; + + /** + * The property a lake split carries to say which bucket it holds. Written by the paimon connector + * (from {@code DataSplit.bucket()}) for this connector's sake, and part of the contract between the + * two: a fluss table's lake table is bucketed identically, so a lake split can be matched with the + * log tail of the SAME bucket. Its absence is never treated as "this bucket has no tail" — that would + * turn a version mismatch into duplicated rows — see {@link #lakeSplitBucket}. + */ + private static final String LAKE_BUCKET_PROPERTY = "paimon.bucket"; + + /** The only lake format that can be delegated today; fluss also defines iceberg / lance / hudi. */ + private static final String PAIMON_LAKE_FORMAT = "paimon"; + + private final FlussAdminOps adminOps; + private final Map catalogProperties; + private final Function, Connector> lakeSiblingFactory; + + /** + * What the last {@link #planScan} produced, for the EXPLAIN line. Plain fields, not volatile: the + * engine memoizes one provider instance per scan node and plans that node on the FE planning + * thread, then renders EXPLAIN from the same thread afterwards. This connector declares neither + * batch scan nor streaming splits, which are what would move planning off that thread — enabling + * either means revisiting this (the ES provider carries the same caveat). + */ + private int plannedLogRanges; + private int plannedPkRanges; + private int plannedPkTailRanges; + private int plannedLakeSplits; + private int plannedSuppressedLakeSplits; + private boolean plannedUnionRead; + + /** + * This scan node's lake half, resolved at most once (see {@link #resolveUnionRead}). Same threading + * argument as the counters above; {@code unionResolved} distinguishes "not asked yet" from "asked, and + * this table has no lake half". + */ + private boolean unionResolved; + private UnionRead unionRead; + + /** + * Why this scan gave up its lake half, or null when it did not. Only {@code auto} can get here — the + * same conditions are errors under {@code required} — and the plan that results is the fluss-only read + * {@code disabled} would have produced, which for a primary-key table is the whole table. It shows up + * in EXPLAIN because that is otherwise the only difference between "there is no lake to read" and + * "there is one and this query could not use it". + */ + private String degradedReason; + + /** {@link #degradedReason} when the log no longer holds the tail the lake snapshot stops before. */ + private static final String DEGRADED_TAIL_TRUNCATED = "tail-truncated"; + + /** {@link #degradedReason} when a key column's values cannot be compared exactly across the halves. */ + private static final String DEGRADED_KEY_TYPE = "key-type"; + + /** {@link #degradedReason} when a partition column's values do not render the same way on both sides. */ + private static final String DEGRADED_PARTITION_TYPE = "partition-type"; + + public FlussScanPlanProvider(FlussAdminOps adminOps, Map catalogProperties, + Function, Connector> lakeSiblingFactory) { + this.adminOps = adminOps; + this.catalogProperties = catalogProperties; + this.lakeSiblingFactory = lakeSiblingFactory; + } + + /** + * The lake half of a union read: the sibling connector that owns it, its scan planner, its table handle + * already pinned to {@link #snapshotId}, and where that snapshot left each bucket's log. + */ + private static final class UnionRead { + private final Connector sibling; + private final ConnectorScanPlanProvider siblingProvider; + private final ConnectorTableHandle pinnedLakeHandle; + private final long snapshotId; + private final Map logOffsets; + + private UnionRead(Connector sibling, ConnectorScanPlanProvider siblingProvider, + ConnectorTableHandle pinnedLakeHandle, long snapshotId, + Map logOffsets) { + this.sibling = sibling; + this.siblingProvider = siblingProvider; + this.pinnedLakeHandle = pinnedLakeHandle; + this.snapshotId = snapshotId; + this.logOffsets = logOffsets; + } + } + + @Override + public List planScan(ConnectorSession session, ConnectorScanRequest request) { + FlussTableHandle handle = (FlussTableHandle) request.getTableHandle(); + UnionRead union = resolveUnionRead(session, handle); + + List buckets = allBuckets(handle.getBucketCount()); + List ranges = union != null && handle.hasPrimaryKey() + ? planPrimaryKeyUnion(session, handle, union, buckets, request) + : planWithoutKeyMerging(session, handle, union, buckets, request); + + plannedLogRanges = count(ranges, FlussScanRange.RangeType.LOG); + plannedPkRanges = count(ranges, FlussScanRange.RangeType.PK_FULL); + plannedPkTailRanges = count(ranges, FlussScanRange.RangeType.PK_TAIL); + plannedLakeSplits = countLakeSplits(ranges); + plannedSuppressedLakeSplits = countSuppressedLakeSplits(ranges); + // Read back from the field, not from the local: a primary-key plan may have given up its lake half + // half-way through, and EXPLAIN has to say what was actually planned. + plannedUnionRead = unionRead != null; + return ranges; + } + + /** + * A log table, or any table read from fluss alone: every bucket end to end, with the lake half — when + * there is one — prepended. + */ + private List planWithoutKeyMerging(ConnectorSession session, + FlussTableHandle handle, UnionRead union, List buckets, + ConnectorScanRequest request) { + List ranges = new ArrayList<>(); + // The lake half first, so its ranges lead the list the way they lead the table's history. It is + // planned once for the whole table: the sibling prunes partitions from the pushed-down filter, not + // from the engine's pruned partition list (which it does not consume). + if (union != null) { + ranges.addAll(planLakeRanges(session, union, request)); + } + + if (handle.isPartitioned()) { + for (PartitionInfo partition : selectedPartitions(handle, request.getRequiredPartitions())) { + // fluss's own partition name ("20260101$cn"), not the Doris one: this is a fluss API. + appendPartitionRanges(ranges, handle, union, + FlussPartitions.toScanPartition(partition, handle.getPartitionKeys()), + buckets, partition.getPartitionName()); + } + } else { + appendPartitionRanges(ranges, handle, union, FlussScanRange.Partition.NONE, buckets, null); + } + return ranges; + } + + /** + * A primary-key table read as its lake plus the log written since: lake splits bound to the tail of + * their own bucket, plus one range per bucket for what the lake does not hold. + * + *

The offsets are read BEFORE the lake half is planned, and that ordering carries the last guard of + * the design (D17): a tail the log no longer holds cannot be read at all, and a primary-key table + * cannot fetch it from the lake instead. Finding that out after asking the sibling to plan would mean + * throwing its work away; finding it out here means the fluss-only read that replaces it is planned + * from the very same offsets, and is the plan {@code disabled} would have produced. + */ + private List planPrimaryKeyUnion(ConnectorSession session, + FlussTableHandle handle, UnionRead union, List buckets, + ConnectorScanRequest request) { + List states = new ArrayList<>(); + if (handle.isPartitioned()) { + for (PartitionInfo partition : selectedPartitions(handle, request.getRequiredPartitions())) { + states.add(readPartitionState(handle, union, + FlussPartitions.toScanPartition(partition, handle.getPartitionKeys()), + buckets, partition.getPartitionName())); + } + } else { + states.add(readPartitionState(handle, union, FlussScanRange.Partition.NONE, buckets, null)); + } + + String truncated = firstTruncatedTail(states, buckets); + if (truncated != null) { + if (FlussConnectorProperties.unionReadMode(catalogProperties) + == FlussConnectorProperties.UnionReadMode.REQUIRED) { + throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." + + handle.getTableName() + "' cannot be read as its lake plus its log: " + truncated + + ". Fluss has already deleted part of the log the lake snapshot stops before, and a" + + " primary-key table's log cannot be re-read from the lake. Set '" + + FlussConnectorProperties.UNION_READ_MODE + "' to auto or disabled to read the" + + " table from fluss alone, which still returns every row."); + } + degradeToFlussOnly(DEGRADED_TAIL_TRUNCATED); + return pkRangesFromFlussAlone(states, buckets); + } + + List ranges = new ArrayList<>(); + for (ConnectorScanRange lakeSplit : planLakeRanges(session, union, request)) { + ranges.add(bindTailToLakeSplit(handle, lakeSplit, states)); + } + for (PartitionState state : states) { + for (int bucket : buckets) { + BucketState bucketState = state.buckets.get(bucket); + if (bucketState.lakeEnd == null) { + // Never tiered: the lake holds nothing of this bucket, so it is read exactly as it + // would be with no lake at all, and no lake split of it can be suppressed. + appendPkFullRange(ranges, state, bucket); + } else if (bucketState.lakeEnd < bucketState.stop) { + ranges.add(FlussScanRange.pkTail(state.partition, bucket, + bucketState.lakeEnd, bucketState.stop)); + } + // lakeEnd == stop: the lake holds this bucket entirely and fluss adds nothing. + } + } + return ranges; + } + + /** What planning read about one partition of a primary-key table, or about an unpartitioned one. */ + private static final class PartitionState { + private final FlussScanRange.Partition partition; + private final KvSnapshots snapshots; + private final Map buckets; + + private PartitionState(FlussScanRange.Partition partition, KvSnapshots snapshots, + Map buckets) { + this.partition = partition; + this.snapshots = snapshots; + this.buckets = buckets; + } + } + + /** Where one bucket's log begins, ends, and how much of it the lake already holds. */ + private static final class BucketState { + /** First offset NOT in the lake, or null when the lake snapshot does not mention this bucket. */ + private final Long lakeEnd; + /** Where planning saw the log end; 0 for a bucket nothing has ever been written to. */ + private final long stop; + /** The earliest offset fluss still holds, or null when it did not answer for this bucket. */ + private final Long earliest; + + private BucketState(Long lakeEnd, long stop, Long earliest) { + this.lakeEnd = lakeEnd; + this.stop = stop; + this.earliest = earliest; + } + } + + /** + * Everything planning needs about one partition, read in the one order that is safe: kv snapshots, + * then the offsets that bound them, then the earliest offsets the guard compares against. + * + *

The kv snapshots are read even when every bucket turns out to be tiered and none of them is used. + * That is deliberate: giving up the lake half is decided AFTER the offsets are in hand, and the + * fluss-only plan that replaces it needs the snapshots — read at that point they would be newer than + * the offsets that bound them, which is the one ordering this planner exists to prevent. + */ + private PartitionState readPartitionState(FlussTableHandle handle, UnionRead union, + FlussScanRange.Partition partition, List buckets, String flussPartitionName) { + TablePath tablePath = handle.toTablePath(); + KvSnapshots snapshots = latestKvSnapshots(tablePath, flussPartitionName); + Map stopping = latestOffsets(tablePath, flussPartitionName, buckets); + // Last, and that order is what makes it conservative: the earliest offset only moves forward, so + // one read later can only make the guard stricter, never let a truncated tail through. + Map earliest = earliestOffsets(tablePath, flussPartitionName, buckets); + + Map byBucket = new LinkedHashMap<>(); + for (int bucket : buckets) { + TableBucket tableBucket = partition.isPartitioned() + ? new TableBucket(handle.getTableId(), partition.getId(), bucket) + : new TableBucket(handle.getTableId(), bucket); + Long stop = stopping.get(bucket); + byBucket.put(bucket, new BucketState(union.logOffsets.get(tableBucket), + stop == null ? 0L : stop, earliest.get(bucket))); + } + return new PartitionState(partition, snapshots, byBucket); + } + + /** + * The first bucket whose tail fluss can no longer serve, described for an error message, or null when + * every tail is intact. + * + *

A tail is the log from where the lake snapshot ended up to where planning saw the log. Fluss + * deletes old log segments on a timer that does not wait for tiering (only for the kv snapshot), so on + * a cluster without remote log storage the beginning of that tail can be gone. Reading it anyway + * returns fewer rows than the table holds, and unlike a log table there is nowhere else to get them + * from — the lake's copy stops exactly where the missing tail begins. + * + *

A bucket fluss did not answer for at all counts as truncated: the check is the only thing + * standing between a deleted tail and a silently short answer, and "could not verify" is not "fine". + */ + private static String firstTruncatedTail(List states, List buckets) { + for (PartitionState state : states) { + for (int bucket : buckets) { + BucketState bucketState = state.buckets.get(bucket); + if (bucketState.lakeEnd == null || bucketState.lakeEnd >= bucketState.stop) { + continue; + } + String where = (state.partition.isPartitioned() + ? "partition '" + state.partition.getName() + "', " : "") + "bucket " + bucket; + if (bucketState.earliest == null) { + return where + " (fluss did not report an earliest offset for it)"; + } + if (bucketState.earliest > bucketState.lakeEnd) { + return where + " (the lake ends at offset " + bucketState.lakeEnd + + ", but the log now starts at " + bucketState.earliest + ")"; + } + } + } + return null; + } + + /** Every bucket read whole from fluss: the plan {@code disabled} produces, and the safe fallback. */ + private static List pkRangesFromFlussAlone(List states, + List buckets) { + List ranges = new ArrayList<>(); + for (PartitionState state : states) { + for (int bucket : buckets) { + appendPkFullRange(ranges, state, bucket); + } + } + return ranges; + } + + /** + * Gives up this scan's lake half for {@code reason}. Everything asked afterwards — the node + * properties, the scan-level params, EXPLAIN — then answers as a fluss-only read, because the field + * the answers come from is the one being cleared here. + * + *

One question was asked earlier and cannot be taken back: {@link #getMustReadColumns}, at plan + * translation time, may already have kept the key columns in the scan's tuple. That is harmless in + * this direction — they are read and then dropped by the projection above the scan — and it is the + * reason this guard is allowed to run so late. The opposite order would not be harmless, which is why + * the conditions that CAN be decided before translation (a key column's type) are decided there. + */ + private void degradeToFlussOnly(String reason) { + degradedReason = reason; + unionRead = null; + } + + /** + * The lake split, wrapped with the log tail of its own bucket when that tail holds anything. + * + *

Which bucket a split belongs to is the sibling's answer ({@code paimon.bucket}); which partition + * it belongs to is compared by the rendered partition values, which is sound because a union read of a + * partitioned table is refused unless the partition columns are strings. Three things can go wrong, + * and none of them may be waved through: a split with no bucket at all means the paimon connector does + * not write the property this contract is built on; a bucket number this table does not have means the + * two tables are not bucketed alike; a bucket the lake holds files for but fluss records no tiering + * offset for means their metadata disagrees. Each would silently duplicate rows. + */ + private ConnectorScanRange bindTailToLakeSplit(FlussTableHandle handle, ConnectorScanRange lakeSplit, + List states) { + PartitionState state = matchingPartition(lakeSplit, states); + if (state == null) { + // A partition of the lake that this scan does not read from fluss: either one fluss has since + // dropped, or one the engine pruned away. Nothing of it can be superseded by a tail this plan + // does not read, so the split is passed through untouched. + return lakeSplit; + } + int bucket = lakeSplitBucket(handle, lakeSplit); + BucketState bucketState = state.buckets.get(bucket); + if (bucketState == null) { + throw new DorisConnectorException("The lake table of fluss table '" + handle.getDatabaseName() + + "." + handle.getTableName() + "' has a split in bucket " + bucket + ", but the fluss" + + " table has only " + handle.getBucketCount() + " buckets. The two are not bucketed" + + " alike, so their rows cannot be matched by bucket"); + } + if (bucketState.lakeEnd == null) { + throw new DorisConnectorException("The lake table of fluss table '" + handle.getDatabaseName() + + "." + handle.getTableName() + "' holds data for bucket " + bucket + + (state.partition.isPartitioned() + ? " of partition '" + state.partition.getName() + "'" : "") + + ", but fluss records no tiering offset for that bucket. Their metadata disagrees;" + + " reading them as one would return the rows of that bucket twice"); + } + if (bucketState.lakeEnd >= bucketState.stop) { + // The lake holds this bucket up to where its log ends: nothing can supersede it. + return lakeSplit; + } + return new FlussSuppressedLakeRange(lakeSplit, new FlussSuppressedLakeRange.Tail( + state.partition, bucket, bucketState.lakeEnd, bucketState.stop)); + } + + /** + * The partition this scan planned that {@code lakeSplit} belongs to, or null when it planned none. + * Matched on the partition values as a whole rather than on a rendered name, so that there is exactly + * one place in this connector that turns partition values into a name (see {@link FlussPartitions}). + */ + private static PartitionState matchingPartition(ConnectorScanRange lakeSplit, + List states) { + Map values = lakeSplit.getPartitionValues(); + for (PartitionState state : states) { + if (state.partition.getValues().equals(values)) { + return state; + } + } + return null; + } + + /** The bucket a lake split holds, per the contract in {@link #LAKE_BUCKET_PROPERTY}. */ + private static int lakeSplitBucket(FlussTableHandle handle, ConnectorScanRange lakeSplit) { + String bucket = lakeSplit.getProperties().get(LAKE_BUCKET_PROPERTY); + if (bucket == null) { + throw new DorisConnectorException("A lake split of fluss table '" + handle.getDatabaseName() + + "." + handle.getTableName() + "' does not say which bucket it holds ('" + + LAKE_BUCKET_PROPERTY + "' is missing). Reading a primary-key table together with its" + + " lake needs it, to bind each split to the log tail of the same bucket; the paimon" + + " connector plugin is older than this fluss connector plugin. Split: " + lakeSplit); + } + try { + return Integer.parseInt(bucket); + } catch (NumberFormatException e) { + throw new DorisConnectorException("A lake split of fluss table '" + handle.getDatabaseName() + + "." + handle.getTableName() + "' reports bucket '" + bucket + "', which is not a" + + " number", e); + } + } + + /** + * The lake half, planned by the sibling connector on the handle already pinned to the readable snapshot. + * + *

Only the filter is carried over. The row limit is deliberately dropped — applying it to one half of + * a union would silently drop rows from the other — and so is the {@code COUNT(*)} signal, because a + * per-range row count is not this table's count once the log half is added. The engine's pruned + * partition list is not passed either: the sibling does not consume it (it re-plans from the filter), + * and pretending otherwise would hide that the two halves prune by different means. + */ + private List planLakeRanges(ConnectorSession session, UnionRead union, + ConnectorScanRequest request) { + List lakeColumns = + lakeColumns(session, union, request.getColumns(), request.getTableHandle()); + ConnectorScanRequest lakeRequest = + ConnectorScanRequest.builder(union.pinnedLakeHandle, lakeColumns) + .filter(request.getFilter()) + .build(); + return LakeSibling.call(union.sibling, + () -> union.siblingProvider.planScan(session, lakeRequest)); + } + + /** + * The sibling's own column handles for the columns this scan reads. + * + *

Needed because the sibling projects by ITS handle type and silently ignores anything else: handing + * it fluss's handles would leave it with no projection at all. The lake table's columns are this table's + * columns plus the three fluss system columns appended at the end ({@code __bucket} / {@code __offset} / + * {@code __timestamp}), so every column asked for here exists there under the same name and the extra + * three are simply never asked for. A column that is missing is a real mismatch between the two schemas + * — the lake table was not created by this fluss table's tiering — and fails loud rather than reading a + * silently narrower row. + */ + private List lakeColumns(ConnectorSession session, UnionRead union, + List columns, ConnectorTableHandle handle) { + Map lakeHandles = LakeSibling.forward(session, union.sibling, + metadata -> metadata.getColumnHandles(session, union.pinnedLakeHandle)); + List mapped = new ArrayList<>(columns.size()); + for (ConnectorColumnHandle column : columns) { + String name = ((FlussColumnHandle) column).getName(); + ConnectorColumnHandle lakeHandle = lakeHandles.get(name); + if (lakeHandle == null) { + FlussTableHandle flussHandle = (FlussTableHandle) handle; + throw new DorisConnectorException("Column '" + name + "' of fluss table '" + + flussHandle.getDatabaseName() + "." + flussHandle.getTableName() + + "' does not exist in its lake table, so the two cannot be read as one"); + } + mapped.add(lakeHandle); + } + return mapped; + } + + /** + * This scan node's lake half, or null when the table is read from fluss alone. Resolved at most once per + * scan node because two entry points need it — {@link #planScan} and {@link #getScanNodeProperties} — + * and the engine may call either first. + * + *

Resolving it once is also what keeps the two halves from overlapping. The lake snapshot is read + * HERE, before {@link #planScan} asks fluss where each bucket's log currently ends; log offsets only + * move forward, so the snapshot can only be at or behind those stopping offsets. Read the other way + * round, a snapshot committed in between would cover rows past the point the log half stops at, and the + * bucket's log range would start after it ends. + */ + private UnionRead resolveUnionRead(ConnectorSession session, FlussTableHandle handle) { + if (unionResolved) { + return unionRead; + } + unionResolved = true; + unionRead = resolveUnionReadUncached(session, handle); + return unionRead; + } + + private UnionRead resolveUnionReadUncached(ConnectorSession session, FlussTableHandle handle) { + FlussConnectorProperties.UnionReadMode mode = + FlussConnectorProperties.unionReadMode(catalogProperties); + if (!handle.isDataLakeEnabled() || mode == FlussConnectorProperties.UnionReadMode.DISABLED) { + // Not a lake table, or the user asked for the fluss-only read explicitly. + return null; + } + if (handle.hasPrimaryKey()) { + // Whether the two halves can be matched by key at all is decided HERE, before the lake snapshot + // is asked for, because it depends only on the table's schema — and because this is the one + // decision that must be the same at plan-translation time (when getMustReadColumns keeps the key + // columns in the scan's tuple) as it is later, when the ranges are planned. Everything that can + // only be known from the offsets is decided in planScan, where giving up is still safe. + String rejection = keyColumnRejection(handle); + String reason = DEGRADED_KEY_TYPE; + if (rejection == null) { + rejection = partitionColumnRejection(handle); + reason = DEGRADED_PARTITION_TYPE; + } + if (rejection != null) { + if (mode == FlussConnectorProperties.UnionReadMode.REQUIRED) { + throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." + + handle.getTableName() + "' cannot be read as its lake plus its change log: " + + rejection + ". Reading it from fluss alone still returns every row, so set '" + + FlussConnectorProperties.UNION_READ_MODE + "' to auto or disabled."); + } + degradedReason = reason; + return null; + } + } + LakeSnapshot snapshot; + try { + snapshot = adminOps.getReadableLakeSnapshot(handle.toTablePath()); + } catch (LakeTableSnapshotNotExistException e) { + if (mode == FlussConnectorProperties.UnionReadMode.REQUIRED) { + throw new DorisConnectorException("Table '" + handle.getDatabaseName() + "." + + handle.getTableName() + "' has no readable lake snapshot yet, and '" + + FlussConnectorProperties.UNION_READ_MODE + "=required' forbids falling back to " + + "a fluss-only read. Wait for the tiering service to commit, or set the property " + + "to auto or disabled.", e); + } + // Nothing is in the lake, so the log holds everything: the fluss-only read is the whole table. + return null; + } + String lakeFormat = handle.getDataLakeFormat(); + if (lakeFormat == null || !PAIMON_LAKE_FORMAT.equalsIgnoreCase(lakeFormat)) { + throw new DorisConnectorException("Cannot read table '" + handle.getDatabaseName() + "." + + handle.getTableName() + "': its table.datalake.format is '" + lakeFormat + + "', and the fluss connector currently supports only '" + PAIMON_LAKE_FORMAT + "'"); + } + + Connector sibling = lakeSiblingFactory.apply( + PaimonSiblingProperties.synthesize(handle.getProperties())); + ConnectorTableHandle lakeHandle = LakeSibling.forward(session, sibling, + metadata -> metadata.getTableHandle( + session, handle.getDatabaseName(), handle.getTableName())) + .orElseThrow(() -> new DorisConnectorException("Fluss reports a readable lake snapshot for '" + + handle.getDatabaseName() + "." + handle.getTableName() + "' but its lake table" + + " does not exist. The lake warehouse and the fluss cluster disagree; check the" + + " table's table.datalake.* settings")); + // The lake half's ranges are planned by the sibling and mixed into this node's range list, where + // they are told apart from fluss's by which connector owns them. Checked at birth; see requireOwned. + LakeSibling.requireOwned(sibling, lakeHandle); + // The pin is expressed in the SPI's own terms — a snapshot id and no connector options — so the + // sibling translates it into whatever its SDK calls a snapshot. Nothing paimon-specific is named + // here. The id needs no mapping either: what fluss records as the lake snapshot IS the id the lake + // returned when tiering committed it. + ConnectorMvccSnapshot pin = ConnectorMvccSnapshot.builder() + .snapshotId(snapshot.getSnapshotId()) + .build(); + ConnectorTableHandle pinnedHandle = LakeSibling.forward(session, sibling, + metadata -> metadata.applySnapshot(session, lakeHandle, pin)); + ConnectorScanPlanProvider siblingProvider = LakeSibling.call(sibling, + () -> sibling.getScanPlanProvider(pinnedHandle)); + return new UnionRead(sibling, siblingProvider, pinnedHandle, snapshot.getSnapshotId(), + snapshot.getTableBucketsOffset()); + } + + /** + * Why this table's rows cannot be matched across the two halves by their keys, or null when they can. + * The key is the primary key minus the partition columns, because a bucket lives inside one partition + * and the partition columns are equal for every row in it. + */ + private static String keyColumnRejection(FlussTableHandle handle) { + for (String column : handle.getPhysicalPrimaryKeys()) { + DataType type = handle.getKeyColumnTypes().get(column); + if (type == null) { + // The handle names a key column the schema it was built from does not have. + return "its primary key names column '" + column + "', which the table does not have"; + } + String rejection = FlussUnionKeyTypes.keyColumnRejection(type); + if (rejection != null) { + return "primary-key column '" + column + "' has type " + type + ", and " + rejection; + } + } + return null; + } + + /** The same question for the partition columns, which decide which bucket a lake split belongs to. */ + private static String partitionColumnRejection(FlussTableHandle handle) { + for (String column : handle.getPartitionKeys()) { + DataType type = handle.getKeyColumnTypes().get(column); + if (type == null) { + return "it is partitioned by column '" + column + "', which the table does not have"; + } + String rejection = FlussUnionKeyTypes.partitionColumnRejection(type); + if (rejection != null) { + return "partition column '" + column + "' has type " + type + ", and " + rejection; + } + } + return null; + } + + /** + * The key columns BE has to read whether or not the query asked for them, so that it can tell which + * lake rows the log tail supersedes. Empty for every other read, which is every read but this one. + * + *

Asked by the engine while it is translating the plan, well before {@link #planScan}, and answered + * from the SAME memoized resolution — that is a correctness requirement, not a saving. If the two + * answers could differ, a scan whose tuple was pruned as a fluss-only read could still be planned as a + * union read, and BE would look for a key column that is not in its projection. + * + *

Only a primary-key table is asked further, so that a log table's scan does not pull the lake + * snapshot's round trip forward into plan translation for nothing. + */ + @Override + public Set getMustReadColumns(ConnectorSession session, ConnectorTableHandle handle) { + FlussTableHandle flussHandle = (FlussTableHandle) handle; + if (!flussHandle.hasPrimaryKey() || resolveUnionRead(session, flussHandle) == null) { + return Collections.emptySet(); + } + return new LinkedHashSet<>(flussHandle.getPhysicalPrimaryKeys()); + } + + /** + * The ranges covering one partition of a table, or the whole of an unpartitioned one + * ({@code flussPartitionName} is null, which is also how the two admin overloads are told apart). + */ + private void appendPartitionRanges(List ranges, FlussTableHandle handle, + UnionRead union, FlussScanRange.Partition partition, List buckets, + String flussPartitionName) { + TablePath tablePath = handle.toTablePath(); + if (!handle.hasPrimaryKey()) { + Map stopping = latestOffsets(tablePath, flussPartitionName, buckets); + if (union == null) { + appendLogRanges(ranges, partition, buckets, stopping); + } else { + appendUnionLogRanges(ranges, handle, union, partition, buckets, stopping); + } + return; + } + // Snapshots BEFORE offsets, and the order is load-bearing. A snapshot committed between the two + // calls ends past the offset planning stopped at; that bucket would then be read from a snapshot + // already containing rows written after the query started, while every other bucket stopped + // where planning saw it. Log offsets only move forward, so asking in this order keeps every + // snapshot at or behind the stopping offset. + KvSnapshots snapshots = latestKvSnapshots(tablePath, flussPartitionName); + appendPkRanges(ranges, partition, buckets, snapshots, + latestOffsets(tablePath, flussPartitionName, buckets)); + } + + private KvSnapshots latestKvSnapshots(TablePath tablePath, String flussPartitionName) { + return flussPartitionName == null + ? adminOps.getLatestKvSnapshots(tablePath) + : adminOps.getLatestKvSnapshots(tablePath, flussPartitionName); + } + + private Map latestOffsets(TablePath tablePath, String flussPartitionName, + List buckets) { + return flussPartitionName == null + ? adminOps.listOffsets(tablePath, buckets, new OffsetSpec.LatestSpec()) + : adminOps.listOffsets(tablePath, flussPartitionName, buckets, new OffsetSpec.LatestSpec()); + } + + /** The earliest offset each bucket's log still holds — what a tail has to still begin at or after. */ + private Map earliestOffsets(TablePath tablePath, String flussPartitionName, + List buckets) { + return flussPartitionName == null + ? adminOps.listOffsets(tablePath, buckets, new OffsetSpec.EarliestSpec()) + : adminOps.listOffsets(tablePath, flussPartitionName, buckets, + new OffsetSpec.EarliestSpec()); + } + + /** + * The partitions to scan: those the engine's pruning left, or all of them when it pruned nothing. + * A pruned name that fluss no longer lists is simply absent from the result — the partition was + * dropped between pruning and planning, and there is nothing left to read. + */ + private List selectedPartitions(FlussTableHandle handle, List requiredPartitions) { + List partitions = adminOps.listPartitionInfos(handle.toTablePath()); + if (requiredPartitions.isEmpty()) { + return partitions; + } + Set required = new HashSet<>(requiredPartitions); + List selected = new ArrayList<>(partitions.size()); + for (PartitionInfo partition : partitions) { + // Matched on the DORIS name, which is what the engine pruned over: FlussPartitions renders + // both this and the metadata listing the engine pruned, so the two cannot disagree. + if (required.contains( + FlussPartitions.toScanPartition(partition, handle.getPartitionKeys()).getName())) { + selected.add(partition); + } + } + return selected; + } + + private static void appendLogRanges(List ranges, + FlussScanRange.Partition partition, List buckets, Map stopping) { + for (int bucket : buckets) { + Long stop = stopping.get(bucket); + if (stop == null || stop <= 0) { + // Never written to: no range at all rather than an empty one for BE to open and close. + // A bucket whose records have all aged out of the log is NOT this case — its latest + // offset stayed where it was — and still gets a range that reads nothing, which costs + // one scanner rather than an extra round trip for every bucket to find out. + continue; + } + ranges.add(FlussScanRange.log(partition, bucket, LogScanner.EARLIEST_OFFSET, stop)); + } + } + + /** + * The log half of a union read: each bucket from where the lake snapshot left off, up to where planning + * saw the log. The snapshot's offset is exclusive — it is the first offset NOT in the lake — so the two + * halves meet exactly, with no row read twice and none skipped. + * + *

A bucket the snapshot does not mention has never been tiered, so its log is read from the earliest + * offset fluss still holds; a bucket whose snapshot offset has caught up with the stopping offset has + * nothing left outside the lake and yields no range at all. + */ + private static void appendUnionLogRanges(List ranges, FlussTableHandle handle, + UnionRead union, FlussScanRange.Partition partition, List buckets, + Map stopping) { + for (int bucket : buckets) { + Long stop = stopping.get(bucket); + if (stop == null || stop <= 0) { + continue; + } + TableBucket tableBucket = partition.isPartitioned() + ? new TableBucket(handle.getTableId(), partition.getId(), bucket) + : new TableBucket(handle.getTableId(), bucket); + Long lakeEnd = union.logOffsets.get(tableBucket); + if (lakeEnd == null) { + ranges.add(FlussScanRange.log(partition, bucket, LogScanner.EARLIEST_OFFSET, stop)); + } else if (lakeEnd < stop) { + ranges.add(FlussScanRange.log(partition, bucket, lakeEnd, stop)); + } + } + } + + /** + * One range per bucket that holds anything: its latest kv snapshot, plus the change log from where + * that snapshot ended up to where planning saw the log. A bucket fluss has never snapshotted gets + * {@code -1} and the earliest sentinel, and its state is rebuilt by replaying the whole change log + * — correct, because a primary-key table's log carries every change, just slower. + */ + private static void appendPkRanges(List ranges, + FlussScanRange.Partition partition, List buckets, KvSnapshots snapshots, + Map stopping) { + for (int bucket : buckets) { + Long stop = stopping.get(bucket); + appendPkFullRange(ranges, partition, bucket, snapshots, stop == null ? 0L : stop); + } + } + + /** The same, for a bucket of a table whose lake half is being read too. */ + private static void appendPkFullRange(List ranges, PartitionState state, + int bucket) { + appendPkFullRange(ranges, state.partition, bucket, state.snapshots, state.buckets.get(bucket).stop); + } + + private static void appendPkFullRange(List ranges, + FlussScanRange.Partition partition, int bucket, KvSnapshots snapshots, long logStop) { + long snapshotId = snapshots.getSnapshotId(bucket).orElse(FlussScanRange.NO_KV_SNAPSHOT); + long logStart = snapshots.getLogOffset(bucket).orElse(LogScanner.EARLIEST_OFFSET); + if (snapshotId == FlussScanRange.NO_KV_SNAPSHOT && logStop <= 0) { + // Nothing snapshotted and nothing logged: the bucket is empty. A bucket WITH a snapshot + // is planned even when its log has caught up, because the snapshot still holds rows. + return; + } + ranges.add(FlussScanRange.pkFull(partition, bucket, snapshotId, logStart, logStop)); + } + + /** + * How many of {@code ranges} are fluss ranges of this kind. The type test is not defensive: on a union + * read the list also holds the lake half's ranges, which are the sibling's own type and would fail a + * cast. + */ + private static int count(List ranges, FlussScanRange.RangeType rangeType) { + int found = 0; + for (ConnectorScanRange range : ranges) { + if (range instanceof FlussScanRange && ((FlussScanRange) range).getRangeType() == rangeType) { + found++; + } + } + return found; + } + + /** Ranges the sibling planned, counted by exclusion: a wrapped one is still one of the lake's. */ + private static int countLakeSplits(List ranges) { + int found = 0; + for (ConnectorScanRange range : ranges) { + if (!(range instanceof FlussScanRange)) { + found++; + } + } + return found; + } + + private static int countSuppressedLakeSplits(List ranges) { + int found = 0; + for (ConnectorScanRange range : ranges) { + if (range instanceof FlussSuppressedLakeRange) { + found++; + } + } + return found; + } + + private static List allBuckets(int bucketCount) { + List buckets = new ArrayList<>(bucketCount); + for (int bucket = 0; bucket < bucketCount; bucket++) { + buckets.add(bucket); + } + return buckets; + } + + /** + * What every range of this scan shares. The {@code fluss.}-prefixed entries are the BE half and are + * forwarded by {@link #populateScanLevelParams}; {@code path_partition_keys} is the engine's, and + * declaring it is what keeps the partition columns out of the scanner's projection so BE + * materializes them from each range instead (see {@link FlussScanRange}). + */ + @Override + public Map getScanNodeProperties(ConnectorSession session, ConnectorTableHandle handle, + List columns, Optional filter) { + FlussTableHandle flussHandle = (FlussTableHandle) handle; + Map props = new LinkedHashMap<>(); + if (flussHandle.isPartitioned()) { + props.put(ScanNodePropertyKeys.PATH_PARTITION_KEYS, + String.join(",", flussHandle.getPartitionKeys())); + } + props.put(PROP_DB_NAME, flussHandle.getDatabaseName()); + props.put(PROP_TABLE_NAME, flussHandle.getTableName()); + FlussConnectorProperties.toFlussClientConfig(catalogProperties) + .forEach((key, value) -> props.put(PROP_CLIENT_PREFIX + key, value)); + + UnionRead union = resolveUnionRead(session, flussHandle); + if (union != null) { + if (flussHandle.hasPrimaryKey()) { + // What BE needs to suppress lake rows by key: which columns the key is made of, and how + // large a tail it may hold in memory while doing so. Both are node-level because both are + // the same for every range of the scan. + props.put(PROP_UNION_PK_NAMES, String.join(",", flussHandle.getPhysicalPrimaryKeys())); + props.put(PROP_UNION_MAX_TAIL_ROWS, + String.valueOf(FlussConnectorProperties.maxTailRows(catalogProperties))); + } + List lakeColumns = lakeColumns(session, union, columns, handle); + mergeLakeProperties(props, LakeSibling.call(union.sibling, + () -> union.siblingProvider.getScanNodeProperties( + session, union.pinnedLakeHandle, lakeColumns, filter))); + } + return props; + } + + /** + * Folds the lake half's node properties into this scan node's, which is what makes one node able to + * serve both kinds of range: the engine calls {@code populateScanLevelParams} once, with one map, and + * the sibling reads its own entries back out of it. + * + *

The two connectors share three keys, and all three must already agree, so a difference is a real + * mismatch and is raised rather than resolved by picking a side. {@code path_partition_keys} is the one + * that matters: the split between file columns and partition columns is decided ONCE for the node, so + * two halves that disagree about which columns come from the range would read different columns from + * the same tuple. They cannot legitimately disagree — a fluss table's lake table is created with its + * partition keys — which is exactly why a disagreement means something is wrong upstream. + */ + private static void mergeLakeProperties(Map props, Map lakeProps) { + for (Map.Entry entry : lakeProps.entrySet()) { + String existing = props.get(entry.getKey()); + if (existing == null) { + props.put(entry.getKey(), entry.getValue()); + } else if (!existing.equals(entry.getValue())) { + throw new DorisConnectorException("The fluss table and its lake table disagree about scan" + + " property '" + entry.getKey() + "' ('" + existing + "' vs '" + entry.getValue() + + "'), so they cannot be read as one"); + } + } + } + + @Override + public void populateScanLevelParams(TFileScanRangeParams params, Map nodeProperties) { + Map beProperties = new LinkedHashMap<>(); + for (Map.Entry entry : nodeProperties.entrySet()) { + // Prefix-gated rather than copied wholesale: the map also carries the engine's own keys, the + // synthetic ones it injects for EXPLAIN and (on a union read) the lake half's, none of which + // mean anything to the scanner. + if (entry.getKey().startsWith(BE_PROPERTY_PREFIX)) { + beProperties.put(entry.getKey(), entry.getValue()); + } + } + params.setFlussProperties(beProperties); + + // The lake half's turn at the same params. Its ranges are useless without it — the paimon reader + // fails outright on a missing serialized table — and only the sibling knows which of the merged + // entries are its own. + UnionRead union = unionRead; + if (union != null) { + LakeSibling.call(union.sibling, () -> { + union.siblingProvider.populateScanLevelParams(params, nodeProperties); + return null; + }); + } + } + + /** + * Forwarded so the VERBOSE EXPLAIN counts the lake half's merge-on-read delete files. A fluss range + * carries none, and the sibling reads them off its own per-range descriptor, so asking it about every + * range is both harmless and the only way to ask without knowing which range is whose. + */ + @Override + public List getDeleteFiles(TTableFormatFileDesc tableFormatParams) { + UnionRead union = unionRead; + if (union == null) { + return Collections.emptyList(); + } + return LakeSibling.call(union.sibling, + () -> union.siblingProvider.getDeleteFiles(tableFormatParams)); + } + + /** + * The line a regression test reads to tell which way a scan was actually planned. {@code auto} + * silently falls back to a fluss-only read, so "did this query read the lake?" is otherwise + * invisible in the plan and a union-read test would pass without having tested anything. Log and + * primary-key ranges are counted apart for the same reason: they are read by different code, and a + * single total cannot tell a primary-key table planned the right way from one planned the wrong way. + */ + @Override + public void appendExplainInfo(StringBuilder output, String prefix, Map nodeProperties) { + output.append(prefix) + .append("flussScan: unionRead=").append(plannedUnionRead ? "yes" : "no") + .append(", lakeSplits=").append(plannedLakeSplits) + .append(", suppressedLakeSplits=").append(plannedSuppressedLakeSplits) + .append(", logRanges=").append(plannedLogRanges) + .append(", pkRanges=").append(plannedPkRanges) + .append(", pkTailRanges=").append(plannedPkTailRanges) + .append(", mode=") + .append(FlussConnectorProperties.unionReadMode(catalogProperties).propertyValue()); + if (degradedReason != null) { + // Only reachable under 'auto'. Without it, a table whose lake this query could not use looks + // exactly like a table that has no lake at all. + output.append(", degraded=").append(degradedReason); + } + output.append("\n"); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanRange.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanRange.java new file mode 100644 index 00000000000000..91e30ff81d8e38 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussScanRange.java @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TTableFormatFileDesc; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * One unit of fluss reading: a single bucket of a single partition, read one of three ways. + * + *

The payload is an untyped string map rather than a thrift struct of its own. The C++ layer holds + * no fluss logic — it hands the map straight to the java scanner in be-java-extensions — so a typed + * struct would buy nothing but a transcription step in the middle (which is what paimon pays for, + * {@code paimon_jni_reader.cpp} rebuilding a string map field by field). {@code es_params} and + * {@code jdbc_params} are the same shape for the same reason. + * + *

Only what VARIES per split lives here. Bootstrap servers, table identity, client and table + * options and the projection are identical for every split of a scan, so they are written once at + * scan-node level ({@code TFileScanRangeParams.fluss_properties}); a table with 100 partitions and + * 128 buckets would otherwise serialize them 12800 times. + * + *

Partition columns are NOT part of what the scanner returns. The connector declares the partition + * keys as {@code path_partition_keys}, which makes the engine leave them out of the scanner's + * projection and materialize them from {@link #getPartitionValues()} instead. That is forced by the + * union read: the lake half of a union comes from paimon ranges, and the paimon connector already + * declares its partition keys that way — the file-slot / partition-slot split is decided once per scan + * node, so both halves have to agree on it. + */ +public class FlussScanRange implements ConnectorScanRange { + + private static final long serialVersionUID = 1L; + + /** + * How the scanner reads this range; the wire value is the enum name. + * + *

These are the kinds the java scanner reads. A union read of a primary-key table puts one more + * value on the wire — the suppression descriptor that rides along with a LAKE split, see + * {@link FlussSuppressedLakeRange} — which never reaches the scanner: BE's C++ side reads it to + * build the key set that hides superseded lake rows. + */ + public enum RangeType { + /** Log-only read of one bucket over {@code [logStart, logStop)}. */ + LOG, + /** Full read of one bucket of a primary-key table: kv snapshot plus the log after it. */ + PK_FULL, + /** + * The log tail of one bucket of a primary-key table, replayed by key into the state it ended in. + * The rows before it are the lake's; this range contributes only what the lake does not hold yet. + */ + PK_TAIL + } + + public static final String PROP_RANGE_TYPE = "fluss.range_type"; + public static final String PROP_PARTITION_ID = "fluss.partition_id"; + public static final String PROP_PARTITION_NAME = "fluss.partition_name"; + public static final String PROP_BUCKET_ID = "fluss.bucket_id"; + public static final String PROP_LOG_START_OFFSET = "fluss.log_start_offset"; + public static final String PROP_LOG_STOP_OFFSET = "fluss.log_stop_offset"; + public static final String PROP_KV_SNAPSHOT_ID = "fluss.kv_snapshot_id"; + + /** {@code kv_snapshot_id} for a bucket that has never been snapshotted. */ + public static final long NO_KV_SNAPSHOT = -1L; + + private final RangeType rangeType; + private final Partition partition; + private final int bucketId; + private final Map properties; + + private FlussScanRange(RangeType rangeType, Partition partition, int bucketId, + Map properties) { + this.rangeType = rangeType; + this.partition = partition; + this.bucketId = bucketId; + this.properties = Collections.unmodifiableMap(properties); + } + + /** + * A log range over {@code [logStartOffset, logStopOffset)} of one bucket. {@code logStartOffset} is + * either a real offset or fluss's {@code LogScanner.EARLIEST_OFFSET} sentinel, passed through + * verbatim — the scanner hands it back to fluss, which is the only side that interprets it. + */ + public static FlussScanRange log(Partition partition, int bucketId, + long logStartOffset, long logStopOffset) { + Map props = baseProps(RangeType.LOG, partition, bucketId, + logStartOffset, logStopOffset); + return new FlussScanRange(RangeType.LOG, partition, bucketId, props); + } + + /** + * A full primary-key range: the kv snapshot {@code kvSnapshotId} plus the log from + * {@code logStartOffset} (the offset that snapshot was taken at) up to {@code logStopOffset}. + * {@code kvSnapshotId} is {@link #NO_KV_SNAPSHOT} when the bucket has never been snapshotted, in + * which case the whole state is replayed from the log. + */ + public static FlussScanRange pkFull(Partition partition, int bucketId, long kvSnapshotId, + long logStartOffset, long logStopOffset) { + Map props = baseProps(RangeType.PK_FULL, partition, bucketId, + logStartOffset, logStopOffset); + props.put(PROP_KV_SNAPSHOT_ID, String.valueOf(kvSnapshotId)); + return new FlussScanRange(RangeType.PK_FULL, partition, bucketId, props); + } + + /** + * The log tail of one bucket of a primary-key table: the change log over + * {@code [logStartOffset, logStopOffset)}, which the scanner replays by key and returns as the state + * that range ended in. {@code logStartOffset} is where the lake snapshot this tail follows left off, + * so the lake half holds everything before it. + * + *

An empty range is refused rather than planned: a bucket whose lake has caught up with its log + * contributes nothing, and the caller decides that by not planning a range at all. Reaching here + * with one would mean the two halves were bounded by different offsets. + */ + public static FlussScanRange pkTail(Partition partition, int bucketId, + long logStartOffset, long logStopOffset) { + if (logStartOffset >= logStopOffset) { + throw new IllegalArgumentException("a PK_TAIL range must read something, but bucket " + bucketId + + " was given [" + logStartOffset + ", " + logStopOffset + ")"); + } + Map props = baseProps(RangeType.PK_TAIL, partition, bucketId, + logStartOffset, logStopOffset); + return new FlussScanRange(RangeType.PK_TAIL, partition, bucketId, props); + } + + private static Map baseProps(RangeType rangeType, Partition partition, + int bucketId, long logStartOffset, long logStopOffset) { + Objects.requireNonNull(partition, "partition"); + Map props = new LinkedHashMap<>(); + props.put(PROP_RANGE_TYPE, rangeType.name()); + if (partition.isPartitioned()) { + props.put(PROP_PARTITION_ID, String.valueOf(partition.id)); + props.put(PROP_PARTITION_NAME, partition.name); + } + props.put(PROP_BUCKET_ID, String.valueOf(bucketId)); + props.put(PROP_LOG_START_OFFSET, String.valueOf(logStartOffset)); + props.put(PROP_LOG_STOP_OFFSET, String.valueOf(logStopOffset)); + return props; + } + + public RangeType getRangeType() { + return rangeType; + } + + public Partition getPartition() { + return partition; + } + + @Override + public String getTableFormatType() { + return "fluss"; + } + + @Override + public Map getProperties() { + return properties; + } + + @Override + public Map getPartitionValues() { + return partition.values; + } + + /** + * Always true: a fluss range's partition values come from fluss metadata, never from a file path. + * On an unpartitioned table that keeps the engine from falling back to parsing a path that does not + * exist (the range has no path at all). + */ + @Override + public boolean isPartitionBearing() { + return true; + } + + @Override + public void populateRangeParams(TTableFormatFileDesc formatDesc, TFileRangeDesc rangeDesc) { + formatDesc.setFlussParams(new LinkedHashMap<>(properties)); + Map partitionValues = partition.values; + if (!partitionValues.isEmpty()) { + List keys = new ArrayList<>(partitionValues.size()); + List values = new ArrayList<>(partitionValues.size()); + List isNull = new ArrayList<>(partitionValues.size()); + for (Map.Entry entry : partitionValues.entrySet()) { + keys.add(entry.getKey()); + values.add(entry.getValue()); + // Never null: fluss rejects a null partition value at write time + // (PartitionGetter.getResolvedPartitionSpec), so there is no null sentinel to decode. + isNull.add(false); + } + rangeDesc.setColumnsFromPathKeys(keys); + rangeDesc.setColumnsFromPath(values); + rangeDesc.setColumnsFromPathIsNull(isNull); + } + } + + @Override + public String toString() { + return "FlussScanRange{" + rangeType + + (partition.isPartitioned() ? ", partition=" + partition.name : "") + + ", bucket=" + bucketId + + ", " + properties + "}"; + } + + /** + * Which partition a range belongs to, as the three facts that must agree: fluss's partition id + * (what the scanner subscribes by), its Hive-style {@code k=v/k=v} name (what the user sees) and + * the per-column values (what the engine materializes the partition columns from). Bundled so an + * unpartitioned table cannot be described half-way — {@link #NONE} is the only way to say "none". + */ + public static final class Partition implements java.io.Serializable { + + private static final long serialVersionUID = 1L; + + /** The single pseudo-partition of an unpartitioned table. */ + public static final Partition NONE = new Partition(-1L, null, Collections.emptyMap()); + + private final long id; + private final String name; + private final Map values; + + private Partition(long id, String name, Map values) { + this.id = id; + this.name = name; + this.values = values; + } + + /** + * @param id fluss's partition id + * @param name the Hive-style {@code k=v/k=v} partition name Doris uses + * @param values partition column name to value, in partition-key order + */ + public static Partition of(long id, String name, Map values) { + Objects.requireNonNull(name, "name"); + Objects.requireNonNull(values, "values"); + if (values.isEmpty()) { + throw new IllegalArgumentException( + "partition " + name + " has no column values; use Partition.NONE for an " + + "unpartitioned table"); + } + return new Partition(id, name, Collections.unmodifiableMap(new LinkedHashMap<>(values))); + } + + /** + * Whether this is a real partition rather than {@link #NONE}. Asked of the field, not of + * object identity: a range crosses a java-serialization boundary, which would hand back a + * copy of {@code NONE} that {@code ==} no longer recognizes. + */ + public boolean isPartitioned() { + return name != null; + } + + public long getId() { + return id; + } + + public String getName() { + return name; + } + + public Map getValues() { + return values; + } + + @Override + public String toString() { + return this == NONE ? "Partition.NONE" : name + "(" + id + ")"; + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussStatementScope.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussStatementScope.java new file mode 100644 index 00000000000000..6d0c09839019ca --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussStatementScope.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorStatementScopes; + +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; + +import java.util.function.Supplier; + +/** + * Per-statement sharing of one table's {@link TableInfo}. + * + *

A single statement asks for the same table's metadata several times over — the handle, the + * schema, the column handles, then split planning — and each of those is a coordinator round trip on + * its own. Routing them through the statement scope collapses them to one fetch, and it also makes the + * statement self-consistent: without it, a concurrent ALTER could land between two of those calls and + * leave the plan built from two different schema versions. + * + *

Under a {@code null} session or a statement scope of {@code NONE} (offline, no live statement) + * the loader simply runs every time, which is what an untracked call did before. + */ +final class FlussStatementScope { + + /** + * Namespace for fluss's per-statement {@link TableInfo} memo. Prefixed with this connector's type + * name ("fluss") per the {@link ConnectorStatementScopes} convention, so a gateway statement + * spanning two connectors cannot hand one of them the other's value. + */ + static final String TABLE_INFO_NAMESPACE = "fluss.table_info"; + + private FlussStatementScope() { + } + + static TableInfo sharedTableInfo(ConnectorSession session, TablePath tablePath, + Supplier loader) { + return ConnectorStatementScopes.resolveInStatement( + session, TABLE_INFO_NAMESPACE, + tablePath.getDatabaseName(), tablePath.getTableName(), loader); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussSuppressedLakeRange.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussSuppressedLakeRange.java new file mode 100644 index 00000000000000..e9ddf4e8bbc5e7 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussSuppressedLakeRange.java @@ -0,0 +1,232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TTableFormatFileDesc; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * A lake split of a primary-key table, with the log tail that supersedes part of it. + * + *

A tiered primary-key table is read as its lake plus the change log written since the lake snapshot. + * The two halves overlap by key, not by row: a row the lake holds may since have been updated or deleted + * in the log, and returning both copies is wrong in a way no row count reveals. This range is the lake + * half of that arrangement — the sibling connector's own split, unchanged in every respect that decides + * how it is read and scheduled — carrying the one extra fact BE needs: which slice of which bucket's log + * may contradict it. BE reads that slice's keys once per bucket and drops the lake rows they name; the + * surviving state of the tail is contributed separately, by a {@link FlussScanRange.RangeType#PK_TAIL} + * range, so each row is produced exactly once. + * + *

Delegation is the whole design. Everything except the format name and the extra payload is + * the sibling's answer, because everything else is what makes the lake half worth having: its file path + * and byte extent (native columnar readers), its split weight and target size (scheduling), its deletion + * vectors, its schema id, its partition values. Overriding any of them here would silently reshape the + * sibling's own read. That is also why this class must not grow a "smarter" version of any delegated + * method — the one thing it knows better than the sibling is the tail. + */ +public class FlussSuppressedLakeRange implements ConnectorScanRange { + + private static final long serialVersionUID = 1L; + + /** + * The {@code table_format_type} BE dispatches on. It is neither {@code fluss} nor {@code paimon}: the + * range is read by the sibling's reader stack WRAPPED in the suppression, and BE picks that composite + * reader by this name. Both plain names would land on a reader that ignores half the payload. + */ + public static final String TABLE_FORMAT_TYPE = "fluss_union"; + + /** {@code fluss.range_type} of a wrapped lake split; read by BE's C++ side, never by the scanner. */ + public static final String RANGE_TYPE_LAKE_SUPPRESS = "LAKE_SUPPRESS"; + + /** The log slice whose keys suppress rows of this split, as {@code partitionId:bucket:start:stop}. */ + public static final String PROP_TAIL = "fluss.union.tail"; + + private final ConnectorScanRange inner; + private final Tail tail; + + public FlussSuppressedLakeRange(ConnectorScanRange inner, Tail tail) { + this.inner = Objects.requireNonNull(inner, "inner"); + this.tail = Objects.requireNonNull(tail, "tail"); + } + + /** The wrapped sibling range, for a caller that has to look at what the lake half actually is. */ + public ConnectorScanRange getInner() { + return inner; + } + + public Tail getTail() { + return tail; + } + + /** + * One bucket's log tail: the half-open offset range that the lake snapshot does not cover yet. + * + *

Its encoding is also its identity on the BE side — the same string keys the cache that holds the + * suppression keys, so every split of the same bucket shares one read of the tail. Partition id is + * empty for an unpartitioned table rather than a sentinel number, so that the key of an unpartitioned + * table's bucket cannot collide with a partitioned one's. + */ + public static final class Tail implements java.io.Serializable { + + private static final long serialVersionUID = 1L; + + private final FlussScanRange.Partition partition; + private final int bucketId; + private final long startOffset; + private final long stopOffset; + + public Tail(FlussScanRange.Partition partition, int bucketId, long startOffset, long stopOffset) { + this.partition = Objects.requireNonNull(partition, "partition"); + if (startOffset >= stopOffset) { + throw new IllegalArgumentException("a suppressing tail must contain something, but bucket " + + bucketId + " was given [" + startOffset + ", " + stopOffset + ")"); + } + this.bucketId = bucketId; + this.startOffset = startOffset; + this.stopOffset = stopOffset; + } + + public int getBucketId() { + return bucketId; + } + + public long getStartOffset() { + return startOffset; + } + + public long getStopOffset() { + return stopOffset; + } + + String encode() { + return (partition.isPartitioned() ? String.valueOf(partition.getId()) : "") + + ":" + bucketId + ":" + startOffset + ":" + stopOffset; + } + + @Override + public String toString() { + return "Tail{" + encode() + "}"; + } + } + + // ------------------------------------------------------------------ the two things that differ + + @Override + public String getTableFormatType() { + return TABLE_FORMAT_TYPE; + } + + /** + * The sibling fills its own payload first — serialized split or schema id and deletion files, file + * format, columns from path — and the tail is appended beside it. The two ride in different fields of + * the same descriptor ({@code paimon_params} and {@code fluss_params}), so neither has to know about + * the other. + */ + @Override + public void populateRangeParams(TTableFormatFileDesc formatDesc, TFileRangeDesc rangeDesc) { + inner.populateRangeParams(formatDesc, rangeDesc); + Map params = new LinkedHashMap<>(); + params.put(FlussScanRange.PROP_RANGE_TYPE, RANGE_TYPE_LAKE_SUPPRESS); + params.put(PROP_TAIL, tail.encode()); + formatDesc.setFlussParams(params); + } + + // ------------------------------------------------------------------ everything else is the sibling's + + @Override + public Optional getPath() { + return inner.getPath(); + } + + @Override + public long getStart() { + return inner.getStart(); + } + + @Override + public long getLength() { + return inner.getLength(); + } + + @Override + public long getFileSize() { + return inner.getFileSize(); + } + + @Override + public String getFileFormat() { + return inner.getFileFormat(); + } + + @Override + public long getModificationTime() { + return inner.getModificationTime(); + } + + @Override + public long getSelfSplitWeight() { + return inner.getSelfSplitWeight(); + } + + @Override + public long getTargetSplitSize() { + return inner.getTargetSplitSize(); + } + + @Override + public List getHosts() { + return inner.getHosts(); + } + + @Override + public Map getProperties() { + return inner.getProperties(); + } + + @Override + public Map getPartitionValues() { + return inner.getPartitionValues(); + } + + @Override + public boolean isPartitionBearing() { + return inner.isPartitionBearing(); + } + + @Override + public long getPushDownRowCount() { + return inner.getPushDownRowCount(); + } + + @Override + public boolean isNativeReadRange() { + return inner.isNativeReadRange(); + } + + @Override + public String toString() { + return "FlussSuppressedLakeRange{" + tail + ", " + inner + "}"; + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTableHandle.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTableHandle.java new file mode 100644 index 00000000000000..c4513c97e8a1c8 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTableHandle.java @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.handle.ConnectorTableHandle; + +import org.apache.fluss.metadata.DataLakeFormat; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.RowType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * A fluss table, as everything downstream of {@code getTableHandle} needs to know about it. + * + *

The fields are the ones split planning reads, snapshotted from the {@link TableInfo} the handle + * was built from: whether the table has a primary key (which of the two scanner families reads it), + * the bucket count and keys (how many splits and how they line up with the lake's), the partition keys, + * and whether the table is tiered into a lake at all. They are copied rather than looked up again + * because planning must see one coherent view of the table, not a field from before an ALTER and a + * field from after it. + * + *

{@link #getProperties()} is the fluss table's own property map, and it carries more than the user + * wrote: for a datalake-enabled table the fluss coordinator merges the cluster's lake-catalog + * connection settings into it under {@code table.datalake..*}, which is where the lake side of + * a union read gets its catalog configuration. Nothing else in Doris supplies it — the Doris catalog is + * configured with fluss bootstrap servers only. + * + *

The column schema is deliberately NOT here — with one exception. It is the one part that a + * statement re-reads through {@link FlussStatementScope}, so the handle stays a small, serializable + * identity object; {@link #getKeyColumnTypes()} carries the types of the primary-key and partition-key + * columns alone, because planning has to reason about those by type (see its javadoc) and re-reading + * the whole schema for them would both cost a round trip and risk answering from a different schema + * version than the rest of this handle describes. + */ +public class FlussTableHandle implements ConnectorTableHandle { + + private static final long serialVersionUID = 1L; + + private final String databaseName; + private final String tableName; + private final long tableId; + private final int schemaId; + private final boolean hasPrimaryKey; + private final List primaryKeys; + private final List bucketKeys; + private final int bucketCount; + private final List partitionKeys; + private final boolean dataLakeEnabled; + /** The lake format's fluss name ({@code "paimon"}), or {@code null} when the table declares none. */ + private final String dataLakeFormat; + private final Map properties; + private final Map keyColumnTypes; + + public FlussTableHandle(String databaseName, String tableName, long tableId, int schemaId, + boolean hasPrimaryKey, List primaryKeys, List bucketKeys, int bucketCount, + List partitionKeys, boolean dataLakeEnabled, String dataLakeFormat, + Map properties, Map keyColumnTypes) { + this.databaseName = Objects.requireNonNull(databaseName, "databaseName"); + this.tableName = Objects.requireNonNull(tableName, "tableName"); + this.tableId = tableId; + this.schemaId = schemaId; + this.hasPrimaryKey = hasPrimaryKey; + this.primaryKeys = copyOf(primaryKeys); + this.bucketKeys = copyOf(bucketKeys); + this.bucketCount = bucketCount; + this.partitionKeys = copyOf(partitionKeys); + this.dataLakeEnabled = dataLakeEnabled; + this.dataLakeFormat = dataLakeFormat; + this.properties = properties == null + ? Collections.emptyMap() + : Collections.unmodifiableMap(new LinkedHashMap<>(properties)); + this.keyColumnTypes = keyColumnTypes == null + ? Collections.emptyMap() + : Collections.unmodifiableMap(new LinkedHashMap<>(keyColumnTypes)); + } + + /** Snapshots {@code tableInfo} into a handle. */ + public static FlussTableHandle of(TableInfo tableInfo) { + TablePath path = tableInfo.getTablePath(); + DataLakeFormat lakeFormat = tableInfo.getTableConfig().getDataLakeFormat().orElse(null); + return new FlussTableHandle( + path.getDatabaseName(), + path.getTableName(), + tableInfo.getTableId(), + tableInfo.getSchemaId(), + tableInfo.hasPrimaryKey(), + tableInfo.getPrimaryKeys(), + tableInfo.getBucketKeys(), + tableInfo.getNumBuckets(), + tableInfo.getPartitionKeys(), + tableInfo.getTableConfig().isDataLakeEnabled(), + lakeFormat == null ? null : lakeFormat.toString(), + tableInfo.getProperties().toMap(), + keyColumnTypes(tableInfo)); + } + + /** + * The types of the columns that are part of the primary key or of the partition key, taken from the + * same {@link TableInfo} as every other field here. + */ + private static Map keyColumnTypes(TableInfo tableInfo) { + RowType rowType = tableInfo.getRowType(); + Map types = new LinkedHashMap<>(); + List keyColumns = new ArrayList<>(tableInfo.getPrimaryKeys()); + keyColumns.addAll(tableInfo.getPartitionKeys()); + for (String column : keyColumns) { + int index = rowType.getFieldIndex(column); + if (index >= 0) { + types.put(column, rowType.getTypeAt(index)); + } + } + return types; + } + + public TablePath toTablePath() { + return TablePath.of(databaseName, tableName); + } + + public String getDatabaseName() { + return databaseName; + } + + public String getTableName() { + return tableName; + } + + public long getTableId() { + return tableId; + } + + public int getSchemaId() { + return schemaId; + } + + public boolean hasPrimaryKey() { + return hasPrimaryKey; + } + + public List getPrimaryKeys() { + return primaryKeys; + } + + public List getBucketKeys() { + return bucketKeys; + } + + public int getBucketCount() { + return bucketCount; + } + + public List getPartitionKeys() { + return partitionKeys; + } + + public boolean isPartitioned() { + return !partitionKeys.isEmpty(); + } + + public boolean isDataLakeEnabled() { + return dataLakeEnabled; + } + + public String getDataLakeFormat() { + return dataLakeFormat; + } + + public Map getProperties() { + return properties; + } + + /** + * The fluss types of the primary-key and partition-key columns, by column name. + * + *

Split planning needs these two, and only these two, by type. A primary-key table read as the + * union of its lake and its log tail identifies rows across the two halves BY KEY, so a key column + * whose values do not compare exactly the same way on both sides (a float, a timestamp Doris rounds) + * cannot be read that way at all. A partition column is matched the same way one level up: a lake + * split is bound to a fluss partition by comparing the rendered partition values, which is only + * sound for a type both sides render identically. + * + *

Everything else about the schema stays out of the handle — the point is not "some of the + * schema", it is the columns whose type decides how the table can be PLANNED. + */ + public Map getKeyColumnTypes() { + return keyColumnTypes; + } + + /** The primary-key columns that are not partition columns — what a bucket's rows are keyed by. */ + public List getPhysicalPrimaryKeys() { + List physical = new ArrayList<>(primaryKeys); + physical.removeAll(partitionKeys); + return Collections.unmodifiableList(physical); + } + + /** + * Identity is the table plus the schema version it was read at: two handles for the same table at + * different schema versions describe different column sets and must not compare equal. The + * remaining fields all derive from that pair. + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof FlussTableHandle)) { + return false; + } + FlussTableHandle that = (FlussTableHandle) o; + return tableId == that.tableId + && schemaId == that.schemaId + && databaseName.equals(that.databaseName) + && tableName.equals(that.tableName); + } + + @Override + public int hashCode() { + return Objects.hash(databaseName, tableName, tableId, schemaId); + } + + @Override + public String toString() { + return "FlussTableHandle{" + databaseName + "." + tableName + + ", tableId=" + tableId + ", schemaId=" + schemaId + + ", primaryKey=" + hasPrimaryKey + ", buckets=" + bucketCount + + ", partitionKeys=" + partitionKeys + + ", dataLake=" + (dataLakeEnabled ? dataLakeFormat : "disabled") + "}"; + } + + private static List copyOf(List values) { + return values == null || values.isEmpty() + ? Collections.emptyList() + : Collections.unmodifiableList(new ArrayList<>(values)); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTypeMapping.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTypeMapping.java new file mode 100644 index 00000000000000..e8d5d723c815e6 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussTypeMapping.java @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.ConnectorType; + +import org.apache.fluss.types.ArrayType; +import org.apache.fluss.types.BigIntType; +import org.apache.fluss.types.BinaryType; +import org.apache.fluss.types.BooleanType; +import org.apache.fluss.types.BytesType; +import org.apache.fluss.types.CharType; +import org.apache.fluss.types.DataField; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypeVisitor; +import org.apache.fluss.types.DateType; +import org.apache.fluss.types.DecimalType; +import org.apache.fluss.types.DoubleType; +import org.apache.fluss.types.FloatType; +import org.apache.fluss.types.IntType; +import org.apache.fluss.types.LocalZonedTimestampType; +import org.apache.fluss.types.MapType; +import org.apache.fluss.types.RowType; +import org.apache.fluss.types.SmallIntType; +import org.apache.fluss.types.StringType; +import org.apache.fluss.types.TimeType; +import org.apache.fluss.types.TimestampType; +import org.apache.fluss.types.TinyIntType; + +import java.util.ArrayList; +import java.util.List; + +/** + * Maps a fluss {@link DataType} to a Doris {@link ConnectorType}. + * + *

Why a visitor and not a switch. Fluss's {@link DataTypeVisitor} has one method per type, + * so a type added by a future fluss release breaks this compile instead of falling into a + * {@code default:} branch and degrading a column silently. + * + *

The parity invariant. A datalake-enabled fluss table is reachable through two doors: the + * table itself (this mapping) and its lake sibling {@code tbl$lake}, which is served by the paimon + * connector over the paimon table fluss's tiering service writes. Both doors must describe the same + * column the same way, so every rule below is the composition of fluss's own + * {@code FlussDataTypeToPaimonDataType} with the paimon connector's {@code PaimonTypeMapping} — + * including the ones that look arbitrary in isolation (CHAR over 255 collapsing to STRING, the + * microsecond clamp, the two mapping switches). Changing a rule here without checking that + * composition makes one table show two schemas. + * + *

Unsupported types degrade, they do not throw. A type Doris cannot represent becomes the + * {@code UNSUPPORTED} marker, which fe-core turns into {@code Type.UNSUPPORTED}: the table still + * loads and still answers queries over its other columns, and only a query that projects that column + * fails. Throwing here would make one exotic column cost the whole table. + */ +public final class FlussTypeMapping implements DataTypeVisitor { + + /** Doris DATETIMEV2 stops at microseconds; fluss timestamps go to nanoseconds (precision 9). */ + private static final int MAX_TIMESTAMP_SCALE = 6; + + /** Doris CHAR stops at 255 characters; a longer fluss CHAR becomes STRING. */ + private static final int MAX_CHAR_LENGTH = 255; + + /** The type name fe-core's converter turns into {@code Type.UNSUPPORTED}. */ + private static final String UNSUPPORTED = "UNSUPPORTED"; + + private final Options options; + + private FlussTypeMapping(Options options) { + this.options = options; + } + + /** Convert one fluss type, top level or nested, under the given catalog mapping switches. */ + public static ConnectorType toConnectorType(DataType dataType, Options options) { + return dataType.accept(new FlussTypeMapping(options)); + } + + @Override + public ConnectorType visit(CharType charType) { + int length = charType.getLength(); + if (length > MAX_CHAR_LENGTH) { + return ConnectorType.of("STRING"); + } + return ConnectorType.of("CHAR", length, 0); + } + + @Override + public ConnectorType visit(StringType stringType) { + // Fluss STRING carries no declared length, so there is no VARCHAR(n) to preserve. + return ConnectorType.of("STRING"); + } + + @Override + public ConnectorType visit(BooleanType booleanType) { + return ConnectorType.of("BOOLEAN"); + } + + @Override + public ConnectorType visit(BinaryType binaryType) { + if (options.isMapBinaryToVarbinary()) { + // Doris has no fixed-length binary; the declared length survives as the VARBINARY bound. + return ConnectorType.of("VARBINARY", binaryType.getLength(), 0); + } + return ConnectorType.of("STRING"); + } + + @Override + public ConnectorType visit(BytesType bytesType) { + if (options.isMapBinaryToVarbinary()) { + // Unbounded in fluss: leave the length unset so fe-core's converter fills in the Doris + // VARBINARY maximum, which is what the paimon side (BYTES = VARBINARY(Integer.MAX_VALUE)) + // resolves to as well. + return ConnectorType.of("VARBINARY"); + } + return ConnectorType.of("STRING"); + } + + @Override + public ConnectorType visit(DecimalType decimalType) { + return ConnectorType.of("DECIMALV3", decimalType.getPrecision(), decimalType.getScale()); + } + + @Override + public ConnectorType visit(TinyIntType tinyIntType) { + return ConnectorType.of("TINYINT"); + } + + @Override + public ConnectorType visit(SmallIntType smallIntType) { + return ConnectorType.of("SMALLINT"); + } + + @Override + public ConnectorType visit(IntType intType) { + return ConnectorType.of("INT"); + } + + @Override + public ConnectorType visit(BigIntType bigIntType) { + return ConnectorType.of("BIGINT"); + } + + @Override + public ConnectorType visit(FloatType floatType) { + return ConnectorType.of("FLOAT"); + } + + @Override + public ConnectorType visit(DoubleType doubleType) { + return ConnectorType.of("DOUBLE"); + } + + @Override + public ConnectorType visit(DateType dateType) { + return ConnectorType.of("DATEV2"); + } + + @Override + public ConnectorType visit(TimeType timeType) { + // Doris has no storable TIME column type. Mapping it to STRING or to the elapsed-millis INT + // that other engines use would hand back a value whose meaning differs from the source, so the + // column is marked unsupported instead. The paimon and iceberg connectors mark their own TIME + // the same way, which also keeps tbl and tbl$lake agreeing. + return ConnectorType.of(UNSUPPORTED); + } + + @Override + public ConnectorType visit(TimestampType timestampType) { + return ConnectorType.of("DATETIMEV2", clampScale(timestampType.getPrecision()), 0); + } + + @Override + public ConnectorType visit(LocalZonedTimestampType localZonedTimestampType) { + int scale = clampScale(localZonedTimestampType.getPrecision()); + if (options.isMapTimestampTz()) { + return ConnectorType.of("TIMESTAMPTZ", scale, 0); + } + return ConnectorType.of("DATETIMEV2", scale, 0); + } + + @Override + public ConnectorType visit(ArrayType arrayType) { + // Element nullability is deliberately not carried: the paimon connector's read path leaves it + // at the default, and the two must not disagree (see the parity invariant). + return ConnectorType.arrayOf(arrayType.getElementType().accept(this)); + } + + @Override + public ConnectorType visit(MapType mapType) { + // MAP is a first-class fluss type on every path Doris uses: the ARROW log format, the compacted + // KV format behind primary-key tables, and the paimon table the tiering service writes. Marking + // it unsupported here would leave tbl$lake showing a MAP that tbl refuses to project. + return ConnectorType.mapOf( + mapType.getKeyType().accept(this), + mapType.getValueType().accept(this)); + } + + @Override + public ConnectorType visit(RowType rowType) { + List fields = rowType.getFields(); + List names = new ArrayList<>(fields.size()); + List types = new ArrayList<>(fields.size()); + List nullable = new ArrayList<>(fields.size()); + List comments = new ArrayList<>(fields.size()); + for (DataField field : fields) { + names.add(field.getName()); + types.add(field.getType().accept(this)); + // Nested NOT NULL and COMMENT are carried so DESCRIBE / SHOW CREATE TABLE report what the + // fluss schema actually declares for the field. + nullable.add(field.getType().isNullable()); + comments.add(field.getDescription().orElse(null)); + } + return ConnectorType.structOf(names, types, nullable, comments); + } + + private static int clampScale(int precision) { + return Math.min(precision, MAX_TIMESTAMP_SCALE); + } + + /** + * The catalog-level switches that change a mapping. Both default to off and both carry the same + * name and meaning they have on other Doris catalogs (see {@link FlussConnectorProperties}). + */ + public static final class Options { + + public static final Options DEFAULT = new Options(false, false); + + private final boolean mapBinaryToVarbinary; + private final boolean mapTimestampTz; + + public Options(boolean mapBinaryToVarbinary, boolean mapTimestampTz) { + this.mapBinaryToVarbinary = mapBinaryToVarbinary; + this.mapTimestampTz = mapTimestampTz; + } + + public boolean isMapBinaryToVarbinary() { + return mapBinaryToVarbinary; + } + + public boolean isMapTimestampTz() { + return mapTimestampTz; + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussUnionKeyTypes.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussUnionKeyTypes.java new file mode 100644 index 00000000000000..40b6051694e9f8 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussUnionKeyTypes.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypeRoot; +import org.apache.fluss.types.LocalZonedTimestampType; +import org.apache.fluss.types.TimestampType; + +/** + * Which column types a primary-key table can be read by, when its lake and its log tail are read as one. + * + *

Such a read identifies rows across two halves that were written by different systems and are read + * by different code: the lake half arrives through paimon's readers, the log tail through fluss's. A + * row of the lake is dropped when its KEY appears in the tail, so the two sides must agree on what + * "the same key" means, exactly, for every key column. Where they might not, the table is not read that + * way at all — the fluss-only read still returns every row of a primary-key table, so refusing to + * combine the halves costs speed, never correctness. + * + *

The same question one level up decides which partition a lake split belongs to: the split's + * partition VALUES are compared, as text, with the ones fluss reports. That comparison is only sound + * for a type the two sides render identically, which is why partition columns are held to a stricter + * rule than key columns. + * + *

Neither rule is a fluss restriction — fluss accepts a DOUBLE primary key and an INT partition key + * quite happily ({@code TableDescriptorValidation}, whose only key-type ban is ARRAY/MAP/ROW). They are + * this connector's, and they exist because a wrong answer here is silent: an over-matched key drops a + * row that should have been returned, an under-matched one returns a superseded row twice. + */ +final class FlussUnionKeyTypes { + + /** Doris DATETIMEV2 stops at microseconds, so a finer fluss timestamp is rounded on the way in. */ + private static final int MAX_TIMESTAMP_PRECISION = 6; + + private FlussUnionKeyTypes() { + } + + /** + * Why {@code type} cannot be a key column of a union read, or null when it can. + * + *

The switch is exhaustive over fluss's type roots on purpose: a type a future fluss release adds + * lands in the default branch and is refused, rather than being waved through by a rule written + * before it existed. {@link FlussUnionKeyTypesTest} fails the build when that happens, so the + * refusal is a stopgap, not the answer. + */ + static String keyColumnRejection(DataType type) { + switch (type.getTypeRoot()) { + case BOOLEAN: + case TINYINT: + case SMALLINT: + case INTEGER: + case BIGINT: + case DECIMAL: + case CHAR: + case STRING: + case BINARY: + case BYTES: + case DATE: + return null; + case TIMESTAMP_WITHOUT_TIME_ZONE: + return timestampRejection(((TimestampType) type).getPrecision()); + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return timestampRejection(((LocalZonedTimestampType) type).getPrecision()); + case FLOAT: + case DOUBLE: + // Two encodings of the same number (0.0 and -0.0) compare equal while NaN compares equal + // to nothing, so "the key is in the tail" has no stable answer. Doris's own key-based + // table types refuse floating-point keys for the same reason. + return "its equality is not exact for floating-point values"; + case TIME_WITHOUT_TIME_ZONE: + // Doris has no type for it at all: the column already reads as UNSUPPORTED. + return "Doris cannot represent a fluss TIME column"; + default: + return "the fluss connector does not know how to compare values of this type"; + } + } + + private static String timestampRejection(int precision) { + return precision <= MAX_TIMESTAMP_PRECISION ? null + : "Doris rounds it to microseconds, which can make two different keys look like one"; + } + + /** + * Why {@code type} cannot be a partition column of a union read, or null when it can. + * + *

Only STRING. A partition value reaches this connector already rendered — fluss hands back the + * text it stored, paimon renders its own typed partition value — and nothing guarantees the two + * spellings of a number, a date or a padded CHAR agree. They cannot be normalized here either, + * without this connector taking on how paimon renders every type it has. + */ + static String partitionColumnRejection(DataType type) { + return type.getTypeRoot() == DataTypeRoot.STRING ? null + : "a lake split is matched to a fluss partition by its rendered partition value, which" + + " only STRING is guaranteed to render the same way on both sides"; + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/LakeSibling.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/LakeSibling.java new file mode 100644 index 00000000000000..03e333c10d69f1 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/LakeSibling.java @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.ConnectorMetadata; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; + +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * Runs a call into the lake sibling connector with the context classloader pinned to the sibling's own. + * + *

The engine pins the context classloader to the plugin whose object it is about to call + * ({@code PluginDrivenScanNode.onPluginClassLoader}), so everything this connector runs sees the FLUSS + * plugin's loader. That is right until this connector calls the sibling itself: the sibling's SDK is + * loaded child-first by ITS plugin, and the parts of it that discover implementations by + * {@code ServiceLoader} (catalog factories, file IO, file formats) look them up through the context + * classloader — which would be this plugin, where none of them exist. The failure is a + * {@code NoClassDefFoundError} or an empty factory list at the first lake table read, not at wiring time. + * + *

Every call that crosses into the sibling goes through here. The pin target is derived from an object + * the SIBLING created (its connector or its metadata), never named as a class: the sibling's types are + * invisible from this loader. + */ +final class LakeSibling { + + private LakeSibling() { + } + + /** + * Runs {@code body} with the context classloader set to the one that loaded {@code sibling}. + * + * @param sibling an object created by the sibling plugin (its {@code Connector} or its metadata) + * @param body the call to make into the sibling + */ + static T call(Object sibling, Supplier body) { + ClassLoader previous = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(sibling.getClass().getClassLoader()); + try { + return body.get(); + } finally { + Thread.currentThread().setContextClassLoader(previous); + } + } + + /** + * Calls the sibling's metadata, pinned, through the per-statement funnel: the first call in a statement + * builds the metadata and every later one reuses that instance, mirroring what fe-core's own metadata + * funnel does for a plain connector. Building it is inside the pin too — that call already runs the + * sibling's code (it opens its catalog). + * + *

This is the only route to the sibling's metadata, so the pin and the memo cannot be forgotten by a + * new caller, and the metadata gateway and the scan planner cannot end up on two different instances + * (and therefore two different views of the lake table) within one statement. + * + *

The memo key carries the catalog id AND a role, because a fluss catalog runs two connectors + * (itself and the paimon sibling) under ONE catalog id: keying on the id alone would collapse them onto + * one metadata and misroute every call. + */ + static T forward(ConnectorSession session, Connector sibling, + Function call) { + String key = "metadata:" + session.getCatalogId() + ":lake"; + return call(sibling, () -> call.apply(session.getStatementScope() + .getOrCreateMetadata(key, () -> sibling.getMetadata(session)))); + } + + /** + * Returns {@code handle} after checking the sibling recognizes it as its own, which is what every guard + * on this side relies on to tell a lake handle from a fluss one. + * + *

Checked where the handle is born rather than trusted, because {@link Connector#ownsHandle} defaults + * to {@code false}: a sibling that does not override it answers "not mine" about handles it just made, + * every guard silently fails open, and the first cast throws a ClassCastException naming two connectors + * and no cause. That is a real failure mode, not a hypothetical — the paimon connector had no override + * until it was first used as a sibling, and no unit test could see it (a hand-written stand-in + * implements the method precisely because it has to). + * + * @throws DorisConnectorException naming the sibling's own class, so the fix is on the reader's screen + */ + static T requireOwned(Connector sibling, T handle) { + if (!call(sibling, () -> sibling.ownsHandle(handle))) { + throw new DorisConnectorException("The lake connector " + + sibling.getClass().getName() + " does not recognize the table handle it just" + + " produced: it does not implement Connector.ownsHandle, which a connector must do" + + " to be usable as a sibling. Without it the fluss connector cannot tell its handles" + + " apart from its own"); + } + return handle; + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/PaimonSiblingProperties.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/PaimonSiblingProperties.java new file mode 100644 index 00000000000000..4781f131c157ff --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/PaimonSiblingProperties.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.DorisConnectorException; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Synthesizes the catalog-property map for the embedded paimon sibling connector that serves a + * fluss lake table ({@code tbl$lake}). Mirrors the hive gateway's {@code HudiSiblingProperties} / + * {@code IcebergSiblingProperties}: the sibling is built through + * {@code ConnectorContext.createSiblingConnector("paimon", synthesize(...))} so its classes come from the + * paimon plugin's own loader — the fluss plugin bundles no paimon at all. + * + *

The one structural difference from hive: hive's lake configuration is part of the CATALOG properties + * the user typed, so its synthesis is a verbatim copy. Fluss's lives in the TABLE properties instead — the + * fluss coordinator merges the cluster-level {@code datalake.paimon.*} settings into every datalake table's + * properties under a {@code table.} prefix, and only while {@code table.datalake.enabled} is true + * ({@code TableRegistration#toTableInfo}). So the input here is a table's property map and the mapping has + * to strip that prefix: + * + * 

+ *   fluss cluster config      datalake.paimon.warehouse = /lake
+ *   fluss table property      table.datalake.paimon.warehouse = /lake
+ *   paimon catalog property   warehouse = /lake
+ *
+ * + *

Two keys are renamed because Doris's paimon connector spells them differently from paimon itself; + * everything else is forwarded with only the prefix removed. That verbatim tail is what carries a real + * deployment's storage keys ({@code fs.*} / {@code dfs.*} / {@code hadoop.*}), which the paimon connector + * reads under exactly those names. + */ +final class PaimonSiblingProperties { + + /** + * The prefix the fluss coordinator gives its injected paimon lake settings inside a table's property + * map. + */ + private static final String LAKE_OPTION_PREFIX = "table.datalake.paimon."; + + /** Paimon's own name for the metastore flavor; Doris's paimon connector calls it paimon.catalog.type. */ + private static final String FLUSS_METASTORE = "metastore"; + private static final String PAIMON_CATALOG_TYPE = "paimon.catalog.type"; + + /** Same name on both sides, but required, so it is handled explicitly rather than by the tail. */ + private static final String WAREHOUSE = "warehouse"; + + /** The only metastore flavor this connector serves today (see the class comment of the gate). */ + private static final String FILESYSTEM = "filesystem"; + + private PaimonSiblingProperties() { + } + + /** + * Returns a NEW paimon catalog-property map derived from one fluss table's properties. The input is + * never mutated and is not aliased into the result. + * + *

Fails loud on a lake configuration this connector cannot serve, rather than handing the paimon + * connector a half-built map and letting it fail with a message about a catalog nobody created. + */ + static Map synthesize(Map flussTableProperties) { + Map lakeOptions = new LinkedHashMap<>(); + for (Map.Entry entry : flussTableProperties.entrySet()) { + if (entry.getKey().startsWith(LAKE_OPTION_PREFIX)) { + lakeOptions.put(entry.getKey().substring(LAKE_OPTION_PREFIX.length()), entry.getValue()); + } + } + + String metastore = lakeOptions.remove(FLUSS_METASTORE); + // Absent means paimon's own default, which is filesystem — the same thing this connector supports. + if (metastore != null && !FILESYSTEM.equalsIgnoreCase(metastore)) { + throw new DorisConnectorException( + "Cannot read the lake table: its fluss cluster configures a '" + metastore + + "' paimon catalog (datalake.paimon." + FLUSS_METASTORE + + "), and the fluss connector currently supports only '" + FILESYSTEM + "'"); + } + + String warehouse = lakeOptions.remove(WAREHOUSE); + if (warehouse == null || warehouse.isEmpty()) { + throw new DorisConnectorException( + "Cannot read the lake table: the fluss table carries no '" + + LAKE_OPTION_PREFIX + WAREHOUSE + + "'. The fluss cluster must configure datalake.paimon." + WAREHOUSE + + " for its lake tables to be readable"); + } + + // LinkedHashMap so a failure message or a log line renders the same order every time. + Map siblingProperties = new LinkedHashMap<>(); + siblingProperties.put(PAIMON_CATALOG_TYPE, FILESYSTEM); + siblingProperties.put(WAREHOUSE, warehouse); + siblingProperties.putAll(lakeOptions); + return siblingProperties; + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/main/resources/META-INF/services/org.apache.doris.connector.spi.ConnectorProvider b/fe/fe-connector/fe-connector-fluss/src/main/resources/META-INF/services/org.apache.doris.connector.spi.ConnectorProvider new file mode 100644 index 00000000000000..5a743650899591 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/main/resources/META-INF/services/org.apache.doris.connector.spi.ConnectorProvider @@ -0,0 +1 @@ +org.apache.doris.connector.fluss.FlussConnectorProvider diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorLakeSiblingTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorLakeSiblingTest.java new file mode 100644 index 00000000000000..b3f498fd9ee91c --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorLakeSiblingTest.java @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.spi.ConnectorContext; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * The connector's ownership of its lake sibling: how it is built, cached, routed to and closed. + * + *

None of this touches a fluss cluster, which is also what the production code guarantees: a query that + * never reads a lake table must not build a paimon catalog, and reaching a lake table's scan provider must + * not open a fluss connection. + */ +public class FlussConnectorLakeSiblingTest { + + /** A lake configuration, i.e. what {@code PaimonSiblingProperties} hands the factory. */ + private static Map lakeProperties(String warehouse) { + Map properties = new HashMap<>(); + properties.put("paimon.catalog.type", "filesystem"); + properties.put("warehouse", warehouse); + return properties; + } + + /** Records what the engine was asked to build, and answers with whatever the test staged. */ + private static final class RecordingContext implements ConnectorContext { + + private final List requestedTypes = new ArrayList<>(); + private final List> requestedProperties = new ArrayList<>(); + private boolean providerAvailable = true; + + @Override + public String getCatalogName() { + return "fluss_catalog"; + } + + @Override + public long getCatalogId() { + return 1L; + } + + @Override + public Connector createSiblingConnector(String catalogType, Map properties) { + requestedTypes.add(catalogType); + requestedProperties.add(properties); + return providerAvailable ? new RecordingLakeSibling(properties) : null; + } + } + + private static FlussConnector connector(ConnectorContext context) { + Map catalogProperties = new HashMap<>(); + catalogProperties.put("fluss.bootstrap.servers", "127.0.0.1:9123"); + return new FlussConnector(catalogProperties, context); + } + + @Test + public void siblingIsBuiltAsAPaimonConnectorFromTheGivenProperties() { + RecordingContext context = new RecordingContext(); + FlussConnector connector = connector(context); + + Map properties = lakeProperties("/lake"); + Connector sibling = connector.getOrCreateLakeSibling(properties); + + Assertions.assertNotNull(sibling); + // The type string is what the engine matches against the installed plugins; the fluss plugin + // bundles no paimon class at all, so this literal is the whole contract. + Assertions.assertEquals(Collections.singletonList("paimon"), context.requestedTypes); + Assertions.assertEquals(properties, context.requestedProperties.get(0)); + } + + @Test + public void oneLakeConfigurationBuildsOneSibling() { + RecordingContext context = new RecordingContext(); + FlussConnector connector = connector(context); + + Connector first = connector.getOrCreateLakeSibling(lakeProperties("/lake")); + Connector second = connector.getOrCreateLakeSibling(lakeProperties("/lake")); + + // Equal-by-value properties, not the same map instance: every statement synthesizes a fresh map + // from the table's properties, so an identity-keyed cache would build a paimon catalog per query. + Assertions.assertSame(first, second); + Assertions.assertEquals(1, context.requestedTypes.size()); + } + + @Test + public void secondLakeConfigurationFailsLoud() { + RecordingContext context = new RecordingContext(); + FlussConnector connector = connector(context); + connector.getOrCreateLakeSibling(lakeProperties("/lake")); + + // Two paimon siblings answer "is this handle yours?" with the same class test, so the second could + // never be routed apart from the first — a table would silently read the wrong warehouse. Refusing + // is the only honest answer; the message points at the fix (refresh the catalog). + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> connector.getOrCreateLakeSibling(lakeProperties("/other"))); + Assertions.assertTrue(failure.getMessage().contains("refresh the catalog"), failure.getMessage()); + // Storage credentials live in these maps; a message that dumps them ends up in the FE audit log. + Assertions.assertFalse(failure.getMessage().contains("/other"), failure.getMessage()); + Assertions.assertEquals(1, context.requestedTypes.size(), "the refused one must not be built"); + } + + @Test + public void missingPaimonPluginFailsLoudAndIsNotCached() { + RecordingContext context = new RecordingContext(); + context.providerAvailable = false; + FlussConnector connector = connector(context); + + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> connector.getOrCreateLakeSibling(lakeProperties("/lake"))); + Assertions.assertTrue(failure.getMessage().contains("paimon"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("fluss_catalog"), failure.getMessage()); + + // Not memoizing the failure is what lets a later-installed plugin recover without a catalog + // refresh; a cached null would keep answering "not available" forever. + context.providerAvailable = true; + Assertions.assertNotNull(connector.getOrCreateLakeSibling(lakeProperties("/lake"))); + } + + @Test + public void lakeHandleIsPlannedByTheSibling() { + RecordingContext context = new RecordingContext(); + FlussConnector connector = connector(context); + RecordingLakeSibling sibling = + (RecordingLakeSibling) connector.getOrCreateLakeSibling(lakeProperties("/lake")); + + // Routing is by handle OWNERSHIP, asked of the sibling — the fluss side cannot recognize a foreign + // handle by class across the plugin split, so "is it yours?" is the only question available. + Assertions.assertSame(sibling.scanPlanProvider, + connector.getScanPlanProvider(new RecordingLakeSibling.Handle("db", "t"))); + } + + @Test + public void onlyFlussHandlesAreOwnedByThisConnector() { + RecordingContext context = new RecordingContext(); + FlussConnector connector = connector(context); + + // The engine asks this to route a handle back to the connector that made it. Claiming a sibling's + // handle would make the engine plan a paimon table with fluss's planner. + Assertions.assertFalse(connector.ownsHandle(new RecordingLakeSibling.Handle("db", "t"))); + } + + @Test + public void closingTheConnectorClosesTheSibling() throws IOException { + RecordingContext context = new RecordingContext(); + FlussConnector connector = connector(context); + RecordingLakeSibling sibling = + (RecordingLakeSibling) connector.getOrCreateLakeSibling(lakeProperties("/lake")); + + connector.close(); + + Assertions.assertTrue(sibling.closed); + // And the reference must be dropped, or a reopened catalog would keep routing to a closed sibling. + // A now-different lake configuration being accepted is what proves the slot is free again. + Assertions.assertNotSame(sibling, connector.getOrCreateLakeSibling(lakeProperties("/other"))); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java new file mode 100644 index 00000000000000..3a836c41e877aa --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorMetadataTest.java @@ -0,0 +1,568 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.ConnectorColumn; +import org.apache.doris.connector.api.ConnectorPartitionInfo; +import org.apache.doris.connector.api.ConnectorTableSchema; +import org.apache.doris.connector.api.ConnectorTableStatistics; +import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.thrift.TTableDescriptor; +import org.apache.doris.thrift.TTableType; + +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.ResolvedPartitionSpec; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.metadata.TableStats; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Covers the metadata surface a fluss table is described through: the handle planning later reads, + * the Doris schema DESCRIBE shows, the partition list pruning depends on, and the statistics. + * + *

Everything here runs against {@link RecordingFlussAdminOps} rather than a cluster, which is what + * makes the awkward cases testable at all — a table that vanished, a coordinator that is down, a + * partition spec whose map iterates in a different order than the partition columns. The mapping is + * checked against a live cluster in {@link FlussMetadataClusterTest}. + */ +public class FlussConnectorMetadataTest { + + private static final TablePath LOG_TABLE = TablePath.of("db", "log_table"); + private static final TablePath PK_TABLE = TablePath.of("db", "pk_table"); + private static final TablePath PART_TABLE = TablePath.of("db", "part_table"); + + private static FlussConnectorMetadata metadata(RecordingFlussAdminOps adminOps) { + return metadata(adminOps, FlussTypeMapping.Options.DEFAULT); + } + + /** + * A metadata with the lake seams wired to "there is no lake": the sibling owner never claims a handle + * and the factory refuses to build one. A test in this class that unexpectedly took the lake path + * therefore fails loudly instead of quietly exercising a half-built sibling. + */ + private static FlussConnectorMetadata metadata( + RecordingFlussAdminOps adminOps, FlussTypeMapping.Options options) { + return new FlussConnectorMetadata(adminOps, options, + properties -> { + throw new AssertionError("no lake sibling is expected in this test"); + }, + handle -> null); + } + + /** A non-partitioned log table: two columns, one of them commented, three buckets. */ + private static RecordingFlussAdminOps withLogTable() { + RecordingFlussAdminOps adminOps = new RecordingFlussAdminOps(); + adminOps.tableInfos.put(LOG_TABLE, FlussTestTables.builder(LOG_TABLE) + .column("id", DataTypes.BIGINT()) + .column("payload", DataTypes.STRING(), "what happened") + .buckets(3) + .comment("the log") + .build()); + return adminOps; + } + + /** A partitioned primary-key table that is tiered into a paimon lake. */ + private static RecordingFlussAdminOps withDataLakePkTable() { + RecordingFlussAdminOps adminOps = new RecordingFlussAdminOps(); + adminOps.tableInfos.put(PK_TABLE, FlussTestTables.builder(PK_TABLE) + .column("dt", DataTypes.STRING()) + .column("region", DataTypes.STRING()) + .column("id", DataTypes.BIGINT()) + .column("amount", DataTypes.DECIMAL(20, 4)) + .primaryKey("dt", "region", "id") + .partitionedBy("dt", "region") + .buckets(2, "id") + .property("table.datalake.enabled", "true") + .property("table.datalake.format", "paimon") + .property("table.datalake.paimon.metastore", "filesystem") + .property("table.datalake.paimon.warehouse", "/tmp/lake") + .build()); + return adminOps; + } + + @Test + public void listingIsPassedThroughToTheClusterUnchanged() { + // Fluss already has database/table names in Doris's own shape, so nothing here may invent, + // filter or re-case a name: what the cluster reports is what SHOW DATABASES / SHOW TABLES show. + RecordingFlussAdminOps adminOps = new RecordingFlussAdminOps(); + adminOps.databases = Arrays.asList("fluss_db", "MixedCase"); + adminOps.tablesByDatabase.put("fluss_db", Arrays.asList("log_table", "pk_table")); + adminOps.tablesByDatabase.put("empty_db", Collections.emptyList()); + + FlussConnectorMetadata metadata = metadata(adminOps); + + Assertions.assertEquals(Arrays.asList("fluss_db", "MixedCase"), metadata.listDatabaseNames(null)); + Assertions.assertEquals(Arrays.asList("log_table", "pk_table"), metadata.listTableNames(null, "fluss_db")); + Assertions.assertEquals(Collections.emptyList(), metadata.listTableNames(null, "empty_db")); + Assertions.assertTrue(metadata.databaseExists(null, "fluss_db")); + Assertions.assertFalse(metadata.databaseExists(null, "absent_db")); + + Assertions.assertEquals( + Arrays.asList("listDatabases()", "listTables(fluss_db)", "listTables(empty_db)", + "databaseExists(fluss_db)", "databaseExists(absent_db)"), + adminOps.calls); + } + + @Test + public void theHandleSnapshotsWhatSplitPlanningWillNeed() { + // Planning must not have to re-read the table to learn which scanner family reads it, how many + // buckets to split into, or where the lake is; and reading those fields one by one later could + // straddle an ALTER. Everything below is therefore captured at handle time. + FlussTableHandle handle = (FlussTableHandle) metadata(withDataLakePkTable()) + .getTableHandle(null, "db", "pk_table").orElseThrow(AssertionError::new); + + Assertions.assertEquals("db", handle.getDatabaseName()); + Assertions.assertEquals("pk_table", handle.getTableName()); + Assertions.assertTrue(handle.hasPrimaryKey()); + Assertions.assertEquals(Arrays.asList("dt", "region", "id"), handle.getPrimaryKeys()); + Assertions.assertEquals(Arrays.asList("dt", "region"), handle.getPartitionKeys()); + Assertions.assertTrue(handle.isPartitioned()); + Assertions.assertEquals(Collections.singletonList("id"), handle.getBucketKeys()); + Assertions.assertEquals(2, handle.getBucketCount()); + Assertions.assertTrue(handle.isDataLakeEnabled()); + Assertions.assertEquals("paimon", handle.getDataLakeFormat()); + // The lake catalog's own connection settings ride on the fluss table properties (the fluss + // coordinator merges them in); the Doris catalog is told nothing but bootstrap servers, so this + // map is the only place a union read can learn where the lake lives. + Assertions.assertEquals("filesystem", handle.getProperties().get("table.datalake.paimon.metastore")); + Assertions.assertEquals("/tmp/lake", handle.getProperties().get("table.datalake.paimon.warehouse")); + } + + @Test + public void logTableIsNotMistakenForPkOrLakeTable() { + // The negative side of the handle: these three flags pick the scan strategy, so a default that + // drifted to "true" would silently route a plain log table down the merge-on-read path. + FlussTableHandle handle = (FlussTableHandle) metadata(withLogTable()) + .getTableHandle(null, "db", "log_table").orElseThrow(AssertionError::new); + + Assertions.assertFalse(handle.hasPrimaryKey()); + Assertions.assertFalse(handle.isPartitioned()); + Assertions.assertFalse(handle.isDataLakeEnabled()); + Assertions.assertNull(handle.getDataLakeFormat()); + Assertions.assertEquals(3, handle.getBucketCount()); + } + + @Test + public void missingTableIsEmptyButUnreachableClusterIsAnError() { + RecordingFlussAdminOps adminOps = withLogTable(); + Assertions.assertEquals(Optional.empty(), metadata(adminOps).getTableHandle(null, "db", "absent")); + + // The distinction that matters: reporting "no such table" for a cluster that is merely down + // turns a broken catalog into an empty-looking one — the user never sees the real cause, and a + // CREATE TABLE IF NOT EXISTS would believe the name is free. + adminOps.failure = new IllegalStateException("coordinator unreachable"); + IllegalStateException e = Assertions.assertThrows(IllegalStateException.class, + () -> metadata(adminOps).getTableHandle(null, "db", "log_table")); + Assertions.assertEquals("coordinator unreachable", e.getMessage()); + } + + @Test + public void theSchemaCarriesColumnsCommentsAndTheirTypes() { + FlussConnectorMetadata metadata = metadata(withLogTable()); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "log_table") + .orElseThrow(AssertionError::new); + + ConnectorTableSchema schema = metadata.getTableSchema(null, handle); + + Assertions.assertEquals("log_table", schema.getTableName()); + Assertions.assertEquals("FLUSS", schema.getTableFormatType()); + Assertions.assertEquals(Arrays.asList("id", "payload"), names(schema.getColumns())); + Assertions.assertEquals(ConnectorType.of("BIGINT"), schema.getColumns().get(0).getType()); + // Column comments come from the fluss schema, NOT from the table's row type: fluss rebuilds that + // row type from the schema and drops every description on the way, so a mapping that read + // getRowType() would report every column as undocumented. + Assertions.assertEquals("what happened", schema.getColumns().get(1).getComment()); + Assertions.assertEquals("the log", metadata.getTableComment(null, "db", "log_table")); + } + + @Test + public void everyColumnIsReportedNullableEvenWhenFlussSaysNotNull() { + // Fluss marks primary-key columns NOT NULL. Propagating that would let the planner fold + // null-rejecting predicates on this path only, while the same table read through its lake + // sibling (served by the paimon connector, which reports everything nullable) would keep them — + // one table, two plans. Doris external tables report nullable throughout for this reason. + FlussConnectorMetadata metadata = metadata(withDataLakePkTable()); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "pk_table") + .orElseThrow(AssertionError::new); + + for (ConnectorColumn column : metadata.getTableSchema(null, handle).getColumns()) { + Assertions.assertTrue(column.isNullable(), column.getName() + " must be reported nullable"); + } + } + + @Test + public void partitionedTableAdvertisesItsPartitionColumns() { + // Without this property fe-core treats the table as unpartitioned: no error, just partition + // pruning quietly gone. The value keeps the partition-column ORDER, which is what the partition + // names below are zipped against. + FlussConnectorMetadata metadata = metadata(withDataLakePkTable()); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "pk_table") + .orElseThrow(AssertionError::new); + + Map properties = metadata.getTableSchema(null, handle).getProperties(); + Assertions.assertEquals("dt,region", + properties.get(ConnectorTableSchema.PARTITION_COLUMNS_KEY)); + // The fluss table's own properties stay visible so SHOW CREATE TABLE can render them. + Assertions.assertEquals("true", properties.get("table.datalake.enabled")); + + FlussConnectorMetadata logMetadata = metadata(withLogTable()); + ConnectorTableHandle logHandle = logMetadata.getTableHandle(null, "db", "log_table") + .orElseThrow(AssertionError::new); + Assertions.assertFalse( + logMetadata.getTableSchema(null, logHandle).getProperties() + .containsKey(ConnectorTableSchema.PARTITION_COLUMNS_KEY), + "an unpartitioned table must not claim partition columns"); + } + + @Test + public void timestampWithLocalTimeZoneKeepsItsMarkerUnderEitherMapping() { + // The marker is what DESCRIBE prints in Extra, and it describes the SOURCE column, so it must + // not depend on whether enable.mapping.timestamp_tz mapped the column to TIMESTAMPTZ or to a + // plain DATETIME. Losing it under one mapping would make the same column self-describe + // differently depending on a catalog switch. + RecordingFlussAdminOps adminOps = new RecordingFlussAdminOps(); + adminOps.tableInfos.put(LOG_TABLE, FlussTestTables.builder(LOG_TABLE) + .column("event_time", DataTypes.TIMESTAMP_LTZ(6)) + .column("local_time", DataTypes.TIMESTAMP(6)) + .buckets(1) + .build()); + + for (boolean mapTimestampTz : new boolean[] {false, true}) { + FlussConnectorMetadata metadata = metadata( + adminOps, new FlussTypeMapping.Options(false, mapTimestampTz)); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "log_table") + .orElseThrow(AssertionError::new); + List columns = metadata.getTableSchema(null, handle).getColumns(); + Assertions.assertTrue(columns.get(0).isWithTimeZone(), + "TIMESTAMP_LTZ must keep the marker with timestamp_tz mapping = " + mapTimestampTz); + Assertions.assertFalse(columns.get(1).isWithTimeZone(), + "a plain TIMESTAMP must not be marked as zoned"); + } + } + + @Test + public void columnHandlesAreKeyedByNameAndCarryTheFieldIndex() { + // The index is the projection fluss is asked for later; keying by name is how the engine's slots + // find their handle. A mismatch between the two would project the wrong column, not fail. + FlussConnectorMetadata metadata = metadata(withDataLakePkTable()); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "pk_table") + .orElseThrow(AssertionError::new); + + Map handles = metadata.getColumnHandles(null, handle); + + Assertions.assertEquals(Arrays.asList("dt", "region", "id", "amount"), + new ArrayList<>(handles.keySet())); + Assertions.assertEquals(new FlussColumnHandle("id", 2), handles.get("id")); + Assertions.assertEquals(3, ((FlussColumnHandle) handles.get("amount")).getFieldIndex()); + } + + @Test + public void partitionsAreRenderedWithDorisNamesInPartitionColumnOrder() { + // Fluss spells a partition "2026_08_02$eu"; Doris spells it "dt=2026_08_02/region=eu", which is + // what fe-core parses back for SHOW PARTITIONS and the partition_values function. The ordered + // values are supplied explicitly because fe-core zips them positionally against the partition + // columns — and it does that silently, so a wrong order mis-assigns values rather than failing. + RecordingFlussAdminOps adminOps = withDataLakePkTable(); + adminOps.partitionsByTable.put(PK_TABLE, Arrays.asList( + partition(1L, "region", "eu", "dt", "2026_08_02"), + partition(2L, "dt", "2026_08_03", "region", "us"))); + + FlussConnectorMetadata metadata = metadata(adminOps); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "pk_table") + .orElseThrow(AssertionError::new); + + List partitions = metadata.listPartitions(null, handle, Optional.empty()); + Assertions.assertEquals(2, partitions.size()); + // The first fixture's spec deliberately lists region before dt: the rendering must follow the + // partition COLUMNS, not whatever order the spec happens to iterate in. + Assertions.assertEquals("dt=2026_08_02/region=eu", partitions.get(0).getPartitionName()); + Assertions.assertEquals(Arrays.asList("2026_08_02", "eu"), partitions.get(0).getOrderedPartitionValues()); + Assertions.assertEquals("2026_08_02", partitions.get(0).getPartitionValues().get("dt")); + Assertions.assertEquals("eu", partitions.get(0).getPartitionValues().get("region")); + Assertions.assertEquals("dt=2026_08_03/region=us", partitions.get(1).getPartitionName()); + + Assertions.assertEquals(Arrays.asList("dt=2026_08_02/region=eu", "dt=2026_08_03/region=us"), + metadata.listPartitionNames(null, handle)); + } + + @Test + public void anUnpartitionedTableNeverAsksTheClusterForPartitions() { + // Asking fluss for the partitions of an unpartitioned table is an error there, and the handle + // already knows the answer is "none" — so this must be decided locally, not round-tripped. + RecordingFlussAdminOps adminOps = withLogTable(); + FlussConnectorMetadata metadata = metadata(adminOps); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "log_table") + .orElseThrow(AssertionError::new); + + Assertions.assertEquals(Collections.emptyList(), metadata.listPartitions(null, handle, Optional.empty())); + Assertions.assertEquals(Collections.emptyList(), metadata.listPartitionNames(null, handle)); + Assertions.assertFalse(adminOps.calls.stream().anyMatch(call -> call.startsWith("listPartitionInfos")), + "no partition call should have been made, calls were: " + adminOps.calls); + } + + /** + * A partition column of every type fluss allows AND Doris can read back, in one table. The values + * below are the ones a fluss cluster really names these partitions with — verified against one — + * because the whole question here is whether the name survives the round trip. + */ + private static RecordingFlussAdminOps withEveryReadablePartitionType() { + RecordingFlussAdminOps adminOps = new RecordingFlussAdminOps(); + adminOps.tableInfos.put(PART_TABLE, FlussTestTables.builder(PART_TABLE) + .column("id", DataTypes.INT()) + .column("p_str", DataTypes.STRING()) + .column("p_char", DataTypes.CHAR(2)) + .column("p_bool", DataTypes.BOOLEAN()) + .column("p_tiny", DataTypes.TINYINT()) + .column("p_small", DataTypes.SMALLINT()) + .column("p_int", DataTypes.INT()) + .column("p_big", DataTypes.BIGINT()) + .column("p_date", DataTypes.DATE()) + .column("p_bin", DataTypes.BINARY(2)) + .partitionedBy("p_str", "p_char", "p_bool", "p_tiny", "p_small", "p_int", "p_big", + "p_date", "p_bin") + .buckets(1) + .build()); + adminOps.partitionsByTable.put(PART_TABLE, Collections.singletonList( + partition(1L, "p_str", "cn", "p_char", "c1", "p_bool", "true", "p_tiny", "1", + "p_small", "10", "p_int", "100", "p_big", "1000", "p_date", "2026-01-01", + "p_bin", "0102"))); + return adminOps; + } + + /** A table partitioned by one column of the given type, and nothing else of interest. */ + private static RecordingFlussAdminOps withPartitionColumnOfType( + org.apache.fluss.types.DataType type) { + RecordingFlussAdminOps adminOps = new RecordingFlussAdminOps(); + adminOps.tableInfos.put(PART_TABLE, FlussTestTables.builder(PART_TABLE) + .column("id", DataTypes.INT()) + .column("p", type) + .partitionedBy("p") + .buckets(1) + .build()); + adminOps.partitionsByTable.put(PART_TABLE, + Collections.singletonList(partition(1L, "p", "2026-01-01-01-02-03"))); + return adminOps; + } + + @Test + public void everyPartitionTypeWhoseNameSurvivesIsListed() { + RecordingFlussAdminOps adminOps = withEveryReadablePartitionType(); + FlussConnectorMetadata metadata = metadata(adminOps); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "part_table") + .orElseThrow(AssertionError::new); + + List partitions = metadata.listPartitions(null, handle, Optional.empty()); + Assertions.assertEquals(1, partitions.size()); + Assertions.assertEquals("p_str=cn/p_char=c1/p_bool=true/p_tiny=1/p_small=10/p_int=100/" + + "p_big=1000/p_date=2026-01-01/p_bin=0102", partitions.get(0).getPartitionName()); + } + + /** + * Fluss stores a partition's value only in its name, and rewrites the characters a name may not + * hold: a TIMESTAMP {@code 2026-01-01 01:02:03} is named {@code 2026-01-01-01-02-03}. Handing that + * on gets it as far as fe-core's partition parser, which fails with the name and nothing else — + * no column, no type, no fluss. So the refusal belongs here, where all three are still known. + */ + @Test + public void partitionValueFlussRewroteIsRefusedByName() { + RecordingFlussAdminOps adminOps = withPartitionColumnOfType(DataTypes.TIMESTAMP(3)); + FlussConnectorMetadata metadata = metadata(adminOps); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "part_table") + .orElseThrow(AssertionError::new); + + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.listPartitions(null, handle, Optional.empty())); + Assertions.assertTrue(failure.getMessage().contains("db.part_table"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("'p'"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("TIMESTAMP(3)"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("cannot be read back"), failure.getMessage()); + // Refused from the schema alone, so a table with no partitions yet is refused the same way and + // the cluster is not asked a question whose answer could not be used. + Assertions.assertFalse(adminOps.calls.stream().anyMatch(call -> call.startsWith("listPartitionInfos")), + "the partitions should not have been fetched, calls were: " + adminOps.calls); + // listPartitionNames goes through the same funnel; a second entrance would be a way around it. + Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.listPartitionNames(null, handle)); + } + + /** + * A handle that names a partition column it carries no type for has come apart from the schema it + * was built from — an ALTER between the two reads, a handle assembled by hand. Assuming such a + * column readable would put exactly the names this guard exists to stop back on the path to + * fe-core's parser, so the unknown is refused and says which column it was. + */ + @Test + public void partitionColumnWithNoTypeIsRefusedRatherThanAssumed() { + FlussTableHandle handle = new FlussTableHandle("db", "part_table", 1L, 1, false, + Collections.emptyList(), Collections.emptyList(), 1, + Collections.singletonList("p"), false, null, + Collections.emptyMap(), Collections.emptyMap()); + RecordingFlussAdminOps adminOps = withPartitionColumnOfType(DataTypes.STRING()); + FlussConnectorMetadata metadata = metadata(adminOps); + + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.listPartitions(null, handle, Optional.empty())); + Assertions.assertTrue(failure.getMessage().contains("'p'"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("no type"), failure.getMessage()); + Assertions.assertFalse(adminOps.calls.stream().anyMatch(call -> call.startsWith("listPartitionInfos")), + "the partitions should not have been fetched, calls were: " + adminOps.calls); + } + + @Test + public void binaryPartitionIsRefusedOnlyWhenTheColumnIsNotText() { + // Fluss names such a partition with the hex text of the bytes. Read as a string that is exactly + // what was written; asked for as a VARBINARY it is not a literal of anything. + FlussConnectorMetadata asText = metadata(withPartitionColumnOfType(DataTypes.BINARY(2))); + Assertions.assertEquals(1, asText.listPartitions(null, + asText.getTableHandle(null, "db", "part_table").orElseThrow(AssertionError::new), + Optional.empty()).size()); + + RecordingFlussAdminOps adminOps = withPartitionColumnOfType(DataTypes.BINARY(2)); + FlussConnectorMetadata asVarbinary = metadata(adminOps, new FlussTypeMapping.Options(true, false)); + ConnectorTableHandle handle = asVarbinary.getTableHandle(null, "db", "part_table") + .orElseThrow(AssertionError::new); + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> asVarbinary.listPartitions(null, handle, Optional.empty())); + Assertions.assertTrue( + failure.getMessage().contains(FlussConnectorProperties.ENABLE_MAPPING_VARBINARY), + "the fix is a property, so name it: " + failure.getMessage()); + } + + @Test + public void statisticsAreARowCountOrNothingAtAll() { + RecordingFlussAdminOps adminOps = withLogTable(); + adminOps.statsByTable.put(LOG_TABLE, new TableStats(4200L)); + FlussConnectorMetadata metadata = metadata(adminOps); + ConnectorTableHandle handle = metadata.getTableHandle(null, "db", "log_table") + .orElseThrow(AssertionError::new); + + ConnectorTableStatistics stats = metadata.getTableStatistics(null, handle) + .orElseThrow(AssertionError::new); + Assertions.assertEquals(4200L, stats.getRowCount()); + // Fluss reports no data size; -1 (unknown) says so instead of implying zero bytes, which would + // make the optimizer treat the table as free to scan. + Assertions.assertEquals(-1L, stats.getDataSize()); + + // A table whose statistics are off reports zero, which is not the same as "empty" — report + // unknown so the estimate falls back instead of pinning the table at zero rows. + adminOps.statsByTable.put(LOG_TABLE, new TableStats(0L)); + Assertions.assertEquals(Optional.empty(), metadata.getTableStatistics(null, handle)); + + // Statistics are best effort: analysis and SHOW must not fail because the coordinator hiccuped. + adminOps.failure = new IllegalStateException("coordinator unreachable"); + Assertions.assertEquals(Optional.empty(), metadata.getTableStatistics(null, handle)); + } + + @Test + public void oneStatementReadsTheTableOnceAndTheNextStatementReadsItAgain() { + // Four metadata questions about one table are one coordinator round trip. Beyond the cost, it is + // what keeps a statement self-consistent: a concurrent ALTER between two of these calls would + // otherwise build a plan from two different schema versions. + RecordingFlussAdminOps adminOps = withLogTable(); + FlussConnectorMetadata metadata = metadata(adminOps); + FlussTestSession session = new FlussTestSession(7L, "query-1"); + + ConnectorTableHandle handle = metadata.getTableHandle(session, "db", "log_table") + .orElseThrow(AssertionError::new); + metadata.getTableSchema(session, handle); + metadata.getColumnHandles(session, handle); + metadata.getTableComment(session, "db", "log_table"); + + Assertions.assertEquals(1, countCalls(adminOps, "getTableInfo"), + "expected one fetch per statement, calls were: " + adminOps.calls); + + // A second statement must NOT reuse the first one's view — this memo is per statement, never a + // cache, or an external ALTER would stay invisible until the FE restarts. + FlussTestSession next = new FlussTestSession(7L, "query-2"); + metadata.getTableHandle(next, "db", "log_table"); + Assertions.assertEquals(2, countCalls(adminOps, "getTableInfo")); + + // And with no session at all (offline planning) nothing is shared: load every time. + metadata.getTableHandle(null, "db", "log_table"); + metadata.getTableHandle(null, "db", "log_table"); + Assertions.assertEquals(4, countCalls(adminOps, "getTableInfo")); + } + + @Test + public void theTableDescriptorIsTheGenericFileScanShape() { + // A fluss scan reaches the BE through the same file scan node the lake connectors use, so the + // descriptor is the hive-shaped one they send. It has to name the right table: the BE looks the + // descriptor up by id and reports the names in profiles. + TTableDescriptor descriptor = metadata(withLogTable()) + .buildTableDescriptor(null, 42L, "log_table", "db", "log_table", 2, 9L); + + Assertions.assertEquals(TTableType.HIVE_TABLE, descriptor.getTableType()); + Assertions.assertEquals(42L, descriptor.getId()); + Assertions.assertEquals(2, descriptor.getNumCols()); + Assertions.assertEquals("log_table", descriptor.getTableName()); + Assertions.assertEquals("db", descriptor.getDbName()); + Assertions.assertEquals("db", descriptor.getHiveTable().getDbName()); + Assertions.assertEquals("log_table", descriptor.getHiveTable().getTableName()); + } + + @Test + public void tableThatVanishesIsReportedMissingNotBroken() { + // getTableHandle discriminates on fluss's own not-exists exception, so it must keep working when + // that exception arrives from a memoized loader rather than straight from the admin call. + RecordingFlussAdminOps adminOps = new RecordingFlussAdminOps(); + FlussTestSession session = new FlussTestSession(1L, "query-1"); + Assertions.assertEquals(Optional.empty(), + metadata(adminOps).getTableHandle(session, "db", "gone")); + Assertions.assertThrows(TableNotExistException.class, + () -> adminOps.getTableInfo(TablePath.of("db", "gone"))); + } + + private static long countCalls(RecordingFlussAdminOps adminOps, String method) { + return adminOps.calls.stream().filter(call -> call.startsWith(method + "(")).count(); + } + + private static List names(List columns) { + List names = new ArrayList<>(columns.size()); + for (ConnectorColumn column : columns) { + names.add(column.getName()); + } + return names; + } + + /** A fluss partition whose spec is given as key/value pairs, in the caller's order. */ + private static PartitionInfo partition(long partitionId, String... keyValues) { + Map spec = new LinkedHashMap<>(); + for (int i = 0; i < keyValues.length; i += 2) { + spec.put(keyValues[i], keyValues[i + 1]); + } + return new PartitionInfo(partitionId, + ResolvedPartitionSpec.fromPartitionSpec( + new ArrayList<>(spec.keySet()), new PartitionSpec(spec)), + null); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorPropertiesTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorPropertiesTest.java new file mode 100644 index 00000000000000..c1a74f5b38bd2b --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorPropertiesTest.java @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.fluss.FlussConnectorProperties.UnionReadMode; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +/** + * Pins the catalog-property contract of a fluss catalog: what CREATE CATALOG must reject, and exactly + * which properties reach the fluss client. + */ +public class FlussConnectorPropertiesTest { + + private static Map props(String... keyValues) { + Map map = new HashMap<>(); + for (int i = 0; i < keyValues.length; i += 2) { + map.put(keyValues[i], keyValues[i + 1]); + } + return map; + } + + @Test + public void bootstrapServersIsRequired() { + // A catalog with no bootstrap servers can never answer a query, so it must fail at CREATE + // CATALOG rather than at the user's first SELECT. + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> FlussConnectorProperties.validate(props())); + Assertions.assertTrue(e.getMessage().contains(FlussConnectorProperties.BOOTSTRAP_SERVERS), + "message should name the missing property, was: " + e.getMessage()); + + Assertions.assertThrows(IllegalArgumentException.class, + () -> FlussConnectorProperties.validate(props(FlussConnectorProperties.BOOTSTRAP_SERVERS, " "))); + } + + @Test + public void bootstrapServersMustBeHostPortPairs() { + // Each rejected form is one a user actually writes: a bare host, a non-numeric port, a port out + // of range, and a trailing empty element from a stray comma. + for (String bad : new String[] {"localhost", "localhost:", "localhost:abc", "localhost:0", + "localhost:65536", "host1:9123,"}) { + Assertions.assertThrows(IllegalArgumentException.class, + () -> FlussConnectorProperties.validate( + props(FlussConnectorProperties.BOOTSTRAP_SERVERS, bad)), + "expected '" + bad + "' to be rejected"); + } + } + + @Test + public void bootstrapServersAcceptsListsAndIpv6() { + // The IPv6 case is why the port is split at the LAST colon, not the first. + for (String good : new String[] {"localhost:9123", " host1:9123 , host2:9124 ", "[::1]:9123"}) { + FlussConnectorProperties.validate(props(FlussConnectorProperties.BOOTSTRAP_SERVERS, good)); + } + } + + @Test + public void unionReadModeDefaultsToAutoAndIsCaseInsensitive() { + Assertions.assertEquals(UnionReadMode.AUTO, FlussConnectorProperties.unionReadMode(props())); + Assertions.assertEquals(UnionReadMode.REQUIRED, + FlussConnectorProperties.unionReadMode(props(FlussConnectorProperties.UNION_READ_MODE, "ReQuIrEd"))); + Assertions.assertEquals(UnionReadMode.DISABLED, + FlussConnectorProperties.unionReadMode(props(FlussConnectorProperties.UNION_READ_MODE, " disabled "))); + } + + @Test + public void unionReadModeRejectsUnknownValueAtCreateCatalog() { + // A typo here would otherwise degrade silently to whatever the default is, and the difference + // between auto and required is only visible as "the query returned fewer rows than it should". + Map properties = props( + FlussConnectorProperties.BOOTSTRAP_SERVERS, "localhost:9123", + FlussConnectorProperties.UNION_READ_MODE, "enabled"); + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> FlussConnectorProperties.validate(properties)); + Assertions.assertTrue(e.getMessage().contains("auto, required, disabled"), + "message should list the accepted values, was: " + e.getMessage()); + } + + /** + * The ceiling on how much log tail BE may hold while reading a primary-key table together with its + * lake. Zero and negative are rejected rather than read as "no limit": that limit exists because BE + * is a long-lived process shared by every other query, and there is deliberately no way to say + * "unbounded". + */ + @Test + public void theTailCeilingDefaultsToTwoMillionRowsAndMustBePositive() { + Assertions.assertEquals(2_000_000L, FlussConnectorProperties.maxTailRows(props())); + Assertions.assertEquals(500L, FlussConnectorProperties.maxTailRows( + props(FlussConnectorProperties.UNION_READ_MAX_TAIL_ROWS, " 500 "))); + + for (String bad : new String[] {"0", "-1", "lots", ""}) { + Map properties = props( + FlussConnectorProperties.BOOTSTRAP_SERVERS, "localhost:9123", + FlussConnectorProperties.UNION_READ_MAX_TAIL_ROWS, bad); + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> FlussConnectorProperties.validate(properties), + "accepted '" + bad + "' as a row ceiling"); + Assertions.assertTrue( + e.getMessage().contains(FlussConnectorProperties.UNION_READ_MAX_TAIL_ROWS), + e.getMessage()); + } + } + + @Test + public void clientConfigIsThePrefixedPropertiesMinusTheDorisOnlyOnes() { + Map properties = props( + FlussConnectorProperties.BOOTSTRAP_SERVERS, "localhost:9123", + "fluss.client.security.protocol", "sasl", + FlussConnectorProperties.UNION_READ_MODE, "required", + FlussConnectorProperties.UNION_READ_MAX_TAIL_ROWS, "10", + FlussConnectorProperties.ENABLE_MAPPING_VARBINARY, "true", + FlussConnectorProperties.ENABLE_MAPPING_TIMESTAMP_TZ, "true", + "type", "fluss", + "warehouse", "s3://ignored"); + + Map config = FlussConnectorProperties.toFlussClientConfig(properties); + + // bootstrap.servers and client.* arrive under fluss's own names; the Doris-only union-read + // switch and every non-fluss catalog property stay behind. The engine's own keys ("type") and + // other connectors' keys ("warehouse") are not fluss options and must not be handed over as if + // they were — the fluss config is not a place to dump whatever the catalog happened to carry. + // The two type-mapping switches are in the input for the same reason: they steer Doris's own + // schema rendering and fluss has no idea what they mean. + Map expected = new HashMap<>(); + expected.put("bootstrap.servers", "localhost:9123"); + expected.put("client.security.protocol", "sasl"); + Assertions.assertEquals(expected, config); + } + + @Test + public void typeMappingSwitchesDefaultToOffAndUseTheEngineWideNames() { + // The names are deliberately the unprefixed, engine-wide ones the hive/paimon/iceberg catalogs + // already answer to: a user who knows enable.mapping.varbinary must not have to discover a + // fluss-specific spelling, and a misspelling here degrades silently to "switch is off". + FlussTypeMapping.Options off = FlussConnectorProperties.typeMappingOptions(props()); + Assertions.assertFalse(off.isMapBinaryToVarbinary()); + Assertions.assertFalse(off.isMapTimestampTz()); + + FlussTypeMapping.Options on = FlussConnectorProperties.typeMappingOptions(props( + "enable.mapping.varbinary", "true", + "enable.mapping.timestamp_tz", "TRUE")); + Assertions.assertTrue(on.isMapBinaryToVarbinary()); + Assertions.assertTrue(on.isMapTimestampTz()); + + // Anything that is not "true" is off, matching how every other catalog reads these. + FlussTypeMapping.Options garbage = FlussConnectorProperties.typeMappingOptions(props( + FlussConnectorProperties.ENABLE_MAPPING_VARBINARY, "yes")); + Assertions.assertFalse(garbage.isMapBinaryToVarbinary()); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorProviderTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorProviderTest.java new file mode 100644 index 00000000000000..8223f1e2544e80 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorProviderTest.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.spi.ConnectorContext; +import org.apache.doris.connector.spi.ConnectorProvider; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.ServiceLoader; + +/** + * Guards the fluss connector's identity and its discovery wiring. + */ +public class FlussConnectorProviderTest { + + private static ConnectorContext context() { + return new ConnectorContext() { + @Override + public String getCatalogName() { + return "fluss_catalog"; + } + + @Override + public long getCatalogId() { + return 1L; + } + }; + } + + @Test + public void typeIsFluss() { + // The type name is what CREATE CATALOG routes on and what every statement-scope namespace is + // prefixed with, so it is API: renaming it breaks existing catalogs. + Assertions.assertEquals("fluss", new FlussConnectorProvider().getType()); + Assertions.assertTrue(new FlussConnectorProvider().isStandaloneCatalogType()); + } + + @Test + public void providerIsDiscoverableThroughServiceLoader() { + // The META-INF/services line is what makes the plugin loadable at all; a typo in it is invisible + // until FE starts and the catalog type simply does not exist. + boolean found = false; + for (ConnectorProvider provider : ServiceLoader.load(ConnectorProvider.class, + FlussConnectorProviderTest.class.getClassLoader())) { + if (provider instanceof FlussConnectorProvider) { + found = true; + } + } + Assertions.assertTrue(found, "FlussConnectorProvider must be registered in META-INF/services"); + } + + @Test + public void createValidatesPropertiesBeforeHandingBackACatalog() { + // Validation must also happen on the create path, not only in validateProperties: a catalog + // restored from the FE image at startup goes straight through create. + Assertions.assertThrows(IllegalArgumentException.class, + () -> new FlussConnectorProvider().create(new HashMap<>(), context())); + } + + @Test + public void createBuildsAFlussConnectorWithoutTouchingTheCluster() throws IOException { + // Constructing a catalog must stay offline — FE creates connectors while replaying its image, + // where an unreachable cluster must not block startup. The fluss connection is therefore lazy, + // and closing a connector that never connected must be a no-op rather than an NPE. + Map properties = new HashMap<>(); + properties.put(FlussConnectorProperties.BOOTSTRAP_SERVERS, "localhost:9123"); + + Connector connector = new FlussConnectorProvider().create(properties, context()); + Assertions.assertTrue(connector instanceof FlussConnector); + connector.close(); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLakeTableTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLakeTableTest.java new file mode 100644 index 00000000000000..819efde568a3c7 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLakeTableTest.java @@ -0,0 +1,343 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.ConnectorPartitionInfo; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorTableSchema; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; + +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * The lake gateway: {@code tbl$lake} is served by an embedded paimon sibling, and every per-handle call + * about that table has to reach the sibling instead of fluss. + * + *

The sibling is a hand-written {@link RecordingLakeSibling} (this module bans mock frameworks) whose + * answers are values fluss could never produce, so an assertion that finds them proves the forward really + * happened rather than that both sides agree by coincidence. + */ +public class FlussLakeTableTest { + + private static final TablePath LAKE_TABLE = TablePath.of("db", "lake_table"); + private static final TablePath PLAIN_TABLE = TablePath.of("db", "plain_table"); + + /** The sibling connectors this test's factory built, in build order. */ + private final List builtSiblings = new ArrayList<>(); + + private final ConnectorSession session = new FlussTestSession(7L, "q1"); + + /** + * A fluss table with a paimon lake, exactly as a coordinator reports one: the lake switch, the format, + * and the cluster's paimon settings injected under {@code table.datalake.paimon.}. + */ + private RecordingFlussAdminOps withLakeTable() { + return withLakeTable("paimon"); + } + + private RecordingFlussAdminOps withLakeTable(String lakeFormat) { + RecordingFlussAdminOps adminOps = new RecordingFlussAdminOps(); + adminOps.tableInfos.put(LAKE_TABLE, FlussTestTables.builder(LAKE_TABLE) + .column("id", DataTypes.BIGINT()) + .column("name", DataTypes.STRING()) + .buckets(2) + .property("table.datalake.enabled", "true") + .property("table.datalake.format", lakeFormat) + .property("table.datalake.paimon.metastore", "filesystem") + .property("table.datalake.paimon.warehouse", "/lake/warehouse") + .build()); + adminOps.tableInfos.put(PLAIN_TABLE, FlussTestTables.builder(PLAIN_TABLE) + .column("id", DataTypes.BIGINT()) + .buckets(1) + .build()); + return adminOps; + } + + /** + * Metadata whose lake seams are the real ones a connector supplies: a factory that builds (and + * records) a sibling, and an owner resolver that answers exactly the way {@code FlussConnector}'s does + * — by asking each built sibling whether the handle is its own. + */ + private FlussConnectorMetadata metadata(RecordingFlussAdminOps adminOps) { + return new FlussConnectorMetadata(adminOps, FlussTypeMapping.Options.DEFAULT, + properties -> { + RecordingLakeSibling sibling = new RecordingLakeSibling(properties); + builtSiblings.add(sibling); + return sibling; + }, + handle -> { + for (RecordingLakeSibling sibling : builtSiblings) { + if (sibling.ownsHandle(handle)) { + return sibling; + } + } + return null; + }); + } + + private ConnectorTableHandle baseHandle(FlussConnectorMetadata metadata, TablePath tablePath) { + return metadata.getTableHandle(session, tablePath.getDatabaseName(), tablePath.getTableName()) + .orElseThrow(AssertionError::new); + } + + private ConnectorTableHandle lakeHandle(FlussConnectorMetadata metadata) { + return metadata.getSysTableHandle(session, baseHandle(metadata, LAKE_TABLE), "lake") + .orElseThrow(AssertionError::new); + } + + @Test + public void datalakeTableOffersTheLakeSystemTable() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + // This listing is what makes fe-core resolve the name "lake_table$lake" at all. + Assertions.assertEquals(Collections.singletonList("lake"), + metadata.listSupportedSysTables(session, baseHandle(metadata, LAKE_TABLE))); + } + + @Test + public void tableWithoutALakeOffersNoSystemTable() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + // Advertising $lake on a table that has none would offer a sub-table whose only possible outcome + // is an error; fe-core's "no such table" is the honest answer. + Assertions.assertTrue( + metadata.listSupportedSysTables(session, baseHandle(metadata, PLAIN_TABLE)).isEmpty()); + } + + @Test + public void theLakeHandleIsTheSiblingsHandleForTheSameName() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + ConnectorTableHandle handle = lakeHandle(metadata); + + // The lake table lives under the SAME db.table name in paimon — that is where fluss's tiering + // service writes it. A handle of the sibling's own type is also what routes every later call. + Assertions.assertTrue(handle instanceof RecordingLakeSibling.Handle); + Assertions.assertEquals("db", ((RecordingLakeSibling.Handle) handle).dbName); + Assertions.assertEquals("lake_table", ((RecordingLakeSibling.Handle) handle).tableName); + Assertions.assertEquals(1, builtSiblings.size(), "one lake configuration, one sibling"); + Assertions.assertEquals(Collections.singletonList("getTableHandle:db.lake_table"), + builtSiblings.get(0).calls); + } + + @Test + public void theSiblingIsConfiguredFromTheTablesLakeProperties() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + lakeHandle(metadata); + + Map expected = new HashMap<>(); + expected.put("paimon.catalog.type", "filesystem"); + expected.put("warehouse", "/lake/warehouse"); + // Whole-map equality: the sibling is a real catalog and a leaked fluss property would either be + // rejected by it or silently reinterpreted as a paimon option. + Assertions.assertEquals(expected, builtSiblings.get(0).properties); + } + + @Test + public void anUnknownSystemTableNameIsNotServed() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + // Only "lake" is this connector's; asking for anything else must not build a sibling. + Assertions.assertEquals(Optional.empty(), + metadata.getSysTableHandle(session, baseHandle(metadata, LAKE_TABLE), "snapshots")); + Assertions.assertTrue(builtSiblings.isEmpty()); + } + + @Test + public void tableWithoutALakeFailsLoudOnTheLakeHandle() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + // Reachable when the table's lake is turned off between the listing and the resolution: two round + // trips, so the gate is re-checked rather than assumed. + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.getSysTableHandle(session, baseHandle(metadata, PLAIN_TABLE), "lake")); + Assertions.assertTrue(failure.getMessage().contains("table.datalake.enabled"), + failure.getMessage()); + } + + @Test + public void anUnsupportedLakeFormatFailsLoud() { + FlussConnectorMetadata metadata = metadata(withLakeTable("iceberg")); + // The reason $lake is offered for ANY lake format: this precise message beats fe-core's generic + // "no such table" for a user whose table is tiered to a lake this connector cannot read yet. + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.getSysTableHandle(session, baseHandle(metadata, LAKE_TABLE), "lake")); + Assertions.assertTrue(failure.getMessage().contains("iceberg"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("paimon"), failure.getMessage()); + Assertions.assertTrue(builtSiblings.isEmpty(), "an unreadable format must not build a catalog"); + } + + @Test + public void anUntieredLakeTableFailsLoud() { + FlussConnectorMetadata metadata = new FlussConnectorMetadata(withLakeTable(), + FlussTypeMapping.Options.DEFAULT, + properties -> { + RecordingLakeSibling sibling = new RecordingLakeSibling(properties); + sibling.lakeTableExists = false; + builtSiblings.add(sibling); + return sibling; + }, + handle -> null); + + // The lake table is only created on the tiering service's first commit. Answering "no such table" + // would send the user after a name that is in fact correct; naming tiering is the actionable form. + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.getSysTableHandle(session, baseHandle(metadata, LAKE_TABLE), "lake")); + Assertions.assertTrue(failure.getMessage().contains("tiering"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("db.lake_table"), failure.getMessage()); + } + + @Test + public void siblingThatDisownsItsHandleFailsLoud() { + // Connector.ownsHandle defaults to false, so a connector that never overrode it disowns the handles + // it just produced. Every guard on this side then silently fails open and the first cast throws a + // ClassCastException naming two class loaders and no cause -- which is exactly what the paimon + // connector did the first time it was used as a sibling for real. Caught where the handle is born, + // the message names the class that has to change. + FlussConnectorMetadata metadata = new FlussConnectorMetadata(withLakeTable(), + FlussTypeMapping.Options.DEFAULT, + properties -> { + RecordingLakeSibling sibling = new RecordingLakeSibling(properties); + sibling.claimsItsOwnHandles = false; + builtSiblings.add(sibling); + return sibling; + }, + handle -> null); + + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> metadata.getSysTableHandle(session, baseHandle(metadata, LAKE_TABLE), "lake")); + Assertions.assertTrue(failure.getMessage().contains("ownsHandle"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains(RecordingLakeSibling.class.getName()), + failure.getMessage()); + } + + @Test + public void theSchemaOfALakeTableComesFromTheSibling() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + ConnectorTableSchema schema = metadata.getTableSchema(session, lakeHandle(metadata)); + + // Fluss's own schema for this table is (id, name) with format FLUSS; every value here is the + // sibling's, so the forward is what produced them. + Assertions.assertEquals(RecordingLakeSibling.SCHEMA_TABLE_NAME, schema.getTableName()); + Assertions.assertEquals(RecordingLakeSibling.FORMAT_TYPE, schema.getTableFormatType()); + Assertions.assertEquals(1, schema.getColumns().size()); + Assertions.assertEquals(RecordingLakeSibling.COLUMN_NAME, schema.getColumns().get(0).getName()); + } + + @Test + public void theColumnHandlesOfALakeTableComeFromTheSibling() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + Assertions.assertEquals( + Collections.singleton(RecordingLakeSibling.COLUMN_NAME), + metadata.getColumnHandles(session, lakeHandle(metadata)).keySet()); + } + + @Test + public void thePartitionsOfALakeTableComeFromTheSibling() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + ConnectorTableHandle handle = lakeHandle(metadata); + + // Fluss's own answer for this (unpartitioned) table is an empty list, so a missing guard would + // silently report "no partitions" for a partitioned lake table and read the whole thing. + List partitions = + metadata.listPartitions(session, handle, Optional.empty()); + Assertions.assertEquals(1, partitions.size()); + Assertions.assertEquals(RecordingLakeSibling.PARTITION_NAME, partitions.get(0).getPartitionName()); + + Assertions.assertEquals( + Collections.singletonList(RecordingLakeSibling.PARTITION_NAME), + metadata.listPartitionNames(session, handle)); + // listPartitionNames must reach the sibling's own method, not derive names from listPartitions: + // the sibling may answer the two differently (and more cheaply). + Assertions.assertTrue(builtSiblings.get(0).calls.contains("listPartitionNames")); + } + + @Test + public void theStatisticsOfALakeTableComeFromTheSibling() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + // Fluss's own path would call getTableStats on the fluss table (an unprogrammed recording call, + // which throws), so this asserts the forward AND that fluss's remote call never happened. + Assertions.assertEquals(RecordingLakeSibling.ROW_COUNT, + metadata.getTableStatistics(session, lakeHandle(metadata)) + .orElseThrow(AssertionError::new).getRowCount()); + } + + @Test + public void systemTableQuestionsAboutALakeTableGoToTheSibling() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + ConnectorTableHandle handle = lakeHandle(metadata); + + // A lake handle reaching fluss's own bodies would ClassCastException, so these guards are about + // not crashing as much as about delegating: paimon owns its table's system tables. + Assertions.assertEquals(Collections.singletonList("snapshots"), + metadata.listSupportedSysTables(session, handle)); + Assertions.assertEquals(Optional.empty(), + metadata.getSysTableHandle(session, handle, "snapshots")); + Assertions.assertTrue(metadata.isPartitionValuesSysTable(session, handle, "partitions")); + } + + @Test + public void flussHandleStillGetsFlussAnswers() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + // Build the sibling first, so the guards have a sibling to (wrongly) route to if they matched on + // anything other than handle ownership. + lakeHandle(metadata); + + ConnectorTableSchema schema = metadata.getTableSchema(session, baseHandle(metadata, LAKE_TABLE)); + Assertions.assertEquals("lake_table", schema.getTableName()); + Assertions.assertEquals("FLUSS", schema.getTableFormatType()); + Assertions.assertEquals(2, schema.getColumns().size()); + } + + @Test + public void oneStatementBuildsOneSiblingMetadata() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + ConnectorTableHandle handle = lakeHandle(metadata); + + metadata.getTableSchema(session, handle); + metadata.getColumnHandles(session, handle); + metadata.listPartitions(session, handle, Optional.empty()); + metadata.getTableStatistics(session, handle); + + // The per-statement funnel: the sibling's metadata is built once and every forward reuses it, + // exactly as fe-core's own funnel does for a plain connector. Rebuilding per call would re-open + // whatever that metadata memoizes (a schema, a snapshot) and let one statement see two versions. + Assertions.assertEquals(1, builtSiblings.get(0).metadataBuilds); + } + + @Test + public void twoStatementsDoNotShareTheSiblingMetadata() { + FlussConnectorMetadata metadata = metadata(withLakeTable()); + ConnectorTableHandle handle = lakeHandle(metadata); + metadata.getTableSchema(session, handle); + + // A different statement scope must not reuse the previous statement's metadata: the funnel is + // keyed inside the scope, so this proves the key is not accidentally global. + ConnectorSession other = new FlussTestSession(7L, "q2"); + metadata.getTableSchema(other, handle); + + Assertions.assertEquals(2, builtSiblings.get(0).metadataBuilds); + } + +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLogScanPlanClusterTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLogScanPlanClusterTest.java new file mode 100644 index 00000000000000..c4c8f81218d135 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussLogScanPlanClusterTest.java @@ -0,0 +1,262 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.connector.api.scan.ConnectorScanRequest; +import org.apache.doris.connector.spi.ConnectorContext; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.writer.AppendWriter; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Split planning against a real fluss cluster started in this JVM. + * + *

{@link FlussSplitPlanTest} proves the planning logic over recorded answers; this proves the + * premise those recordings encode — that a cluster really reports a bucket nobody wrote to as offset + * 0, that the stopping offset planning takes is the one a reader would see, and that a partition's + * buckets are addressed the way this connector addresses them. A recorded answer that has drifted from + * the real one makes every unit test above pass while nothing works. + * + *

Named {@code ...Test}, not {@code ...ITCase}: surefire's default includes do not match + * {@code *ITCase}, so that name would leave the class silently unexecuted with a green build. + */ +public class FlussLogScanPlanClusterTest { + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER = FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .build(); + + private static final int BUCKETS = 3; + + /** The cluster extension drops every non-built-in database after each test, so fixtures are per-test. */ + private static int databaseCounter; + + private static Connection connection; + private static Admin admin; + private static Connector connector; + + private String db; + + @BeforeAll + public static void connectToCluster() throws Exception { + Configuration clientConf = FLUSS_CLUSTER.getClientConfig(); + connection = ConnectionFactory.createConnection(clientConf); + admin = connection.getAdmin(); + + Map catalogProperties = new HashMap<>(); + catalogProperties.put(FlussConnectorProperties.BOOTSTRAP_SERVERS, + FLUSS_CLUSTER.getBootstrapServers()); + connector = new FlussConnectorProvider().create(catalogProperties, new ConnectorContext() { + @Override + public String getCatalogName() { + return "fluss_catalog"; + } + + @Override + public long getCatalogId() { + return 1L; + } + }); + } + + @BeforeEach + public void createTables() throws Exception { + db = "doris_scan_plan_test_" + (++databaseCounter); + admin.createDatabase(db, DatabaseDescriptor.EMPTY, true).get(); + + admin.createTable(TablePath.of(db, "log_table"), + TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .column("name", DataTypes.STRING()) + .build()) + .distributedBy(BUCKETS, "id") + .build(), + true).get(); + + admin.createTable(TablePath.of(db, "log_part"), + TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .column("dt", DataTypes.STRING()) + .build()) + .partitionedBy("dt") + .distributedBy(BUCKETS, "id") + .build(), + true).get(); + admin.createPartition(TablePath.of(db, "log_part"), + new PartitionSpec(Collections.singletonMap("dt", "2026_08_02")), true).get(); + admin.createPartition(TablePath.of(db, "log_part"), + new PartitionSpec(Collections.singletonMap("dt", "2026_08_03")), true).get(); + } + + @AfterAll + public static void disconnect() throws Exception { + if (connector != null) { + connector.close(); + } + if (connection != null) { + connection.close(); + } + } + + /** + * The stopping offset a range carries has to be the offset the log had actually reached, asked of + * the same cluster the reader will read. Anything else silently truncates the scan or runs past the + * end. + */ + @Test + public void rangesStopAtTheOffsetsTheClusterReports() throws Exception { + TablePath tablePath = TablePath.of(db, "log_table"); + appendRows(tablePath, 20); + + Map latest = admin.listOffsets(tablePath, allBuckets(), new OffsetSpec.LatestSpec()) + .all().get(); + List ranges = plan("log_table"); + + Map planned = new HashMap<>(); + for (ConnectorScanRange range : ranges) { + Map props = range.getProperties(); + Assertions.assertEquals("LOG", props.get("fluss.range_type")); + // The earliest sentinel goes out verbatim; only fluss resolves it. + Assertions.assertEquals("-2", props.get("fluss.log_start_offset")); + planned.put(Integer.parseInt(props.get("fluss.bucket_id")), + Long.parseLong(props.get("fluss.log_stop_offset"))); + } + + // Every non-empty bucket, and only those, with exactly the cluster's offset. + Map expected = new HashMap<>(); + latest.forEach((bucket, offset) -> { + if (offset > 0) { + expected.put(bucket, offset); + } + }); + Assertions.assertEquals(expected, planned); + Assertions.assertFalse(expected.isEmpty(), "the fixture wrote no rows anywhere"); + long total = 0; + for (long offset : expected.values()) { + total += offset; + } + Assertions.assertEquals(20, total, "planned ranges must cover every appended row"); + } + + /** The premise behind skipping empty buckets: a cluster reports a never-written bucket as 0. */ + @Test + public void tableNothingWasWrittenToPlansNoRanges() { + Assertions.assertTrue(plan("log_table").isEmpty()); + } + + /** + * A partitioned table is planned partition by partition, addressed by fluss's own partition name + * and reported to Doris under the Hive-style one. Getting either wrong is invisible until a query + * returns the wrong partition's rows. + */ + @Test + public void partitionedRangesCarryTheClusterAssignedPartitionIdentity() throws Exception { + TablePath tablePath = TablePath.of(db, "log_part"); + appendPartitionedRows(tablePath, "2026_08_02", 6); + + List ranges = plan("log_part"); + + Assertions.assertFalse(ranges.isEmpty()); + long partitionId = admin.listPartitionInfos(tablePath).get().stream() + .filter(p -> p.getPartitionName().equals("2026_08_02")) + .findFirst().orElseThrow(AssertionError::new) + .getPartitionId(); + long rows = 0; + for (ConnectorScanRange range : ranges) { + Map props = range.getProperties(); + // Only the written partition produced ranges; the empty one has nothing to read. + Assertions.assertEquals("dt=2026_08_02", props.get("fluss.partition_name")); + Assertions.assertEquals(String.valueOf(partitionId), props.get("fluss.partition_id")); + Assertions.assertEquals(Collections.singletonMap("dt", "2026_08_02"), + range.getPartitionValues()); + rows += Long.parseLong(props.get("fluss.log_stop_offset")); + } + Assertions.assertEquals(6, rows); + } + + private List plan(String tableName) { + FlussTestSession session = new FlussTestSession(1L, "cluster-plan"); + ConnectorTableHandle handle = connector.getMetadata(session) + .getTableHandle(session, db, tableName).orElseThrow(AssertionError::new); + ConnectorScanPlanProvider provider = connector.getScanPlanProvider(); + return provider.planScan(session, + ConnectorScanRequest.builder(handle, Collections.emptyList()).build()); + } + + private static void appendRows(TablePath tablePath, int rows) throws Exception { + try (Table table = connection.getTable(tablePath)) { + AppendWriter writer = table.newAppend().createWriter(); + for (int i = 0; i < rows; i++) { + writer.append(GenericRow.of((long) i, BinaryString.fromString("name-" + i))); + } + writer.flush(); + } + } + + private static void appendPartitionedRows(TablePath tablePath, String partition, int rows) + throws Exception { + try (Table table = connection.getTable(tablePath)) { + AppendWriter writer = table.newAppend().createWriter(); + for (int i = 0; i < rows; i++) { + writer.append(GenericRow.of((long) i, BinaryString.fromString(partition))); + } + writer.flush(); + } + } + + private static List allBuckets() { + List buckets = new ArrayList<>(BUCKETS); + for (int bucket = 0; bucket < BUCKETS; bucket++) { + buckets.add(bucket); + } + return buckets; + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussMetadataClusterTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussMetadataClusterTest.java new file mode 100644 index 00000000000000..d13a4c499d9cbf --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussMetadataClusterTest.java @@ -0,0 +1,287 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.ConnectorColumn; +import org.apache.doris.connector.api.ConnectorMetadata; +import org.apache.doris.connector.api.ConnectorPartitionInfo; +import org.apache.doris.connector.api.ConnectorTableSchema; +import org.apache.doris.connector.api.ConnectorTestResult; +import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.ConnectorContext; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * The metadata path against a real fluss cluster started in this JVM. + * + *

{@link FlussConnectorMetadataTest} pins the mapping against canned {@code TableInfo}s, which + * proves the logic but not the premise: it cannot tell whether a real cluster actually reports column + * comments where we read them, orders schema columns the way we assume, or answers the calls we make + * at all. This starts a coordinator, a tablet server and an embedded ZooKeeper, creates tables through + * the fluss Java client, and reads them back through the connector — every hop real except Doris's own + * plugin classloading. + * + *

It is deliberately named {@code ...Test} rather than {@code ...ITCase}: surefire's default + * includes do not match {@code *ITCase}, so that name would leave the whole class silently unexecuted + * with a green build. + */ +public class FlussMetadataClusterTest { + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER = FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .build(); + + /** + * Each test gets its own database. The cluster extension drops every non-built-in database after + * each test, so a shared fixture would vanish after the first one; a fresh NAME each time (rather + * than re-creating the same one) also keeps the fluss client's metadata cache from ever holding a + * dropped table's id for a path that came back. + */ + private static int databaseCounter; + + private static Connection connection; + private static Admin admin; + private static Connector connector; + + private String db; + + @BeforeAll + public static void connectToCluster() throws Exception { + Configuration clientConf = FLUSS_CLUSTER.getClientConfig(); + connection = ConnectionFactory.createConnection(clientConf); + admin = connection.getAdmin(); + + Map catalogProperties = new HashMap<>(); + catalogProperties.put(FlussConnectorProperties.BOOTSTRAP_SERVERS, + FLUSS_CLUSTER.getBootstrapServers()); + connector = new FlussConnectorProvider().create(catalogProperties, new ConnectorContext() { + @Override + public String getCatalogName() { + return "fluss_catalog"; + } + + @Override + public long getCatalogId() { + return 1L; + } + }); + } + + @BeforeEach + public void createTables() throws Exception { + db = "doris_metadata_test_" + (++databaseCounter); + admin.createDatabase(db, DatabaseDescriptor.EMPTY, true).get(); + + admin.createTable(TablePath.of(db, "log_table"), + TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .withComment("the row id") + .column("name", DataTypes.STRING()) + .column("price", DataTypes.DECIMAL(20, 4)) + .column("event_time", DataTypes.TIMESTAMP_LTZ(6)) + .column("started", DataTypes.TIME(3)) + .column("tags", DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())) + .column("nested", DataTypes.ROW( + DataTypes.FIELD("a", DataTypes.INT(), "inner doc"))) + .build()) + .distributedBy(3) + .comment("a log table") + .build(), + true).get(); + + admin.createTable(TablePath.of(db, "pk_table"), + TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("dt", DataTypes.STRING()) + .column("id", DataTypes.BIGINT()) + .column("amount", DataTypes.DOUBLE()) + .primaryKey("dt", "id") + .build()) + .partitionedBy("dt") + .distributedBy(2, "id") + .build(), + true).get(); + admin.createPartition(TablePath.of(db, "pk_table"), + new PartitionSpec(Collections.singletonMap("dt", "2026_08_02")), true).get(); + admin.createPartition(TablePath.of(db, "pk_table"), + new PartitionSpec(Collections.singletonMap("dt", "2026_08_03")), true).get(); + } + + @AfterAll + public static void disconnect() throws Exception { + if (connector != null) { + connector.close(); + } + if (connection != null) { + connection.close(); + } + } + + private static ConnectorMetadata metadata() { + return connector.getMetadata(new FlussTestSession(1L, "cluster-query")); + } + + @Test + public void theCatalogSeesTheClusterAndItsNamespaces() { + ConnectorTestResult result = connector.testConnection(null); + Assertions.assertTrue(result.isSuccess(), () -> "connectivity test failed: " + result); + + ConnectorMetadata metadata = metadata(); + Assertions.assertTrue(metadata.listDatabaseNames(null).contains(db)); + Assertions.assertTrue(metadata.databaseExists(null, db)); + Assertions.assertFalse(metadata.databaseExists(null, "no_such_db")); + List tables = metadata.listTableNames(null, db); + Assertions.assertTrue(tables.containsAll(Arrays.asList("log_table", "pk_table")), + "expected both tables, got " + tables); + Assertions.assertEquals(Optional.empty(), metadata.getTableHandle(null, db, "no_such_table")); + } + + @Test + public void schemaComesBackFromTheClusterWithTypesAndComments() { + // The premise the unit tests cannot check: that a real cluster reports these facts where this + // connector reads them. Column comments are the sharpest case — fluss keeps them on the schema + // and drops them from the table's row type, so reading the wrong one loses every comment. + ConnectorMetadata metadata = metadata(); + ConnectorTableHandle handle = metadata.getTableHandle(null, db, "log_table") + .orElseThrow(AssertionError::new); + + ConnectorTableSchema schema = metadata.getTableSchema(null, handle); + Map columns = byName(schema.getColumns()); + + Assertions.assertEquals(ConnectorType.of("BIGINT"), columns.get("id").getType()); + Assertions.assertEquals("the row id", columns.get("id").getComment()); + Assertions.assertEquals(ConnectorType.of("STRING"), columns.get("name").getType()); + Assertions.assertEquals(ConnectorType.of("DECIMALV3", 20, 4), columns.get("price").getType()); + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 6, 0), columns.get("event_time").getType()); + Assertions.assertTrue(columns.get("event_time").isWithTimeZone()); + // TIME survives the round trip as an unsupported column rather than making the table unloadable. + Assertions.assertEquals(ConnectorType.of("UNSUPPORTED"), columns.get("started").getType()); + Assertions.assertEquals( + ConnectorType.mapOf(ConnectorType.of("STRING"), ConnectorType.of("INT")), + columns.get("tags").getType()); + ConnectorType nested = columns.get("nested").getType(); + Assertions.assertEquals("STRUCT", nested.getTypeName()); + Assertions.assertEquals(Collections.singletonList("a"), nested.getFieldNames()); + Assertions.assertEquals("inner doc", nested.getChildComment(0)); + + Assertions.assertEquals("a log table", metadata.getTableComment(null, db, "log_table")); + Assertions.assertEquals("FLUSS", schema.getTableFormatType()); + + // Column handles are positional, and the position must be the cluster's column order. + Map handles = metadata.getColumnHandles(null, handle); + Assertions.assertEquals(new FlussColumnHandle("id", 0), handles.get("id")); + Assertions.assertEquals(new FlussColumnHandle("nested", 6), handles.get("nested")); + } + + @Test + public void theHandleReflectsHowTheClusterCreatedTheTable() { + ConnectorMetadata metadata = metadata(); + FlussTableHandle logHandle = (FlussTableHandle) metadata.getTableHandle(null, db, "log_table") + .orElseThrow(AssertionError::new); + Assertions.assertFalse(logHandle.hasPrimaryKey()); + Assertions.assertFalse(logHandle.isPartitioned()); + Assertions.assertFalse(logHandle.isDataLakeEnabled()); + Assertions.assertEquals(3, logHandle.getBucketCount()); + // A real cluster assigns the ids; all the fixtures can say is that they were carried through. + Assertions.assertTrue(logHandle.getTableId() > 0, "expected a cluster-assigned table id"); + + FlussTableHandle pkHandle = (FlussTableHandle) metadata.getTableHandle(null, db, "pk_table") + .orElseThrow(AssertionError::new); + Assertions.assertTrue(pkHandle.hasPrimaryKey()); + Assertions.assertEquals(Arrays.asList("dt", "id"), pkHandle.getPrimaryKeys()); + Assertions.assertEquals(Collections.singletonList("dt"), pkHandle.getPartitionKeys()); + Assertions.assertEquals(Collections.singletonList("id"), pkHandle.getBucketKeys()); + Assertions.assertEquals(2, pkHandle.getBucketCount()); + } + + @Test + public void partitionsAreListedAsTheClusterCreatedThem() { + ConnectorMetadata metadata = metadata(); + ConnectorTableHandle handle = metadata.getTableHandle(null, db, "pk_table") + .orElseThrow(AssertionError::new); + + List partitions = metadata.listPartitions(null, handle, Optional.empty()); + List names = new ArrayList<>(); + for (ConnectorPartitionInfo partition : partitions) { + names.add(partition.getPartitionName()); + Assertions.assertEquals(1, partition.getOrderedPartitionValues().size()); + } + Collections.sort(names); + Assertions.assertEquals(Arrays.asList("dt=2026_08_02", "dt=2026_08_03"), names); + + Assertions.assertEquals("dt", + metadata.getTableSchema(null, handle).getProperties() + .get(ConnectorTableSchema.PARTITION_COLUMNS_KEY)); + + // The unpartitioned table must come back empty, not fail: fluss rejects the partition call for + // such a table, so this also proves the connector never makes it. + ConnectorTableHandle logHandle = metadata.getTableHandle(null, db, "log_table") + .orElseThrow(AssertionError::new); + Assertions.assertEquals(Collections.emptyList(), + metadata.listPartitions(null, logHandle, Optional.empty())); + } + + @Test + public void statisticsOfAFreshTableAreUnknownRatherThanZero() { + // Nothing has been written and statistics are off by default, so fluss reports no rows. Reporting + // that as a row count of zero would tell the optimizer the table is empty. + ConnectorMetadata metadata = metadata(); + ConnectorTableHandle handle = metadata.getTableHandle(null, db, "log_table") + .orElseThrow(AssertionError::new); + Assertions.assertEquals(Optional.empty(), metadata.getTableStatistics(null, handle)); + } + + private static Map byName(List columns) { + Map byName = new LinkedHashMap<>(); + for (ConnectorColumn column : columns) { + byName.put(column.getName(), column); + } + return byName; + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussPartitionColumnTypesTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussPartitionColumnTypesTest.java new file mode 100644 index 00000000000000..0982e7fb4302f8 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussPartitionColumnTypesTest.java @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypeRoot; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Which column types a fluss table may be partitioned by and still be readable. + * + *

The verdicts here were not reasoned out from fluss's naming rules and then trusted: every one of + * them was run against a fluss cluster through a Doris catalog first, and the split below is where the + * two sides actually part company. A type that renders into a partition name and back unchanged is + * allowed; one that fluss rewrites on the way in is refused, because the rewriting loses which character + * was there ({@code 1_5} was {@code 1.5}, {@code 01-02-03} was {@code 01:02:03}). + * + *

Completeness is the property worth more than any single verdict: every fluss type root is answered, + * so a type a future release adds to fluss's partition-key whitelist cannot arrive here as "allowed by + * omission" and reach fe-core's partition parser, where the failure names neither fluss nor the column. + */ +public class FlussPartitionColumnTypesTest { + + private static final FlussTypeMapping.Options VARBINARY = + new FlussTypeMapping.Options(true, false); + + /** One type per fluss type root, and the verdict each is meant to get as a PARTITION column. */ + private static Map verdicts() { + Map expected = new LinkedHashMap<>(); + // Stored in the partition name exactly as written, and read back the same way. + expected.put(DataTypes.CHAR(2), true); + expected.put(DataTypes.STRING(), true); + expected.put(DataTypes.BOOLEAN(), true); + expected.put(DataTypes.TINYINT(), true); + expected.put(DataTypes.SMALLINT(), true); + expected.put(DataTypes.INT(), true); + expected.put(DataTypes.BIGINT(), true); + expected.put(DataTypes.DATE(), true); + // Named with the hex text of the bytes, which is a string; readable as long as the catalog is + // mapping such a column to a string too. The other half of that pair is its own test below. + expected.put(DataTypes.BINARY(2), true); + expected.put(DataTypes.BYTES(), true); + // Fluss allows all five as partition keys and rewrites the '.', ':' and ' ' in their values. + expected.put(DataTypes.FLOAT(), false); + expected.put(DataTypes.DOUBLE(), false); + expected.put(DataTypes.TIME(), false); + expected.put(DataTypes.TIMESTAMP(3), false); + expected.put(DataTypes.TIMESTAMP_LTZ(3), false); + // Fluss refuses these as partition keys itself; refused here as well so that its refusal is not + // the only thing between them and fe-core's parser. + expected.put(DataTypes.DECIMAL(20, 4), false); + expected.put(DataTypes.ARRAY(DataTypes.INT()), false); + expected.put(DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), false); + expected.put(DataTypes.ROW(DataTypes.FIELD("f", DataTypes.INT())), false); + return expected; + } + + @Test + public void everyFlussTypeRootHasADeliberateVerdict() { + Set covered = EnumSet.noneOf(DataTypeRoot.class); + for (Map.Entry entry : verdicts().entrySet()) { + DataType type = entry.getKey(); + covered.add(type.getTypeRoot()); + boolean readable = + FlussPartitionColumnTypes.rejection(type, FlussTypeMapping.Options.DEFAULT) == null; + Assertions.assertEquals(entry.getValue(), readable, "partition verdict for " + type); + } + Assertions.assertEquals(EnumSet.allOf(DataTypeRoot.class), covered, + "a fluss type root has no verdict here; decide whether a partition of it can be read"); + } + + /** + * The one verdict the catalog's own settings can flip. Fluss names a BINARY partition with the hex + * text of its bytes: readable while the column is a Doris string, unreadable the moment + * {@code enable.mapping.varbinary} turns it into a VARBINARY, which no hex text is a literal of. + */ + @Test + public void binaryPartitionsAreReadableOnlyWhileTheColumnIsText() { + for (DataType type : Arrays.asList(DataTypes.BINARY(2), DataTypes.BYTES())) { + Assertions.assertNull( + FlussPartitionColumnTypes.rejection(type, FlussTypeMapping.Options.DEFAULT), + type + " is readable while it maps to a string"); + String rejection = FlussPartitionColumnTypes.rejection(type, VARBINARY); + Assertions.assertNotNull(rejection, type + " under varbinary mapping"); + // The property that caused it is named, because turning it off is the fix and nothing else + // about the table changed. + Assertions.assertTrue( + rejection.contains(FlussConnectorProperties.ENABLE_MAPPING_VARBINARY), + "the rejection should name the property that caused it: " + rejection); + } + } + + /** + * The catalog's settings must not reach any other verdict. A property about binary columns deciding + * whether a DATE partition can be read would be a rule nobody could predict from its name. + */ + @Test + public void noOtherVerdictDependsOnTheCatalogSettings() { + for (DataType type : verdicts().keySet()) { + if (type.getTypeRoot() == DataTypeRoot.BINARY || type.getTypeRoot() == DataTypeRoot.BYTES) { + continue; + } + Assertions.assertEquals( + FlussPartitionColumnTypes.rejection(type, FlussTypeMapping.Options.DEFAULT) == null, + FlussPartitionColumnTypes.rejection(type, VARBINARY) == null, + "the verdict for " + type + " changed with the catalog's mapping options"); + } + } + + /** + * A refused type says why in words that name the loss, not just that something is unsupported: the + * reader of this error has a table fluss created happily and a DESC that looks ordinary. + */ + @Test + public void refusalsExplainWhatWasLost() { + List rewritten = Arrays.asList(DataTypes.FLOAT(), DataTypes.DOUBLE(), + DataTypes.TIME(), DataTypes.TIMESTAMP(3), DataTypes.TIMESTAMP_LTZ(3)); + for (DataType type : rewritten) { + String rejection = FlussPartitionColumnTypes.rejection(type, FlussTypeMapping.Options.DEFAULT); + Assertions.assertNotNull(rejection, type.toString()); + Assertions.assertTrue(rejection.contains("cannot be read back"), + "the rejection for " + type + " should say the value is lost, was: " + rejection); + } + } + + /** + * A timestamp is refused at every precision. Doris can hold the value at six digits or fewer, so a + * rule written around precision — the one the union-read key gate needs — would let those through + * and back into the parser that cannot read the name they arrive under. + */ + @Test + public void timestampsAreRefusedAtEveryPrecision() { + for (int precision = 0; precision <= 9; precision++) { + Assertions.assertNotNull( + FlussPartitionColumnTypes.rejection(DataTypes.TIMESTAMP(precision), + FlussTypeMapping.Options.DEFAULT), + "TIMESTAMP(" + precision + ")"); + Assertions.assertNotNull( + FlussPartitionColumnTypes.rejection(DataTypes.TIMESTAMP_LTZ(precision), + FlussTypeMapping.Options.DEFAULT), + "TIMESTAMP_LTZ(" + precision + ")"); + } + } + + /** + * Readability is the weaker of the two questions asked about a partition column: whatever cannot be + * read at all cannot be matched to a lake split either. Were that ever the other way round, a table + * would be planned as a lake-plus-tail read on partition values nothing can parse. + */ + @Test + public void everythingReadableIsAlsoAskedTheStricterUnionQuestion() { + for (DataType type : verdicts().keySet()) { + if (FlussUnionKeyTypes.partitionColumnRejection(type) == null) { + Assertions.assertNull( + FlussPartitionColumnTypes.rejection(type, FlussTypeMapping.Options.DEFAULT), + type + " may be matched across the halves but cannot be read at all"); + } + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussPkScanPlanClusterTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussPkScanPlanClusterTest.java new file mode 100644 index 00000000000000..1620080d6bd871 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussPkScanPlanClusterTest.java @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.connector.api.scan.ConnectorScanRequest; +import org.apache.doris.connector.spi.ConnectorContext; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.client.metadata.KvSnapshots; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.writer.UpsertWriter; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Primary-key split planning against a real fluss cluster started in this JVM. + * + *

{@link FlussSplitPlanTest} proves the planning logic over recorded snapshot answers; this proves + * the premise those recordings encode — that a cluster really reports a never-snapshotted bucket as + * "no snapshot at all" rather than snapshot 0, and that the log offset it reports alongside a snapshot + * is the one a reader has to resume from. A recording that has drifted from the real answer leaves + * every unit test above green while a primary-key query returns superseded rows. + * + *

Snapshots are triggered, not waited for: the cluster's periodic interval is ten minutes, + * so nothing snapshots on its own and every test states exactly which buckets have one. That is what + * makes "this bucket has a snapshot and a log tail after it" and "this bucket has none" both + * deterministic in the same class. + * + *

Named {@code ...Test}, not {@code ...ITCase}: surefire's default includes do not match + * {@code *ITCase}, so that name would leave the class silently unexecuted with a green build. + */ +public class FlussPkScanPlanClusterTest { + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER = FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .build(); + + /** One bucket, so every fixture row lands where the test can trigger a snapshot for it. */ + private static final int BUCKETS = 1; + + /** The cluster extension drops every non-built-in database after each test. */ + private static int databaseCounter; + + private static Connection connection; + private static Admin admin; + private static Connector connector; + + private String db; + + @BeforeAll + public static void connectToCluster() { + connection = ConnectionFactory.createConnection(FLUSS_CLUSTER.getClientConfig()); + admin = connection.getAdmin(); + + Map catalogProperties = new HashMap<>(); + catalogProperties.put(FlussConnectorProperties.BOOTSTRAP_SERVERS, + FLUSS_CLUSTER.getBootstrapServers()); + connector = new FlussConnectorProvider().create(catalogProperties, new ConnectorContext() { + @Override + public String getCatalogName() { + return "fluss_catalog"; + } + + @Override + public long getCatalogId() { + return 1L; + } + }); + } + + @AfterAll + public static void disconnect() throws Exception { + if (connector != null) { + connector.close(); + } + if (connection != null) { + connection.close(); + } + } + + @BeforeEach + public void createTables() throws Exception { + db = "doris_pk_scan_plan_test_" + (++databaseCounter); + admin.createDatabase(db, DatabaseDescriptor.EMPTY, true).get(); + + admin.createTable(TablePath.of(db, "pk_table"), + TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.BIGINT().copy(false)) + .column("name", DataTypes.STRING()) + .primaryKey("id") + .build()) + .distributedBy(BUCKETS, "id") + .build(), + true).get(); + } + + /** + * The two halves of a primary-key range have to line up with each other: the snapshot the cluster + * reports, and the change log resumed at exactly where that snapshot ended. Resuming earlier + * replays changes the snapshot already holds; resuming later drops the ones it does not. + */ + @Test + public void snapshottedBucketResumesTheChangeLogWhereTheSnapshotEnded() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_table"); + upsert(tablePath, 1, 2, 3); + FLUSS_CLUSTER.triggerAndWaitSnapshot(new TableBucket(tableId(tablePath), 0)); + // A tail the snapshot does not cover. Nothing snapshots it: the periodic interval is ten + // minutes and this test does not trigger again. + upsert(tablePath, 4, 5); + + List ranges = plan("pk_table"); + + KvSnapshots snapshots = admin.getLatestKvSnapshots(tablePath).get(); + long latest = admin.listOffsets(tablePath, Collections.singletonList(0), + new OffsetSpec.LatestSpec()).all().get().get(0); + Assertions.assertEquals(1, ranges.size()); + Map props = ranges.get(0).getProperties(); + Assertions.assertEquals("PK_FULL", props.get("fluss.range_type")); + Assertions.assertEquals("0", props.get("fluss.bucket_id")); + Assertions.assertEquals(String.valueOf(snapshots.getSnapshotId(0).orElse(-1L)), + props.get("fluss.kv_snapshot_id")); + Assertions.assertEquals(String.valueOf(snapshots.getLogOffset(0).orElse(-2L)), + props.get("fluss.log_start_offset")); + Assertions.assertEquals(String.valueOf(latest), props.get("fluss.log_stop_offset")); + + // And the fixture really did produce the shape this test is about, rather than passing because + // there was no snapshot and no tail to disagree about. + long logStart = Long.parseLong(props.get("fluss.log_start_offset")); + Assertions.assertTrue(Long.parseLong(props.get("fluss.kv_snapshot_id")) >= 0, props.toString()); + Assertions.assertTrue(logStart > 0, props.toString()); + Assertions.assertTrue(logStart < latest, props.toString()); + } + + /** + * With no snapshot the state has to be rebuilt from the whole change log, and the cluster says so + * by reporting no snapshot id AND no log offset for the bucket. Reading either as a number — 0 for + * the snapshot, 0 for the offset — would silently read a snapshot that does not exist. + */ + @Test + public void bucketFlussHasNeverSnapshottedReplaysItsWholeChangeLog() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_table"); + upsert(tablePath, 1, 2, 3); + + List ranges = plan("pk_table"); + + KvSnapshots snapshots = admin.getLatestKvSnapshots(tablePath).get(); + Assertions.assertFalse(snapshots.getSnapshotId(0).isPresent(), "the fixture snapshotted"); + Assertions.assertFalse(snapshots.getLogOffset(0).isPresent(), "the fixture snapshotted"); + Assertions.assertEquals(1, ranges.size()); + Map props = ranges.get(0).getProperties(); + Assertions.assertEquals("PK_FULL", props.get("fluss.range_type")); + Assertions.assertEquals("-1", props.get("fluss.kv_snapshot_id")); + Assertions.assertEquals("-2", props.get("fluss.log_start_offset")); + long latest = admin.listOffsets(tablePath, Collections.singletonList(0), + new OffsetSpec.LatestSpec()).all().get().get(0); + Assertions.assertEquals(String.valueOf(latest), props.get("fluss.log_stop_offset")); + } + + @Test + public void primaryKeyTableNothingWasWrittenToPlansNoRanges() { + Assertions.assertTrue(plan("pk_table").isEmpty()); + } + + /** + * A partitioned primary-key table is snapshotted per partition, so its snapshots have to be asked + * for per partition too. Asking at table level returns the wrong partition's offsets, which reads + * the change log from the wrong place. + */ + @Test + public void partitionedPrimaryKeyRangesCarryTheirOwnPartitionSnapshots() throws Exception { + TablePath tablePath = TablePath.of(db, "pk_part"); + admin.createTable(tablePath, + TableDescriptor.builder() + .schema(Schema.newBuilder() + .column("id", DataTypes.BIGINT().copy(false)) + .column("dt", DataTypes.STRING().copy(false)) + .primaryKey("id", "dt") + .build()) + .partitionedBy("dt") + .distributedBy(BUCKETS, "id") + .build(), + true).get(); + admin.createPartition(tablePath, + new PartitionSpec(Collections.singletonMap("dt", "2026_08_03")), true).get(); + admin.createPartition(tablePath, + new PartitionSpec(Collections.singletonMap("dt", "2026_08_04")), true).get(); + long partitionId = partitionId(tablePath, "2026_08_03"); + upsertPartitioned(tablePath, "2026_08_03", 1, 2); + FLUSS_CLUSTER.triggerAndWaitSnapshot(new TableBucket(tableId(tablePath), partitionId, 0)); + upsertPartitioned(tablePath, "2026_08_03", 3); + + List ranges = plan("pk_part"); + + // Only the written partition has anything to read; the empty one yields no range. + Assertions.assertEquals(1, ranges.size()); + Map props = ranges.get(0).getProperties(); + Assertions.assertEquals("PK_FULL", props.get("fluss.range_type")); + Assertions.assertEquals("dt=2026_08_03", props.get("fluss.partition_name")); + Assertions.assertEquals(String.valueOf(partitionId), props.get("fluss.partition_id")); + KvSnapshots snapshots = admin.getLatestKvSnapshots(tablePath, "2026_08_03").get(); + Assertions.assertEquals(String.valueOf(snapshots.getSnapshotId(0).orElse(-1L)), + props.get("fluss.kv_snapshot_id")); + Assertions.assertEquals(String.valueOf(snapshots.getLogOffset(0).orElse(-2L)), + props.get("fluss.log_start_offset")); + Assertions.assertTrue(Long.parseLong(props.get("fluss.kv_snapshot_id")) >= 0, props.toString()); + } + + private List plan(String tableName) { + FlussTestSession session = new FlussTestSession(1L, "cluster-pk-plan"); + ConnectorTableHandle handle = connector.getMetadata(session) + .getTableHandle(session, db, tableName).orElseThrow(AssertionError::new); + return connector.getScanPlanProvider().planScan(session, + ConnectorScanRequest.builder(handle, Collections.emptyList()).build()); + } + + private static long tableId(TablePath tablePath) throws Exception { + return admin.getTableInfo(tablePath).get().getTableId(); + } + + private static long partitionId(TablePath tablePath, String partitionName) throws Exception { + return admin.listPartitionInfos(tablePath).get().stream() + .filter(p -> p.getPartitionName().equals(partitionName)) + .findFirst().orElseThrow(AssertionError::new) + .getPartitionId(); + } + + private static void upsert(TablePath tablePath, long... ids) throws Exception { + try (Table table = connection.getTable(tablePath)) { + UpsertWriter writer = table.newUpsert().createWriter(); + for (long id : ids) { + writer.upsert(GenericRow.of(id, BinaryString.fromString("name-" + id))); + } + writer.flush(); + } + } + + private static void upsertPartitioned(TablePath tablePath, String partition, long... ids) + throws Exception { + try (Table table = connection.getTable(tablePath)) { + UpsertWriter writer = table.newUpsert().createWriter(); + for (long id : ids) { + writer.upsert(GenericRow.of(id, BinaryString.fromString(partition))); + } + writer.flush(); + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussScanRangeTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussScanRangeTest.java new file mode 100644 index 00000000000000..dc53842dc59ad7 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussScanRangeTest.java @@ -0,0 +1,306 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TTableFormatFileDesc; + +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The range is the FE half of the wire contract with the be-java-extensions scanner: an untyped string + * map that nothing between here and there type-checks. So these assert the map WHOLE — every key, its + * exact rendering, and the absence of keys that do not belong to the range type — rather than spot + * checking. A key that is quietly renamed, dropped or leaked across range types is exactly the failure + * this file exists to catch. + */ +public class FlussScanRangeTest { + + private static final FlussScanRange.Partition DT_20260101 = + FlussScanRange.Partition.of(77L, "dt=20260101", + Collections.singletonMap("dt", "20260101")); + + @Test + public void logRangeCarriesOffsetsAndNothingElse() { + FlussScanRange range = FlussScanRange.log(FlussScanRange.Partition.NONE, 3, 10L, 42L); + + Map expected = new LinkedHashMap<>(); + expected.put("fluss.range_type", "LOG"); + expected.put("fluss.bucket_id", "3"); + expected.put("fluss.log_start_offset", "10"); + expected.put("fluss.log_stop_offset", "42"); + Assertions.assertEquals(expected, range.getProperties()); + Assertions.assertEquals(FlussScanRange.RangeType.LOG, range.getRangeType()); + } + + @Test + public void pkFullRangeCarriesTheKvSnapshotButNoLakeFields() { + FlussScanRange range = + FlussScanRange.pkFull(FlussScanRange.Partition.NONE, 0, 900L, 120L, 500L); + + Map expected = new LinkedHashMap<>(); + expected.put("fluss.range_type", "PK_FULL"); + expected.put("fluss.bucket_id", "0"); + expected.put("fluss.log_start_offset", "120"); + expected.put("fluss.log_stop_offset", "500"); + expected.put("fluss.kv_snapshot_id", "900"); + Assertions.assertEquals(expected, range.getProperties()); + } + + /** + * A bucket that has never been snapshotted still produces a range: the scanner replays its whole + * state from the log. The sentinel has to survive as {@code -1}, because "key missing" and + * "no snapshot" would otherwise be indistinguishable on the scanner side. + */ + @Test + public void pkFullWithoutASnapshotSendsTheSentinel() { + FlussScanRange range = FlussScanRange.pkFull(FlussScanRange.Partition.NONE, 0, + FlussScanRange.NO_KV_SNAPSHOT, LogScanner.EARLIEST_OFFSET, 500L); + + Assertions.assertEquals("-1", range.getProperties().get("fluss.kv_snapshot_id")); + Assertions.assertEquals("-2", range.getProperties().get("fluss.log_start_offset")); + } + + /** + * EARLIEST is fluss's own sentinel, not a Doris one: the scanner hands the value straight back to + * fluss. If fluss ever renumbers it, FE and the scanner move together — but the rendering must stay + * the raw number, never a name the scanner would have to translate. + */ + @Test + public void earliestOffsetGoesOutVerbatim() { + FlussScanRange range = FlussScanRange.log(FlussScanRange.Partition.NONE, 1, + LogScanner.EARLIEST_OFFSET, 7L); + + Assertions.assertEquals(String.valueOf(LogScanner.EARLIEST_OFFSET), + range.getProperties().get("fluss.log_start_offset")); + Assertions.assertEquals("-2", range.getProperties().get("fluss.log_start_offset")); + } + + /** + * A tail range is a log range in everything but name — same offsets, same keys — and that is the + * point: what differs is how the scanner treats what it reads (replay by key rather than row by row), + * which is exactly what the range type says. Carrying a kv snapshot id would be the bug: the rows + * before the tail come from the lake, not from a snapshot. + */ + @Test + public void tailRangeCarriesTheLogWindowAndNoKvSnapshot() { + FlussScanRange range = FlussScanRange.pkTail(DT_20260101, 2, 300L, 305L); + + Map expected = new LinkedHashMap<>(); + expected.put("fluss.range_type", "PK_TAIL"); + expected.put("fluss.partition_id", "77"); + expected.put("fluss.partition_name", "dt=20260101"); + expected.put("fluss.bucket_id", "2"); + expected.put("fluss.log_start_offset", "300"); + expected.put("fluss.log_stop_offset", "305"); + Assertions.assertEquals(expected, range.getProperties()); + } + + /** + * An empty tail must not become a range. The bucket it would describe is one the lake already holds + * entirely, and planning says so by not producing a range at all; one that got here would mean the + * lake half and the fluss half were bounded by different offsets, which is how rows get lost. + */ + @Test + public void emptyTailRangeIsRejected() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> FlussScanRange.pkTail(DT_20260101, 0, 5L, 5L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> FlussScanRange.pkTail(DT_20260101, 0, 6L, 5L)); + } + + @Test + public void unpartitionedRangeOmitsThePartitionKeys() { + FlussScanRange range = FlussScanRange.log(FlussScanRange.Partition.NONE, 0, 0L, 1L); + + Assertions.assertFalse(range.getProperties().containsKey("fluss.partition_id")); + Assertions.assertFalse(range.getProperties().containsKey("fluss.partition_name")); + Assertions.assertTrue(range.getPartitionValues().isEmpty()); + } + + /** + * An unpartitioned range still reports itself partition-bearing: the engine reads a null partition + * value list as "parse the values out of the file path", and a fluss range has no path to parse. + */ + @Test + public void rangesAreAlwaysPartitionBearing() { + Assertions.assertTrue( + FlussScanRange.log(FlussScanRange.Partition.NONE, 0, 0L, 1L).isPartitionBearing()); + Assertions.assertTrue( + FlussScanRange.log(DT_20260101, 0, 0L, 1L).isPartitionBearing()); + } + + @Test + public void tableFormatTypeSelectsTheFlussReader() { + FlussScanRange range = FlussScanRange.log(FlussScanRange.Partition.NONE, 0, 0L, 1L); + + Assertions.assertEquals("fluss", range.getTableFormatType()); + // Default of the SPI: everything fluss reads goes through the JNI scanner, there is no + // native path to downgrade to. + Assertions.assertEquals("jni", range.getFileFormat()); + } + + @Test + public void populateRangeParamsWritesTheWholeMapIntoFlussParams() { + FlussScanRange range = FlussScanRange.log(FlussScanRange.Partition.NONE, 5, 0L, 9L); + TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + + range.populateRangeParams(formatDesc, rangeDesc); + + Assertions.assertEquals(range.getProperties(), formatDesc.getFlussParams()); + // No other format's params may be touched — BE dispatches on table_format_type and would + // read a stale struct if one were half-filled. + Assertions.assertFalse(formatDesc.isSetPaimonParams()); + Assertions.assertFalse(formatDesc.isSetIcebergParams()); + Assertions.assertFalse(formatDesc.isSetJdbcParams()); + Assertions.assertFalse(formatDesc.isSetEsParams()); + // Unpartitioned: nothing for BE to materialize from the range. + Assertions.assertFalse(rangeDesc.isSetColumnsFromPathKeys()); + Assertions.assertFalse(rangeDesc.isSetColumnsFromPath()); + } + + /** + * Partition columns are not in what the scanner returns — the connector declares them as + * path partition keys — so BE materializes them from here. Order matters: BE pairs the value list + * with the key list positionally. + */ + @Test + public void populateRangeParamsHandsPartitionColumnsToBe() { + Map values = new LinkedHashMap<>(); + values.put("dt", "20260101"); + values.put("region", "cn"); + FlussScanRange range = FlussScanRange.log( + FlussScanRange.Partition.of(9L, "dt=20260101/region=cn", values), 0, 0L, 1L); + TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + + range.populateRangeParams(formatDesc, rangeDesc); + + Assertions.assertEquals(Arrays.asList("dt", "region"), rangeDesc.getColumnsFromPathKeys()); + Assertions.assertEquals(Arrays.asList("20260101", "cn"), rangeDesc.getColumnsFromPath()); + // fluss refuses a null partition value at write time, so there is no null to signal. + Assertions.assertEquals(Arrays.asList(false, false), rangeDesc.getColumnsFromPathIsNull()); + } + + /** Half-describing a partition is the bug this rules out; NONE is the only way to say "none". */ + @Test + public void partitionWithoutColumnValuesIsRejected() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> FlussScanRange.Partition.of(1L, "dt=20260101", Collections.emptyMap())); + } + + /** + * The SPI declares a scan range {@link java.io.Serializable} and the engine takes it at its word. + * Round-tripping also pins that "is this partitioned?" survives — answering it by identity against + * the NONE singleton would come back wrong on the far side of a deserialization. + */ + @Test + public void rangeSurvivesJavaSerialization() throws Exception { + for (FlussScanRange original : Arrays.asList( + FlussScanRange.log(FlussScanRange.Partition.NONE, 3, 10L, 42L), + FlussScanRange.pkTail(DT_20260101, 2, 300L, 305L))) { + FlussScanRange restored = roundTrip(original); + + Assertions.assertEquals(original.getProperties(), restored.getProperties()); + Assertions.assertEquals(original.getPartitionValues(), restored.getPartitionValues()); + Assertions.assertEquals(original.getRangeType(), restored.getRangeType()); + + // The restored NONE is a different object than the singleton; the range must still be + // able to tell it is unpartitioned. + TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + restored.populateRangeParams(formatDesc, rangeDesc); + Assertions.assertEquals(original.getProperties(), formatDesc.getFlussParams()); + } + } + + /** + * {@code NONE} is a singleton but has no {@code readResolve}, so deserializing gives back a + * different object. Anything that answers "is this partitioned?" by comparing against the singleton + * would say yes here, and the range would then claim a partition id of -1 with no columns. + */ + @Test + public void restoredNonePartitionStillReportsUnpartitioned() throws Exception { + FlussScanRange restored = roundTrip( + FlussScanRange.log(FlussScanRange.Partition.NONE, 0, 0L, 1L)); + + Assertions.assertNotSame(FlussScanRange.Partition.NONE, restored.getPartition()); + Assertions.assertFalse(restored.getPartition().isPartitioned()); + Assertions.assertTrue(restored.getPartitionValues().isEmpty()); + } + + @Test + public void partitionExposesItsIdentity() { + Assertions.assertTrue(DT_20260101.isPartitioned()); + Assertions.assertEquals(77L, DT_20260101.getId()); + Assertions.assertEquals("dt=20260101", DT_20260101.getName()); + Assertions.assertEquals(Collections.singletonMap("dt", "20260101"), DT_20260101.getValues()); + Assertions.assertFalse(FlussScanRange.Partition.NONE.isPartitioned()); + } + + private static FlussScanRange roundTrip(FlussScanRange range) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream out = new ObjectOutputStream(bytes)) { + out.writeObject(range); + } + try (ObjectInputStream in = + new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + return (FlussScanRange) in.readObject(); + } + } + + /** Guards the fixture itself: an unmodifiable view is still a live view of a caller's map. */ + @Test + public void partitionValuesAreCopiedFromTheCaller() { + Map mutable = new LinkedHashMap<>(); + mutable.put("dt", "20260101"); + FlussScanRange.Partition partition = FlussScanRange.Partition.of(1L, "dt=20260101", mutable); + + mutable.put("dt", "20260102"); + + Assertions.assertEquals("20260101", partition.getValues().get("dt")); + } + + /** The property map is the wire payload; a caller must not be able to edit it after the fact. */ + @Test + public void propertiesAreImmutable() { + List ranges = Arrays.asList( + FlussScanRange.log(FlussScanRange.Partition.NONE, 0, 0L, 1L), + FlussScanRange.pkFull(DT_20260101, 0, 1L, 0L, 1L), + FlussScanRange.pkTail(DT_20260101, 0, 0L, 1L)); + for (FlussScanRange range : ranges) { + Assertions.assertThrows(UnsupportedOperationException.class, + () -> range.getProperties().put("fluss.bucket_id", "999")); + Assertions.assertThrows(UnsupportedOperationException.class, + () -> range.getPartitionValues().put("dt", "x")); + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java new file mode 100644 index 00000000000000..d465dad42d14aa --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSplitPlanTest.java @@ -0,0 +1,1633 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.ConnectorPartitionInfo; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.DorisConnectorException; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.connector.api.scan.ConnectorScanRequest; +import org.apache.doris.connector.api.scan.ScanNodePropertyKeys; +import org.apache.doris.thrift.TFileScanRangeParams; + +import org.apache.fluss.client.metadata.KvSnapshots; +import org.apache.fluss.client.metadata.LakeSnapshot; +import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.ResolvedPartitionSpec; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Split planning driven entirely off recorded admin answers, which is the point: the states worth + * asserting — a bucket nobody ever wrote to, a partition the engine pruned away, a lake table whose + * tiering has not committed once — are a nuisance to produce on a live cluster and trivial here. The + * cluster test alongside this one covers that the recorded answers match what a real cluster gives. + */ +public class FlussSplitPlanTest { + + private static final TablePath LOG_TABLE = TablePath.of("db", "log_tbl"); + private static final TablePath PK_TABLE = TablePath.of("db", "pk_tbl"); + + /** Fixture shorthand: this bucket has never been snapshotted. */ + private static final long NO_SNAPSHOT = -1L; + + private RecordingFlussAdminOps adminOps; + private ConnectorSession session; + + /** + * The lake sibling, built only when a test asks for it. A test that never registers a lake table must + * never reach it, and {@link #lakeSibling} fails loud if it does — a fluss-only plan that quietly + * consulted the lake would still look correct in every other assertion. + */ + private RecordingLakeSibling sibling; + private boolean siblingExpected; + + @BeforeEach + public void setUp() { + adminOps = new RecordingFlussAdminOps(); + session = new FlussTestSession(1L, "q1"); + sibling = null; + siblingExpected = false; + } + + // ---------------------------------------------------------------- unpartitioned log table + + @Test + public void everyNonEmptyBucketGetsARangeFromEarliestToTheOffsetPlanningSaw() { + registerLogTable(LOG_TABLE, 3); + latestOffsets(null, 10L, 25L, 7L); + + List ranges = plan(LOG_TABLE, catalog()); + + Assertions.assertEquals(3, ranges.size()); + assertLogRange(ranges.get(0), 0, -2L, 10L); + assertLogRange(ranges.get(1), 1, -2L, 25L); + assertLogRange(ranges.get(2), 2, -2L, 7L); + } + + /** + * All of a partition's buckets stop at one view of the table. Taking the offsets per bucket instead + * would let a bucket planned later include rows written after the query started. + */ + @Test + public void offsetsAreTakenOncePerPartition() { + registerLogTable(LOG_TABLE, 4); + latestOffsets(null, 1L, 1L, 1L, 1L); + + plan(LOG_TABLE, catalog()); + + long offsetCalls = adminOps.calls.stream().filter(c -> c.startsWith("listOffsets(")).count(); + Assertions.assertEquals(1, offsetCalls, adminOps.calls.toString()); + Assertions.assertTrue(adminOps.calls.get(0).contains("[0, 1, 2, 3]"), adminOps.calls.toString()); + Assertions.assertTrue(adminOps.calls.get(0).contains("LatestSpec"), adminOps.calls.toString()); + } + + /** A bucket nobody has written to yields no range: an empty scanner would only cost a round trip. */ + @Test + public void neverWrittenBucketsAreSkipped() { + registerLogTable(LOG_TABLE, 3); + latestOffsets(null, 0L, 12L, 0L); + + List ranges = plan(LOG_TABLE, catalog()); + + Assertions.assertEquals(1, ranges.size()); + assertLogRange(ranges.get(0), 1, -2L, 12L); + } + + @Test + public void anEmptyTablePlansNothing() { + registerLogTable(LOG_TABLE, 2); + latestOffsets(null, 0L, 0L); + + Assertions.assertTrue(plan(LOG_TABLE, catalog()).isEmpty()); + } + + // ---------------------------------------------------------------- partitioned log table + + @Test + public void eachPartitionContributesItsOwnBuckets() { + registerPartitionedLogTable(2, "20260101", "20260102"); + latestOffsets("20260101", 5L, 6L); + latestOffsets("20260102", 7L, 0L); + + List ranges = plan(LOG_TABLE, catalog()); + + Assertions.assertEquals(3, ranges.size()); + assertPartition(ranges.get(0), "dt=20260101", 100L, 0, 5L); + assertPartition(ranges.get(1), "dt=20260101", 100L, 1, 6L); + assertPartition(ranges.get(2), "dt=20260102", 101L, 0, 7L); + } + + @Test + public void onlyThePartitionsTheEnginePrunedToAreScanned() { + registerPartitionedLogTable(1, "20260101", "20260102", "20260103"); + latestOffsets("20260102", 9L); + + List ranges = plan(LOG_TABLE, catalog(), + Collections.singletonList("dt=20260102")); + + Assertions.assertEquals(1, ranges.size()); + assertPartition(ranges.get(0), "dt=20260102", 101L, 0, 9L); + // The other two partitions must not even be asked for their offsets. + Assertions.assertEquals(1, + adminOps.calls.stream().filter(c -> c.startsWith("listOffsets(")).count(), + adminOps.calls.toString()); + } + + /** + * The pruned names the engine hands back are the ones the metadata listing produced, so planning has + * to render partition names exactly the same way. Rendering them separately is how "prune to one + * partition, then match nothing and scan zero rows" happens, and it looks like an empty table. + */ + @Test + public void plannedPartitionNamesMatchTheOnesPruningSaw() { + registerPartitionedLogTable(1, "20260101", "20260102"); + latestOffsets("20260101", 3L); + latestOffsets("20260102", 3L); + ConnectorTableHandle handle = handle(LOG_TABLE); + + List listedNames = new ArrayList<>(); + for (ConnectorPartitionInfo partition + : metadata().listPartitions(session, handle, Optional.empty())) { + listedNames.add(partition.getPartitionName()); + } + // Feeding the listing's own names back as the pruned set must select every partition. + List ranges = plan(LOG_TABLE, catalog(), listedNames); + + Assertions.assertEquals(Arrays.asList("dt=20260101", "dt=20260102"), listedNames); + Assertions.assertEquals(2, ranges.size()); + } + + /** Dropped between pruning and planning: nothing left to read, and nothing to fail about. */ + @Test + public void prunedPartitionThatNoLongerExistsIsSkipped() { + registerPartitionedLogTable(1, "20260101"); + latestOffsets("20260101", 3L); + + List ranges = plan(LOG_TABLE, catalog(), + Arrays.asList("dt=20260101", "dt=19990101")); + + Assertions.assertEquals(1, ranges.size()); + assertPartition(ranges.get(0), "dt=20260101", 100L, 0, 3L); + } + + @Test + public void partitionColumnsAreDeclaredToTheEngineNotReadBackFromTheScanner() { + registerPartitionedLogTable(1, "20260101"); + + Map props = nodeProperties(LOG_TABLE, catalog()); + + Assertions.assertEquals("dt", props.get(ScanNodePropertyKeys.PATH_PARTITION_KEYS)); + } + + @Test + public void anUnpartitionedTableDeclaresNoPartitionKeys() { + registerLogTable(LOG_TABLE, 1); + + Map props = nodeProperties(LOG_TABLE, catalog()); + + Assertions.assertFalse(props.containsKey(ScanNodePropertyKeys.PATH_PARTITION_KEYS)); + } + + // ---------------------------------------------------------------- primary-key table + + /** + * A primary-key bucket is read as "the kv snapshot, then the change log that followed it". The + * starting offset therefore has to be the one the snapshot ended at: starting anywhere earlier + * replays changes the snapshot already contains, and anywhere later drops changes it does not. + */ + @Test + public void primaryKeyBucketsAreReadFromTheirSnapshotForward() { + registerPkTable(PK_TABLE, 3); + kvSnapshots(null, new long[] {4L, 9L, 2L}, new long[] {40L, 90L, 20L}); + latestOffsets(null, 55L, 90L, 31L); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(3, ranges.size()); + assertPkRange(ranges.get(0), 0, 4L, 40L, 55L); + // Nothing new since the snapshot: still planned, and the scanner reads the snapshot alone. + assertPkRange(ranges.get(1), 1, 9L, 90L, 90L); + assertPkRange(ranges.get(2), 2, 2L, 20L, 31L); + } + + /** + * With no snapshot the whole state is rebuilt by replaying the change log from the beginning, + * which is correct because a primary-key table's log carries every change. The two facts that say + * so — {@code -1} and the earliest sentinel — travel separately, and a bucket that got one without + * the other would read a snapshot that does not exist or start from the wrong place. + */ + @Test + public void bucketsWithoutSnapshotsReplayTheirWholeChangeLog() { + registerPkTable(PK_TABLE, 2); + kvSnapshots(null, new long[] {NO_SNAPSHOT, 3L}, new long[] {0L, 30L}); + latestOffsets(null, 12L, 44L); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + assertPkRange(ranges.get(0), 0, -1L, -2L, 12L); + assertPkRange(ranges.get(1), 1, 3L, 30L, 44L); + } + + /** Neither snapshotted nor written to: nothing to read, so nothing for BE to open a scanner for. */ + @Test + public void neverWrittenPrimaryKeyBucketsAreSkipped() { + registerPkTable(PK_TABLE, 3); + kvSnapshots(null, new long[] {NO_SNAPSHOT, 7L, NO_SNAPSHOT}, new long[] {0L, 70L, 0L}); + latestOffsets(null, 0L, 88L, 5L); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + assertPkRange(ranges.get(0), 1, 7L, 70L, 88L); + assertPkRange(ranges.get(1), 2, -1L, -2L, 5L); + } + + /** + * A bucket the offsets answer says nothing about still has to be read when it has a snapshot: the + * snapshot holds rows regardless of what the log is doing. Skipping on "no stopping offset" alone + * would drop them, and the query would succeed with a bucket's worth of rows missing. + */ + @Test + public void snapshottedBucketMissingFromTheOffsetsAnswerIsStillRead() { + registerPkTable(PK_TABLE, 2); + kvSnapshots(null, new long[] {6L, NO_SNAPSHOT}, new long[] {60L, 0L}); + Map partialOffsets = new LinkedHashMap<>(); + partialOffsets.put(1, 9L); + adminOps.latestOffsetsByPartition.put(null, partialOffsets); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + // No log to read, so the range covers the snapshot alone. + assertPkRange(ranges.get(0), 0, 6L, 60L, 0L); + assertPkRange(ranges.get(1), 1, -1L, -2L, 9L); + } + + /** + * Snapshots are asked for BEFORE offsets, and the order is the whole point. A snapshot committed + * between the two calls ends past the offset planning stopped at; that bucket would then be read + * from its snapshot — which already contains rows written after the query started — while every + * other bucket stopped where planning saw them. Log offsets only move forward, so asking in this + * order keeps every snapshot at or behind the stopping offset. + */ + @Test + public void snapshotsAreAskedForBeforeTheOffsetsThatBoundThem() { + registerPkTable(PK_TABLE, 1); + kvSnapshots(null, new long[] {1L}, new long[] {10L}); + latestOffsets(null, 20L); + + plan(PK_TABLE, catalog()); + + int snapshotCall = indexOfCall("getLatestKvSnapshots("); + int offsetCall = indexOfCall("listOffsets("); + Assertions.assertTrue(snapshotCall >= 0 && snapshotCall < offsetCall, adminOps.calls.toString()); + } + + @Test + public void partitionedPrimaryKeyTablesTakeSnapshotsAndOffsetsPerPartition() { + registerPartitionedPkTable(1, "20260101", "20260102"); + kvSnapshots("20260101", new long[] {5L}, new long[] {50L}); + latestOffsets("20260101", 60L); + kvSnapshots("20260102", new long[] {NO_SNAPSHOT}, new long[] {0L}); + latestOffsets("20260102", 8L); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + Assertions.assertEquals("dt=20260101", ranges.get(0).getProperties().get("fluss.partition_name")); + assertPkRange(ranges.get(0), 0, 5L, 50L, 60L); + Assertions.assertEquals("dt=20260102", ranges.get(1).getProperties().get("fluss.partition_name")); + assertPkRange(ranges.get(1), 0, -1L, -2L, 8L); + } + + /** Pruning applies to a primary-key table exactly as it does to a log table. */ + @Test + public void prunedPartitionsOfPrimaryKeyTablesAreNotEvenAskedAbout() { + registerPartitionedPkTable(1, "20260101", "20260102"); + kvSnapshots("20260102", new long[] {2L}, new long[] {20L}); + latestOffsets("20260102", 25L); + + List ranges = plan(PK_TABLE, catalog(), + Collections.singletonList("dt=20260102")); + + Assertions.assertEquals(1, ranges.size()); + Assertions.assertEquals(1, + adminOps.calls.stream().filter(c -> c.startsWith("getLatestKvSnapshots(")).count(), + adminOps.calls.toString()); + } + + // ------------------------------------------- tiered primary-key tables: read from fluss alone + + /** + * A primary-key table whose tiering has never committed is read from fluss alone, and what replaces the + * union is NOT the "whatever has not been tiered away" fallback a log table would get: fluss keeps such + * a table's state in full, so its kv snapshot plus the log after it is every row. + */ + @Test + public void tieredPrimaryKeyTableWithoutALakeSnapshotIsReadFromFlussAlone() { + registerTieredPkTable(2); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 12L, 25L); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + assertPkRange(ranges.get(0), 0, 4L, 10L, 12L); + assertPkRange(ranges.get(1), 1, 5L, 20L, 25L); + } + + /** + * The fallback is the very read {@code disabled} asks for outright, down to the ranges — not a third + * code path that happens to look similar. + */ + @Test + public void theFallbackIsTheReadDisabledModeAsksForOutright() { + registerTieredPkTable(2); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 12L, 25L); + + List> auto = rangeProperties(plan(PK_TABLE, catalog())); + List> disabled = rangeProperties( + plan(PK_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "disabled"))); + + Assertions.assertEquals(auto, disabled); + } + + /** + * {@code required} is what a regression test sets so that it cannot pass without the lake having been + * read, so a primary-key table whose tiering has not committed fails loud rather than falling back — + * the same answer a log table gets, for the same reason. + */ + @Test + public void requiredModeRefusesAPrimaryKeyTableWithNothingInItsLake() { + registerTieredPkTable(2); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> plan(PK_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "required"))); + Assertions.assertTrue(e.getMessage().contains("no readable lake snapshot yet"), e.getMessage()); + } + + /** + * The plan's own account of not having read the lake. Nothing else in the plan of a tiered primary-key + * table distinguishes it from one that was read as a union. + */ + @Test + public void explainShowsATieredPrimaryKeyTableWasReadWithoutItsLake() { + registerTieredPkTable(1); + kvSnapshots(null, new long[] {4L}, new long[] {10L}); + latestOffsets(null, 12L); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); + provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList())); + + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, "", Collections.emptyMap()); + + Assertions.assertEquals("flussScan: unionRead=no, lakeSplits=0, suppressedLakeSplits=0," + + " logRanges=0, pkRanges=1, pkTailRanges=0, mode=auto\n", output.toString()); + } + + + // ---------------------------------------------------------------- union read: lake + log + + /** + * The two halves have to MEET: the lake holds everything up to the offset its snapshot recorded, and + * the log range starts at exactly that offset. One off in either direction is a wrong answer that no + * row count catches — reading the log from the earliest offset instead duplicates every tiered row, + * and starting one later drops one. + */ + @Test + public void theLogIsReadFromWhereTheLakeSnapshotLeftOff() { + registerLakeTable(2); + lakeSnapshotAt(7L, offsets(5L, 3L)); + latestOffsets(null, 9L, 3L); + lakeRanges(2); + + List ranges = plan(LOG_TABLE, catalog()); + + // Two lake ranges, then the one bucket whose log went past the snapshot. Bucket 1's log has not + // moved since it was tiered, so everything it holds is already in the lake and it yields nothing. + Assertions.assertEquals(3, ranges.size()); + assertLogRange(ranges.get(2), 0, 5L, 9L); + } + + /** + * A bucket the lake snapshot never mentions has never been tiered, so nothing of it is in the lake + * and its whole log is still the truth. Starting it at the snapshot's (absent) offset would be the + * bug: there is no offset to start at, and treating that as zero rows would drop the bucket. + */ + @Test + public void bucketTheLakeNeverSawIsReadFromTheEarliestOffset() { + registerLakeTable(2); + lakeSnapshotAt(7L, offsets(4L, null)); + latestOffsets(null, 6L, 8L); + + List ranges = plan(LOG_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + assertLogRange(ranges.get(0), 0, 4L, 6L); + assertLogRange(ranges.get(1), 1, -2L, 8L); + } + + /** A bucket whose log has not moved past the lake needs no log range at all. */ + @Test + public void bucketTheLakeHasCaughtUpWithYieldsNoLogRange() { + registerLakeTable(2); + lakeSnapshotAt(7L, offsets(6L, 6L)); + latestOffsets(null, 6L, 6L); + + Assertions.assertTrue(plan(LOG_TABLE, catalog()).isEmpty()); + } + + /** + * The lake is pinned to the snapshot FLUSS says is readable, not to whatever the lake calls latest. + * Unpinned, the lake half would drift ahead of the offsets the log half stops at and the rows in + * between would be read twice. + */ + @Test + public void theLakeIsPinnedToTheSnapshotFlussReportsAsReadable() { + registerLakeTable(1); + lakeSnapshotAt(7L, offsets(2L)); + latestOffsets(null, 5L); + + plan(LOG_TABLE, catalog()); + + Assertions.assertEquals(7L, sibling.plannedHandle.pinnedSnapshotId); + // The pin is expressed in the SPI's terms only: a snapshot id and no connector-specific options. + Assertions.assertTrue(sibling.calls.contains("applySnapshot:7:{}"), sibling.calls.toString()); + } + + /** + * The lake snapshot must be read BEFORE the log's stopping offsets. Log offsets only move forward, so + * asking in this order keeps the snapshot at or behind where the log half stops; asked the other way + * round, a snapshot committed in between would cover rows past that point and the bucket's log range + * would start after it ended. + */ + @Test + public void theLakeSnapshotIsReadBeforeTheStoppingOffsets() { + registerLakeTable(1); + lakeSnapshotAt(7L, offsets(2L)); + latestOffsets(null, 5L); + + plan(LOG_TABLE, catalog()); + + int snapshotCall = indexOfCall("getReadableLakeSnapshot"); + int offsetsCall = indexOfCall("listOffsets"); + Assertions.assertTrue(snapshotCall >= 0 && snapshotCall < offsetsCall, adminOps.calls.toString()); + } + + /** + * The lake half is planned on the LAKE's own column handles. The sibling projects by its own handle + * type and ignores anything else, so handing it fluss's handles would leave it reading every column — + * including the three system columns tiering appends — and no assertion about rows would notice. + */ + @Test + public void theLakeHalfIsPlannedOnTheLakeTablesOwnColumnHandles() { + registerLakeTable(1); + lakeTableColumns("id", "__bucket", "__offset", "__timestamp"); + lakeSnapshotAt(7L, offsets(0L)); + latestOffsets(null, 5L); + + new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling).planScan(session, + ConnectorScanRequest.builder(handle(LOG_TABLE), + Collections.singletonList(new FlussColumnHandle("id", 0))).build()); + + Assertions.assertEquals( + Collections.singletonList(new RecordingLakeSibling.LakeColumn("id")), + sibling.plannedColumns); + } + + /** A column the lake table does not have means the two schemas are not the same table's. */ + @Test + public void columnMissingFromTheLakeTableIsRefused() { + registerLakeTable(1); + lakeTableColumns("id"); + lakeSnapshotAt(7L, offsets(0L)); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling).planScan(session, + ConnectorScanRequest.builder(handle(LOG_TABLE), + Collections.singletonList(new FlussColumnHandle("gone", 1))).build())); + Assertions.assertTrue(e.getMessage().contains("does not exist in its lake table"), e.getMessage()); + } + + /** + * One scan node, one property map: the engine calls {@code populateScanLevelParams} once, so the lake + * half's entries have to travel in the same map or its ranges arrive at BE unreadable. + */ + @Test + public void nodePropertiesCarryBothHalves() { + registerLakeTable(1); + lakeSnapshotAt(7L, offsets(0L)); + sibling(); // built up-front so the node properties can be set before they are asked for + sibling.lakeNodeProperties = Collections.singletonMap("paimon.serialized_table", "encoded"); + + Map props = nodeProperties(LOG_TABLE, catalog()); + + Assertions.assertEquals("db", props.get("fluss.db_name")); + Assertions.assertEquals("encoded", props.get("paimon.serialized_table")); + } + + /** The lake half gets its turn at the same thrift params, and only it knows which entries are its. */ + @Test + public void scanLevelParamsAreOfferedToTheLakeHalfToo() { + registerLakeTable(1); + lakeSnapshotAt(7L, offsets(0L)); + sibling(); + sibling.lakeNodeProperties = Collections.singletonMap("paimon.serialized_table", "encoded"); + Map catalog = catalog(); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog, this::lakeSibling); + Map props = provider.getScanNodeProperties( + session, handle(LOG_TABLE), Collections.emptyList(), Optional.empty()); + + TFileScanRangeParams params = new TFileScanRangeParams(); + provider.populateScanLevelParams(params, props); + + // fluss took only its own keys; the sibling saw the whole map, its own entries included. + Assertions.assertFalse(params.getFlussProperties().containsKey("paimon.serialized_table")); + Assertions.assertEquals("encoded", + sibling.populatedNodeProperties.get("paimon.serialized_table")); + } + + /** + * The split between file columns and partition columns is decided ONCE for the scan node, so two + * halves that disagree about it would read different columns out of the same tuple. They cannot + * legitimately disagree, which is why a disagreement is raised rather than resolved by picking one. + */ + @Test + public void halvesThatDisagreeAboutTheSharedPropertiesAreRefused() { + registerPartitionedLakeTable(1, "20260101"); + lakeSnapshotAt(7L, offsets(0L)); + sibling(); + sibling.lakeNodeProperties = + Collections.singletonMap(ScanNodePropertyKeys.PATH_PARTITION_KEYS, "not_dt"); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> nodeProperties(LOG_TABLE, catalog())); + Assertions.assertTrue(e.getMessage().contains(ScanNodePropertyKeys.PATH_PARTITION_KEYS), + e.getMessage()); + } + + /** A partitioned table's lake offsets are per (partition, bucket), not per bucket. */ + @Test + public void partitionedTablesLogIsResumedPerPartitionAndBucket() { + registerPartitionedLakeTable(1, "20260101", "20260102"); + Map lakeOffsets = new HashMap<>(); + lakeOffsets.put(new TableBucket(FlussTestTables.TABLE_ID, 100L, 0), 4L); + lakeOffsets.put(new TableBucket(FlussTestTables.TABLE_ID, 101L, 0), 1L); + lakeSnapshotAt(7L, lakeOffsets); + latestOffsets("20260101", 9L); + latestOffsets("20260102", 6L); + + List ranges = plan(LOG_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + assertLogRange(ranges.get(0), 0, 4L, 9L); + assertLogRange(ranges.get(1), 0, 1L, 6L); + } + + /** The line a union-read regression test reads to know the lake was actually part of the answer. */ + @Test + public void explainReportsTheUnionAndTheSizeOfEachHalf() { + registerLakeTable(2); + lakeSnapshotAt(7L, offsets(1L, 9L)); + latestOffsets(null, 5L, 9L); + lakeRanges(3); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); + provider.planScan(session, request(handle(LOG_TABLE), Collections.emptyList())); + + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, "", Collections.emptyMap()); + + Assertions.assertEquals( + "flussScan: unionRead=yes, lakeSplits=3, suppressedLakeSplits=0, logRanges=1," + + " pkRanges=0, pkTailRanges=0, mode=auto\n", output.toString()); + } + + /** Nothing in the lake means the log holds the whole table, so the fluss-only read IS complete. */ + @Test + public void lakeTableThatHasNeverTieredFallsBackToTheLogAlone() { + registerLakeTable(2); + latestOffsets(null, 4L, 4L); + + List ranges = plan(LOG_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + } + + @Test + public void requiredModeRefusesTheFallbackSoATestCannotPassByAccident() { + registerLakeTable(2); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> plan(LOG_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "required"))); + Assertions.assertTrue(e.getMessage().contains("no readable lake snapshot"), e.getMessage()); + } + + /** Disabled is the user asking for the fluss-only read on purpose; the lake is not even consulted. */ + @Test + public void disabledModeReadsTheLogWithoutAskingAboutTheLake() { + registerLakeTable(2); + latestOffsets(null, 4L, 4L); + + List ranges = + plan(LOG_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "disabled")); + + Assertions.assertEquals(2, ranges.size()); + Assertions.assertTrue(adminOps.calls.stream().noneMatch(c -> c.startsWith("getReadableLakeSnapshot")), + adminOps.calls.toString()); + } + + // ------------------------------------------- union read of a primary-key table: lake + log tail + + /** + * The shape of the whole arrangement, in one plan. Bucket 0 has been tiered and has written more + * since, so its lake split is bound to the tail that supersedes part of it AND the tail is planned as + * its own range — the lake split only hides rows, it never produces the new ones. Bucket 1 has been + * tiered and written nothing since, so its lake split is passed through untouched and fluss + * contributes nothing at all. + */ + @Test + public void bucketWithATailBindsItToItsLakeSplitsAndPlansTheTailOnce() { + registerPkLakeTable(2); + lakeSnapshotAt(9L, offsets(100L, 200L)); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 105L, 200L); + earliestOffsets(null, 0L, 0L); + lakeSplits(RecordingLakeSibling.LakeRange.inBucket(0), + RecordingLakeSibling.LakeRange.inBucket(1)); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(3, ranges.size()); + assertSuppressed(ranges.get(0), 0, 100L, 105L); + Assertions.assertTrue(ranges.get(1) instanceof RecordingLakeSibling.LakeRange, + "bucket 1 has no tail and must be passed through: " + ranges.get(1)); + assertTailRange(ranges.get(2), 0, 100L, 105L); + } + + /** + * The suppressing tail and the range that produces the tail's rows must be the SAME window. Any gap + * between them is a wrong answer in one direction or the other: a suppression window wider than the + * produced one hides rows nothing brings back, a narrower one lets a superseded row through beside + * its replacement. + */ + @Test + public void theSuppressedWindowIsExactlyTheWindowTheTailRangeReads() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + kvSnapshots(null, new long[] {4L}, new long[] {10L}); + latestOffsets(null, 130L); + earliestOffsets(null, 7L); + lakeSplits(RecordingLakeSibling.LakeRange.inBucket(0)); + + List ranges = plan(PK_TABLE, catalog()); + + FlussSuppressedLakeRange.Tail tail = ((FlussSuppressedLakeRange) ranges.get(0)).getTail(); + Map tailRange = ranges.get(1).getProperties(); + Assertions.assertEquals(String.valueOf(tail.getStartOffset()), + tailRange.get("fluss.log_start_offset")); + Assertions.assertEquals(String.valueOf(tail.getStopOffset()), + tailRange.get("fluss.log_stop_offset")); + } + + /** + * A bucket the lake has never seen is read whole from fluss, exactly as it would be with no lake at + * all — and nothing of it can be suppressed, because the lake holds none of it. + */ + @Test + public void bucketTheLakeHasNeverSeenIsReadWholeFromFluss() { + registerPkLakeTable(2); + lakeSnapshotAt(9L, offsets(100L, null)); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 100L, 25L); + earliestOffsets(null, 0L, 0L); + lakeSplits(RecordingLakeSibling.LakeRange.inBucket(0)); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(2, ranges.size()); + Assertions.assertTrue(ranges.get(0) instanceof RecordingLakeSibling.LakeRange, "bucket 0 is tiered" + + " up to where its log ends and must be passed through: " + ranges.get(0)); + assertPkRange(ranges.get(1), 1, 5L, 20L, 25L); + } + + /** + * The tail of one bucket must not be bound to another bucket's split. Bucket-blind binding still + * produces a plausible plan — every split suppressed by SOME tail — and returns wrong rows: the keys + * of bucket 0's tail say nothing about the rows of bucket 1. + */ + @Test + public void eachBucketsSplitsAreBoundToThatBucketsTail() { + registerPkLakeTable(2); + lakeSnapshotAt(9L, offsets(100L, 200L)); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 105L, 250L); + earliestOffsets(null, 0L, 0L); + lakeSplits(RecordingLakeSibling.LakeRange.inBucket(1), + RecordingLakeSibling.LakeRange.inBucket(0)); + + List ranges = plan(PK_TABLE, catalog()); + + assertSuppressed(ranges.get(0), 1, 200L, 250L); + assertSuppressed(ranges.get(1), 0, 100L, 105L); + } + + /** The same, one level up: a partition's splits are bound to the tails of THAT partition's buckets. */ + @Test + public void eachPartitionsSplitsAreBoundToThatPartitionsTails() { + registerPartitionedPkLakeTable(1, "20260101", "20260102"); + adminOps.readableLakeSnapshot = new LakeSnapshot(9L, partitionedOffsets( + new long[] {100L}, new long[] {700L})); + kvSnapshots("20260101", new long[] {1L}, new long[] {10L}); + kvSnapshots("20260102", new long[] {2L}, new long[] {20L}); + latestOffsets("20260101", 105L); + latestOffsets("20260102", 750L); + earliestOffsets("20260101", 0L); + earliestOffsets("20260102", 0L); + lakeSplits( + RecordingLakeSibling.LakeRange.inBucket(0, Collections.singletonMap("dt", "20260102")), + RecordingLakeSibling.LakeRange.inBucket(0, Collections.singletonMap("dt", "20260101"))); + + List ranges = plan(PK_TABLE, catalog()); + + assertSuppressed(ranges.get(0), 0, 700L, 750L); + assertSuppressed(ranges.get(1), 0, 100L, 105L); + } + + /** + * A lake split of a partition this scan does not read from fluss — one fluss has dropped, or one the + * engine pruned away — has no tail to be bound to and is passed through. It cannot be dropped either: + * partition pruning removes the fluss half of a partition, not the predicate that pruned it. + */ + @Test + public void lakeSplitOfAPartitionThisScanDoesNotReadIsPassedThrough() { + registerPartitionedPkLakeTable(1, "20260101"); + adminOps.readableLakeSnapshot = new LakeSnapshot(9L, partitionedOffsets(new long[] {100L})); + kvSnapshots("20260101", new long[] {1L}, new long[] {10L}); + latestOffsets("20260101", 105L); + earliestOffsets("20260101", 0L); + lakeSplits( + RecordingLakeSibling.LakeRange.inBucket(0, Collections.singletonMap("dt", "20251231"))); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertTrue(ranges.get(0) instanceof RecordingLakeSibling.LakeRange, ranges.toString()); + assertTailRange(ranges.get(1), 0, 100L, 105L); + } + + /** + * The bucket a split holds is the one fact this connector cannot work out for itself, and an older + * paimon plugin does not report it. Reading that absence as "this bucket has no tail" would return + * every superseded row a second time, so it is an error instead. + */ + @Test + public void lakeSplitThatDoesNotSayWhichBucketItHoldsIsRefused() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + kvSnapshots(null, new long[] {4L}, new long[] {10L}); + latestOffsets(null, 105L); + earliestOffsets(null, 0L); + lakeSplits(RecordingLakeSibling.LakeRange.withoutABucket()); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> plan(PK_TABLE, catalog())); + Assertions.assertTrue(e.getMessage().contains("paimon.bucket"), e.getMessage()); + } + + /** A lake table bucketed differently from the fluss table cannot be matched to it bucket by bucket. */ + @Test + public void lakeSplitInABucketThisTableDoesNotHaveIsRefused() { + registerPkLakeTable(2); + lakeSnapshotAt(9L, offsets(100L, 200L)); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 105L, 205L); + earliestOffsets(null, 0L, 0L); + lakeSplits(RecordingLakeSibling.LakeRange.inBucket(7)); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> plan(PK_TABLE, catalog())); + Assertions.assertTrue(e.getMessage().contains("bucketed alike"), e.getMessage()); + } + + /** + * The lake holding data for a bucket fluss has no tiering offset for means the two disagree about what + * has been tiered. Passing the split through would return that bucket's rows twice — once from the + * lake, once from the {@code PK_FULL} range the missing offset produces. + */ + @Test + public void lakeSplitOfABucketFlussRecordsNoTieringOffsetForIsRefused() { + registerPkLakeTable(2); + lakeSnapshotAt(9L, offsets(100L, null)); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 105L, 25L); + earliestOffsets(null, 0L, 0L); + lakeSplits(RecordingLakeSibling.LakeRange.inBucket(1)); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> plan(PK_TABLE, catalog())); + Assertions.assertTrue(e.getMessage().contains("metadata disagrees"), e.getMessage()); + } + + /** + * Fluss deletes old log segments on a timer that does not wait for tiering, so the beginning of a tail + * can be gone. There is nowhere else to read it from — the lake's copy stops exactly where the missing + * tail begins — so {@code auto} gives up the lake half rather than return fewer rows than the table + * holds, and says so in EXPLAIN. + */ + @Test + public void tailTheLogNoLongerHoldsGivesUpTheLakeHalf() { + registerPkLakeTable(2); + lakeSnapshotAt(9L, offsets(100L, 200L)); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 105L, 205L); + earliestOffsets(null, 0L, 201L); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); + + List ranges = + provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList())); + + Assertions.assertEquals(2, ranges.size()); + assertPkRange(ranges.get(0), 0, 4L, 10L, 105L); + assertPkRange(ranges.get(1), 1, 5L, 20L, 205L); + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, "", Collections.emptyMap()); + Assertions.assertTrue(output.toString().contains("unionRead=no"), output.toString()); + Assertions.assertTrue(output.toString().contains("degraded=tail-truncated"), output.toString()); + } + + /** A bucket fluss will not report an earliest offset for is not "fine", it is unverifiable. */ + @Test + public void tailThatCannotBeVerifiedGivesUpTheLakeHalf() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + kvSnapshots(null, new long[] {4L}, new long[] {10L}); + latestOffsets(null, 105L); + adminOps.earliestOffsetsByPartition.put(null, Collections.emptyMap()); + + List ranges = plan(PK_TABLE, catalog()); + + Assertions.assertEquals(1, ranges.size()); + assertPkRange(ranges.get(0), 0, 4L, 10L, 105L); + } + + /** + * What replaces the lake half is the very read {@code disabled} asks for outright, down to the ranges. + * That is what makes it safe to give up so late: a primary-key table read from fluss alone is the whole + * table. + */ + @Test + public void whatReplacesTheLakeHalfIsTheReadDisabledModeAsksForOutright() { + registerPkLakeTable(2); + lakeSnapshotAt(9L, offsets(100L, 200L)); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 105L, 205L); + earliestOffsets(null, 104L, 0L); + + List> degraded = rangeProperties(plan(PK_TABLE, catalog())); + List> disabled = rangeProperties( + plan(PK_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "disabled"))); + + Assertions.assertEquals(disabled, degraded); + } + + /** Under {@code required} the same truncated tail is an error: there is nothing to fall back to. */ + @Test + public void requiredModeRefusesATailTheLogNoLongerHolds() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + kvSnapshots(null, new long[] {4L}, new long[] {10L}); + latestOffsets(null, 105L); + earliestOffsets(null, 101L); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> plan(PK_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "required"))); + Assertions.assertTrue(e.getMessage().contains("the log now starts at 101"), e.getMessage()); + } + + /** + * A key column Doris cannot compare exactly is a permanent property of the table, so it is settled + * before anything is asked of the lake — that is what lets the answer be the same at plan-translation + * time, when the key columns are kept in the scan's tuple, as it is here. + */ + @Test + public void keyColumnThatCannotBeComparedExactlyGivesUpTheLakeHalf() { + registerPkLakeTableKeyedBy(DataTypes.DOUBLE()); + kvSnapshots(null, new long[] {4L}, new long[] {10L}); + latestOffsets(null, 105L); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); + + List ranges = + provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList())); + + Assertions.assertEquals(1, ranges.size()); + assertPkRange(ranges.get(0), 0, 4L, 10L, 105L); + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, "", Collections.emptyMap()); + Assertions.assertTrue(output.toString().contains("degraded=key-type"), output.toString()); + // Settled without asking the lake anything at all. + Assertions.assertTrue(adminOps.calls.stream().noneMatch(c -> c.startsWith("getReadableLakeSnapshot")), + adminOps.calls.toString()); + } + + @Test + public void requiredModeRefusesAKeyColumnThatCannotBeComparedExactly() { + registerPkLakeTableKeyedBy(DataTypes.DOUBLE()); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> plan(PK_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "required"))); + Assertions.assertTrue(e.getMessage().contains("primary-key column 'id'"), e.getMessage()); + Assertions.assertTrue(e.getMessage().contains("floating-point"), e.getMessage()); + } + + /** + * A lake split is matched to a fluss partition by comparing rendered partition values, so a partition + * column of a type the two sides may spell differently is refused the same way. Fluss itself allows + * such a column — INT, DATE, BOOLEAN are all legal partition keys — so this is not a case the schema + * makes impossible. + */ + @Test + public void partitionColumnThatMayNotRenderAlikeGivesUpTheLakeHalf() { + registerPartitionedPkLakeTable(1, DataTypes.INT(), "20260101"); + kvSnapshots("20260101", new long[] {1L}, new long[] {10L}); + latestOffsets("20260101", 105L); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); + + List ranges = + provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList())); + + Assertions.assertEquals(1, ranges.size()); + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, "", Collections.emptyMap()); + Assertions.assertTrue(output.toString().contains("degraded=partition-type"), output.toString()); + } + + @Test + public void requiredModeRefusesAPartitionColumnThatMayNotRenderAlike() { + registerPartitionedPkLakeTable(1, DataTypes.INT(), "20260101"); + + DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, + () -> plan(PK_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "required"))); + Assertions.assertTrue(e.getMessage().contains("partition column 'dt'"), e.getMessage()); + } + + /** + * The kv snapshots are read for every partition even when the lake half turns out to cover it, because + * the fallback needs them and reading them after the offsets would leave a bucket bounded by an offset + * older than the snapshot it is read from. The order is the assertion. + */ + @Test + public void snapshotsAreReadBeforeTheOffsetsEvenWhenTheLakeCoversEverything() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + kvSnapshots(null, new long[] {4L}, new long[] {10L}); + latestOffsets(null, 105L); + earliestOffsets(null, 0L); + lakeSplits(RecordingLakeSibling.LakeRange.inBucket(0)); + + plan(PK_TABLE, catalog()); + + int snapshotCall = indexOfCall("getLatestKvSnapshots"); + int latestCall = indexOfCall("listOffsets(db.pk_tbl, [0], LatestSpec)"); + int earliestCall = indexOfCall("listOffsets(db.pk_tbl, [0], EarliestSpec)"); + Assertions.assertTrue(snapshotCall >= 0 && snapshotCall < latestCall, adminOps.calls.toString()); + Assertions.assertTrue(latestCall < earliestCall, adminOps.calls.toString()); + } + + /** EXPLAIN accounts for all three parts, so a regression test can tell which of them did the work. */ + @Test + public void explainCountsTheSuppressedSplitsAndTheTailsApart() { + registerPkLakeTable(2); + lakeSnapshotAt(9L, offsets(100L, 200L)); + kvSnapshots(null, new long[] {4L, 5L}, new long[] {10L, 20L}); + latestOffsets(null, 105L, 200L); + earliestOffsets(null, 0L, 0L); + lakeSplits(RecordingLakeSibling.LakeRange.inBucket(0), + RecordingLakeSibling.LakeRange.inBucket(0), + RecordingLakeSibling.LakeRange.inBucket(1)); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); + provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList())); + + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, "", Collections.emptyMap()); + + Assertions.assertEquals("flussScan: unionRead=yes, lakeSplits=3, suppressedLakeSplits=2," + + " logRanges=0, pkRanges=0, pkTailRanges=1, mode=auto\n", output.toString()); + } + + // ------------------------------------------- the key columns BE has to read either way + + /** + * BE identifies the rows a tail supersedes by their keys, so those columns have to be read whether or + * not the query asked for them. The engine keeps them in the scan's tuple on the strength of this + * answer; the projection above the scan drops them again before the user sees a row. + */ + @Test + public void unionReadKeepsTheKeyColumnsInTheScan() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + + Assertions.assertEquals(Collections.singleton("id"), mustReadColumns(PK_TABLE, catalog())); + } + + /** Only the physical key: a partition column is the same for every row of a bucket. */ + @Test + public void thePartitionColumnsAreNotPartOfTheKeyBeMustRead() { + registerPartitionedPkLakeTable(1, "20260101"); + adminOps.readableLakeSnapshot = new LakeSnapshot(9L, partitionedOffsets(new long[] {100L})); + + Assertions.assertEquals(Collections.singleton("id"), mustReadColumns(PK_TABLE, catalog())); + } + + /** + * Nothing else keeps a column it was not asked for. Each of these reads is served by one scanner that + * needs no key at all, so keeping one would be a column read for nobody — and, for a log table, would + * pull the lake snapshot's round trip forward into plan translation for nothing. + */ + @Test + public void everyOtherReadKeepsNothing() { + registerLakeTable(2); + lakeSnapshotAt(7L, offsets(1L, 1L)); + Assertions.assertEquals(Collections.emptySet(), mustReadColumns(LOG_TABLE, catalog())); + Assertions.assertTrue(adminOps.calls.isEmpty(), adminOps.calls.toString()); + + registerPkTable(PK_TABLE, 1); + Assertions.assertEquals(Collections.emptySet(), mustReadColumns(PK_TABLE, catalog())); + + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + Assertions.assertEquals(Collections.emptySet(), + mustReadColumns(PK_TABLE, catalog(FlussConnectorProperties.UNION_READ_MODE, "disabled"))); + + registerPkLakeTableKeyedBy(DataTypes.DOUBLE()); + Assertions.assertEquals(Collections.emptySet(), mustReadColumns(PK_TABLE, catalog())); + } + + /** + * The engine asks for the key columns while translating the plan and plans the ranges later, and the + * two answers must come from ONE resolution. Were they resolved twice, a snapshot committed in between + * could make the first say "no lake" (so the key columns are pruned away) and the second say "lake", + * leaving BE to look for a key column that is not in its projection. + */ + @Test + public void bothQuestionsAreAnsweredByTheSameResolution() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + kvSnapshots(null, new long[] {4L}, new long[] {10L}); + latestOffsets(null, 105L); + earliestOffsets(null, 0L); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); + + provider.getMustReadColumns(session, handle(PK_TABLE)); + provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList())); + + Assertions.assertEquals(1, + adminOps.calls.stream().filter(c -> c.startsWith("getReadableLakeSnapshot")).count(), + adminOps.calls.toString()); + } + + // ---------------------------------------------------------------- what BE and EXPLAIN receive + + /** + * What BE needs to suppress by key, and only on the read that suppresses: the key columns by name (the + * types travel as ordinary slot descriptors, because the columns are ordinary projected columns) and + * the ceiling on how much tail it may hold while doing it. + */ + @Test + public void primaryKeyUnionTellsBeWhichColumnsTheKeyIsMadeOf() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + + Map props = nodeProperties(PK_TABLE, catalog()); + + Assertions.assertEquals("id", props.get("fluss.union.pk_names")); + Assertions.assertEquals("2000000", props.get("fluss.union.max_tail_rows")); + } + + @Test + public void theTailCeilingIsTheOneTheCatalogDeclares() { + registerPkLakeTable(1); + lakeSnapshotAt(9L, offsets(100L)); + + Map props = nodeProperties(PK_TABLE, + catalog(FlussConnectorProperties.UNION_READ_MAX_TAIL_ROWS, "500")); + + Assertions.assertEquals("500", props.get("fluss.union.max_tail_rows")); + } + + /** A read that suppresses nothing must not describe a key: there is no reader on the far side. */ + @Test + public void readThatSuppressesNothingSendsNoKey() { + registerLakeTable(1); + lakeSnapshotAt(7L, offsets(1L)); + + Map props = nodeProperties(LOG_TABLE, catalog()); + + Assertions.assertFalse(props.containsKey("fluss.union.pk_names"), props.toString()); + Assertions.assertFalse(props.containsKey("fluss.union.max_tail_rows"), props.toString()); + } + + @Test + public void scanLevelParamsCarryTheClientConfigAndTableIdentity() { + registerLogTable(LOG_TABLE, 1); + Map catalog = catalog(); + catalog.put("fluss.client.writer.batch-size", "2mb"); + + TFileScanRangeParams params = new TFileScanRangeParams(); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog, this::lakeSibling); + provider.populateScanLevelParams(params, nodeProperties(LOG_TABLE, catalog)); + + Map expected = new LinkedHashMap<>(); + expected.put("fluss.db_name", "db"); + expected.put("fluss.table_name", "log_tbl"); + expected.put("fluss.client.bootstrap.servers", "localhost:9123"); + expected.put("fluss.client.client.writer.batch-size", "2mb"); + Assertions.assertEquals(expected, params.getFlussProperties()); + } + + /** The engine's own keys are not the scanner's; forwarding them wholesale would be noise at best. */ + @Test + public void scanLevelParamsDropTheEngineOnlyKeys() { + registerPartitionedLogTable(1, "20260101"); + Map nodeProps = new LinkedHashMap<>(nodeProperties(LOG_TABLE, catalog())); + nodeProps.put(ScanNodePropertyKeys.SYNTHETIC_TOTAL_READ_SPLITS, "3"); + + TFileScanRangeParams params = new TFileScanRangeParams(); + new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling).populateScanLevelParams(params, nodeProps); + + Assertions.assertFalse(params.getFlussProperties() + .containsKey(ScanNodePropertyKeys.PATH_PARTITION_KEYS)); + Assertions.assertFalse(params.getFlussProperties() + .containsKey(ScanNodePropertyKeys.SYNTHETIC_TOTAL_READ_SPLITS)); + } + + /** + * {@code auto} falls back to a fluss-only read without saying so anywhere else in the plan, so a + * union-read regression test has nothing to assert on but this line. + */ + @Test + public void explainReportsHowTheScanWasActuallyPlanned() { + registerLogTable(LOG_TABLE, 3); + latestOffsets(null, 1L, 0L, 5L); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); + provider.planScan(session, request(handle(LOG_TABLE), Collections.emptyList())); + + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, " ", Collections.emptyMap()); + + Assertions.assertEquals( + " flussScan: unionRead=no, lakeSplits=0, suppressedLakeSplits=0, logRanges=2," + + " pkRanges=0, pkTailRanges=0, mode=auto\n", output.toString()); + } + + /** + * Counted apart from log ranges rather than lumped together: the two are read by different code + * paths, and a test that only sees a total cannot tell a primary-key table that was planned the + * wrong way from one that was planned the right way. + */ + @Test + public void explainCountsPrimaryKeyRangesApartFromLogRanges() { + registerPkTable(PK_TABLE, 3); + kvSnapshots(null, new long[] {1L, NO_SNAPSHOT, NO_SNAPSHOT}, new long[] {10L, 0L, 0L}); + latestOffsets(null, 12L, 4L, 0L); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog(), this::lakeSibling); + provider.planScan(session, request(handle(PK_TABLE), Collections.emptyList())); + + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, "", Collections.emptyMap()); + + Assertions.assertEquals( + "flussScan: unionRead=no, lakeSplits=0, suppressedLakeSplits=0, logRanges=0," + + " pkRanges=2, pkTailRanges=0, mode=auto\n", output.toString()); + } + + @Test + public void explainReportsTheConfiguredMode() { + registerLogTable(LOG_TABLE, 1); + latestOffsets(null, 1L); + Map catalog = catalog(FlussConnectorProperties.UNION_READ_MODE, "disabled"); + FlussScanPlanProvider provider = new FlussScanPlanProvider(adminOps, catalog, this::lakeSibling); + provider.planScan(session, request(handle(LOG_TABLE), Collections.emptyList())); + + StringBuilder output = new StringBuilder(); + provider.appendExplainInfo(output, "", Collections.emptyMap()); + + Assertions.assertTrue(output.toString().contains("mode=disabled"), output.toString()); + } + + // ---------------------------------------------------------------- helpers + + private Map catalog() { + Map properties = new HashMap<>(); + properties.put(FlussConnectorProperties.BOOTSTRAP_SERVERS, "localhost:9123"); + return properties; + } + + private Map catalog(String key, String value) { + Map properties = catalog(); + properties.put(key, value); + return properties; + } + + private FlussConnectorMetadata metadata() { + return new FlussConnectorMetadata(adminOps, new FlussTypeMapping.Options(false, false), + properties -> { + throw new AssertionError("no lake sibling is expected in this test"); + }, + handle -> null); + } + + private ConnectorTableHandle handle(TablePath tablePath) { + return FlussTableHandle.of(adminOps.tableInfos.get(tablePath)); + } + + private List plan(TablePath tablePath, Map catalogProperties) { + return plan(tablePath, catalogProperties, Collections.emptyList()); + } + + private List plan(TablePath tablePath, Map catalogProperties, + List requiredPartitions) { + return new FlussScanPlanProvider(adminOps, catalogProperties, this::lakeSibling) + .planScan(session, request(handle(tablePath), requiredPartitions)); + } + + private Map nodeProperties(TablePath tablePath, Map catalogProperties) { + return new FlussScanPlanProvider(adminOps, catalogProperties, this::lakeSibling).getScanNodeProperties( + session, handle(tablePath), Collections.emptyList(), Optional.empty()); + } + + private Set mustReadColumns(TablePath tablePath, Map catalogProperties) { + return new FlussScanPlanProvider(adminOps, catalogProperties, this::lakeSibling) + .getMustReadColumns(session, handle(tablePath)); + } + + private static ConnectorScanRequest request(ConnectorTableHandle handle, List requiredPartitions) { + return ConnectorScanRequest.builder(handle, Collections.emptyList()) + .requiredPartitions(requiredPartitions) + .build(); + } + + private void registerLogTable(TablePath tablePath, int buckets) { + adminOps.tableInfos.put(tablePath, FlussTestTables.builder(tablePath) + .column("id", DataTypes.INT()) + .column("v", DataTypes.STRING()) + .buckets(buckets) + .build()); + } + + /** A log table partitioned by {@code dt}, with partition ids 100, 101, ... in the order given. */ + private void registerPartitionedLogTable(int buckets, String... partitionValues) { + adminOps.tableInfos.put(LOG_TABLE, FlussTestTables.builder(LOG_TABLE) + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionedBy("dt") + .buckets(buckets) + .build()); + List partitions = new ArrayList<>(); + for (int i = 0; i < partitionValues.length; i++) { + partitions.add(new PartitionInfo(100L + i, + ResolvedPartitionSpec.fromPartitionValue("dt", partitionValues[i]), null)); + } + adminOps.partitionsByTable.put(LOG_TABLE, partitions); + } + + private void registerLakeTable(int buckets) { + adminOps.tableInfos.put(LOG_TABLE, FlussTestTables.builder(LOG_TABLE) + .column("id", DataTypes.INT()) + .buckets(buckets) + .property("table.datalake.enabled", "true") + .property("table.datalake.format", "paimon") + .property("table.datalake.paimon.metastore", "filesystem") + .property("table.datalake.paimon.warehouse", "/lake/warehouse") + .build()); + siblingExpected = true; + } + + private void registerPkTable(TablePath tablePath, int buckets) { + adminOps.tableInfos.put(tablePath, FlussTestTables.builder(tablePath) + .column("id", DataTypes.INT().copy(false)) + .column("v", DataTypes.STRING()) + .primaryKey("id") + .buckets(buckets, "id") + .build()); + } + + /** + * A primary-key table tiered into a lake. Deliberately does NOT mark a sibling as expected: its lake is + * never read, so reaching the sibling factory at all is a failure. + */ + private void registerTieredPkTable(int buckets) { + adminOps.tableInfos.put(PK_TABLE, FlussTestTables.builder(PK_TABLE) + .column("id", DataTypes.INT().copy(false)) + .column("v", DataTypes.STRING()) + .primaryKey("id") + .buckets(buckets, "id") + .property("table.datalake.enabled", "true") + .property("table.datalake.format", "paimon") + .property("table.datalake.paimon.metastore", "filesystem") + .property("table.datalake.paimon.warehouse", "/lake/warehouse") + .build()); + } + + /** A primary-key table partitioned by {@code dt}, with partition ids 100, 101, ... in order. */ + private void registerPartitionedPkTable(int buckets, String... partitionValues) { + adminOps.tableInfos.put(PK_TABLE, FlussTestTables.builder(PK_TABLE) + .column("id", DataTypes.INT().copy(false)) + .column("dt", DataTypes.STRING().copy(false)) + .primaryKey("id", "dt") + .partitionedBy("dt") + .buckets(buckets, "id") + .build()); + List partitions = new ArrayList<>(); + for (int i = 0; i < partitionValues.length; i++) { + partitions.add(new PartitionInfo(100L + i, + ResolvedPartitionSpec.fromPartitionValue("dt", partitionValues[i]), null)); + } + adminOps.partitionsByTable.put(PK_TABLE, partitions); + } + + /** + * Latest kv snapshot per bucket 0..n-1 of {@code partitionName} ({@code null} = unpartitioned); + * {@link #NO_SNAPSHOT} for a bucket that has never been snapshotted. Such a bucket gets a null + * snapshot id AND a null log offset, which is the only shape a cluster produces — the client + * asserts the two are both set or both absent when it decodes the response. + */ + private void kvSnapshots(String partitionName, long[] snapshotIds, long[] logOffsets) { + Map ids = new LinkedHashMap<>(); + Map offsets = new LinkedHashMap<>(); + for (int bucket = 0; bucket < snapshotIds.length; bucket++) { + boolean snapshotted = snapshotIds[bucket] != NO_SNAPSHOT; + ids.put(bucket, snapshotted ? snapshotIds[bucket] : null); + offsets.put(bucket, snapshotted ? logOffsets[bucket] : null); + } + adminOps.kvSnapshotsByPartition.put(partitionName, new KvSnapshots(1L, null, ids, offsets)); + } + + /** Latest offsets for buckets 0..n-1 of {@code partitionName} ({@code null} = unpartitioned). */ + private void latestOffsets(String partitionName, long... offsets) { + adminOps.latestOffsetsByPartition.put(partitionName, byBucket(offsets)); + } + + /** + * Earliest offsets for buckets 0..n-1 — how far back fluss can still serve. Only a union read of a + * primary-key table asks for these, which is why the log-table fixtures do not set them: a test that + * needed them and did not say so gets an error from the recorder, not a default. + */ + private void earliestOffsets(String partitionName, long... offsets) { + adminOps.earliestOffsetsByPartition.put(partitionName, byBucket(offsets)); + } + + private static Map byBucket(long... offsets) { + Map byBucket = new LinkedHashMap<>(); + for (int bucket = 0; bucket < offsets.length; bucket++) { + byBucket.put(bucket, offsets[bucket]); + } + return byBucket; + } + + private static void assertLogRange(ConnectorScanRange range, int bucket, long start, long stop) { + Map props = range.getProperties(); + Assertions.assertEquals("LOG", props.get("fluss.range_type")); + Assertions.assertEquals(String.valueOf(bucket), props.get("fluss.bucket_id")); + Assertions.assertEquals(String.valueOf(start), props.get("fluss.log_start_offset")); + Assertions.assertEquals(String.valueOf(stop), props.get("fluss.log_stop_offset")); + } + + private static void assertPkRange(ConnectorScanRange range, int bucket, long snapshotId, + long start, long stop) { + Map props = range.getProperties(); + Assertions.assertEquals("PK_FULL", props.get("fluss.range_type")); + Assertions.assertEquals(String.valueOf(bucket), props.get("fluss.bucket_id")); + Assertions.assertEquals(String.valueOf(snapshotId), props.get("fluss.kv_snapshot_id")); + Assertions.assertEquals(String.valueOf(start), props.get("fluss.log_start_offset")); + Assertions.assertEquals(String.valueOf(stop), props.get("fluss.log_stop_offset")); + } + + /** The ranges' payloads, in order — what two plans have to agree on to be the same read. */ + private static List> rangeProperties(List ranges) { + List> properties = new ArrayList<>(ranges.size()); + for (ConnectorScanRange range : ranges) { + properties.add(range.getProperties()); + } + return properties; + } + + /** Position of the first recorded call starting with {@code prefix}, or -1. */ + private int indexOfCall(String prefix) { + for (int i = 0; i < adminOps.calls.size(); i++) { + if (adminOps.calls.get(i).startsWith(prefix)) { + return i; + } + } + return -1; + } + + private static void assertPartition(ConnectorScanRange range, String partitionName, + long partitionId, int bucket, long stop) { + Map props = range.getProperties(); + Assertions.assertEquals(partitionName, props.get("fluss.partition_name")); + Assertions.assertEquals(String.valueOf(partitionId), props.get("fluss.partition_id")); + assertLogRange(range, bucket, -2L, stop); + } + + // ---------------------------------------------------------------- union-read fixtures + + /** + * The lake sibling this catalog would build. Fails loud when a test that registered no lake table + * reaches it: a fluss-only plan that quietly consulted the lake still passes every other assertion. + */ + private Connector lakeSibling(Map siblingProperties) { + if (!siblingExpected) { + throw new AssertionError("no lake sibling is expected in this test"); + } + if (sibling == null) { + sibling = new RecordingLakeSibling(siblingProperties); + } + return sibling; + } + + /** The sibling, built now so a test can set what it answers before planning asks. */ + private RecordingLakeSibling sibling() { + lakeSibling(Collections.emptyMap()); + return sibling; + } + + /** The readable lake snapshot fluss reports, with the log offset it recorded for each bucket. */ + private void lakeSnapshotAt(long snapshotId, Map bucketOffsets) { + adminOps.readableLakeSnapshot = new LakeSnapshot(snapshotId, bucketOffsets); + } + + /** + * Lake offsets for buckets 0..n-1 of an unpartitioned table; a {@code null} entry is a bucket the + * snapshot does not mention at all (never tiered), which is not the same as offset 0. + */ + private static Map offsets(Long... byBucket) { + Map offsets = new HashMap<>(); + for (int bucket = 0; bucket < byBucket.length; bucket++) { + if (byBucket[bucket] != null) { + offsets.put(new TableBucket(FlussTestTables.TABLE_ID, bucket), byBucket[bucket]); + } + } + return offsets; + } + + /** The exact splits the sibling's scan planner returns, in order. */ + private void lakeSplits(RecordingLakeSibling.LakeRange... splits) { + sibling().lakeRanges = new ArrayList<>(Arrays.asList(splits)); + } + + /** Lake offsets for buckets 0..n-1 of each partition, partition ids 100, 101, ... in order. */ + private static Map partitionedOffsets(long[]... byPartition) { + Map offsets = new HashMap<>(); + for (int partition = 0; partition < byPartition.length; partition++) { + for (int bucket = 0; bucket < byPartition[partition].length; bucket++) { + offsets.put(new TableBucket(FlussTestTables.TABLE_ID, 100L + partition, bucket), + byPartition[partition][bucket]); + } + } + return offsets; + } + + /** + * A primary-key table tiered into a lake whose lake IS read. The same table as + * {@link #registerTieredPkTable}, but declaring that reaching the sibling is expected — the other + * fixture keeps that guard on, for the tests that assert the lake was never consulted. + */ + private void registerPkLakeTable(int buckets) { + registerTieredPkTable(buckets); + siblingExpected = true; + } + + /** The same, keyed by a column of {@code keyType} — for the types a union read cannot compare. */ + private void registerPkLakeTableKeyedBy(DataType keyType) { + adminOps.tableInfos.put(PK_TABLE, FlussTestTables.builder(PK_TABLE) + .column("id", keyType.copy(false)) + .column("v", DataTypes.STRING()) + .primaryKey("id") + .buckets(1, "id") + .property("table.datalake.enabled", "true") + .property("table.datalake.format", "paimon") + .property("table.datalake.paimon.metastore", "filesystem") + .property("table.datalake.paimon.warehouse", "/lake/warehouse") + .build()); + siblingExpected = true; + } + + /** A partitioned primary-key lake table, partitioned by a STRING {@code dt}. */ + private void registerPartitionedPkLakeTable(int buckets, String... partitionValues) { + registerPartitionedPkLakeTable(buckets, DataTypes.STRING(), partitionValues); + } + + private void registerPartitionedPkLakeTable(int buckets, DataType partitionType, + String... partitionValues) { + adminOps.tableInfos.put(PK_TABLE, FlussTestTables.builder(PK_TABLE) + .column("id", DataTypes.INT().copy(false)) + .column("dt", partitionType.copy(false)) + .primaryKey("id", "dt") + .partitionedBy("dt") + .buckets(buckets, "id") + .property("table.datalake.enabled", "true") + .property("table.datalake.format", "paimon") + .property("table.datalake.paimon.metastore", "filesystem") + .property("table.datalake.paimon.warehouse", "/lake/warehouse") + .build()); + List partitions = new ArrayList<>(); + for (int i = 0; i < partitionValues.length; i++) { + partitions.add(new PartitionInfo(100L + i, + ResolvedPartitionSpec.fromPartitionValue("dt", partitionValues[i]), null)); + } + adminOps.partitionsByTable.put(PK_TABLE, partitions); + siblingExpected = true; + } + + /** A lake split bound to the tail of {@code bucket} over {@code [start, stop)}. */ + private static void assertSuppressed(ConnectorScanRange range, int bucket, long start, long stop) { + Assertions.assertTrue(range instanceof FlussSuppressedLakeRange, + "expected a suppressed lake split but got " + range); + FlussSuppressedLakeRange.Tail tail = ((FlussSuppressedLakeRange) range).getTail(); + Assertions.assertEquals(bucket, tail.getBucketId()); + Assertions.assertEquals(start, tail.getStartOffset()); + Assertions.assertEquals(stop, tail.getStopOffset()); + } + + private static void assertTailRange(ConnectorScanRange range, int bucket, long start, long stop) { + Map props = range.getProperties(); + Assertions.assertEquals("PK_TAIL", props.get("fluss.range_type")); + Assertions.assertEquals(String.valueOf(bucket), props.get("fluss.bucket_id")); + Assertions.assertEquals(String.valueOf(start), props.get("fluss.log_start_offset")); + Assertions.assertEquals(String.valueOf(stop), props.get("fluss.log_stop_offset")); + } + + /** {@code count} ranges for the sibling's scan planner to return as the lake half. */ + private void lakeRanges(int count) { + List ranges = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + ranges.add(new RecordingLakeSibling.LakeRange()); + } + sibling().lakeRanges = ranges; + } + + /** The columns the lake table reports, in order. */ + private void lakeTableColumns(String... names) { + Map columns = new LinkedHashMap<>(); + for (String name : names) { + columns.put(name, new RecordingLakeSibling.LakeColumn(name)); + } + sibling().lakeColumns = columns; + } + + /** A lake-enabled log table partitioned by {@code dt}, with partition ids 100, 101, ... in order. */ + private void registerPartitionedLakeTable(int buckets, String... partitionValues) { + adminOps.tableInfos.put(LOG_TABLE, FlussTestTables.builder(LOG_TABLE) + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionedBy("dt") + .buckets(buckets) + .property("table.datalake.enabled", "true") + .property("table.datalake.format", "paimon") + .property("table.datalake.paimon.metastore", "filesystem") + .property("table.datalake.paimon.warehouse", "/lake/warehouse") + .build()); + List partitions = new ArrayList<>(); + for (int i = 0; i < partitionValues.length; i++) { + partitions.add(new PartitionInfo(100L + i, + ResolvedPartitionSpec.fromPartitionValue("dt", partitionValues[i]), null)); + } + adminOps.partitionsByTable.put(LOG_TABLE, partitions); + siblingExpected = true; + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussStatementScopeTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussStatementScopeTest.java new file mode 100644 index 00000000000000..79f5b41562726e --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussStatementScopeTest.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; + +/** + * Guards the statement-scope naming norm: every namespace this connector declares is prefixed with its + * own connector type. The scope is keyed by {@code (catalogId, db, table, queryId)} and shared across + * connectors, so an unprefixed namespace could hand a gateway statement another connector's value for + * the same table coordinate — which surfaces as a ClassCastException, not as a wrong answer, but only + * on the gateway path nobody runs by default. + */ +public class FlussStatementScopeTest { + + @Test + public void allNamespacesArePrefixedWithConnectorType() throws Exception { + // Reflective on purpose: a namespace added later is covered without anyone remembering to come + // back here, and a drift in getType() turns this red on its own. + String prefix = new FlussConnectorProvider().getType() + "."; + int checked = 0; + for (Field field : FlussStatementScope.class.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers()) && field.getType() == String.class + && field.getName().endsWith("_NAMESPACE")) { + field.setAccessible(true); + String namespace = (String) field.get(null); + Assertions.assertTrue(namespace.startsWith(prefix), + field.getName() + " (\"" + namespace + "\") must start with \"" + prefix + "\""); + checked++; + } + } + Assertions.assertTrue(checked > 0, "expected at least one *_NAMESPACE constant to guard"); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSuppressedLakeRangeTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSuppressedLakeRangeTest.java new file mode 100644 index 00000000000000..8fa40dcde1c67f --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussSuppressedLakeRangeTest.java @@ -0,0 +1,217 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TPaimonFileDesc; +import org.apache.doris.thrift.TTableFormatFileDesc; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * The wrapper that carries a log tail alongside a lake split. + * + *

Everything it does NOT change is what makes the lake half worth having — its file extent, its split + * weight, its deletion vectors — so the delegation is asserted method by method against a stand-in that + * answers something distinctive for each. A silently swallowed answer (a zero weight, an empty path) + * would not fail any assertion about rows; it would just make the lake half read or schedule badly. + */ +public class FlussSuppressedLakeRangeTest { + + private static final FlussScanRange.Partition DT_20260101 = + FlussScanRange.Partition.of(77L, "dt=20260101", + Collections.singletonMap("dt", "20260101")); + + /** A sibling range whose every answer is distinctive, so a dropped delegation is visible. */ + private static final class LakeSplit implements ConnectorScanRange { + private static final long serialVersionUID = 1L; + + @Override + public Optional getPath() { + return Optional.of("/warehouse/db/tbl/bucket-2/data-1.parquet"); + } + + @Override + public long getStart() { + return 4096L; + } + + @Override + public long getLength() { + return 8192L; + } + + @Override + public long getFileSize() { + return 16384L; + } + + @Override + public String getFileFormat() { + return "parquet"; + } + + @Override + public long getModificationTime() { + return 1735689600L; + } + + @Override + public long getSelfSplitWeight() { + return 7L; + } + + @Override + public long getTargetSplitSize() { + return 134217728L; + } + + @Override + public List getHosts() { + return Arrays.asList("be-1", "be-2"); + } + + @Override + public Map getProperties() { + Map props = new LinkedHashMap<>(); + props.put("paimon.bucket", "2"); + props.put("paimon.schema_id", "5"); + return props; + } + + @Override + public Map getPartitionValues() { + return Collections.singletonMap("dt", "20260101"); + } + + @Override + public boolean isPartitionBearing() { + return true; + } + + @Override + public long getPushDownRowCount() { + return 42L; + } + + @Override + public boolean isNativeReadRange() { + return true; + } + + @Override + public String getTableFormatType() { + return "paimon"; + } + + @Override + public void populateRangeParams(TTableFormatFileDesc formatDesc, TFileRangeDesc rangeDesc) { + formatDesc.setPaimonParams(new TPaimonFileDesc()); + rangeDesc.setColumnsFromPathKeys(Collections.singletonList("dt")); + rangeDesc.setColumnsFromPath(Collections.singletonList("20260101")); + } + } + + private static FlussSuppressedLakeRange wrap() { + return new FlussSuppressedLakeRange(new LakeSplit(), + new FlussSuppressedLakeRange.Tail(DT_20260101, 2, 100L, 105L)); + } + + @Test + public void everythingThatDecidesHowTheSplitIsReadStaysTheSiblingsAnswer() { + FlussSuppressedLakeRange wrapped = wrap(); + ConnectorScanRange inner = new LakeSplit(); + + Assertions.assertEquals(inner.getPath(), wrapped.getPath()); + Assertions.assertEquals(inner.getStart(), wrapped.getStart()); + Assertions.assertEquals(inner.getLength(), wrapped.getLength()); + Assertions.assertEquals(inner.getFileSize(), wrapped.getFileSize()); + Assertions.assertEquals(inner.getFileFormat(), wrapped.getFileFormat()); + Assertions.assertEquals(inner.getModificationTime(), wrapped.getModificationTime()); + Assertions.assertEquals(inner.getSelfSplitWeight(), wrapped.getSelfSplitWeight()); + Assertions.assertEquals(inner.getTargetSplitSize(), wrapped.getTargetSplitSize()); + Assertions.assertEquals(inner.getHosts(), wrapped.getHosts()); + Assertions.assertEquals(inner.getProperties(), wrapped.getProperties()); + Assertions.assertEquals(inner.getPartitionValues(), wrapped.getPartitionValues()); + Assertions.assertEquals(inner.isPartitionBearing(), wrapped.isPartitionBearing()); + Assertions.assertEquals(inner.getPushDownRowCount(), wrapped.getPushDownRowCount()); + Assertions.assertEquals(inner.isNativeReadRange(), wrapped.isNativeReadRange()); + } + + /** + * The one answer that changes. BE picks the reader by this name, and both plain names would be wrong: + * {@code paimon} reads the split and ignores the tail (superseded rows come back), {@code fluss} reads + * the tail and cannot read the split at all. + */ + @Test + public void theFormatNameIsTheCompositeReadersOwn() { + Assertions.assertEquals("fluss_union", wrap().getTableFormatType()); + } + + /** + * The payload is both halves, in their own thrift fields: whatever the sibling wrote, untouched, plus + * the tail. Asserted whole rather than by spot check — this is an untyped string map that nothing + * between here and BE type-checks. + */ + @Test + public void theDescriptorCarriesTheSiblingsPayloadAndTheTail() { + TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + + wrap().populateRangeParams(formatDesc, rangeDesc); + + Map expected = new LinkedHashMap<>(); + expected.put("fluss.range_type", "LAKE_SUPPRESS"); + expected.put("fluss.union.tail", "77:2:100:105"); + Assertions.assertEquals(expected, formatDesc.getFlussParams()); + // The sibling's own payload has to survive beside it: the paimon reader fails outright without it. + Assertions.assertTrue(formatDesc.isSetPaimonParams()); + Assertions.assertEquals(Collections.singletonList("20260101"), rangeDesc.getColumnsFromPath()); + } + + /** + * An unpartitioned table's tail leaves the partition segment empty rather than writing a sentinel id. + * The string is also the cache key BE builds the suppression set under, so "no partition" and + * "partition -1" must not be the same key. + */ + @Test + public void anUnpartitionedTailLeavesThePartitionSegmentEmpty() { + TTableFormatFileDesc formatDesc = new TTableFormatFileDesc(); + new FlussSuppressedLakeRange(new LakeSplit(), + new FlussSuppressedLakeRange.Tail(FlussScanRange.Partition.NONE, 3, 0L, 9L)) + .populateRangeParams(formatDesc, new TFileRangeDesc()); + + Assertions.assertEquals(":3:0:9", formatDesc.getFlussParams().get("fluss.union.tail")); + } + + /** An empty tail suppresses nothing; producing one would mean planning wrapped a split for no reason. */ + @Test + public void anEmptyTailIsRejected() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new FlussSuppressedLakeRange.Tail(DT_20260101, 0, 5L, 5L)); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTableHandleTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTableHandleTest.java new file mode 100644 index 00000000000000..e1b1ff3593dc3c --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTableHandleTest.java @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * The handle is the connector's contract with the engine, and the SPI declares it {@link + * java.io.Serializable}. These pin the two ways that contract can break silently. + */ +public class FlussTableHandleTest { + + private static FlussTableHandle handle(String tableName, long tableId, int schemaId) { + Map properties = new LinkedHashMap<>(); + properties.put("table.datalake.enabled", "true"); + properties.put("table.datalake.paimon.warehouse", "/tmp/lake"); + Map keyTypes = new LinkedHashMap<>(); + keyTypes.put("dt", DataTypes.STRING()); + keyTypes.put("id", DataTypes.INT()); + return new FlussTableHandle("db", tableName, tableId, schemaId, true, + Arrays.asList("dt", "id"), Collections.singletonList("id"), 4, + Collections.singletonList("dt"), true, "paimon", properties, keyTypes); + } + + @Test + public void survivesJavaSerializationWithEveryFieldIntact() { + // A field that fails to serialize does not fail loudly here — it comes back null and the plan + // built from it is silently wrong (no lake configuration, zero buckets), so every field the + // scan side reads is checked on the far side of the round trip. + FlussTableHandle restored = roundTrip(handle("pk_table", 7L, 3)); + + Assertions.assertEquals("db", restored.getDatabaseName()); + Assertions.assertEquals("pk_table", restored.getTableName()); + Assertions.assertEquals(7L, restored.getTableId()); + Assertions.assertEquals(3, restored.getSchemaId()); + Assertions.assertTrue(restored.hasPrimaryKey()); + Assertions.assertEquals(Arrays.asList("dt", "id"), restored.getPrimaryKeys()); + Assertions.assertEquals(Collections.singletonList("id"), restored.getBucketKeys()); + Assertions.assertEquals(4, restored.getBucketCount()); + Assertions.assertEquals(Collections.singletonList("dt"), restored.getPartitionKeys()); + Assertions.assertTrue(restored.isPartitioned()); + Assertions.assertTrue(restored.isDataLakeEnabled()); + Assertions.assertEquals("paimon", restored.getDataLakeFormat()); + Assertions.assertEquals("/tmp/lake", restored.getProperties().get("table.datalake.paimon.warehouse")); + // The key column types decide whether the table can be read as its lake plus its log at all, and + // they are the one part of the schema the handle carries, so they have to survive the trip too. + Assertions.assertEquals(DataTypes.INT(), restored.getKeyColumnTypes().get("id")); + Assertions.assertEquals(DataTypes.STRING(), restored.getKeyColumnTypes().get("dt")); + Assertions.assertEquals(handle("pk_table", 7L, 3), restored); + } + + /** + * A bucket's rows are keyed by the primary key MINUS the partition columns: a bucket lives inside one + * partition, so the partition columns are the same for every row in it and carrying them would make + * the key wider than it is. Order follows the primary key's own. + */ + @Test + public void physicalPrimaryKeyDropsThePartitionColumns() { + Assertions.assertEquals(Collections.singletonList("id"), + handle("t", 1L, 1).getPhysicalPrimaryKeys()); + } + + @Test + public void identityIsTheTableAtItsSchemaVersion() { + Assertions.assertEquals(handle("t", 1L, 1), handle("t", 1L, 1)); + Assertions.assertEquals(handle("t", 1L, 1).hashCode(), handle("t", 1L, 1).hashCode()); + + // A different schema version describes a different set of columns; treating the two as the same + // handle would let a cached value from before an ALTER answer for the table after it. + Assertions.assertNotEquals(handle("t", 1L, 1), handle("t", 1L, 2)); + // A table dropped and recreated under the same name is a different table. + Assertions.assertNotEquals(handle("t", 1L, 1), handle("t", 2L, 1)); + Assertions.assertNotEquals(handle("t", 1L, 1), handle("other", 1L, 1)); + } + + private static FlussTableHandle roundTrip(FlussTableHandle handle) { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream out = new ObjectOutputStream(bytes)) { + out.writeObject(handle); + } + try (ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + return (FlussTableHandle) in.readObject(); + } + } catch (Exception e) { + throw new AssertionError("the handle must be serializable", e); + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestSession.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestSession.java new file mode 100644 index 00000000000000..7a36ecddbcd06d --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestSession.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorStatementScope; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; + +/** + * A session with a real per-statement scope, for tests that assert what a statement shares. + * + *

The scope is a plain map keyed exactly as the engine's is, so the sharing (and the isolation + * between two query ids) is the connector's own, not something this class arranges. + */ +final class FlussTestSession implements ConnectorSession { + + private final long catalogId; + private final String queryId; + private final Map values = new ConcurrentHashMap<>(); + private final ConnectorStatementScope scope = new ConnectorStatementScope() { + @SuppressWarnings("unchecked") + @Override + public T computeIfAbsent(String key, Supplier loader) { + return (T) values.computeIfAbsent(key, ignored -> loader.get()); + } + }; + + FlussTestSession(long catalogId, String queryId) { + this.catalogId = catalogId; + this.queryId = queryId; + } + + @Override + public long getCatalogId() { + return catalogId; + } + + @Override + public String getQueryId() { + return queryId; + } + + @Override + public ConnectorStatementScope getStatementScope() { + return scope; + } + + @Override + public String getCatalogName() { + return "fluss_catalog"; + } + + @Override + public String getUser() { + return "test_user"; + } + + @Override + public String getTimeZone() { + return "UTC"; + } + + @Override + public String getLocale() { + return "en_US"; + } + + @Override + public T getProperty(String name, Class type) { + return null; + } + + @Override + public Map getCatalogProperties() { + return Collections.emptyMap(); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestTables.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestTables.java new file mode 100644 index 00000000000000..0eb2271a641820 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTestTables.java @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Builds the {@link TableInfo} a fluss cluster would hand back, for tests that do not start one. + * + *

It goes through fluss's own {@link TableDescriptor} and {@link TableInfo#of} rather than + * assembling a {@link TableInfo} field by field, so a fixture cannot describe a table fluss would + * refuse to create (a partition key outside the primary key, a missing bucket count) and quietly make + * the connector look correct on input it will never see. + */ +final class FlussTestTables { + + /** Ids a fixture does not care about; a real cluster assigns its own. */ + static final long TABLE_ID = 1L; + private static final int SCHEMA_ID = 1; + + private FlussTestTables() { + } + + static Builder builder(TablePath tablePath) { + return new Builder(tablePath); + } + + static final class Builder { + + private final TablePath tablePath; + private final Schema.Builder schema = Schema.newBuilder(); + private final Map properties = new LinkedHashMap<>(); + private final List partitionKeys = new ArrayList<>(); + private List bucketKeys = new ArrayList<>(); + private int bucketCount = 1; + private String comment; + + private Builder(TablePath tablePath) { + this.tablePath = tablePath; + } + + Builder column(String name, DataType type) { + schema.column(name, type); + return this; + } + + Builder column(String name, DataType type, String columnComment) { + schema.column(name, type).withComment(columnComment); + return this; + } + + Builder primaryKey(String... columns) { + schema.primaryKey(columns); + return this; + } + + Builder partitionedBy(String... columns) { + partitionKeys.addAll(Arrays.asList(columns)); + return this; + } + + Builder buckets(int count, String... keys) { + this.bucketCount = count; + this.bucketKeys = new ArrayList<>(Arrays.asList(keys)); + return this; + } + + Builder property(String key, String value) { + properties.put(key, value); + return this; + } + + Builder comment(String tableComment) { + this.comment = tableComment; + return this; + } + + TableInfo build() { + TableDescriptor descriptor = TableDescriptor.builder() + .schema(schema.build()) + .partitionedBy(partitionKeys) + .distributedBy(bucketCount, bucketKeys) + .properties(properties) + .comment(comment) + .build(); + return TableInfo.of(tablePath, TABLE_ID, SCHEMA_ID, descriptor, null, 0L, 0L); + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTypeMappingTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTypeMappingTest.java new file mode 100644 index 00000000000000..545131e2199de0 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussTypeMappingTest.java @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.ConnectorType; + +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypeRoot; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Pins what a fluss column looks like once Doris describes it. + * + *

Two properties are worth more than the individual rows. The first is completeness: a fluss + * release that adds a type must not slip through as a silently degraded column, so + * {@link #everyFlussTypeRootIsMapped} fails the build when a {@link DataTypeRoot} has no case here. + * The second is that these rules are not free choices — they are the composition of fluss's own + * fluss-to-paimon conversion with the paimon connector's mapping, because a datalake table is + * readable both as {@code tbl} (this mapping) and as {@code tbl$lake} (the paimon connector), and one + * table must not show two schemas. + */ +public class FlussTypeMappingTest { + + private static ConnectorType map(DataType type) { + return FlussTypeMapping.toConnectorType(type, FlussTypeMapping.Options.DEFAULT); + } + + private static ConnectorType map(DataType type, boolean varbinary, boolean timestampTz) { + return FlussTypeMapping.toConnectorType(type, new FlussTypeMapping.Options(varbinary, timestampTz)); + } + + @Test + public void everyFlussTypeRootIsMapped() { + Map expected = new LinkedHashMap<>(); + expected.put(DataTypes.CHAR(10), ConnectorType.of("CHAR", 10, 0)); + expected.put(DataTypes.STRING(), ConnectorType.of("STRING")); + expected.put(DataTypes.BOOLEAN(), ConnectorType.of("BOOLEAN")); + expected.put(DataTypes.BINARY(16), ConnectorType.of("STRING")); + expected.put(DataTypes.BYTES(), ConnectorType.of("STRING")); + expected.put(DataTypes.DECIMAL(20, 4), ConnectorType.of("DECIMALV3", 20, 4)); + expected.put(DataTypes.TINYINT(), ConnectorType.of("TINYINT")); + expected.put(DataTypes.SMALLINT(), ConnectorType.of("SMALLINT")); + expected.put(DataTypes.INT(), ConnectorType.of("INT")); + expected.put(DataTypes.BIGINT(), ConnectorType.of("BIGINT")); + expected.put(DataTypes.FLOAT(), ConnectorType.of("FLOAT")); + expected.put(DataTypes.DOUBLE(), ConnectorType.of("DOUBLE")); + expected.put(DataTypes.DATE(), ConnectorType.of("DATEV2")); + expected.put(DataTypes.TIME(3), ConnectorType.of("UNSUPPORTED")); + expected.put(DataTypes.TIMESTAMP(6), ConnectorType.of("DATETIMEV2", 6, 0)); + expected.put(DataTypes.TIMESTAMP_LTZ(6), ConnectorType.of("DATETIMEV2", 6, 0)); + expected.put(DataTypes.ARRAY(DataTypes.INT()), ConnectorType.arrayOf(ConnectorType.of("INT"))); + expected.put(DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), + ConnectorType.mapOf(ConnectorType.of("STRING"), ConnectorType.of("INT"))); + expected.put(DataTypes.ROW(DataTypes.FIELD("f", DataTypes.INT())), + ConnectorType.structOf(singleton("f"), singleton(ConnectorType.of("INT")))); + + Set covered = EnumSet.noneOf(DataTypeRoot.class); + for (Map.Entry entry : expected.entrySet()) { + covered.add(entry.getKey().getTypeRoot()); + Assertions.assertEquals(entry.getValue(), map(entry.getKey()), + "mapping of fluss " + entry.getKey()); + } + // The gate: fluss owns this enum, so a new root appearing here means fluss grew a type that + // nobody has decided how to show in Doris yet. Decide it, then add the row above. + Assertions.assertEquals(EnumSet.allOf(DataTypeRoot.class), covered, + "every fluss type root needs a case; missing: " + + complement(covered)); + } + + @Test + public void charLongerThanTheDorisMaximumBecomesString() { + // Doris CHAR stops at 255. Truncating would lose data and failing would make the table + // unloadable, so the column widens to STRING - the same relief valve the paimon connector uses, + // which is what keeps tbl and tbl$lake agreeing on a CHAR(1000) column. + Assertions.assertEquals(ConnectorType.of("CHAR", 255, 0), map(DataTypes.CHAR(255))); + Assertions.assertEquals(ConnectorType.of("STRING"), map(DataTypes.CHAR(256))); + } + + @Test + public void timestampPrecisionIsClampedToMicroseconds() { + // Fluss keeps up to nanoseconds, Doris DATETIMEV2 up to microseconds. The clamp must be a clamp + // and not an error: a nanosecond column still reads, it just reads with microsecond scale. + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 0, 0), map(DataTypes.TIMESTAMP(0))); + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 3, 0), map(DataTypes.TIMESTAMP(3))); + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 6, 0), map(DataTypes.TIMESTAMP(9))); + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 6, 0), map(DataTypes.TIMESTAMP_LTZ(9))); + Assertions.assertEquals(ConnectorType.of("TIMESTAMPTZ", 6, 0), + map(DataTypes.TIMESTAMP_LTZ(9), false, true)); + } + + @Test + public void timeIsMarkedUnsupportedRatherThanReinterpreted() { + // Doris has no storable TIME column. The two tempting substitutes both lie about the value - + // STRING changes its type and the elapsed-millis INT other engines use changes its meaning - so + // the column is marked unsupported and only a query that projects it fails. + Assertions.assertEquals(ConnectorType.of("UNSUPPORTED"), map(DataTypes.TIME())); + Assertions.assertEquals(ConnectorType.of("UNSUPPORTED"), map(DataTypes.TIME(3))); + } + + @Test + public void anUnsupportedLeafDoesNotSinkTheColumnsAroundIt() { + // The marker degrades exactly one leaf. A wide table with one TIME field inside a struct still + // loads, and its neighbouring fields stay readable - that is the whole reason this is a marker + // and not an exception. + ConnectorType struct = map(DataTypes.ROW( + DataTypes.FIELD("started", DataTypes.TIME(3)), + DataTypes.FIELD("id", DataTypes.BIGINT()))); + + Assertions.assertEquals("STRUCT", struct.getTypeName()); + Assertions.assertEquals(ConnectorType.of("UNSUPPORTED"), struct.getChildren().get(0)); + Assertions.assertEquals(ConnectorType.of("BIGINT"), struct.getChildren().get(1)); + } + + @Test + public void theBinaryFamilyFollowsTheVarbinarySwitch() { + // Off by default (STRING), because that is what every other Doris catalog does with binary + // columns and flipping the default would change what existing queries return. + Assertions.assertEquals(ConnectorType.of("STRING"), map(DataTypes.BINARY(16))); + Assertions.assertEquals(ConnectorType.of("STRING"), map(DataTypes.BYTES())); + + // On: fixed-length BINARY(n) keeps n as the VARBINARY bound; unbounded BYTES declares no length + // so fe-core fills in the Doris VARBINARY maximum. + Assertions.assertEquals(ConnectorType.of("VARBINARY", 16, 0), map(DataTypes.BINARY(16), true, false)); + Assertions.assertEquals(ConnectorType.of("VARBINARY"), map(DataTypes.BYTES(), true, false)); + } + + @Test + public void timestampLtzFollowsTheTimestampTzSwitch() { + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 3, 0), map(DataTypes.TIMESTAMP_LTZ(3))); + Assertions.assertEquals(ConnectorType.of("TIMESTAMPTZ", 3, 0), + map(DataTypes.TIMESTAMP_LTZ(3), false, true)); + + // The two switches are independent: neither reads the other's property. + Assertions.assertEquals(ConnectorType.of("DATETIMEV2", 3, 0), + map(DataTypes.TIMESTAMP_LTZ(3), true, false)); + Assertions.assertEquals(ConnectorType.of("VARBINARY", 4, 0), + map(DataTypes.BINARY(4), true, true)); + } + + @Test + public void nestedTypesRecurseWithTheSameRulesAndSwitches() { + // ARRAY> and MAP>: the rules that apply to a top-level column apply + // at any depth, switches included - a BINARY buried three levels down must not quietly ignore + // enable.mapping.varbinary while the top-level one honours it. + ConnectorType arrayOfStruct = map(DataTypes.ARRAY(DataTypes.ROW( + DataTypes.FIELD("k", DataTypes.CHAR(300)), + DataTypes.FIELD("v", DataTypes.BINARY(8)))), true, false); + Assertions.assertEquals( + ConnectorType.arrayOf(ConnectorType.structOf( + pair("k", "v"), + pair(ConnectorType.of("STRING"), ConnectorType.of("VARBINARY", 8, 0)))), + arrayOfStruct); + + Assertions.assertEquals( + ConnectorType.mapOf(ConnectorType.of("STRING"), + ConnectorType.arrayOf(ConnectorType.of("INT"))), + map(DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(DataTypes.INT())))); + + // MAP is mapped, not marked unsupported: fluss reads it in the ARROW log format and in the + // compacted KV format, and the lake sibling served by the paimon connector shows it as a MAP, + // so refusing it here would give one table two different schemas. + Assertions.assertEquals( + ConnectorType.mapOf(ConnectorType.of("INT"), + ConnectorType.structOf(singleton("n"), singleton(ConnectorType.of("DOUBLE")))), + map(DataTypes.MAP(DataTypes.INT(), + DataTypes.ROW(DataTypes.FIELD("n", DataTypes.DOUBLE()))))); + } + + @Test + public void structFieldsKeepTheirDeclaredNullabilityAndComment() { + // These two never affect the type's identity, only what DESCRIBE and SHOW CREATE TABLE print. + // Dropping them would silently report every nested field as a nullable, undocumented column. + ConnectorType struct = map(DataTypes.ROW( + DataTypes.FIELD("required", DataTypes.INT().copy(false), "the id"), + DataTypes.FIELD("optional", DataTypes.STRING()))); + + Assertions.assertEquals(pair("required", "optional"), struct.getFieldNames()); + Assertions.assertFalse(struct.isChildNullable(0), "a NOT NULL fluss field must not read as nullable"); + Assertions.assertEquals("the id", struct.getChildComment(0)); + Assertions.assertTrue(struct.isChildNullable(1)); + Assertions.assertNull(struct.getChildComment(1), "a field with no description has no comment"); + } + + private static Set complement(Set covered) { + Set missing = EnumSet.allOf(DataTypeRoot.class); + missing.removeAll(covered); + return missing; + } + + private static List singleton(T value) { + return Collections.singletonList(value); + } + + private static List pair(T first, T second) { + return Arrays.asList(first, second); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussUnionKeyTypesTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussUnionKeyTypesTest.java new file mode 100644 index 00000000000000..8056cf543ecf06 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussUnionKeyTypesTest.java @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypeRoot; +import org.apache.fluss.types.DataTypes; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Which columns a primary-key table can be read by when its lake and its log tail are read as one. + * + *

The property worth more than any single row is completeness: every one of fluss's type roots has a + * deliberate verdict here, so a type a future fluss release adds cannot slip in as "allowed by omission". + * The rules themselves are one-directional in their risk — allowing a type that does not compare + * identically on both sides returns wrong rows silently, refusing one that would have been fine only + * costs the lake's speed — so the verdicts below are written to be defensible in that direction. + */ +public class FlussUnionKeyTypesTest { + + /** One type per fluss type root, and the verdict each is meant to get as a KEY column. */ + private static Map keyVerdicts() { + Map expected = new LinkedHashMap<>(); + expected.put(DataTypes.BOOLEAN(), true); + expected.put(DataTypes.TINYINT(), true); + expected.put(DataTypes.SMALLINT(), true); + expected.put(DataTypes.INT(), true); + expected.put(DataTypes.BIGINT(), true); + expected.put(DataTypes.DECIMAL(20, 4), true); + expected.put(DataTypes.CHAR(10), true); + expected.put(DataTypes.STRING(), true); + expected.put(DataTypes.BINARY(16), true); + expected.put(DataTypes.BYTES(), true); + expected.put(DataTypes.DATE(), true); + expected.put(DataTypes.TIMESTAMP(6), true); + expected.put(DataTypes.TIMESTAMP_LTZ(6), true); + // Refused, each for its own reason: two encodings of one number and a NaN that equals nothing; + // a type Doris cannot represent at all; and the three fluss itself already refuses as a key, + // covered here so the switch stays exhaustive rather than relying on that. + expected.put(DataTypes.FLOAT(), false); + expected.put(DataTypes.DOUBLE(), false); + expected.put(DataTypes.TIME(), false); + expected.put(DataTypes.ARRAY(DataTypes.INT()), false); + expected.put(DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), false); + expected.put(DataTypes.ROW(DataTypes.FIELD("f", DataTypes.INT())), false); + return expected; + } + + @Test + public void everyFlussTypeRootHasADeliberateVerdict() { + Set covered = EnumSet.noneOf(DataTypeRoot.class); + for (Map.Entry entry : keyVerdicts().entrySet()) { + DataType type = entry.getKey(); + covered.add(type.getTypeRoot()); + boolean allowed = FlussUnionKeyTypes.keyColumnRejection(type) == null; + Assertions.assertEquals(entry.getValue(), allowed, "key verdict for " + type); + // Every root is also asked the partition question, so neither switch can be the one that + // silently falls through on a type fluss adds. + Assertions.assertEquals(type.getTypeRoot() == DataTypeRoot.STRING, + FlussUnionKeyTypes.partitionColumnRejection(type) == null, + "partition verdict for " + type); + } + Assertions.assertEquals(EnumSet.allOf(DataTypeRoot.class), covered, + "a fluss type root has no verdict here; decide whether it can be a union-read key"); + } + + /** + * Doris stops at microseconds, so a finer fluss timestamp arrives rounded — and two keys that differ + * only in the digits it rounded away become one. Over-matching drops a row that should have been + * returned, which no row count reveals. + */ + @Test + public void timestampFinerThanDorisCanHoldIsRefused() { + for (int precision = 0; precision <= 6; precision++) { + Assertions.assertNull(FlussUnionKeyTypes.keyColumnRejection(DataTypes.TIMESTAMP(precision)), + "TIMESTAMP(" + precision + ")"); + Assertions.assertNull(FlussUnionKeyTypes.keyColumnRejection(DataTypes.TIMESTAMP_LTZ(precision)), + "TIMESTAMP_LTZ(" + precision + ")"); + } + for (int precision = 7; precision <= 9; precision++) { + Assertions.assertNotNull(FlussUnionKeyTypes.keyColumnRejection(DataTypes.TIMESTAMP(precision)), + "TIMESTAMP(" + precision + ")"); + Assertions.assertNotNull( + FlussUnionKeyTypes.keyColumnRejection(DataTypes.TIMESTAMP_LTZ(precision)), + "TIMESTAMP_LTZ(" + precision + ")"); + } + } + + /** + * Fluss accepts every one of these as a partition key ({@code PartitionUtils}), so refusing all but + * STRING is this connector's own rule, not a restriction inherited from below: a partition value + * arrives here already rendered, by two different systems, and only a string is guaranteed to come + * out the same on both sides. + */ + @Test + public void everyPartitionKeyTypeFlussAllowsButStringIsRefused() { + List flussAllows = Arrays.asList(DataTypes.CHAR(8), DataTypes.BOOLEAN(), + DataTypes.BINARY(4), DataTypes.BYTES(), DataTypes.TINYINT(), DataTypes.SMALLINT(), + DataTypes.INT(), DataTypes.BIGINT(), DataTypes.DATE(), DataTypes.TIME()); + for (DataType type : flussAllows) { + Assertions.assertNotNull(FlussUnionKeyTypes.partitionColumnRejection(type), type.toString()); + } + Assertions.assertNull(FlussUnionKeyTypes.partitionColumnRejection(DataTypes.STRING())); + } + + /** Nullability is not part of the question — a key column is NOT NULL in fluss either way. */ + @Test + public void nullabilityDoesNotChangeTheVerdict() { + Assertions.assertNull(FlussUnionKeyTypes.keyColumnRejection(DataTypes.INT().copy(false))); + Assertions.assertNull(FlussUnionKeyTypes.partitionColumnRejection(DataTypes.STRING().copy(false))); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/LakeSiblingTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/LakeSiblingTest.java new file mode 100644 index 00000000000000..fa462837ac2bf0 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/LakeSiblingTest.java @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.ConnectorMetadata; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Collections; + +/** + * The classloader pin every call into the lake sibling goes through. + * + *

Asserted against a classloader that is deliberately NOT the one the sibling was loaded by, because + * in a unit test both this class and the stand-in sibling come from the same loader: without a third, + * distinguishable one in play, "the pin happened" and "nothing happened" look identical. + */ +public class LakeSiblingTest { + + /** Stands in for the loader some other part of the engine had pinned before the call. */ + private ClassLoader callerLoader; + private ClassLoader previous; + + @BeforeEach + public void setUp() { + previous = Thread.currentThread().getContextClassLoader(); + callerLoader = new URLClassLoader(new URL[0], previous); + Thread.currentThread().setContextClassLoader(callerLoader); + } + + @AfterEach + public void tearDown() { + Thread.currentThread().setContextClassLoader(previous); + } + + /** + * The sibling's SDK discovers its catalogs, file systems and file formats through the context + * classloader. Left on the caller's, it looks for them in a plugin where none of them exist, and the + * failure surfaces at the first lake read rather than at wiring time. + */ + @Test + public void theCallRunsUnderTheSiblingsOwnClassLoader() { + RecordingLakeSibling sibling = new RecordingLakeSibling(Collections.emptyMap()); + + ClassLoader seen = LakeSibling.call(sibling, + () -> Thread.currentThread().getContextClassLoader()); + + Assertions.assertSame(sibling.getClass().getClassLoader(), seen); + Assertions.assertNotSame(callerLoader, seen); + } + + /** What the caller had pinned is what the caller gets back; the pin is ours only for the call. */ + @Test + public void theCallersClassLoaderIsRestoredAfterwards() { + RecordingLakeSibling sibling = new RecordingLakeSibling(Collections.emptyMap()); + + LakeSibling.call(sibling, () -> null); + + Assertions.assertSame(callerLoader, Thread.currentThread().getContextClassLoader()); + } + + /** + * Restored on the way out of a FAILING call too. A connector's errors are routine (a missing lake + * table, an unreachable warehouse), so leaking the pin on that path would leave every later call in + * the query resolving against the wrong plugin — with nothing to point at. + */ + @Test + public void theCallersClassLoaderIsRestoredWhenTheCallThrows() { + RecordingLakeSibling sibling = new RecordingLakeSibling(Collections.emptyMap()); + + Assertions.assertThrows(IllegalStateException.class, + () -> LakeSibling.call(sibling, () -> { + throw new IllegalStateException("boom"); + })); + + Assertions.assertSame(callerLoader, Thread.currentThread().getContextClassLoader()); + } + + /** + * Building the sibling's metadata is itself a call into the sibling — it opens the lake catalog — so + * it has to be inside the pin, not before it. + */ + @Test + public void buildingTheSiblingsMetadataIsPinnedToo() { + LoaderProbe sibling = new LoaderProbe(); + + LakeSibling.forward(new FlussTestSession(1L, "q1"), sibling, + metadata -> metadata.getTableHandle(null, "db", "tbl")); + + Assertions.assertSame(sibling.getClass().getClassLoader(), sibling.loaderWhileBuilding); + } + + /** + * One metadata per statement, shared by every caller. Two instances would mean the scan planner and + * the metadata gateway could see two different versions of the same lake table within one statement. + */ + @Test + public void theSiblingsMetadataIsBuiltOncePerStatement() { + RecordingLakeSibling sibling = new RecordingLakeSibling(Collections.emptyMap()); + ConnectorSession session = new FlussTestSession(1L, "q1"); + + LakeSibling.forward(session, sibling, metadata -> metadata.getTableHandle(null, "db", "tbl")); + LakeSibling.forward(session, sibling, metadata -> metadata.getTableHandle(null, "db", "tbl")); + + Assertions.assertEquals(1, sibling.metadataBuilds); + } + + /** A sibling that records the context classloader in force while its metadata is being built. */ + private static final class LoaderProbe implements Connector { + + private ClassLoader loaderWhileBuilding; + + @Override + public ConnectorMetadata getMetadata(ConnectorSession session) { + loaderWhileBuilding = Thread.currentThread().getContextClassLoader(); + return new ConnectorMetadata() { + }; + } + + @Override + public ConnectorScanPlanProvider getScanPlanProvider() { + throw new UnsupportedOperationException("not needed by this test"); + } + + @Override + public void close() { + } + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/PaimonSiblingPropertiesTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/PaimonSiblingPropertiesTest.java new file mode 100644 index 00000000000000..5f4bbdcabd6121 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/PaimonSiblingPropertiesTest.java @@ -0,0 +1,157 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.DorisConnectorException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * The translation from a fluss table's properties to the paimon sibling's catalog properties. + * + *

Assertions compare the WHOLE map, not a key or two: the sibling is handed exactly this map and + * nothing else, so a leaked fluss property or a dropped storage key is a bug either way, and only whole-map + * equality catches both. The keys are written out as literals rather than referenced from the class under + * test — a renamed constant must break these tests, since the names are what the paimon connector reads. + */ +public class PaimonSiblingPropertiesTest { + + /** The properties a fluss coordinator injects for a filesystem-backed paimon lake. */ + private static Map flussTableProperties() { + Map properties = new LinkedHashMap<>(); + properties.put("table.datalake.enabled", "true"); + properties.put("table.datalake.format", "paimon"); + properties.put("table.datalake.paimon.metastore", "filesystem"); + properties.put("table.datalake.paimon.warehouse", "/lake/warehouse"); + return properties; + } + + @Test + public void lakeOptionsBecomeThePaimonCatalogProperties() { + Map expected = new HashMap<>(); + // Paimon calls the flavor "metastore"; Doris's paimon connector calls it paimon.catalog.type. + expected.put("paimon.catalog.type", "filesystem"); + expected.put("warehouse", "/lake/warehouse"); + + Assertions.assertEquals(expected, + PaimonSiblingProperties.synthesize(flussTableProperties())); + } + + @Test + public void nonLakePropertiesAreNotForwarded() { + Map properties = flussTableProperties(); + // The kind of thing a fluss table carries alongside its lake settings. Handing these to the paimon + // connector would make it reject the catalog (or, worse, silently treat them as paimon options). + properties.put("bucket.num", "8"); + properties.put("table.datalake.freshness", "3min"); + properties.put("table.log.ttl", "7d"); + + Map expected = new HashMap<>(); + expected.put("paimon.catalog.type", "filesystem"); + expected.put("warehouse", "/lake/warehouse"); + + Assertions.assertEquals(expected, PaimonSiblingProperties.synthesize(properties), + "only the table.datalake.paimon.* namespace describes the lake catalog"); + } + + @Test + public void remainingLakeOptionsKeepTheirNamesWithoutThePrefix() { + Map properties = flussTableProperties(); + // A real object-store deployment: the paimon connector reads these storage keys under exactly + // these names, so the prefix has to come off and nothing may be added. + properties.put("table.datalake.paimon.fs.s3a.endpoint", "http://minio:9000"); + properties.put("table.datalake.paimon.fs.s3a.access.key", "ak"); + + Map expected = new HashMap<>(); + expected.put("paimon.catalog.type", "filesystem"); + expected.put("warehouse", "/lake/warehouse"); + expected.put("fs.s3a.endpoint", "http://minio:9000"); + expected.put("fs.s3a.access.key", "ak"); + + Assertions.assertEquals(expected, PaimonSiblingProperties.synthesize(properties)); + } + + @Test + public void anAbsentMetastoreIsTheFilesystemDefault() { + Map properties = flussTableProperties(); + properties.remove("table.datalake.paimon.metastore"); + + // Paimon's own default is filesystem, so silence means the flavor this connector supports. The + // catalog type must still be stated explicitly: the paimon connector defaults on its own, and + // leaving it unset would make the sibling depend on that default staying put. + Map expected = new HashMap<>(); + expected.put("paimon.catalog.type", "filesystem"); + expected.put("warehouse", "/lake/warehouse"); + + Assertions.assertEquals(expected, PaimonSiblingProperties.synthesize(properties)); + } + + @Test + public void anUnsupportedMetastoreFailsLoud() { + Map properties = flussTableProperties(); + properties.put("table.datalake.paimon.metastore", "hive"); + + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> PaimonSiblingProperties.synthesize(properties)); + // Naming both the flavor found and the one supported is what makes this actionable: the fix is in + // the fluss cluster's configuration, not in Doris. + Assertions.assertTrue(failure.getMessage().contains("hive"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("filesystem"), failure.getMessage()); + Assertions.assertTrue(failure.getMessage().contains("datalake.paimon.metastore"), + failure.getMessage()); + } + + @Test + public void missingWarehouseFailsLoud() { + Map properties = flussTableProperties(); + properties.remove("table.datalake.paimon.warehouse"); + + // Without a warehouse the paimon catalog cannot be built at all. Failing here names the fluss + // setting to fix; letting it through would surface as a paimon error about a catalog nobody created. + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> PaimonSiblingProperties.synthesize(properties)); + Assertions.assertTrue(failure.getMessage().contains("table.datalake.paimon.warehouse"), + failure.getMessage()); + } + + @Test + public void anEmptyWarehouseIsTreatedAsMissing() { + Map properties = flussTableProperties(); + properties.put("table.datalake.paimon.warehouse", ""); + + Assertions.assertThrows(DorisConnectorException.class, + () -> PaimonSiblingProperties.synthesize(properties)); + } + + @Test + public void theInputMapIsNeverMutated() { + Map properties = flussTableProperties(); + Map before = new LinkedHashMap<>(properties); + + PaimonSiblingProperties.synthesize(properties); + + // The caller's map is a table handle's property map, shared and unmodifiable in production; a + // synthesis that consumed entries from it would corrupt every later read of that table. + Assertions.assertEquals(before, properties); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java new file mode 100644 index 00000000000000..35676e1f7b4ca6 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingFlussAdminOps.java @@ -0,0 +1,214 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.fluss.client.admin.OffsetSpec; +import org.apache.fluss.client.metadata.KvSnapshots; +import org.apache.fluss.client.metadata.LakeSnapshot; +import org.apache.fluss.exception.LakeTableSnapshotNotExistException; +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.metadata.PartitionInfo; +import org.apache.fluss.metadata.PartitionSpec; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.metadata.TableStats; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Programmable {@link FlussAdminOps} for unit tests: canned answers plus a log of the calls made. + * + *

This is the connector's substitute for a mocking framework (which the fe-connector modules do not + * carry). Two things it buys that a live cluster cannot: a test can put the cluster in states that are + * awkward or slow to reach for real — a bucket whose lake offset has caught up with its log, a partition + * present only in the lake — and it can assert which remote calls were made, which is how + * per-statement memoization and partition pushdown are pinned down. + * + *

Anything a test has not programmed throws rather than returning a neutral value: silently answering + * "empty" would let a test pass while the code under test called something nobody meant it to call. + */ +class RecordingFlussAdminOps implements FlussAdminOps { + + /** Every call made, in order, as {@code method(args)} — the assertion target for call-count tests. */ + final List calls = new ArrayList<>(); + + List databases = Collections.emptyList(); + final Map> tablesByDatabase = new HashMap<>(); + final Map tableInfos = new HashMap<>(); + final Map> partitionsByTable = new HashMap<>(); + final Map statsByTable = new HashMap<>(); + /** + * Latest log offset per bucket, keyed by fluss's own partition name ({@code null} for an + * unpartitioned table) — what split planning stops each bucket at. + */ + final Map> latestOffsetsByPartition = new HashMap<>(); + /** + * Earliest log offset per bucket, keyed the same way — how far back fluss can still serve. Kept apart + * from the latest offsets rather than sharing one map: a union read of a primary-key table asks for + * both, and a fixture that answered the same numbers to both questions would make the guard that + * compares them ("is the tail still there?") pass no matter what it was given. + */ + final Map> earliestOffsetsByPartition = new HashMap<>(); + /** + * Latest kv snapshot per bucket, keyed by fluss's own partition name ({@code null} for an + * unpartitioned table) — where primary-key planning starts each bucket's change log. + */ + final Map kvSnapshotsByPartition = new HashMap<>(); + /** The readable lake snapshot, or {@code null} to answer as a table nothing has been tiered from. */ + LakeSnapshot readableLakeSnapshot; + /** When set, every call throws this instead of answering — the "cluster is unreachable" case. */ + RuntimeException failure; + + @Override + public List listDatabases() { + calls.add("listDatabases()"); + return databases; + } + + @Override + public boolean databaseExists(String databaseName) { + calls.add("databaseExists(" + databaseName + ")"); + return databases.contains(databaseName); + } + + @Override + public List listTables(String databaseName) { + calls.add("listTables(" + databaseName + ")"); + List tables = tablesByDatabase.get(databaseName); + if (tables == null) { + throw new IllegalStateException("no tables programmed for database '" + databaseName + "'"); + } + return tables; + } + + @Override + public boolean tableExists(TablePath tablePath) { + throw notProgrammed("tableExists"); + } + + @Override + public TableInfo getTableInfo(TablePath tablePath) { + calls.add("getTableInfo(" + tablePath + ")"); + if (failure != null) { + throw failure; + } + TableInfo tableInfo = tableInfos.get(tablePath); + if (tableInfo == null) { + // What a real fluss cluster answers for an unknown table; the connector discriminates on it. + throw new TableNotExistException("Table '" + tablePath + "' does not exist."); + } + return tableInfo; + } + + @Override + public List listPartitionInfos(TablePath tablePath) { + calls.add("listPartitionInfos(" + tablePath + ")"); + List partitions = partitionsByTable.get(tablePath); + if (partitions == null) { + throw new IllegalStateException("no partitions programmed for table '" + tablePath + "'"); + } + return partitions; + } + + @Override + public List listPartitionInfos(TablePath tablePath, PartitionSpec partialPartitionSpec) { + throw notProgrammed("listPartitionInfos"); + } + + @Override + public TableStats getTableStats(TablePath tablePath) { + calls.add("getTableStats(" + tablePath + ")"); + if (failure != null) { + throw failure; + } + TableStats stats = statsByTable.get(tablePath); + if (stats == null) { + throw new IllegalStateException("no stats programmed for table '" + tablePath + "'"); + } + return stats; + } + + @Override + public KvSnapshots getLatestKvSnapshots(TablePath tablePath) { + return recordedKvSnapshots(tablePath, null); + } + + @Override + public KvSnapshots getLatestKvSnapshots(TablePath tablePath, String partitionName) { + return recordedKvSnapshots(tablePath, partitionName); + } + + private KvSnapshots recordedKvSnapshots(TablePath tablePath, String partitionName) { + calls.add("getLatestKvSnapshots(" + tablePath + + (partitionName == null ? "" : ", " + partitionName) + ")"); + KvSnapshots snapshots = kvSnapshotsByPartition.get(partitionName); + if (snapshots == null) { + throw new IllegalStateException( + "no kv snapshots programmed for partition '" + partitionName + "'"); + } + return snapshots; + } + + @Override + public LakeSnapshot getReadableLakeSnapshot(TablePath tablePath) { + calls.add("getReadableLakeSnapshot(" + tablePath + ")"); + if (readableLakeSnapshot == null) { + // What a real cluster answers for a table tiering has never committed for; the connector + // discriminates on this exact type to decide whether a union read is possible at all. + throw new LakeTableSnapshotNotExistException( + "Lake snapshot for table '" + tablePath + "' does not exist."); + } + return readableLakeSnapshot; + } + + @Override + public Map listOffsets(TablePath tablePath, Collection buckets, OffsetSpec offsetSpec) { + return recordedOffsets(tablePath, null, buckets, offsetSpec); + } + + @Override + public Map listOffsets(TablePath tablePath, String partitionName, + Collection buckets, OffsetSpec offsetSpec) { + return recordedOffsets(tablePath, partitionName, buckets, offsetSpec); + } + + private Map recordedOffsets(TablePath tablePath, String partitionName, + Collection buckets, OffsetSpec offsetSpec) { + calls.add("listOffsets(" + tablePath + (partitionName == null ? "" : ", " + partitionName) + + ", " + buckets + ", " + offsetSpec.getClass().getSimpleName() + ")"); + boolean earliest = offsetSpec instanceof OffsetSpec.EarliestSpec; + Map offsets = earliest + ? earliestOffsetsByPartition.get(partitionName) + : latestOffsetsByPartition.get(partitionName); + if (offsets == null) { + throw new IllegalStateException("no " + (earliest ? "earliest" : "latest") + + " offsets programmed for partition '" + partitionName + "'"); + } + return offsets; + } + + private static UnsupportedOperationException notProgrammed(String method) { + return new UnsupportedOperationException( + method + " was called but this test programmed no answer for it"); + } +} diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java new file mode 100644 index 00000000000000..ac98e23ca7e875 --- /dev/null +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/RecordingLakeSibling.java @@ -0,0 +1,363 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.fluss; + +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.ConnectorColumn; +import org.apache.doris.connector.api.ConnectorMetadata; +import org.apache.doris.connector.api.ConnectorPartitionInfo; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.ConnectorTableSchema; +import org.apache.doris.connector.api.ConnectorTableStatistics; +import org.apache.doris.connector.api.ConnectorType; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.mvcc.ConnectorMvccSnapshot; +import org.apache.doris.connector.api.pushdown.ConnectorExpression; +import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.connector.api.scan.ConnectorScanRequest; +import org.apache.doris.thrift.TFileScanRangeParams; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * A stand-in for the embedded paimon sibling connector, hand written because this module bans mock + * frameworks. It plays the part that matters for the gateway: it owns a handle type of its OWN + * ({@link Handle}) that the fluss code can never recognize by class, and it records every call it is + * forwarded so a test can assert the forward actually happened — and reached the sibling, not fluss. + * + *

Every answer is a distinctive constant, so an assertion that finds it can only have come through the + * sibling; a test that passed because fluss answered instead would see fluss's own values. + */ +final class RecordingLakeSibling implements Connector { + + /** Fixture shorthand: this handle has not been pinned to a snapshot. */ + static final long UNPINNED = Long.MIN_VALUE; + + /** + * The sibling's own handle type: the analogue of {@code PaimonTableHandle}. It carries the pin the + * same way a real one does — as a NEW handle returned by {@code applySnapshot} — so a test can tell a + * scan that was planned on the pinned handle from one planned on the raw one. + */ + static final class Handle implements ConnectorTableHandle { + private static final long serialVersionUID = 1L; + + final String dbName; + final String tableName; + final long pinnedSnapshotId; + + Handle(String dbName, String tableName) { + this(dbName, tableName, UNPINNED); + } + + Handle(String dbName, String tableName, long pinnedSnapshotId) { + this.dbName = dbName; + this.tableName = tableName; + this.pinnedSnapshotId = pinnedSnapshotId; + } + } + + /** The table name reported in the schema; a value fluss's own path could never produce. */ + static final String SCHEMA_TABLE_NAME = "lake_side_table"; + static final String FORMAT_TYPE = "PAIMON"; + static final String COLUMN_NAME = "lake_only_column"; + static final String PARTITION_NAME = "dt=lake"; + static final long ROW_COUNT = 4242L; + + /** The scan plan provider this sibling hands out; identity is what the routing test asserts. */ + final ConnectorScanPlanProvider scanPlanProvider = new ScanPlanProvider(); + + final Map properties; + final List calls = new ArrayList<>(); + + /** When false, the sibling reports the lake table as absent (nothing tiered to it yet). */ + boolean lakeTableExists = true; + + /** + * The columns the lake table reports, by name. Defaults to the single lake-only column the gateway + * tests assert on; a union test replaces it with the fluss table's columns plus the three system + * columns tiering appends, which is what a real lake table looks like. + */ + Map lakeColumns = defaultLakeColumns(); + + /** The ranges this sibling's scan planner returns, so a union test can recognize the lake half. */ + List lakeRanges = Collections.emptyList(); + + /** The node properties this sibling's scan planner reports, to be merged into the scan node's. */ + Map lakeNodeProperties = Collections.emptyMap(); + + /** The handle the scan planner was last asked to plan; proves WHICH handle reached the lake half. */ + Handle plannedHandle; + + /** The columns the scan planner was last asked to plan, in order. */ + List plannedColumns = Collections.emptyList(); + + /** The node properties last handed to {@code populateScanLevelParams}. */ + Map populatedNodeProperties; + + private static Map defaultLakeColumns() { + Map handles = new LinkedHashMap<>(); + handles.put(COLUMN_NAME, new LakeColumn(COLUMN_NAME)); + return handles; + } + + /** The sibling's own column-handle type, which fluss can only pass through, never build. */ + static final class LakeColumn implements ConnectorColumnHandle { + private static final long serialVersionUID = 1L; + + final String name; + + LakeColumn(String name) { + this.name = name; + } + + @Override + public boolean equals(Object other) { + return other instanceof LakeColumn && name.equals(((LakeColumn) other).name); + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + @Override + public String toString() { + return "LakeColumn{" + name + "}"; + } + } + + /** + * The sibling's own range type. Deliberately NOT a {@link FlussScanRange}: on a union read both kinds + * share one list, and a fixture that handed back fluss ranges would hide every place that assumes the + * list is homogeneous. + */ + static final class LakeRange implements ConnectorScanRange { + private static final long serialVersionUID = 1L; + + /** What the paimon connector calls the bucket a split holds; the cross-connector contract. */ + static final String BUCKET_PROPERTY = "paimon.bucket"; + + private final Map properties; + private final Map partitionValues; + + LakeRange() { + this(Collections.emptyMap(), Collections.emptyMap()); + } + + private LakeRange(Map properties, Map partitionValues) { + this.properties = properties; + this.partitionValues = partitionValues; + } + + /** A split of {@code bucket} of an unpartitioned table. */ + static LakeRange inBucket(int bucket) { + return inBucket(bucket, Collections.emptyMap()); + } + + /** A split of {@code bucket} of the partition with these values. */ + static LakeRange inBucket(int bucket, Map partitionValues) { + return new LakeRange( + Collections.singletonMap(BUCKET_PROPERTY, String.valueOf(bucket)), partitionValues); + } + + /** A split that does not say which bucket it holds — what an older paimon plugin produces. */ + static LakeRange withoutABucket() { + return new LakeRange(); + } + + @Override + public String getTableFormatType() { + return "paimon"; + } + + @Override + public Map getProperties() { + return properties; + } + + @Override + public Map getPartitionValues() { + return partitionValues; + } + + @Override + public String toString() { + return "LakeRange" + properties + partitionValues; + } + } + + /** Records what the lake half was asked to plan and answers with this sibling's canned values. */ + private final class ScanPlanProvider implements ConnectorScanPlanProvider { + + @Override + public List planScan(ConnectorSession session, ConnectorScanRequest request) { + calls.add("planScan"); + plannedHandle = (Handle) request.getTableHandle(); + plannedColumns = new ArrayList<>(request.getColumns()); + return lakeRanges; + } + + @Override + public Map getScanNodeProperties(ConnectorSession session, + ConnectorTableHandle handle, List columns, + Optional filter) { + calls.add("getScanNodeProperties"); + plannedColumns = new ArrayList<>(columns); + return lakeNodeProperties; + } + + @Override + public void populateScanLevelParams(TFileScanRangeParams params, + Map nodeProperties) { + calls.add("populateScanLevelParams"); + populatedNodeProperties = new LinkedHashMap<>(nodeProperties); + } + } + + /** How many metadata instances this sibling was asked to build (the per-statement funnel's proof). */ + int metadataBuilds; + boolean closed; + + RecordingLakeSibling(Map properties) { + this.properties = properties; + } + + @Override + public ConnectorMetadata getMetadata(ConnectorSession session) { + metadataBuilds++; + return new Metadata(); + } + + @Override + public ConnectorScanPlanProvider getScanPlanProvider() { + return scanPlanProvider; + } + + @Override + public ConnectorScanPlanProvider getScanPlanProvider(ConnectorTableHandle handle) { + calls.add("getScanPlanProvider"); + return scanPlanProvider; + } + + /** + * Set false to imitate a sibling that never overrode {@link Connector#ownsHandle} and so inherits the + * SPI default. That is what a real connector looks like before it is first used as a sibling, and it is + * invisible to a stand-in that always answers correctly. + */ + boolean claimsItsOwnHandles = true; + + @Override + public boolean ownsHandle(ConnectorTableHandle handle) { + return claimsItsOwnHandles && handle instanceof Handle; + } + + @Override + public void close() { + closed = true; + } + + /** The sibling's metadata: records the forward, then answers with its own distinctive values. */ + private final class Metadata implements ConnectorMetadata { + + @Override + public Optional getTableHandle( + ConnectorSession session, String dbName, String tableName) { + calls.add("getTableHandle:" + dbName + "." + tableName); + return lakeTableExists ? Optional.of(new Handle(dbName, tableName)) : Optional.empty(); + } + + @Override + public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle handle) { + calls.add("getTableSchema"); + List columns = Collections.singletonList( + new ConnectorColumn(COLUMN_NAME, ConnectorType.of("INT"), "", true, null, true)); + return new ConnectorTableSchema( + SCHEMA_TABLE_NAME, columns, FORMAT_TYPE, Collections.emptyMap()); + } + + @Override + public Map getColumnHandles( + ConnectorSession session, ConnectorTableHandle handle) { + calls.add("getColumnHandles"); + return lakeColumns; + } + + /** + * Threads the pin the way a real MVCC connector does: a NEW handle carrying the snapshot. The + * empty-properties latest-pin form is the only one fluss can send, so that is the only one + * answered here — a caller that invented connector-specific options would go unnoticed otherwise. + */ + @Override + public ConnectorTableHandle applySnapshot(ConnectorSession session, + ConnectorTableHandle handle, ConnectorMvccSnapshot snapshot) { + calls.add("applySnapshot:" + snapshot.getSnapshotId() + ":" + snapshot.getProperties()); + Handle lakeHandle = (Handle) handle; + return new Handle(lakeHandle.dbName, lakeHandle.tableName, snapshot.getSnapshotId()); + } + + @Override + public List listPartitionNames(ConnectorSession session, ConnectorTableHandle handle) { + calls.add("listPartitionNames"); + return Collections.singletonList(PARTITION_NAME); + } + + @Override + public List listPartitions(ConnectorSession session, + ConnectorTableHandle handle, Optional filter) { + calls.add("listPartitions"); + return Collections.singletonList(new ConnectorPartitionInfo( + PARTITION_NAME, Collections.singletonMap("dt", "lake"), Collections.emptyMap(), + Collections.singletonList("lake"), Collections.emptyList())); + } + + @Override + public Optional getTableStatistics( + ConnectorSession session, ConnectorTableHandle handle) { + calls.add("getTableStatistics"); + return Optional.of(new ConnectorTableStatistics(ROW_COUNT, -1)); + } + + @Override + public List listSupportedSysTables(ConnectorSession session, + ConnectorTableHandle baseTableHandle) { + calls.add("listSupportedSysTables"); + return Collections.singletonList("snapshots"); + } + + @Override + public Optional getSysTableHandle(ConnectorSession session, + ConnectorTableHandle baseTableHandle, String sysName) { + calls.add("getSysTableHandle:" + sysName); + return Optional.empty(); + } + + @Override + public boolean isPartitionValuesSysTable(ConnectorSession session, + ConnectorTableHandle baseTableHandle, String sysName) { + calls.add("isPartitionValuesSysTable:" + sysName); + return true; + } + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java index 79f15eae474fd7..34990f28fc25f7 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java @@ -23,6 +23,7 @@ import org.apache.doris.connector.api.ConnectorPartitionInfo; import org.apache.doris.connector.api.ConnectorSession; import org.apache.doris.connector.api.ConnectorValidationContext; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; import org.apache.doris.connector.cache.ConnectorMetadataCache; import org.apache.doris.connector.metastore.HmsMetaStoreProperties; @@ -254,6 +255,22 @@ public ConnectorMetadata getMetadata(ConnectorSession session) { properties, context, schemaAtMemo, latestSnapshotCache, partitionViewCache); } + /** + * True for a handle this connector produced (a {@link PaimonTableHandle}). Tested against this connector's + * OWN in-loader type, so a gateway connector that embeds this one as a sibling can route a foreign paimon + * handle here without casting it across the plugin classloader split. Returns false for any other + * connector's handle, so the gateway keeps looking. + * + *

The default is {@code false}, which for a sibling means every one of the gateway's guards silently + * fails open and the first cast throws a ClassCastException instead — so this is required of any connector + * used as a sibling, not an optimization. Same implementation as the iceberg and hudi siblings behind the + * hms gateway. + */ + @Override + public boolean ownsHandle(ConnectorTableHandle handle) { + return handle instanceof PaimonTableHandle; + } + @Override public void invalidateTable(String dbName, String tableName) { // REFRESH TABLE (and, via the generic PluginDrivenExternalCatalog DDL hook, a Doris-issued diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java index ef5e4650d88f9c..3f6213ec339cc6 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java @@ -761,7 +761,8 @@ private List planScanInternal( (optDeletionFiles.isPresent() && i < optDeletionFiles.get().size()) ? optDeletionFiles.get().get(i) : null; ranges.addAll(buildNativeRanges(file, deletionFile, defaultFileFormat, - partitionValues, vendedToken, effectiveSplitSize, weightDenominator)); + partitionValues, vendedToken, effectiveSplitSize, weightDenominator, + dataSplit.bucket())); } } else { // JNI reader path @@ -798,7 +799,8 @@ private List planScanInternal( */ PaimonScanRange buildNativeRange(RawFile file, DeletionFile deletionFile, String defaultFileFormat, Map partitionValues, - Map vendedToken, long start, long length, long weightDenominator) { + Map vendedToken, long start, long length, long weightDenominator, + int bucket) { String fileFormat = getFileFormatBySuffix(file.path()).orElse(defaultFileFormat); // FIX-A1: native sub-split FE weight = the sub-range byte length, + the deletion-vector length when // attached (legacy PaimonSplit(LocationPath,...).selfSplitWeight = length, setDeletionFile += DV). @@ -814,7 +816,8 @@ PaimonScanRange buildNativeRange(RawFile file, DeletionFile deletionFile, .partitionValues(partitionValues) .selfSplitWeight(selfSplitWeight) .targetSplitSize(weightDenominator) - .schemaId(file.schemaId()); + .schemaId(file.schemaId()) + .bucket(bucket); if (deletionFile != null) { builder.deletionFile( normalizeUri(deletionFile.path(), vendedToken), @@ -836,11 +839,12 @@ PaimonScanRange buildNativeRange(RawFile file, DeletionFile deletionFile, */ List buildNativeRanges(RawFile file, DeletionFile deletionFile, String defaultFileFormat, Map partitionValues, - Map vendedToken, long targetSplitSize, long weightDenominator) { + Map vendedToken, long targetSplitSize, long weightDenominator, + int bucket) { List result = new ArrayList<>(); for (long[] offset : computeFileSplitOffsets(file.length(), targetSplitSize)) { result.add(buildNativeRange(file, deletionFile, defaultFileFormat, - partitionValues, vendedToken, offset[0], offset[1], weightDenominator)); + partitionValues, vendedToken, offset[0], offset[1], weightDenominator, bucket)); } return result; } @@ -1372,13 +1376,18 @@ private PaimonScanRange buildJniScanRange(Split split, String defaultFileFormat, String fileFormat = isDataSplit ? dataSplitFileFormat((DataSplit) split, defaultFileFormat) : defaultFileFormat; - return new PaimonScanRange.Builder() + PaimonScanRange.Builder builder = new PaimonScanRange.Builder() .fileFormat(fileFormat) .paimonSplit(serializedSplit) .partitionValues(partitionValues) .selfSplitWeight(splitWeight) - .targetSplitSize(weightDenominator) - .build(); + .targetSplitSize(weightDenominator); + if (isDataSplit) { + // Same bucket property as the native arm: which reader BE ends up using must not change + // what a sibling connector can learn about the split (see PaimonScanRange's props). + builder.bucket(((DataSplit) split).bucket()); + } + return builder.build(); } /** diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanRange.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanRange.java index e6097aae4f5920..2a237762115741 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanRange.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanRange.java @@ -91,6 +91,19 @@ private PaimonScanRange(Builder builder) { if (builder.rowCount != null) { props.put("paimon.row_count", String.valueOf(builder.rowCount)); } + // FE-ONLY (never reaches BE, see populateRangeParams): the paimon bucket this range's data + // belongs to, = DataSplit.bucket(). Read by a sibling connector that plans paimon splits on + // behalf of its own table and has to line them up with its own per-bucket state — today the + // fluss connector, whose lake half is planned here (it binds a fluss log tail to the lake + // splits of the SAME bucket; a lake table tiered from fluss has bucket-identical layout). + // Set on every DataSplit-backed range, native and JNI alike. NOT set on the collapsed + // COUNT(*) range (it stands for splits from many buckets, so any single number would be a + // lie) nor on a non-DataSplit system split (no bucket exists). Consumers must fail loud when + // it is absent on a range they expected to bind — silently treating that as "no state for + // this bucket" is a wrong-results bug, not a degradation. + if (builder.bucket != null) { + props.put("paimon.bucket", String.valueOf(builder.bucket)); + } // FIX-A3: emit the self-split-weight for every JNI split, incl. weight 0. Legacy // PaimonScanNode.setPaimonParams:274 sets it unconditionally on the JNI branch (never on // native); the old `selfSplitWeight > 0` gate was a buggy is-set proxy that dropped a genuine @@ -307,6 +320,9 @@ public static class Builder { // COUNT pushdown private Long rowCount; + // Bucket of the backing DataSplit; null for splits that have none (see the props comment). + private Integer bucket; + public Builder path(String path) { this.path = path; return this; @@ -369,6 +385,11 @@ public Builder rowCount(long rowCount) { return this; } + public Builder bucket(int bucket) { + this.bucket = bucket; + return this; + } + public PaimonScanRange build() { return new PaimonScanRange(this); } diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorOwnsHandleTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorOwnsHandleTest.java new file mode 100644 index 00000000000000..a3f7955a8e49f3 --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorOwnsHandleTest.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.ConnectorContext; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; + +/** + * Whether this connector claims the handles it produces, which is what lets a gateway connector + * embed it as a sibling: the gateway cannot name this module's handle type across the plugin classloader + * split, so it routes a handle by asking each sibling to test its own in-loader type. + * + *

Asserted rather than left to the SPI default, because the default is {@code false} and the failure it + * causes points away from here. A sibling that disowns its own handles makes every guard on the gateway + * side fail open, and the first cast throws a ClassCastException naming the GATEWAY's handle type and two + * class loaders — with nothing to suggest that the missing piece is a method this connector never + * overrode. That is not hypothetical: it is what happened the first time the fluss connector read a lake + * table through this one end to end. + */ +public class PaimonConnectorOwnsHandleTest { + + @Test + public void claimsItsOwnTableHandle() { + PaimonConnector connector = new PaimonConnector(Collections.emptyMap(), context()); + + Assertions.assertTrue(connector.ownsHandle(new PaimonTableHandle( + "db1", "t1", Collections.emptyList(), Collections.emptyList()))); + } + + @Test + public void disownsAnotherConnectorsHandle() { + // The gateway asks its siblings in turn, so answering yes to a foreign handle would route it to the + // wrong connector instead of leaving the gateway to keep looking. + PaimonConnector connector = new PaimonConnector(Collections.emptyMap(), context()); + + Assertions.assertFalse(connector.ownsHandle(new ForeignHandle())); + } + + /** Stands in for whatever another connector's handle happens to be; only its type matters here. */ + private static final class ForeignHandle implements ConnectorTableHandle { + private static final long serialVersionUID = 1L; + } + + /** The connector wraps whatever context it is given, so it cannot be null; nothing here reads it. */ + private static ConnectorContext context() { + return new ConnectorContext() { + @Override + public String getCatalogName() { + return "test_catalog"; + } + + @Override + public long getCatalogId() { + return 1L; + } + }; + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java index 1c3dc7f6460ff3..3a1a6442a0bd42 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java @@ -351,7 +351,7 @@ public void nativeRangeNormalizesBothDataAndDeletionVectorPaths() { "oss://bkt/warehouse/db/t/index/dv-0.index", 8L, 16L, 4L); PaimonScanRange range = provider.buildNativeRange( - file, dv, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 100L, 64L * 1024 * 1024); + file, dv, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 100L, 64L * 1024 * 1024, 0); // WHY: BE's scheme-dispatched S3 file factory only opens canonical s3://. An un-normalized // oss:// DATA-file path fails the native ORC/Parquet read outright; an un-normalized oss:// DV @@ -376,7 +376,7 @@ public void nativeRangeWithoutDeletionVectorNormalizesOnlyDataPath() { PaimonScanRange range = provider.buildNativeRange( parquetRawFile("oss://bkt/a/part-0.parquet"), null, "parquet", - Collections.emptyMap(), Collections.emptyMap(), 0L, 100L, 64L * 1024 * 1024); + Collections.emptyMap(), Collections.emptyMap(), 0L, 100L, 64L * 1024 * 1024, 0); // WHY: a DV-less native split must still normalize its data-file path and must NOT emit a DV // descriptor. MUTATION: emitting a deletion_file for a null DV, or skipping data normalization -> red. @@ -396,7 +396,7 @@ public void nativeRangeWithoutContextPreservesRawPath() { PaimonScanRange range = provider.buildNativeRange( parquetRawFile("oss://bkt/a/part-0.parquet"), null, "parquet", - Collections.emptyMap(), Collections.emptyMap(), 0L, 100L, 64L * 1024 * 1024); + Collections.emptyMap(), Collections.emptyMap(), 0L, 100L, 64L * 1024 * 1024, 0); // MUTATION: NPE on null context, or fabricating a normalized path from nothing -> red. Assertions.assertEquals("oss://bkt/a/part-0.parquet", range.getPath().orElse(null)); @@ -421,7 +421,7 @@ public void buildNativeRangeThreadsVendedTokenToBothPaths() { "oss://bkt/warehouse/db/t/index/dv-0.index", 8L, 16L, 4L); PaimonScanRange range = provider.buildNativeRange( - file, dv, "parquet", Collections.emptyMap(), vendedToken, 0L, 100L, 64L * 1024 * 1024); + file, dv, "parquet", Collections.emptyMap(), vendedToken, 0L, 100L, 64L * 1024 * 1024, 0); // WHY: the engine seam normalizes against the VENDED map (the REST static map is empty). If the // connector dropped the token (reverting to the 1-arg seam) or substituted an empty map, a REST @@ -1733,7 +1733,7 @@ public void buildNativeRangesAttachesSameDeletionVectorToEverySubRange() { long target = Math.max(1L, file.length() / 3); // force the file to sub-split into >=2 ranges List ranges = provider.buildNativeRanges( - file, dv, "parquet", Collections.emptyMap(), Collections.emptyMap(), target, 64L * 1024 * 1024); + file, dv, "parquet", Collections.emptyMap(), Collections.emptyMap(), target, 64L * 1024 * 1024, 0); // WHY: the load-bearing correctness claim of FIX-NATIVE-SUBSPLIT — a paimon deletion vector is a // bitmap of GLOBAL file row positions, so EVERY sub-range of a DV-bearing file must carry the @@ -1761,7 +1761,7 @@ public void buildNativeRangesKeepsFileWholeWhenTargetNonPositive() { RawFile file = parquetRawFile("oss://bkt/a/part-0.parquet"); List ranges = provider.buildNativeRanges( - file, null, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 64L * 1024 * 1024); + file, null, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 64L * 1024 * 1024, 0); Assertions.assertEquals(1, ranges.size(), "a non-positive target (COUNT(*) pushdown) must keep the file as one whole-file range"); @@ -2572,14 +2572,14 @@ public void buildNativeRangeSetsProportionalWeightFromLengthAndDv() { DeletionFile dv = new DeletionFile("/data/dv-0.index", 8L, 16L, 4L); PaimonScanRange withDv = provider.buildNativeRange( - file, dv, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 64L, 64 * MB); + file, dv, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 64L, 64 * MB, 0); Assertions.assertEquals(64L + dv.length(), withDv.getSelfSplitWeight(), "native weight = sub-range length + the deletion-vector length"); Assertions.assertEquals(64 * MB, withDv.getTargetSplitSize(), "native range must carry the weight denominator"); PaimonScanRange noDv = provider.buildNativeRange( - file, null, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 70L, 64 * MB); + file, null, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 70L, 64 * MB, 0); Assertions.assertEquals(70L, noDv.getSelfSplitWeight(), "a DV-less native range weight is just the sub-range length"); } @@ -2597,7 +2597,7 @@ public void buildNativeRangesThreadsDenominatorDistinctFromFileSplitTarget() { List ranges = provider.buildNativeRanges( file, null, "parquet", Collections.emptyMap(), Collections.emptyMap(), - fileSplitTarget, denominator); + fileSplitTarget, denominator, 0); Assertions.assertEquals( PaimonScanPlanProvider.computeFileSplitOffsets(file.length(), fileSplitTarget).size(), @@ -2620,7 +2620,7 @@ public void buildNativeRangesCarriesDenominatorEvenWhenFileSplitSizeZero() { RawFile file = parquetRawFile("/data/part-0.parquet"); List ranges = provider.buildNativeRanges( - file, null, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 64 * MB); + file, null, "parquet", Collections.emptyMap(), Collections.emptyMap(), 0L, 64 * MB, 0); Assertions.assertEquals(1, ranges.size(), "a non-positive target keeps the file whole"); Assertions.assertEquals(64 * MB, ranges.get(0).getTargetSplitSize(), diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeBucketTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeBucketTest.java new file mode 100644 index 00000000000000..24de90b2ad4571 --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeBucketTest.java @@ -0,0 +1,296 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.handle.ConnectorColumnHandle; +import org.apache.doris.connector.api.scan.ConnectorScanRange; +import org.apache.doris.connector.api.scan.ConnectorScanRequest; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.FileSystemCatalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.table.source.Split; +import org.apache.paimon.types.DataTypes; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +/** + * Pins the {@code paimon.bucket} scan-range property (P4-2-a0). + * + *

WHY this property exists: a sibling connector can plan paimon splits on behalf of its OWN table + * and then has to line each split up with its own per-bucket state. Today that sibling is the fluss + * connector: a fluss table tiered into paimon keeps bucket-identical layout, and the fluss connector + * binds the un-tiered log tail of bucket b to the lake splits of bucket b. Without the + * bucket on the range there is nothing in a {@link PaimonScanRange} that says which bucket it came + * from (the JNI arm carries only an opaque serialized split; the native arm only a file path), so the + * sibling would have to fall back to whole-table binding — correct but wasteful — or parse the + * sibling's internal directory layout, which the JNI arm does not even expose. + * + *

The fixture is deliberately MULTI-bucket: with a single bucket every range would carry "0" and a + * hard-coded constant would pass. Each test therefore asserts the ranges reproduce the split-side + * bucket SET, which a constant cannot. + */ +public class PaimonScanRangeBucketTest { + + /** + * A two-bucket PK table with rows in BOTH buckets. PK {@code id} hashes into + * {@code bucket = hash(id) % 2}; ids 1..8 cover both buckets for paimon's hash function. + */ + private static Table createTwoBucketTable(Catalog catalog) throws Exception { + catalog.createDatabase("db", false); + Identifier id = Identifier.create("db", "t"); + catalog.createTable(id, Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("val", DataTypes.BIGINT()) + .primaryKey("id") + .option("bucket", "2") + .build(), false); + Table table = catalog.getTable(id); + + BatchWriteBuilder wb = table.newBatchWriteBuilder(); + try (BatchTableWrite write = wb.newWrite()) { + for (int i = 1; i <= 8; i++) { + write.write(GenericRow.of(i, (long) i * 100)); + } + List messages = write.prepareCommit(); + try (BatchTableCommit commit = wb.newCommit()) { + commit.commit(messages); + } + } + return table; + } + + /** The buckets paimon's own read plan reports — the reference the ranges must reproduce. */ + private static Set planBuckets(Table table) throws Exception { + Set buckets = new TreeSet<>(); + for (Split s : table.newReadBuilder().newScan().plan().splits()) { + if (s instanceof DataSplit) { + buckets.add(((DataSplit) s).bucket()); + } + } + return buckets; + } + + /** The buckets the planned ranges claim, as ints. Fails the test if any range omits the property. */ + private static Set rangeBuckets(List ranges) { + Set buckets = new TreeSet<>(); + for (ConnectorScanRange r : ranges) { + String bucket = r.getProperties().get("paimon.bucket"); + Assertions.assertNotNull(bucket, + "every DataSplit-backed range must carry paimon.bucket; missing on " + r); + buckets.add(Integer.parseInt(bucket)); + } + return buckets; + } + + private static PaimonScanPlanProvider providerFor(Table table) { + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.table = table; + return new PaimonScanPlanProvider(Collections.emptyMap(), ops); + } + + private static PaimonTableHandle handleFor(String tableName) { + return new PaimonTableHandle("db", tableName, + Collections.emptyList(), Collections.emptyList()); + } + + @Test + public void nativeRangesCarryTheBucketOfTheSplitTheyCameFrom(@TempDir Path warehouse) + throws Exception { + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + Table table = createTwoBucketTable(catalog); + Set expected = planBuckets(table); + Assertions.assertTrue(expected.size() >= 2, + "fixture precondition: the table must really span >=2 buckets, got " + expected); + + List ranges = providerFor(table).planScan( + sessionWithProps(Collections.emptyMap()), + ConnectorScanRequest.builder(handleFor("t"), noColumns()).build()); + + Assertions.assertFalse(ranges.isEmpty(), "the fixture must plan at least one range"); + for (ConnectorScanRange r : ranges) { + Assertions.assertTrue(((PaimonScanRange) r).isNativeReadRange(), + "fixture precondition: this arm must exercise the NATIVE range builder"); + } + // WHY: a native range is one sub-range of one raw file of one DataSplit, so the bucket has + // to be threaded down from the DataSplit loop through buildNativeRanges/buildNativeRange. + // MUTATION: hard-coding 0 (or dropping .bucket() from the native builder) -> {0} instead of + // {0, 1} -> red; the multi-bucket fixture is what makes the constant detectable. + Assertions.assertEquals(expected, rangeBuckets(ranges), + "native ranges must reproduce exactly the buckets paimon's own plan reports"); + } + } + + @Test + public void jniRangesCarryTheBucketOfTheSplitTheyCameFrom(@TempDir Path warehouse) + throws Exception { + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + Table table = createTwoBucketTable(catalog); + Set expected = planBuckets(table); + Assertions.assertTrue(expected.size() >= 2, + "fixture precondition: the table must really span >=2 buckets, got " + expected); + + List ranges = providerFor(table).planScan( + sessionWithProps(Collections.singletonMap("force_jni_scanner", "true")), + ConnectorScanRequest.builder(handleFor("t"), noColumns()).build()); + + Assertions.assertFalse(ranges.isEmpty(), "the fixture must plan at least one range"); + for (ConnectorScanRange r : ranges) { + Assertions.assertTrue(r.getProperties().containsKey("paimon.split"), + "fixture precondition: this arm must exercise the JNI range builder"); + } + // WHY: which BE reader a split ends up on (native vs JNI, a session-level escape hatch the + // sibling does not control) must not change what the sibling can learn about the split. + // If only the native arm carried the bucket, turning on force_jni_scanner would silently + // break the sibling's binding. MUTATION: setting .bucket() only on the native arm -> the + // rangeBuckets assertNotNull fires -> red. + Assertions.assertEquals(expected, rangeBuckets(ranges), + "JNI ranges must reproduce exactly the buckets paimon's own plan reports"); + } + } + + @Test + public void collapsedCountRangeCarriesNoBucket(@TempDir Path warehouse) throws Exception { + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + Table table = createTwoBucketTable(catalog); + Assertions.assertTrue(planBuckets(table).size() >= 2, + "fixture precondition: >=2 buckets, so the collapse really does span buckets"); + + List ranges = providerFor(table).planScan( + sessionWithProps(Collections.emptyMap()), + ConnectorScanRequest.builder(handleFor("t"), noColumns()) + .countPushdown(true).build()); + + // WHY: the count collapse folds the splits of ALL buckets into ONE range carrying the summed + // total, so no single bucket number is true of it. Stamping the representative split's bucket + // would hand a sibling a range that claims to be bucket b while actually standing for every + // bucket — it would suppress/join against the wrong state. Absent is the honest answer, and + // the sibling is required to fail loud rather than guess (it never forwards count pushdown, + // so it must never see one of these). + // MUTATION: adding .bucket() to buildCountRange -> the count range carries one -> red. + int countRanges = 0; + for (ConnectorScanRange r : ranges) { + if (r.getProperties().containsKey("paimon.row_count")) { + ++countRanges; + Assertions.assertFalse(r.getProperties().containsKey("paimon.bucket"), + "the collapsed count range spans every bucket, so it must claim none"); + } + } + Assertions.assertEquals(1, countRanges, + "fixture precondition: count pushdown must produce exactly one collapsed range"); + } + } + + @Test + public void systemTableSplitCarriesNoBucket(@TempDir Path warehouse) throws Exception { + try (Catalog catalog = new FileSystemCatalog(LocalFileIO.create(), + new org.apache.paimon.fs.Path(warehouse.toUri()))) { + createTwoBucketTable(catalog); + Table snapshots = catalog.getTable(Identifier.create("db", "t$snapshots")); + + List ranges = providerFor(snapshots).planScan( + sessionWithProps(Collections.emptyMap()), + ConnectorScanRequest.builder(handleFor("t$snapshots"), noColumns()).build()); + + Assertions.assertFalse(ranges.isEmpty(), "a snapshots system table must plan >=1 range"); + // WHY: a system-table split is not a DataSplit and has no bucket at all — fabricating one + // (say 0) would be a lie a sibling could act on. MUTATION: setting .bucket() unconditionally + // in buildJniScanRange (dropping the isDataSplit gate) -> red. This also documents the shape + // the sibling must reject: it plans only data reads, so a bucket-less range reaching its + // wrapper means the contract broke. + for (ConnectorScanRange r : ranges) { + Assertions.assertFalse(r.getProperties().containsKey("paimon.bucket"), + "a non-DataSplit system split has no bucket, so it must not claim one"); + } + } + } + + private static List noColumns() { + return Collections.emptyList(); + } + + private static ConnectorSession sessionWithProps(Map sessionProps) { + return new ConnectorSession() { + @Override + public String getQueryId() { + return "q"; + } + + @Override + public String getUser() { + return "u"; + } + + @Override + public String getTimeZone() { + return "UTC"; + } + + @Override + public String getLocale() { + return "en_US"; + } + + @Override + public long getCatalogId() { + return 0; + } + + @Override + public String getCatalogName() { + return "c"; + } + + @Override + public T getProperty(String name, Class type) { + return null; + } + + @Override + public Map getCatalogProperties() { + return Collections.emptyMap(); + } + + @Override + public Map getSessionProperties() { + return sessionProps; + } + }; + } +} diff --git a/fe/fe-connector/pom.xml b/fe/fe-connector/pom.xml index 4b3a44c05a726c..c3fd034c363bff 100644 --- a/fe/fe-connector/pom.xml +++ b/fe/fe-connector/pom.xml @@ -85,6 +85,7 @@ under the License. fe-connector-paimon fe-connector-hudi fe-connector-iceberg + fe-connector-fluss diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java index 910aa25e85d610..c5cc0cbaab7e43 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/scan/PluginDrivenScanNode.java @@ -627,6 +627,29 @@ ConnectorColumnCategory classifyColumnByConnector(String columnName) { return onPluginClassLoader(scanProvider, () -> scanProvider.classifyColumn(columnName)); } + /** + * Asks the connector which columns BE must read for this scan even when the query references none of them + * ({@link ConnectorScanPlanProvider#getMustReadColumns}), so the translator can keep their slots instead of + * pruning them ({@code PhysicalPlanTranslator.preserveConnectorMustReadSlots}, the plugin-table counterpart + * of {@code preserveExtraStorageKeySlots} for aggregate / merge-on-read unique-key OLAP tables). + * + *

Asked through the SAME memoized provider the rest of the scan uses, so a connector that memoizes the + * decision on its provider instance answers this question and plans its splits from one decision — the + * whole point, since a column preserved here and a split plan that assumes otherwise disagree silently. + * A connector with no scan provider (no scan capability) needs nothing. Public + overridable because the + * caller is the translator, in another package, and so the preservation is unit-testable without a live + * connector (mirrors {@link #classifyColumnByConnector}, whose caller is this class).

+ */ + public Set mustReadColumnsFromConnector() { + ConnectorScanPlanProvider scanProvider = resolveScanProvider(); + if (scanProvider == null) { + return Collections.emptySet(); + } + Set columns = onPluginClassLoader(scanProvider, + () -> scanProvider.getMustReadColumns(connectorSession, currentHandle)); + return columns == null ? Collections.emptySet() : columns; + } + /** * Lets the owning connector adjust the compression type this node inferred from the split's file path * before it is shipped to BE, WITHOUT any source-specific code here: the base inference runs first, then diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index c09072174ce2d8..60c9a7bfb61914 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -3041,6 +3041,8 @@ private void updateScanSlotsMaterialization(ScanNode scanNode, } if (scanNode instanceof OlapScanNode) { preserveExtraStorageKeySlots((OlapScanNode) scanNode, requiredWithVirtualColumns); + } else if (scanNode instanceof PluginDrivenScanNode) { + preserveConnectorMustReadSlots((PluginDrivenScanNode) scanNode, requiredWithVirtualColumns); } // Find the smallest column, for count(*) or other situation that slot is empty after prune SlotDescriptor smallest = getSmallestSlot(scanNode.getTupleDesc().getSlots()); @@ -3078,6 +3080,43 @@ private void preserveExtraStorageKeySlots(OlapScanNode scanNode, Set req } } + /** + * Keeps the slots of the columns a plugin connector must read for this scan even when the query + * references none of them — the plugin-table counterpart of {@link #preserveExtraStorageKeySlots}, and + * for the same reason: a reader that merges or suppresses rows by key needs the key whether or not the + * user selected it. The connector answers per scan + * ({@code ConnectorScanPlanProvider.getMustReadColumns}, empty by default), so every connector that needs + * nothing beyond the projection prunes exactly as before. + * + *

Only the scan's tuple is widened. The project above it was already given its own output tuple and + * project list a few lines up, so a column preserved here is read and then dropped — it never reaches the + * query's output.

+ * + *

A name that matches no slot fails the query loud rather than being skipped: it means the connector + * and the engine disagree about the table's columns, and the connector's reader would then be handed a + * scan missing a column it said it needs — silently wrong rows, not an error. Static + visible for testing + * so the ask-and-preserve step is pinned without a live connector.

+ */ + @VisibleForTesting + static void preserveConnectorMustReadSlots(PluginDrivenScanNode scanNode, Set requiredSlotIds) { + Set mustRead = scanNode.mustReadColumnsFromConnector(); + if (mustRead.isEmpty()) { + return; + } + Set missing = Sets.newLinkedHashSet(mustRead); + for (SlotDescriptor slot : scanNode.getTupleDesc().getSlots()) { + Column column = slot.getColumn(); + if (column != null && mustRead.contains(column.getName())) { + requiredSlotIds.add(slot.getId()); + missing.remove(column.getName()); + } + } + if (!missing.isEmpty()) { + throw new AnalysisException("connector requires column(s) " + missing + + " to be read, but the scan has no such column"); + } + } + private boolean shouldPreserveStorageKeySlots(OlapScanNode scanNode) { long selectedIndexId = scanNode.getSelectedIndexId() == -1 ? scanNode.getOlapTable().getBaseIndexId() diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeMustReadColumnsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeMustReadColumnsTest.java new file mode 100644 index 00000000000000..a5ff8203b9fe73 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/scan/PluginDrivenScanNodeMustReadColumnsTest.java @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.scan; + +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.api.handle.ConnectorTableHandle; +import org.apache.doris.connector.api.scan.ConnectorScanPlanProvider; + +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.Set; + +/** + * Guards {@link PluginDrivenScanNode#mustReadColumnsFromConnector()} — the seam the translator asks before it + * prunes a plugin table's scan slots ({@code PhysicalPlanTranslator.preserveConnectorMustReadSlots}), so a + * connector whose BE-side reader merges or suppresses rows by key gets that key read even when the query + * never mentions it. + * + *

WHY this matters (Rule 9): the answer is the whole contract between plan translation and split + * planning. Answering for the WRONG handle, or resolving a FRESH provider to ask, would let the connector + * decide "combine the two sources" at split time while the tuple was pruned as if it had said no — and BE + * would be told to suppress rows by a column that is not there. The memo assertion below is what pins + * "asked and planned through one provider instance".

+ * + *

Driven on a {@code CALLS_REAL_METHODS} node with only the connector/session/handle fields injected — + * the same technique as {@link PluginDrivenScanNodeScanProviderSelectionTest}.

+ */ +public class PluginDrivenScanNodeMustReadColumnsTest { + + private static PluginDrivenScanNode nodeWith(ConnectorScanPlanProvider provider, + ConnectorTableHandle handle, ConnectorSession session) { + PluginDrivenScanNode node = Mockito.mock(PluginDrivenScanNode.class, Mockito.CALLS_REAL_METHODS); + Connector connector = Mockito.mock(Connector.class); + Mockito.when(connector.getScanPlanProvider(handle)).thenReturn(provider); + Deencapsulation.setField(node, "connector", connector); + Deencapsulation.setField(node, "currentHandle", handle); + Deencapsulation.setField(node, "connectorSession", session); + return node; + } + + @Test + public void forwardsTheConnectorsAnswerForTheScannedHandle() { + ConnectorTableHandle handle = Mockito.mock(ConnectorTableHandle.class); + ConnectorSession session = Mockito.mock(ConnectorSession.class); + ConnectorScanPlanProvider provider = Mockito.mock(ConnectorScanPlanProvider.class); + Mockito.when(provider.getMustReadColumns(session, handle)).thenReturn(ImmutableSet.of("id", "part")); + PluginDrivenScanNode node = nodeWith(provider, handle, session); + + Set mustRead = node.mustReadColumnsFromConnector(); + + // WHY: the connector answers PER SCAN, from the handle this scan holds. MUTATION: passing a + // different handle (or the table's original one after pushdown refined it) makes the connector + // answer about another read -> the stub returns empty -> red. + Assertions.assertEquals(ImmutableSet.of("id", "part"), mustRead); + Mockito.verify(provider).getMustReadColumns(session, handle); + } + + @Test + public void connectorWithoutScanCapabilityNeedsNothing() { + ConnectorTableHandle handle = Mockito.mock(ConnectorTableHandle.class); + PluginDrivenScanNode node = nodeWith(null, handle, Mockito.mock(ConnectorSession.class)); + + // WHY: getScanPlanProvider() is null for a connector with no scan capability; every other resolver + // in this node degrades to its default rather than throwing. MUTATION: dropping the null check -> + // NPE during plan translation for such a catalog -> red. + Assertions.assertEquals(Collections.emptySet(), node.mustReadColumnsFromConnector()); + } + + @Test + public void nullAnswerIsReadAsNoExtraColumns() { + ConnectorTableHandle handle = Mockito.mock(ConnectorTableHandle.class); + ConnectorSession session = Mockito.mock(ConnectorSession.class); + ConnectorScanPlanProvider provider = Mockito.mock(ConnectorScanPlanProvider.class); + Mockito.when(provider.getMustReadColumns(session, handle)).thenReturn(null); + PluginDrivenScanNode node = nodeWith(provider, handle, session); + + // WHY: a third-party connector may return null where the SPI says "empty". Turning that into an + // NPE inside plan translation would blame the engine for a connector's slip. MUTATION: returning + // the raw answer -> NPE in the translator's isEmpty() -> red. + Assertions.assertEquals(Collections.emptySet(), node.mustReadColumnsFromConnector()); + } + + @Test + public void asksThroughTheSameProviderInstanceThatWillPlanTheSplits() { + ConnectorTableHandle handle = Mockito.mock(ConnectorTableHandle.class); + ConnectorSession session = Mockito.mock(ConnectorSession.class); + ConnectorScanPlanProvider provider = Mockito.mock(ConnectorScanPlanProvider.class); + Mockito.when(provider.getMustReadColumns(session, handle)).thenReturn(ImmutableSet.of("id")); + Connector connector = Mockito.mock(Connector.class); + Mockito.when(connector.getScanPlanProvider(handle)).thenReturn(provider); + PluginDrivenScanNode node = Mockito.mock(PluginDrivenScanNode.class, Mockito.CALLS_REAL_METHODS); + Deencapsulation.setField(node, "connector", connector); + Deencapsulation.setField(node, "currentHandle", handle); + Deencapsulation.setField(node, "connectorSession", session); + + node.mustReadColumnsFromConnector(); + Object providerForSplits = Deencapsulation.invoke(node, "resolveScanProvider"); + + // WHY: the connector is allowed to memoize "do I combine two sources?" on its provider instance, + // and MUST reach the same answer when it plans the splits later — the columns kept here and the + // splits planned there have to come from one decision. A fresh provider per question loses that + // memo and lets the two disagree. MUTATION: asking via connector.getScanPlanProvider(...) directly + // instead of the memoized resolveScanProvider() -> two instances + a second resolve -> red. + Assertions.assertSame(provider, providerForSplits, + "the must-read question must go through the same memoized provider as split planning"); + Mockito.verify(connector, Mockito.times(1)).getScanPlanProvider(handle); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorMustReadSlotsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorMustReadSlotsTest.java new file mode 100644 index 00000000000000..5f1caa9b39d6d2 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorMustReadSlotsTest.java @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.glue.translator; + +import org.apache.doris.analysis.DescriptorTable; +import org.apache.doris.analysis.SlotDescriptor; +import org.apache.doris.analysis.SlotId; +import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.scan.PluginDrivenScanNode; +import org.apache.doris.nereids.exceptions.AnalysisException; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Guards {@code PhysicalPlanTranslator.preserveConnectorMustReadSlots} — the plugin-table branch next to + * {@code preserveExtraStorageKeySlots}, which keeps the slots a connector says its BE-side reader must read + * even when the query references none of them. + * + *

WHY this matters (Rule 9): the scan's tuple is where the columns BE reads are decided + * ({@code FileQueryScanNode.updateRequiredSlots} rebuilds the required-slot list from exactly these slots + * after planning). A reader that suppresses or merges rows by key and does not get the key back reads the + * key-less rows and emits duplicates — no error anywhere. Doris does the same preservation for its own + * aggregate / merge-on-read unique-key tables; this is that mechanism for plugin connectors.

+ * + *

These tests drive the extracted static entry point directly with a real {@link TupleDescriptor} and a + * {@code CALLS_REAL_METHODS} node whose connector answer is stubbed — building a translator over a live + * plugin catalog needs a harness this module does not have. What they do NOT cover, because it is decided + * before this branch runs and by generic code: the project above the scan gets its own output tuple, so a + * column preserved here is read and then dropped rather than returned. The fluss suites' row baselines are + * the end-to-end guard for that.

+ */ +public class PhysicalPlanTranslatorMustReadSlotsTest { + + private static final DescriptorTable DESC_TABLE = new DescriptorTable(); + + /** A scan tuple holding one slot per named column, in order. */ + private static TupleDescriptor tupleOf(String... columnNames) { + TupleDescriptor tuple = DESC_TABLE.createTupleDescriptor(); + for (String name : columnNames) { + SlotDescriptor slot = DESC_TABLE.addSlotDescriptor(tuple); + slot.setColumn(new Column(name, PrimitiveType.INT)); + } + return tuple; + } + + private static PluginDrivenScanNode nodeAnswering(TupleDescriptor tuple, Set mustRead) { + PluginDrivenScanNode node = Mockito.mock(PluginDrivenScanNode.class, Mockito.CALLS_REAL_METHODS); + Mockito.doReturn(tuple).when(node).getTupleDesc(); + Mockito.doReturn(mustRead).when(node).mustReadColumnsFromConnector(); + return node; + } + + private static SlotId slotIdOf(TupleDescriptor tuple, String columnName) { + for (SlotDescriptor slot : tuple.getSlots()) { + if (slot.getColumn().getName().equals(columnName)) { + return slot.getId(); + } + } + throw new IllegalStateException("no slot for " + columnName); + } + + @Test + public void connectorNamedColumnsSurvivePruning() { + TupleDescriptor tuple = tupleOf("id", "name", "amount"); + PluginDrivenScanNode node = nodeAnswering(tuple, ImmutableSet.of("id")); + // "select name": only that slot is required by the project above the scan. + Set required = Sets.newHashSet(slotIdOf(tuple, "name")); + + PhysicalPlanTranslator.preserveConnectorMustReadSlots(node, required); + + // WHY: 'id' is what the connector's reader needs to suppress rows; without it in the required set + // the removeIf below this call drops it from the tuple and BE reads key-less rows. MUTATION: + // dropping the branch (or the add) -> 'id' absent -> red. + Assertions.assertEquals(ImmutableSet.of(slotIdOf(tuple, "name"), slotIdOf(tuple, "id")), required); + } + + @Test + public void connectorThatNeedsNothingChangesNothing() { + TupleDescriptor tuple = tupleOf("id", "name", "amount"); + PluginDrivenScanNode node = nodeAnswering(tuple, Collections.emptySet()); + Set required = Sets.newHashSet(slotIdOf(tuple, "name")); + + PhysicalPlanTranslator.preserveConnectorMustReadSlots(node, required); + + // WHY: this is the gate that keeps the branch inert for every connector that never opted in — and + // for an opted-in connector on a scan it decided NOT to combine (a fluss table read from fluss + // alone), which is exactly the "only when it is really needed" requirement. MUTATION: preserving + // unconditionally (e.g. the whole primary key regardless of the decision) -> an extra slot -> red. + Assertions.assertEquals(Collections.singleton(slotIdOf(tuple, "name")), required); + } + + @Test + public void everyNamedColumnIsPreservedNotJustTheFirst() { + TupleDescriptor tuple = tupleOf("k1", "k2", "payload"); + PluginDrivenScanNode node = nodeAnswering(tuple, ImmutableSet.of("k1", "k2")); + Set required = Sets.newHashSet(slotIdOf(tuple, "payload")); + + PhysicalPlanTranslator.preserveConnectorMustReadSlots(node, required); + + // WHY: composite keys are the normal case for the readers this exists for; keeping only one column + // of a two-column key compares the wrong thing. MUTATION: `break` after the first match -> red. + Assertions.assertEquals( + ImmutableSet.of(slotIdOf(tuple, "payload"), slotIdOf(tuple, "k1"), slotIdOf(tuple, "k2")), + required); + } + + @Test + public void preservedSlotSurvivesThePruneItself() { + TupleDescriptor tuple = tupleOf("id", "name", "amount"); + PluginDrivenScanNode node = nodeAnswering(tuple, ImmutableSet.of("id")); + Set required = Sets.newHashSet(slotIdOf(tuple, "name")); + + // The real prune step, driven end to end: it is what decides which slots the scan reads, and the + // branch under test sits inside it. + Deencapsulation.invoke(new PhysicalPlanTranslator(), "updateScanSlotsMaterialization", + node, required, Sets.newHashSet(), new PlanTranslatorContext()); + + // WHY: this is the only assertion that also pins the DISPATCH — that a plugin-driven scan reaches + // the branch at all. MUTATION: deleting the `else if (scanNode instanceof PluginDrivenScanNode)` + // arm -> 'id' pruned away -> red. MUTATION: preserving AFTER the removeIf -> also red. + Assertions.assertEquals(ImmutableList.of("id", "name"), + tuple.getSlots().stream().map(s -> s.getColumn().getName()).collect(Collectors.toList()), + "the connector's column must be read; the unreferenced one must still be pruned"); + } + + @Test + public void columnTheScanDoesNotHaveFailsLoud() { + TupleDescriptor tuple = tupleOf("id", "name"); + PluginDrivenScanNode node = nodeAnswering(tuple, ImmutableSet.of("id", "ghost")); + Set required = Sets.newHashSet(slotIdOf(tuple, "name")); + + AnalysisException thrown = Assertions.assertThrows(AnalysisException.class, + () -> PhysicalPlanTranslator.preserveConnectorMustReadSlots(node, required)); + + // WHY: a name matching no slot means the connector and the engine disagree about the table. Reading + // on would hand the reader a scan without a column it said it needs — wrong rows, silently. The + // message must name the column, because that is the only clue to which side is stale. MUTATION: + // skipping unknown names instead of throwing -> no exception -> red. + Assertions.assertTrue(thrown.getMessage().contains("ghost"), + "the failure must name the column the scan does not have: " + thrown.getMessage()); + } +} diff --git a/fe/pom.xml b/fe/pom.xml index 4fa658e448b4e9..280129127bd59f 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -407,6 +407,17 @@ under the License. silently breaks that FE->BE deserialization at runtime. These three MUST stay equal — do NOT override paimon.version per-module. --> 1.3.1 + + + 1.0-SNAPSHOT 3.4.4 17.0.0 diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index bbf2c64b05d77e..87331232219f0f 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -478,6 +478,16 @@ struct TTableFormatFileDesc { // ES per-shard parameters (used when table_format_type == "es") // Contains: index, type, shard_id, host_port, es_hosts 13: optional map es_params + // Fluss per-split parameters (used when table_format_type is "fluss" or "fluss_union"). + // Carries ONLY what varies per split: partition/bucket identity, range type, log offsets and + // the kv snapshot id; a "fluss_union" split is a lake split of another connector wrapped in a + // description of the log tail that suppresses its rows, so it carries that tail here and keeps + // the wrapped connector's own params untouched. Everything constant for the whole scan + // (bootstrap servers, table identity, client/table options) lives in + // TFileScanRangeParams.fluss_properties so it is not re-serialized once per bucket. + // Untyped on purpose: BE C++ holds no fluss logic, it hands this map straight to the Java + // scanner, so a typed struct would only add a transcription step (see es_params, jdbc_params). + 14: optional map fluss_params } // Deprecated, hive text talbe is a special format, not a serde type @@ -564,6 +574,9 @@ struct TFileScanRangeParams { // behavior during a BE-first rolling upgrade; version 1 enables file-wide ID projection and // logical initial-default materialization. 34: optional i32 iceberg_scan_semantics_version + // Fluss scan-level properties (bootstrap servers, table identity, client/table options, + // projected columns). Set at ScanNode level to avoid redundant serialization in each split. + 35: optional map fluss_properties } struct TFileRangeDesc { diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index afbe0b1a5c3ba0..19eda7bf560544 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -161,6 +161,12 @@ hive3PgPort=5732 enableKafkaTest=false kafka_port=19193 +// fluss catalog test config +// to enable fluss test, you need firstly to start fluss containers +// See `docker/thirdparties/run-thirdparties-docker.sh -c fluss` +enableFlussTest=false +fluss_coordinator_port=19123 + // elasticsearch catalog test config // See `docker/thirdparties/run-thirdparties-docker.sh` enableEsTest=false diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_big_data.out b/regression-test/data/external_table_p0/fluss/test_fluss_big_data.out new file mode 100644 index 00000000000000..4b107a55b0676a --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_big_data.out @@ -0,0 +1,98 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !log_summary -- +101000 5100550500 1 101000 7 5100550500.00 + +-- !log_distinct -- +101000 + +-- !log_groups -- +0 14428 +1 14429 +2 14429 +3 14429 +4 14429 +5 14428 +6 14428 + +-- !log_seam -- +1 n1 1.00 1 +100000 n100000 100000.00 5 +100001 hot100001 100001.00 6 +100002 hot100002 100002.00 0 +100999 hot100999 100999.00 3 +101000 hot101000 101000.00 4 +2 n2 2.00 2 +50000 n50000 50000.00 6 +99999 n99999 99999.00 4 + +-- !log_range -- +21 2100000 + +-- !log_projection -- +1000 + +-- !log_ordered_head -- +1 n1 +2 n2 +3 n3 +4 n4 +5 n5 + +-- !log_ordered_tail -- +100996 hot100996 +100997 hot100997 +100998 hot100998 +100999 hot100999 +101000 hot101000 + +-- !pk_summary -- +100495 5049927468 1 100500 7 + +-- !pk_distinct -- +100495 + +-- !pk_by_origin -- +499 500 99496 + +-- !pk_deleted -- + +-- !pk_neighbours -- +100000 p100000 +19998 p19998 +20000 p20000 +49999 p49999 +50001 p50001 +6 hot6 +8 hot8 +99998 p99998 + +-- !pk_sample -- +1 hot1 1 +100000 p100000 5 +100001 new1 6 +100500 new500 1 +500 hot500 3 +501 p501 4 + +-- !pk_groups -- +0 14354 +1 14358 +2 14357 +3 14357 +4 14356 +5 14357 +6 14356 + +-- !pk_ordered_head -- +1 hot1 +2 hot2 +3 hot3 +4 hot4 +5 hot5 + +-- !fluss_only_log -- +101000 5100550500 101000 + +-- !fluss_only_pk -- +100495 5049927468 100500 + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_catalog.out b/regression-test/data/external_table_p0/fluss/test_fluss_catalog.out new file mode 100644 index 00000000000000..9afbb0b41657a1 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_catalog.out @@ -0,0 +1,101 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !databases -- +fluss +fluss_test +information_schema +mysql + +-- !tables -- +big_log +big_pk +lake_cold +lake_empty +lake_log +lake_nested +lake_part +lake_part_int +lake_pk +lake_pk_cold +lake_pk_multi +lake_pk_part +lake_pk_part_int +lake_types +log_basic +log_empty +log_nested +log_part +log_time +log_types +part_ts +part_types +pk_basic +pk_empty +pk_nested +pk_part +pk_types + +-- !desc_log_basic -- +id int Yes true \N +name text Yes true \N +price decimal(10,2) Yes true \N + +-- !desc_log_types -- +id int Yes true \N +f_boolean boolean Yes true \N +f_tinyint tinyint Yes true \N +f_smallint smallint Yes true \N +f_int int Yes true \N +f_bigint bigint Yes true \N +f_float float Yes true \N +f_double double Yes true \N +f_decimal decimal(20,4) Yes true \N +f_char char(5) Yes true \N +f_string text Yes true \N +f_binary text Yes true \N +f_bytes text Yes true \N +f_date date Yes true \N +f_timestamp datetime(6) Yes true \N +f_timestamp_ltz datetime(3) Yes true \N WITH_TIMEZONE +f_array array Yes true \N +f_map map Yes true \N +f_row struct Yes true \N + +-- !desc_log_part -- +id int Yes true \N +name text Yes true \N +dt text Yes true \N + +-- !desc_pk_basic -- +id int Yes true \N +name text Yes true \N +score double Yes true \N + +-- !tables_after_refresh -- +big_log +big_pk +lake_cold +lake_empty +lake_log +lake_nested +lake_part +lake_part_int +lake_pk +lake_pk_cold +lake_pk_multi +lake_pk_part +lake_pk_part_int +lake_types +log_basic +log_empty +log_nested +log_part +log_time +log_types +part_ts +part_types +pk_basic +pk_empty +pk_nested +pk_part +pk_types + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_lake_only.out b/regression-test/data/external_table_p0/fluss/test_fluss_lake_only.out new file mode 100644 index 00000000000000..18b6c9051a6a09 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_lake_only.out @@ -0,0 +1,132 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !lake_rows -- +1 lake1 1.10 +2 lake2 2.20 +3 lake3 3.30 +4 lake4 4.40 + +-- !system_columns -- +4 + +-- !desc_lake_log_lake -- +id int Yes true \N NONE +name text Yes true \N NONE +price decimal(10,2) Yes true \N NONE +__bucket int Yes true \N NONE +__offset bigint Yes true \N NONE +__timestamp datetime(3) Yes true \N NONE + +-- !desc_lake_log -- +id int Yes true \N +name text Yes true \N +price decimal(10,2) Yes true \N + +-- !desc_lake_types -- +id int Yes true \N +f_boolean boolean Yes true \N +f_tinyint tinyint Yes true \N +f_smallint smallint Yes true \N +f_int int Yes true \N +f_bigint bigint Yes true \N +f_float float Yes true \N +f_double double Yes true \N +f_decimal decimal(20,4) Yes true \N +f_char char(5) Yes true \N +f_string text Yes true \N +f_binary text Yes true \N +f_bytes text Yes true \N +f_date date Yes true \N +f_timestamp datetime(6) Yes true \N +f_timestamp_ltz datetime(3) Yes true \N WITH_TIMEZONE +f_array array Yes true \N +f_map map Yes true \N +f_row struct Yes true \N + +-- !desc_lake_types_lake -- +id int Yes true \N NONE +f_boolean boolean Yes true \N NONE +f_tinyint tinyint Yes true \N NONE +f_smallint smallint Yes true \N NONE +f_int int Yes true \N NONE +f_bigint bigint Yes true \N NONE +f_float float Yes true \N NONE +f_double double Yes true \N NONE +f_decimal decimal(20,4) Yes true \N NONE +f_char char(5) Yes true \N NONE +f_string text Yes true \N NONE +f_binary text Yes true \N NONE +f_bytes text Yes true \N NONE +f_date date Yes true \N NONE +f_timestamp datetime(6) Yes true \N NONE +f_timestamp_ltz datetime(3) Yes true \N NONE +f_array array Yes true \N NONE +f_map map Yes true \N NONE +f_row struct Yes true \N NONE +__bucket int Yes true \N NONE +__offset bigint Yes true \N NONE +__timestamp datetime(3) Yes true \N NONE + +-- !lake_types_row -- +1 true 1 2 3 4 1.5 2.5 123.4567 char1 string1 010203 0A0B 2026-01-01 2026-01-01T01:02:03.456789 2026-01-01T09:02:03.456 [1, 2, 3] {"k2":2, "k1":1} {"r_int":1, "r_string":"nested1"} + +-- !lake_types_count -- +1 + +-- !cold_rows -- +1 cold1 +2 cold2 +3 cold3 + +-- !part_rows -- +1 lp1a 20260101 +2 lp1b 20260101 +3 lp2a 20260102 + +-- !part_pruned -- +1 +2 + +-- !pk_lake_rows -- +1 lp1 +2 lp2-lake +3 lp3 + +-- !lake_count -- +4 + +-- !lake_sum -- +11.00 + +-- !lake_names -- +lake3 +lake4 + +-- !tables -- +big_log +big_pk +lake_cold +lake_empty +lake_log +lake_nested +lake_part +lake_part_int +lake_pk +lake_pk_cold +lake_pk_multi +lake_pk_part +lake_pk_part_int +lake_types +log_basic +log_empty +log_nested +log_part +log_time +log_types +part_ts +part_types +pk_basic +pk_empty +pk_nested +pk_part +pk_types + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_lake_pk.out b/regression-test/data/external_table_p0/fluss/test_fluss_lake_pk.out new file mode 100644 index 00000000000000..9551d34afc79d0 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_lake_pk.out @@ -0,0 +1,27 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !lake_side -- +1 lp1 +2 lp2-lake +3 lp3 + +-- !front_door -- +2 lp2-lake +3 lp3-hot +4 lp4-hot + +-- !fluss_only -- +2 lp2-lake +3 lp3-hot +4 lp4-hot + +-- !count -- +3 + +-- !projection -- +lp3-hot + +-- !deleted_row -- + +-- !log_only_key -- +4 lp4-hot + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_lake_pk_merge.out b/regression-test/data/external_table_p0/fluss/test_fluss_lake_pk_merge.out new file mode 100644 index 00000000000000..e60da9c7ef2805 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_lake_pk_merge.out @@ -0,0 +1,55 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !multi_lake -- +1 m1 +2 m2 +3 m3 +4 m4 +5 m5-lake +6 m6 +7 m7 +8 m8 +9 m9 + +-- !multi_rows -- +1 m1 +10 m10-hot +2 m2-hot +3 m3 +4 m4 +5 m5-lake +6 m6 +8 m8 +9 m9 + +-- !multi_count -- +9 + +-- !multi_updated -- +m2-hot + +-- !multi_deleted -- + +-- !multi_new -- +m10-hot + +-- !part_rows -- +1 pp1a-hot 20260101 +3 pp2a 20260102 +4 pp2b 20260102 +5 pp3a 20260103 + +-- !part_lake_only -- +3 pp2a +4 pp2b + +-- !part_fluss_only -- +5 pp3a + +-- !part_with_tail -- +1 pp1a-hot + +-- !cold_rows -- +1 c1 +2 c2 +3 c3 + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_log_table.out b/regression-test/data/external_table_p0/fluss/test_fluss_log_table.out new file mode 100644 index 00000000000000..eb56b8a8f21e60 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_log_table.out @@ -0,0 +1,54 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !basic_all -- +1 alice 10.10 +2 bob 20.20 +3 carol 30.30 + +-- !basic_count -- +3 + +-- !basic_names -- +alice +bob +carol + +-- !basic_swapped -- +bob 2 + +-- !basic_filtered -- +2 +3 + +-- !basic_count_filtered -- +1 + +-- !types_all -- +1 true 1 2 3 4 1.5 2.5 123.4567 char1 string1 010203 0A0B 2026-01-01 2026-01-01T01:02:03.456789 2026-01-01T09:02:03.456 [1, 2, 3] {"k2":2, "k1":1} {"r_int":1, "r_string":"nested1"} +2 false -1 -2 -3 -4 -1.5 -2.5 -123.4567 char2 string2 040506 0C0D 2026-01-02 2026-01-02T01:02:03.456789 2026-01-02T09:02:03.456 [4, 5] {"k3":3} {"r_int":2, "r_string":"nested2"} +3 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N + +-- !types_nested -- +1 3 1 1 2 1 nested1 +2 2 4 \N \N 2 nested2 +3 \N \N \N \N \N \N + +-- !part_all -- +1 p1a 20260101 +2 p1b 20260101 +3 p2a 20260102 +4 p3a 20260103 + +-- !part_grouped -- +20260101 2 +20260102 1 +20260103 1 + +-- !part_pruned -- +1 +2 + +-- !empty_count -- +0 + +-- !empty_rows -- + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_misc.out b/regression-test/data/external_table_p0/fluss/test_fluss_misc.out new file mode 100644 index 00000000000000..e742514350e909 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_misc.out @@ -0,0 +1,156 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !pk_empty_count -- +0 + +-- !pk_empty_rows -- + +-- !empty_aggregates -- +0 0 \N \N \N \N + +-- !log_empty_aggregates -- +0 0 \N \N \N + +-- !empty_join -- +0 + +-- !lake_empty_count -- +0 + +-- !desc_log_time -- +id int Yes true \N +name text Yes true \N +f_time unknown type: UNSUPPORTED_TYPE Yes true \N + +-- !time_other_columns -- +1 time1 +2 time2 + +-- !time_count -- +2 + +-- !join_fluss -- +1 alice string1 +2 bob string2 +3 carol \N + +-- !join_pk -- +1 k1 alice +2 k2-updated bob + +-- !join_lake -- +1 lake1 cold1 +2 lake2 cold2 +3 lake3 cold3 +4 lake4 \N +5 hot5 \N +6 hot6 \N + +-- !join_self -- +1 p1a p1b + +-- !subquery_in -- +2 bob + +-- !subquery_exists -- +1 +2 +3 + +-- !subquery_scalar -- +2 bob +3 carol + +-- !cte -- +2 5 + +-- !union_all -- +1 alice +1 cold1 +2 bob +2 cold2 +3 carol +3 cold3 + +-- !union_distinct -- +1 +2 +3 + +-- !except -- +4 +5 +6 + +-- !intersect -- +1 +2 +3 + +-- !aggregates -- +3 3 10.10 30.30 60.60 2 + +-- !group_having -- +20260101 2 + +-- !order_limit -- +2 bob +3 carol + +-- !limit_offset -- +2 + +-- !star -- +1 alice 10.10 +2 bob 20.20 +3 carol 30.30 + +-- !expression -- +2 ALICE +4 BOB + +-- !case -- +1 low +2 high +3 high + +-- !copied -- +1 alice 10.10 +2 bob 20.20 +3 carol 30.30 + +-- !desc_ctas -- +id int Yes true \N +f_boolean boolean Yes true \N +f_tinyint tinyint Yes true \N +f_smallint smallint Yes false \N NONE +f_int int Yes false \N NONE +f_bigint bigint Yes false \N NONE +f_float float Yes false \N NONE +f_double double Yes false \N NONE +f_decimal decimal(20,4) Yes false \N NONE +f_char text Yes false \N NONE +f_string text Yes false \N NONE +f_date date Yes false \N NONE +f_timestamp datetime(6) Yes false \N NONE +f_timestamp_ltz datetime(3) Yes false \N NONE +f_array array Yes false \N NONE +f_map map Yes false \N NONE +f_row struct Yes false \N NONE + +-- !ctas_count -- +3 2 2 + +-- !desc_ctas_nested -- +id int Yes true \N +f_arr_arr array> Yes false \N NONE +f_map_row map> Yes false \N NONE +f_row_deep struct,r_map:map,r_row:struct> Yes false \N NONE + +-- !ctas_nested_rows -- +1 [[1, 2], [3]] {"k1":{"a":9, "b":"z"}} {"r_int":1, "r_arr":[7, 8], "r_map":{"m":3}, "r_row":{"x":5, "y":"deep"}} + +-- !join_internal -- +1 alice string1 +2 bob string2 +3 carol \N + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_nested_types.out b/regression-test/data/external_table_p0/fluss/test_fluss_nested_types.out new file mode 100644 index 00000000000000..50f890722b8a0f --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_nested_types.out @@ -0,0 +1,75 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !desc_log_nested -- +id int Yes true \N +f_arr_arr array> Yes true \N +f_arr_map array> Yes true \N +f_arr_row array> Yes true \N +f_map_arr map> Yes true \N +f_map_row map> Yes true \N +f_row_deep struct,r_map:map,r_row:struct> Yes true \N +f_arr_arr_arr array>> Yes true \N + +-- !desc_pk_nested -- +id int Yes true \N +f_arr_arr array> Yes true \N +f_arr_row array> Yes true \N +f_map_arr map> Yes true \N +f_map_row map> Yes true \N +f_row_deep struct,r_map:map,r_row:struct> Yes true \N + +-- !log_rows -- +1 [[1, 2], [3]] [{"a":1}, {"b":2}] [{"a":1, "b":"x"}, {"a":2, "b":"y"}] {"k2":[3], "k1":[1, 2]} {"k1":{"a":9, "b":"z"}} {"r_int":1, "r_arr":[7, 8], "r_map":{"m":3}, "r_row":{"x":5, "y":"deep"}} [[[1], [2, 3]], [[4]]] +2 [null] [null] [null] {"k1":null} {"k1":null} {"r_int":null, "r_arr":null, "r_map":null, "r_row":null} [null] +3 \N \N \N \N \N \N \N + +-- !log_elements -- +1 2 [1, 2] 2 2 1 1 x [1, 2] 2 9 z +2 1 \N \N \N \N \N \N \N \N \N \N +3 \N \N \N \N \N \N \N \N \N \N \N + +-- !log_deep -- +1 2 1 8 3 5 deep +2 \N \N \N \N \N \N +3 \N \N \N \N \N \N + +-- !log_nested_predicate -- +1 + +-- !pk_rows -- +1 [[10, 20]] [{"a":11, "b":"xx"}] {"k9":[10]} {"k9":{"a":99, "b":"zz"}} {"r_int":2, "r_arr":[70], "r_map":{"mm":30}, "r_row":{"x":50, "y":"deeper"}} +2 \N \N \N \N \N + +-- !pk_elements -- +1 1 10 xx [10] 10 99 70 deeper +2 \N \N \N \N \N \N \N \N + +-- !desc_lake_nested -- +id int Yes true \N +f_arr_arr array> Yes true \N +f_arr_row array> Yes true \N +f_map_arr map> Yes true \N +f_map_row map> Yes true \N +f_row_deep struct,r_map:map,r_row:struct> Yes true \N + +-- !desc_lake_nested_lake -- +id int Yes true \N NONE +f_arr_arr array> Yes true \N NONE +f_arr_row array> Yes true \N NONE +f_map_arr map> Yes true \N NONE +f_map_row map> Yes true \N NONE +f_row_deep struct,r_map:map,r_row:struct> Yes true \N NONE +__bucket int Yes true \N NONE +__offset bigint Yes true \N NONE +__timestamp datetime(3) Yes true \N NONE + +-- !lake_side -- +1 [[1, 2], [3]] [{"a":1, "b":"x"}] {"k1":[1, 2]} {"k1":{"a":9, "b":"z"}} {"r_int":1, "r_arr":[7, 8], "r_map":{"m":3}, "r_row":{"x":5, "y":"deep"}} + +-- !lake_union -- +1 [[1, 2], [3]] [{"a":1, "b":"x"}] {"k1":[1, 2]} {"k1":{"a":9, "b":"z"}} {"r_int":1, "r_arr":[7, 8], "r_map":{"m":3}, "r_row":{"x":5, "y":"deep"}} +2 \N \N \N \N \N + +-- !lake_union_elements -- +1 2 x 1 z deep +2 \N \N \N \N \N + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_partition_types.out b/regression-test/data/external_table_p0/fluss/test_fluss_partition_types.out new file mode 100644 index 00000000000000..71237c385fe989 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_partition_types.out @@ -0,0 +1,75 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !desc_part_types -- +id int Yes true \N +name text Yes true \N +p_str text Yes true \N +p_char char(2) Yes true \N +p_bool boolean Yes true \N +p_tiny tinyint Yes true \N +p_small smallint Yes true \N +p_int int Yes true \N +p_big bigint Yes true \N +p_date date Yes true \N +p_bin text Yes true \N + +-- !part_names -- +p_str=cn/p_char=c1/p_bool=true/p_tiny=1/p_small=10/p_int=100/p_big=1000/p_date=2026-01-01/p_bin=0102 +p_str=us/p_char=c2/p_bool=false/p_tiny=2/p_small=20/p_int=200/p_big=2000/p_date=2026-01-02/p_bin=0304 + +-- !part_rows -- +1 pt1 cn c1 true 1 10 100 1000 2026-01-01 0102 30313032 +2 pt2 us c2 false 2 20 200 2000 2026-01-02 0304 30333034 + +-- !pruned_str -- +1 pt1 + +-- !pruned_int -- +2 pt2 + +-- !pruned_date -- +2 pt2 + +-- !pruned_bool -- +2 pt2 + +-- !pruned_tiny -- +2 pt2 + +-- !pruned_multi -- +1 pt1 + +-- !pruned_impossible -- +0 + +-- !pruned_absent -- +0 + +-- !desc_part_ts -- +id int Yes true \N +name text Yes true \N +p_ts datetime(3) Yes true \N + +-- !lake_part_int -- +1 li1a 1 +2 li1b 1 +3 li2a 2 +4 li1c 1 + +-- !lake_part_int_tail -- +1 li1a +2 li1b +4 li1c + +-- !lake_part_int_tiered -- +3 li2a + +-- !lake_pk_part_int -- +1 pi1a-hot 1 +2 pi1b 1 +3 pi2a 2 +4 pi2b 2 + +-- !lake_pk_part_int_pruned -- +1 pi1a-hot +2 pi1b + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_pk_table.out b/regression-test/data/external_table_p0/fluss/test_fluss_pk_table.out new file mode 100644 index 00000000000000..3e9656acc4a4b9 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_pk_table.out @@ -0,0 +1,50 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !basic_all -- +1 k1 1.5 +2 k2-updated 22.5 +4 k4 4.5 + +-- !basic_count -- +3 + +-- !basic_deleted_key -- +0 + +-- !basic_updated_key -- +1 + +-- !basic_stale_value -- +0 + +-- !basic_names -- +k1 +k2-updated +k4 + +-- !basic_reordered -- +1.5 k1 + +-- !basic_filtered -- +2 + +-- !types_all -- +1 true 1 2 3 4 1.5 2.5 123.4567 char1 string1 010203 0A0B 2026-01-01 2026-01-01T01:02:03.456789 2026-01-01T09:02:03.456 [1, 2, 3] {"k2":2, "k1":1} {"r_int":1, "r_string":"nested1"} +2 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N + +-- !types_nested -- +1 3 1 1 2 1 nested1 +2 \N \N \N \N \N \N + +-- !part_all -- +1 q1a 20260101 +2 q1b-updated 20260101 +3 q2a 20260102 + +-- !part_grouped -- +20260101 2 +20260102 1 + +-- !part_pruned -- +1 q1a +2 q1b-updated + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_predicates.out b/regression-test/data/external_table_p0/fluss/test_fluss_predicates.out new file mode 100644 index 00000000000000..b7bec6243de213 --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_predicates.out @@ -0,0 +1,157 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !and -- +2 bob + +-- !or -- +1 alice +3 carol + +-- !and_or -- +1 alice +3 carol + +-- !or_and -- +3 carol + +-- !not -- +2 bob + +-- !not_between -- +1 + +-- !in -- +1 alice + +-- !not_in -- +3 + +-- !like -- +2 + +-- !always_false -- +0 + +-- !always_true -- +3 + +-- !is_null -- +3 + +-- !is_not_null -- +1 +2 + +-- !ne_excludes_null -- +2 + +-- !not_eq_excludes_null -- +2 + +-- !null_or -- +1 +3 + +-- !null_and -- +3 + +-- !mixed_types -- +1 + +-- !mixed_types_or -- +1 +2 +3 + +-- !nested_elements -- +1 + +-- !part_or -- +1 p1a 20260101 +2 p1b 20260101 +4 p3a 20260103 + +-- !part_in -- +2 20260101 +3 20260102 + +-- !part_ne -- +3 20260102 +4 20260103 + +-- !part_and_data -- +2 p1b + +-- !part_or_data -- +1 p1a 20260101 +2 p1b 20260101 +4 p3a 20260103 + +-- !part_absent -- +0 + +-- !pk_and -- +2 k2-updated +4 k4 + +-- !pk_or -- +1 k1 +2 k2-updated + +-- !pk_stale_value -- +0 + +-- !pk_not -- +2 k2-updated +4 k4 + +-- !pk_in -- +1 1.5 +2 22.5 + +-- !union_and -- +3 lake3 +4 lake4 +5 hot5 + +-- !union_or -- +1 lake1 +6 hot6 + +-- !union_not -- +1 +6 + +-- !union_like -- +1 lake1 +5 hot5 +6 hot6 + +-- !union_lake_only -- +1 lake1 +2 lake2 +3 lake3 +4 lake4 + +-- !union_log_only -- +5 hot5 +6 hot6 + +-- !union_neither -- +0 + +-- !union_part_or -- +1 lp1a 20260101 +2 lp1b 20260101 +3 lp2a 20260102 +4 lp1c 20260101 + +-- !union_part_and -- +4 + +-- !union_pk_and -- +3 lp3-hot +4 lp4-hot + +-- !union_pk_or -- +4 lp4-hot + diff --git a/regression-test/data/external_table_p0/fluss/test_fluss_union_log.out b/regression-test/data/external_table_p0/fluss/test_fluss_union_log.out new file mode 100644 index 00000000000000..99b260d73c5e7c --- /dev/null +++ b/regression-test/data/external_table_p0/fluss/test_fluss_union_log.out @@ -0,0 +1,41 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !log_rows -- +1 lake1 1.10 +2 lake2 2.20 +3 lake3 3.30 +4 lake4 4.40 +5 hot5 5.50 +6 hot6 6.60 + +-- !log_count -- +6 + +-- !log_sum -- +23.10 + +-- !cold_rows -- +1 cold1 +2 cold2 +3 cold3 + +-- !types_all -- +1 true 1 2 3 4 1.5 2.5 123.4567 char1 string1 010203 0A0B 2026-01-01 2026-01-01T01:02:03.456789 2026-01-01T09:02:03.456 [1, 2, 3] {"k2":2, "k1":1} {"r_int":1, "r_string":"nested1"} +2 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N + +-- !part_rows -- +1 lp1a 20260101 +2 lp1b 20260101 +3 lp2a 20260102 +4 lp1c 20260101 + +-- !part_tiered_only -- +3 + +-- !part_with_tail -- +1 +2 +4 + +-- !plain_count -- +3 + diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_big_data.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_big_data.groovy new file mode 100644 index 00000000000000..fd75a2bc56d1e9 --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_big_data.groovy @@ -0,0 +1,230 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The same reads, at a size where nothing can be eyeballed. +// +// Every other fluss suite here works on a handful of rows chosen so that a wrong +// answer is visible by reading it. That leaves a whole class of faults uncovered: +// a scan that loses the last partial batch, a stopping offset applied to the first +// bucket only, a suppression set built per split rather than per bucket, a cache +// keyed too coarsely. None of them can show up on nine rows -- with one batch and +// one bucket's worth of keys, wrong and right are the same arrangement. +// +// Two fixtures of 100000 rows, each read down both paths: through a union-read +// catalog they are lake plus log, through a disabled one they are the pure fluss +// read of the same rows -- a full log replay for big_log, a kv snapshot plus its +// log for big_pk. So one table answers the log, primary-key and union-read cases, +// and the two paths can be compared against each other rather than against a +// number someone wrote down. +// +// What IS written down is arithmetic: the fixtures derive every column from a +// sequence, so 1..101000 sums to 5100550500 and nothing here is a transcription of +// whatever the environment happened to produce. +// +// Fixtures: big_log and big_pk in +// docker/thirdparties/docker-compose/fluss/sql/init.sql and init-lake-tail.sql. +suite("test_fluss_big_data", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String unionCatalog = "test_fluss_big_data" + String flussOnlyCatalog = "test_fluss_big_data_off" + + sql """drop catalog if exists ${unionCatalog}""" + sql """ + create catalog ${unionCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "required" + ); + """ + sql """drop catalog if exists ${flussOnlyCatalog}""" + sql """ + create catalog ${flussOnlyCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "disabled" + ); + """ + + sql """switch ${unionCatalog}""" + sql """use fluss_test""" + sql """set enable_file_scanner_v2 = true""" + + def rowsOf = { String query -> sql(query).collect { row -> row.collect { it.toString() } } } + def planOf = { String query -> + return sql("""explain ${query}""").collect { it[0].toString() }.join("\n") + } + def countIn = { String plan, String field -> + def matcher = (plan =~ /${field}=(\d+)/) + assertTrue(matcher.find(), "plan has no ${field}: ${plan}") + return matcher.group(1) as int + } + // The load-bearing check of this suite. At this size the recorded aggregates say a + // number came back; only the comparison says the OTHER reader agreed with it, and + // the two readers here share no code below the scan node. + def compareModes = { String query -> + def union = rowsOf("""${query}""") + def flussOnly = rowsOf("""${query}""".replace("from ", "from ${flussOnlyCatalog}.fluss_test.")) + assertEquals(flussOnly, union, + "lake+log and fluss-only disagree for: ${query}\nfluss-only=${flussOnly}\nunion=${union}") + } + + // --- a log table of 101000 rows ------------------------------------------- + // 100000 tiered plus 1000 written after, with contiguous ids, so the count, the + // sum and the bounds are all closed forms: 101000 rows, sum 5100550500, ids 1 to + // 101000. A seam that dropped the tail, read it twice or stopped a bucket early + // moves at least one of those. + def logPlan = planOf("""select * from big_log""") + assertTrue(logPlan.contains("unionRead=yes"), "not a union read: ${logPlan}") + assertTrue(countIn(logPlan, "lakeSplits") >= 1, "no lake splits: ${logPlan}") + def logRanges = countIn(logPlan, "logRanges") + assertTrue(logRanges >= 1 && logRanges <= 3, + "big_log planned ${logRanges} log ranges over 3 buckets: ${logPlan}") + + order_qt_log_summary """ + select count(*) as rows_read, sum(id) as id_sum, min(id) as id_min, max(id) as id_max, + count(distinct grp) as groups, sum(price) as price_sum + from big_log + """ + compareModes("""select count(*), sum(id), min(id), max(id), sum(price) from big_log""") + + // Every id exactly once. count(distinct) over 101000 values is what says so, and it + // is a different question from the count: a row read twice keeps the count wrong and + // the distinct count right, and one lost key does the opposite. + order_qt_log_distinct """select count(distinct id) from big_log""" + + // Per-group counts, which is the whole scan grouped rather than aggregated flat: a + // half silently dropped shows up as one group short of its share, not as zero. + order_qt_log_groups """select grp, count(*) from big_log group by grp order by grp""" + compareModes("""select grp, count(*) from big_log group by grp order by grp""") + + // Rows from both halves by name, including the two ids either side of the seam. + order_qt_log_seam """ + select id, name, price, grp from big_log + where id in (1, 2, 50000, 99999, 100000, 100001, 100002, 100999, 101000) + """ + + // A range predicate that spans the seam: the rows it selects come from the lake up + // to 100000 and from the log after it, in one result. + order_qt_log_range """select count(*), sum(id) from big_log where id between 99990 and 100010""" + order_qt_log_projection """select count(*) from big_log where name like 'hot%'""" + order_qt_log_ordered_head """select id, name from big_log order by id limit 5""" + order_qt_log_ordered_tail """select id, name from big_log order by id desc limit 5""" + + // --- a primary-key table of 100495 rows ----------------------------------- + // 100000 keys in the lake; the tail updates 500 of them, adds 500 more and deletes + // five. So the suppression set that filters the lake half holds 1005 keys rather + // than the two or three every other primary-key fixture has, and the rows it must + // drop are scattered through every bucket instead of sitting at the front of one. + // + // 100000 - 5 + 500 = 100495 rows. Sum of the surviving ids is + // 5000050000 - (7 + 19999 + 50000 + 77777 + 99999) + sum(100001..100500) = + // 5000050000 - 247782 + 50125250 = 5049927468. + def pkPlan = planOf("""select * from big_pk""") + assertTrue(pkPlan.contains("unionRead=yes"), "not a union read: ${pkPlan}") + assertEquals(0, countIn(pkPlan, "logRanges"), + "a primary-key table planned a plain log range: ${pkPlan}") + def lakeSplits = countIn(pkPlan, "lakeSplits") + def suppressed = countIn(pkPlan, "suppressedLakeSplits") + assertTrue(lakeSplits >= 3, "three buckets should each contribute a lake split: ${pkPlan}") + // 500 updated keys hash across all three buckets, so every lake split has a tail to + // be filtered by and none may be let through unfiltered. + assertEquals(lakeSplits, suppressed, + "every bucket has a tail, so every lake split must be suppressed: ${pkPlan}") + assertEquals(3, countIn(pkPlan, "pkTailRanges"), "one tail per bucket: ${pkPlan}") + assertEquals(0, countIn(pkPlan, "pkRanges"), + "the lake holds every bucket, so none is read whole: ${pkPlan}") + + order_qt_pk_summary """ + select count(*) as rows_read, sum(id) as id_sum, min(id) as id_min, max(id) as id_max, + count(distinct grp) as groups + from big_pk + """ + compareModes("""select count(*), sum(id), min(id), max(id) from big_pk""") + order_qt_pk_distinct """select count(distinct id) from big_pk""" + + // What the merge had to get right, counted by which half the winning row came from: + // 500 keys the tail replaced (less the one it then deleted), 500 it added, and the + // rest untouched in the lake. A suppression set that missed keys leaves the 'p' + // count too high and the total wrong by the same amount. + order_qt_pk_by_origin """ + select + sum(case when name like 'hot%' then 1 else 0 end) as replaced, + sum(case when name like 'new%' then 1 else 0 end) as added, + sum(case when name like 'p%' then 1 else 0 end) as untouched + from big_pk + """ + compareModes(""" + select + sum(case when name like 'hot%' then 1 else 0 end), + sum(case when name like 'new%' then 1 else 0 end), + sum(case when name like 'p%' then 1 else 0 end) + from big_pk + """) + + // The five deleted keys, one at a time. A delete is the case a merge of surviving + // rows cannot express: the key is absent from the tail's own result, and the lake + // row it removes stays unless something drops it. + order_qt_pk_deleted """ + select id, name from big_pk where id in (7, 19999, 50000, 77777, 99999) + """ + // And the keys either side of them, so "returns nothing" cannot be how the rows + // above came to be missing. + order_qt_pk_neighbours """ + select id, name from big_pk where id in (6, 8, 19998, 20000, 49999, 50001, 99998, 100000) + """ + + // Updated, added and untouched keys by name. + order_qt_pk_sample """ + select id, name, grp from big_pk + where id in (1, 500, 501, 99999, 100000, 100001, 100500) + """ + order_qt_pk_groups """select grp, count(*) from big_pk group by grp order by grp""" + order_qt_pk_ordered_head """select id, name from big_pk order by id limit 5""" + + // --- the same tables with the lake switched off --------------------------- + // A full change-log replay of 101000 records for the log table, and a kv snapshot + // plus its log for the primary-key one. These are the reads the other suites only + // ever ask for a few rows of. + sql """switch ${flussOnlyCatalog}""" + sql """use fluss_test""" + sql """set enable_file_scanner_v2 = true""" + + def flussOnlyLogPlan = planOf("""select * from big_log""") + assertTrue(flussOnlyLogPlan.contains("unionRead=no"), "the lake should be off: ${flussOnlyLogPlan}") + assertEquals(0, countIn(flussOnlyLogPlan, "lakeSplits"), "no lake half: ${flussOnlyLogPlan}") + order_qt_fluss_only_log """ + select count(*) as rows_read, sum(id) as id_sum, max(id) as id_max from big_log + """ + + def flussOnlyPkPlan = planOf("""select * from big_pk""") + assertTrue(flussOnlyPkPlan.contains("unionRead=no"), "the lake should be off: ${flussOnlyPkPlan}") + assertEquals(3, countIn(flussOnlyPkPlan, "pkRanges"), "one range per bucket: ${flussOnlyPkPlan}") + order_qt_fluss_only_pk """ + select count(*) as rows_read, sum(id) as id_sum, max(id) as id_max from big_pk + """ + + sql """switch internal""" + sql """drop catalog if exists ${unionCatalog}""" + sql """drop catalog if exists ${flussOnlyCatalog}""" +} diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy new file mode 100644 index 00000000000000..983dbccbaf472d --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_catalog.groovy @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Catalog-level checks for the fluss connector: what a user sees before any +// data is read. Fixtures come from +// docker/thirdparties/docker-compose/fluss/sql/init.sql. +suite("test_fluss_catalog", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String catalogName = "test_fluss_catalog" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}" + ); + """ + + // --- the fixture database and its tables are visible ------------------- + // The listings are recorded rather than spot-checked: a table that appears + // out of nowhere is as much a bug as one that goes missing, and only the + // whole list catches the first kind. In particular a lake table must be + // listed ONCE, under its own name -- the $lake reader is a way of reading it, + // not a second table, and a listing that showed both would double every lake + // table for anything walking the schema. + order_qt_databases """show databases from ${catalogName}""" + + sql """switch ${catalogName}""" + sql """use fluss_test""" + + order_qt_tables """show tables""" + + // --- schema mapping ---------------------------------------------------- + // desc is recorded UNSORTED: column order is part of what is being checked, + // and sorting the rows would throw it away. Types are pinned here too, so a + // type-mapping change shows up before any query runs. + qt_desc_log_basic """desc log_basic""" + + // One column per fluss type the connector maps, plus the id column. + qt_desc_log_types """desc log_types""" + + // The partition key is an ordinary column of the table, not a hidden one. + qt_desc_log_part """desc log_part""" + + // Primary-key columns keep their position; the connector reports every + // column as a key column, which is how Doris models external tables. + qt_desc_pk_basic """desc pk_basic""" + + // --- comments survive the metadata mapping ----------------------------- + // Column comments live on the fluss schema, not on the row type: reading + // the row type instead would silently drop every one of them. + // + // Not recorded into a .out on purpose: the statement carries the catalog's + // properties, including this environment's bootstrap address, so a recorded + // baseline would be tied to the machine that generated it. + def createTableRows = sql """show create table log_basic""" + def createTable = createTableRows[0][1].toString() + assertTrue(createTable.contains("row id"), "column comment lost: ${createTable}") + assertTrue(createTable.contains("fluss log table for regression"), + "table comment lost: ${createTable}") + + // --- refresh keeps the catalog usable ---------------------------------- + // Recorded again rather than compared with the listing above: the two blocks + // have to stay identical in the baseline, which is the same statement made + // in a way that also survives someone adding a fixture table. + sql """refresh catalog ${catalogName}""" + order_qt_tables_after_refresh """show tables""" + + sql """switch internal""" + sql """drop catalog ${catalogName}""" + + // --- property validation happens at catalog creation ------------------- + test { + sql """ + create catalog test_fluss_no_bootstrap properties ( + "type" = "fluss" + ); + """ + exception "Required property 'fluss.bootstrap.servers' is missing" + } + + test { + sql """ + create catalog test_fluss_bad_port properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${externalEnvIp}:not-a-port" + ); + """ + exception "expected a number between 1 and 65535" + } + + test { + sql """ + create catalog test_fluss_bad_union_mode properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "sometimes" + ); + """ + exception "expected one of auto, required, disabled" + } +} diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_lake_only.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_lake_only.groovy new file mode 100644 index 00000000000000..d073272a9a6c5a --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_lake_only.groovy @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Reading the lake side of a fluss table on its own, through `tbl$lake`. +// +// This is the first place the delegation runs for real. The fluss plugin bundles +// no paimon at all: it asks the plugin manager for a paimon connector, hands it +// synthesized catalog properties, and routes the scan to the handle that +// connector returns. In unit tests the sibling is a stand-in on the same class +// loader, so what only a deployed run can show is that a real plugin is found, +// that a handle crossing the plugin boundary is routed rather than cast, and +// that paimon's own ServiceLoader lookups resolve under the class loader the +// call is pinned to. +// +// Fixtures come from docker/thirdparties/docker-compose/fluss/sql/init.sql and +// are frozen: the tiering service is stopped before the log tail is written, so +// exactly the rows below are in paimon and no others ever will be. +suite("test_fluss_lake_only", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String catalogName = "test_fluss_lake_only" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}" + ); + """ + sql """switch ${catalogName}""" + sql """use fluss_test""" + + // The connector is wired into the v2 file scanner only, and fuzzy sessions + // randomize this variable. + sql """set enable_file_scanner_v2 = true""" + // TIMESTAMP_LTZ renders through the session time zone, and the baselines below + // record what it rendered as. + sql """set time_zone = 'Asia/Shanghai'""" + + // --- the lake holds what was tiered, and nothing that came after --------- + // lake_log got four rows before the tiering service was stopped and two + // after. Reading the lake alone must return the first four: a $lake that + // quietly fell back to the fluss read would return all six. + order_qt_lake_rows """select id, name, price from lake_log\$lake""" + + // --- the three columns fluss adds to every lake table -------------------- + // They belong to the lake table and not to the fluss one, which is the whole + // reason the two are exposed as separate tables rather than one. + // + // Their VALUES are deliberately not recorded: which bucket a log row lands in + // is the writer's choice and __timestamp is a wall clock, so a baseline holding + // them would be rewritten every time the environment is rebuilt. What is + // recorded is that every row has all three within the range they must be in. + order_qt_system_columns """ + select count(*) from lake_log\$lake + where __bucket >= 0 and __bucket < 3 + and __offset >= 0 + and __timestamp is not null + """ + + qt_desc_lake_log_lake """desc lake_log\$lake""" + // The fluss table itself has none of them. + qt_desc_lake_log """desc lake_log""" + + // --- type parity between the two doors ---------------------------------- + // The connector's fluss->Doris mapping has to equal fluss->paimon->Doris, or + // `tbl` and `tbl$lake` present two different schemas for one table. Both + // schemas are recorded, and the equality is ALSO asserted here: a reader + // comparing two recorded blocks by eye is not what should be guarding an + // invariant this quiet. + qt_desc_lake_types """desc lake_types""" + qt_desc_lake_types_lake """desc lake_types\$lake""" + def typesOf = { String table -> + def result = [:] + sql("""desc ${table}""").each { row -> result.put(row[0].toString(), row[1].toString()) } + return result + } + def flussTypes = typesOf("lake_types") + def lakeTypes = typesOf("lake_types\$lake") + flussTypes.each { column, type -> + assertEquals(type, lakeTypes.get(column), + "column ${column} is ${type} on the fluss table but ${lakeTypes.get(column)} on its lake") + } + assertEquals(flussTypes.size() + 3, lakeTypes.size()) + + // Parity of the values, not just of the declared types: the row is recorded + // here read through paimon, and the same row read through fluss is recorded in + // test_fluss_log_table -- two decoders, one row, and the two baselines have to + // agree column for column. + // + // BINARY and BYTES go through hex() for the same reason as everywhere else. Note + // that an equality predicate on the microsecond TIMESTAMP would match nothing: + // it is pushed into paimon, which does not find the row, while the value itself + // and range predicates on it are right. That is the paimon connector's own + // behaviour (a plain paimon catalog over this warehouse does the same), so it is + // not pinned here -- recording the value sidesteps it entirely. + order_qt_lake_types_row """ + select id, f_boolean, f_tinyint, f_smallint, f_int, f_bigint, f_float, f_double, + f_decimal, f_char, f_string, hex(f_binary) as f_binary_hex, + hex(f_bytes) as f_bytes_hex, f_date, f_timestamp, f_timestamp_ltz, + f_array, f_map, f_row + from lake_types\$lake + """ + + // The all-NULL row was written after tiering stopped, so it is not here. + order_qt_lake_types_count """select count(*) from lake_types\$lake""" + + // --- a table the lake holds in full -------------------------------------- + order_qt_cold_rows """select id, name from lake_cold\$lake""" + + // --- partitioning survives the delegation -------------------------------- + // The lake table is partitioned by the same column, so the partition value + // has to come back with its own row and not with a neighbour's. + order_qt_part_rows """select id, name, dt from lake_part\$lake""" + + // Pruning is the sibling's, not fluss's: the predicate is pushed to the + // paimon connector, which owns the plan for this table. + order_qt_part_pruned """select id from lake_part\$lake where dt = '20260101'""" + + // --- a primary-key table's lake is its merged state at the tiering point -- + // Row 2 was updated before tiering, so the lake holds the update, not both + // versions. Row 3's later update and row 1's delete came after and are absent, + // which is exactly how this differs from the fluss-only read of the same table + // (recorded in test_fluss_lake_pk, against this same fixture). + order_qt_pk_lake_rows """select id, name from lake_pk\$lake""" + + // --- projection and aggregation through the sibling ---------------------- + order_qt_lake_count """select count(*) from lake_log\$lake""" + order_qt_lake_sum """select sum(price) from lake_log\$lake""" + order_qt_lake_names """select name from lake_log\$lake where id > 2""" + + // --- tables with no lake ------------------------------------------------- + // A table with no lake never offers the sub-table, so the name does not resolve + // and the engine answers before the connector is asked anything. That is the + // deliberate choice: advertising $lake on every fluss table would offer a + // sub-table whose only possible outcome is an error. The connector still + // re-checks when it IS asked -- discovery and resolution are two round trips, + // and the lake can be switched off in between -- but that guard is unreachable + // from here, which is why it is pinned in the unit tests instead. + test { + sql """select * from log_basic\$lake""" + exception "Unknown sys table" + } + + // $lake is a way to read a table, not a table of its own: it must not appear + // in the catalog listing, or every tool that walks the schema would show each + // lake table twice. The recorded listing is what says so. + order_qt_tables """show tables""" + + sql """drop catalog if exists ${catalogName}""" +} diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk.groovy new file mode 100644 index 00000000000000..678167b8efc802 --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk.groovy @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Reading a PRIMARY-KEY table that is tiered into a lake. +// +// Its two halves cannot be concatenated the way a log table's are: the log +// carries updates and deletes of rows the lake already holds, so reading both +// and adding them up returns superseded and deleted rows. They are merged BY +// KEY instead -- the lake half is read column by column and a lake row is +// dropped when the log tail that follows it touched its key at all, while the +// tail contributes the state it ended in. +// +// The results below were recorded before that merge existed, off a read that +// went to fluss alone. That read is the WHOLE table rather than a part of it, +// because fluss keeps a primary-key table's state in its own kv store and +// tiering copies rows into the lake rather than moving them out -- so it is a +// second, independent answer to every query here, and the merge has to +// reproduce it row for row. Nothing in the RESULTS says which one ran, which is +// exactly what makes them a baseline; the plan says it, and this suite asserts +// both. When these recorded blocks change, the merge has broken -- that is what +// recording them is for. +// +// Fixtures come from docker/thirdparties/docker-compose/fluss/sql/init.sql and +// init-lake-tail.sql, and are frozen -- the tiering service is stopped before +// the tail is written, so the two halves stay apart for as long as the +// environment lives. +suite("test_fluss_lake_pk", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String autoCatalog = "test_fluss_lake_pk" + String flussOnlyCatalog = "test_fluss_lake_pk_off" + String requiredCatalog = "test_fluss_lake_pk_required" + + // Three catalogs, one per read mode, all live at once: the mode is a catalog + // property, and comparing what the modes return is most of the point here. + sql """drop catalog if exists ${autoCatalog}""" + sql """ + create catalog ${autoCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}" + ); + """ + sql """drop catalog if exists ${flussOnlyCatalog}""" + sql """ + create catalog ${flussOnlyCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "disabled" + ); + """ + sql """drop catalog if exists ${requiredCatalog}""" + sql """ + create catalog ${requiredCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "required" + ); + """ + + sql """switch ${autoCatalog}""" + sql """use fluss_test""" + // The C++ glue exists only for the v2 file scanner, and the session variable + // that picks between them is randomised by the fuzzy mode this pipeline runs. + sql """set enable_file_scanner_v2 = true""" + + def rowsOf = { String query -> sql(query).collect { row -> row.collect { it.toString() } } } + def planOf = { String query -> + return sql("""explain ${query}""").collect { it[0].toString() }.join("\n") + } + + // --- the two halves, and what the table IS ------------------------------- + // The lake was frozen mid-history: it holds row 2 already updated (that + // update happened before tiering) and rows 1 and 3 as first written. + order_qt_lake_side """select id, name from lake_pk\$lake""" + + // The log tail that follows disagrees with the lake in all three possible + // ways: row 3 updated, row 1 deleted, row 4 added. The table is therefore + // neither half, and no concatenation of the two produces it -- adding them up + // would return the deleted row 1, two versions of row 3, and row 4. + order_qt_front_door """select id, name from lake_pk""" + + // --- the merge lands on what a fluss-only read returns -------------------- + // The same rows out of an entirely different reader: this block and the one + // above have to stay identical to each other. That is the whole argument for + // the merge being right, because fluss serves a primary-key table in full and + // has no lake half to get wrong. + order_qt_fluss_only """select id, name from ${flussOnlyCatalog}.fluss_test.lake_pk""" + + // --- the plan says the lake really was read ------------------------------ + // Without this the suite would still pass if the merge quietly stopped + // happening and the read fell back to fluss alone -- the rows would be the + // same ones. One bucket: one lake split, suppressed by the tail of that + // bucket, and one range replaying the tail itself. + def plan = planOf("""select * from lake_pk""") + assertTrue( + plan.contains("flussScan: unionRead=yes, lakeSplits=1, suppressedLakeSplits=1, " + + "logRanges=0, pkRanges=0, pkTailRanges=1, mode=auto"), + "not planned as a lake+tail merge of one bucket: ${plan}") + + // --- required merges rather than refusing -------------------------------- + // That mode exists so a union-read test cannot pass by falling back. It used + // to refuse a primary-key table outright; now it has to produce the merge, + // and produce the same rows as the two modes that may fall back. + def requiredPlan = planOf("""select * from ${requiredCatalog}.fluss_test.lake_pk""") + assertTrue(requiredPlan.contains("unionRead=yes"), "required did not merge: ${requiredPlan}") + assertTrue(requiredPlan.contains("mode=required"), "unexpected mode: ${requiredPlan}") + + // Three modes, three catalogs, one answer -- compared here rather than by + // three recorded blocks, because what is being asserted is that they AGREE. + // Two identical recordings only look alike to a reader. + def query = { String catalog -> "select id, name from ${catalog}.fluss_test.lake_pk order by id" } + def merged = rowsOf(query(autoCatalog)) + assertEquals(rowsOf(query(flussOnlyCatalog)), merged, + "auto and disabled disagree on lake_pk") + assertEquals(rowsOf(query(requiredCatalog)), merged, + "required and auto disagree on lake_pk") + + // --- the ordinary things still work on this path ------------------------- + // Projection, predicates and count all run over the merged pair. The + // projection one carries a second load: the merge needs the key column to + // suppress by, and the engine keeps it in the scan for that reason alone -- + // so a single-column result here is also the assertion that the kept column + // stays out of the answer. + order_qt_count """select count(*) from lake_pk""" + order_qt_projection """select name from lake_pk where id = 3""" + // The deleted row stays gone under a predicate too, not only in a full scan: + // the delete is a log record, and applying it on one path but not the other is + // how a "row that only shows up sometimes" bug looks. + order_qt_deleted_row """select name from lake_pk where id = 1""" + order_qt_log_only_key """select id, name from lake_pk where id > 3""" + + sql """switch internal""" + sql """drop catalog if exists ${autoCatalog}""" + sql """drop catalog if exists ${flussOnlyCatalog}""" + sql """drop catalog if exists ${requiredCatalog}""" +} diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk_merge.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk_merge.groovy new file mode 100644 index 00000000000000..437c913c25c662 --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_lake_pk_merge.groovy @@ -0,0 +1,199 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Merging a primary-key table with its log tail, in the shapes where the +// merge can be got wrong in ways one bucket cannot show. +// +// test_fluss_lake_pk covers what the merge IS, over a single-bucket table. The +// fixtures here cover WHERE each half comes from: +// lake_pk_multi three buckets, tail in some of them -- a lake split may only +// be filtered by the tail of its own bucket; +// lake_pk_part three partitions, one lake+tail, one lake only, one that the +// lake has never seen and that is therefore read whole from +// fluss inside the same scan; +// lake_pk_cold nothing left in the log, so nothing to merge and nothing to +// plan for. +// +// Every table is read twice: through the merge, and through the fluss-only read +// that serves a primary-key table in full. The two have no code in common past +// planning, so their agreeing is the argument -- the recorded blocks alone +// cannot say whether both are wrong the same way. +// +// Fixtures come from docker/thirdparties/docker-compose/fluss/sql/init.sql and +// init-lake-tail.sql, and are frozen: tiering is stopped before the tail is +// written, so the halves stay apart for as long as the environment lives. +suite("test_fluss_lake_pk_merge", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String mergeCatalog = "test_fluss_lake_pk_merge" + String flussOnlyCatalog = "test_fluss_lake_pk_merge_off" + + // required, not auto: a fallback here would read the right rows out of fluss + // alone and every recorded block would still match, so the mode that cannot + // fall back is the one that makes this suite about the merge. + sql """drop catalog if exists ${mergeCatalog}""" + sql """ + create catalog ${mergeCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "required" + ); + """ + sql """drop catalog if exists ${flussOnlyCatalog}""" + sql """ + create catalog ${flussOnlyCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "disabled" + ); + """ + + sql """switch ${mergeCatalog}""" + sql """use fluss_test""" + // The C++ glue exists only for the v2 file scanner, and the session variable + // that picks between them is randomised by the fuzzy mode this pipeline runs. + sql """set enable_file_scanner_v2 = true""" + + def rowsOf = { String query -> sql(query).collect { row -> row.collect { it.toString() } } } + def planOf = { String query -> + return sql("""explain ${query}""").collect { it[0].toString() }.join("\n") + } + def countIn = { String plan, String field -> + def matcher = (plan =~ /${field}=(\d+)/) + assertTrue(matcher.find(), "plan has no ${field}: ${plan}") + return matcher.group(1) as int + } + // The load-bearing check: the merge and a reader that never touches the lake + // agree row for row. It stays in the code rather than becoming a second + // recorded block, because what is asserted is the agreement -- two identical + // recordings only look alike to whoever reads them. + def compareModes = { String query -> + def merged = rowsOf("""${query}""") + def flussOnly = rowsOf("""${query}""".replace("from ", "from ${flussOnlyCatalog}.fluss_test.")) + assertEquals(flussOnly, merged, + "merge and fluss-only disagree for: ${query}\nfluss-only=${flussOnly}\nmerged=${merged}") + } + + // --- three buckets, and a tail that reaches only some of them ------------- + // Which bucket a key lands in is fluss's hash of it, so the numbers below are + // bounds rather than values. What they have to say is that the tail was split + // across buckets and applied to those buckets only: some lake splits carry a + // suppression set and some do not. Binding a tail to the wrong bucket + // suppresses nothing at all -- a key lives in exactly one bucket -- and the + // rows it was meant to replace come back beside their replacements. + def multiPlan = planOf("""select * from lake_pk_multi""") + assertTrue(multiPlan.contains("unionRead=yes"), "not a merge: ${multiPlan}") + def multiLakeSplits = countIn(multiPlan, "lakeSplits") + def multiSuppressed = countIn(multiPlan, "suppressedLakeSplits") + def multiTailRanges = countIn(multiPlan, "pkTailRanges") + assertTrue(multiLakeSplits >= 2, + "fixture no longer spreads over buckets, so nothing here tests binding: ${multiPlan}") + assertTrue(multiTailRanges >= 2, + "fixture tail no longer spans buckets, so nothing here tests binding: ${multiPlan}") + assertTrue(multiSuppressed >= 1 && multiSuppressed < multiLakeSplits, + "suppression is not per bucket: ${multiSuppressed} of ${multiLakeSplits} splits: ${multiPlan}") + // Every bucket of this table is in the lake, so none is read whole from fluss. + assertEquals(0, countIn(multiPlan, "pkRanges"), "a bucket was read whole: ${multiPlan}") + + // The lake half on its own: nine keys as tiered, row 5 already updated. + order_qt_multi_lake """select id, name from lake_pk_multi\$lake""" + // The table: row 2 updated by the tail, row 7 deleted by it, row 10 added -- + // each in whatever bucket its key hashes to. + order_qt_multi_rows """select id, name from lake_pk_multi""" + compareModes("select id, name from lake_pk_multi order by id") + order_qt_multi_count """select count(*) from lake_pk_multi""" + // A predicate on a key the tail replaced, and one on a key it deleted: the + // filtered path has to suppress exactly as the full scan does. + order_qt_multi_updated """select name from lake_pk_multi where id = 2""" + order_qt_multi_deleted """select name from lake_pk_multi where id = 7""" + order_qt_multi_new """select name from lake_pk_multi where id = 10""" + + // --- partitions, each standing differently towards the lake --------------- + // 20260101 is lake plus tail, 20260102 is lake alone, 20260103 was created + // after tiering stopped and the lake has never heard of it. All three are read + // by one scan, which therefore mixes suppressed lake splits, plain lake + // splits, and buckets read whole out of fluss. + def partPlan = planOf("""select * from lake_pk_part""") + assertTrue(partPlan.contains("unionRead=yes"), "not a merge: ${partPlan}") + assertTrue(countIn(partPlan, "lakeSplits") >= 2, "lake half lost a partition: ${partPlan}") + assertTrue(countIn(partPlan, "suppressedLakeSplits") >= 1, + "the partition with a tail was not suppressed: ${partPlan}") + assertTrue(countIn(partPlan, "pkTailRanges") >= 1, "no tail replayed: ${partPlan}") + assertTrue(countIn(partPlan, "pkRanges") >= 1, + "the partition the lake never saw was not read from fluss: ${partPlan}") + + order_qt_part_rows """select id, name, dt from lake_pk_part""" + compareModes("select id, name, dt from lake_pk_part order by dt, id") + + // Pruned to the partition the lake holds in full: lake splits, no tail, and + // nothing suppressed. Pruning is per partition on the fluss half and by pushed + // predicate on the paimon half, so a plan that keeps the tail here has bound + // the two halves at table level. + def tieredPlan = planOf("""select * from lake_pk_part where dt = '20260102'""") + assertTrue(countIn(tieredPlan, "lakeSplits") >= 1, "lake half pruned away: ${tieredPlan}") + assertEquals(0, countIn(tieredPlan, "suppressedLakeSplits"), + "a partition with no tail was suppressed: ${tieredPlan}") + assertEquals(0, countIn(tieredPlan, "pkTailRanges"), "tail read for nothing: ${tieredPlan}") + order_qt_part_lake_only """select id, name from lake_pk_part where dt = '20260102'""" + + // Pruned to the partition the lake has never seen: no lake half at all, and + // its buckets read whole from fluss. + def flussOnlyPartPlan = planOf("""select * from lake_pk_part where dt = '20260103'""") + assertEquals(0, countIn(flussOnlyPartPlan, "lakeSplits"), + "a partition the lake never saw got lake splits: ${flussOnlyPartPlan}") + assertTrue(countIn(flussOnlyPartPlan, "pkRanges") >= 1, + "the partition was not read from fluss: ${flussOnlyPartPlan}") + order_qt_part_fluss_only """select id, name from lake_pk_part where dt = '20260103'""" + + // Pruned to the partition that has a tail. + def tailPartPlan = planOf("""select * from lake_pk_part where dt = '20260101'""") + assertTrue(countIn(tailPartPlan, "suppressedLakeSplits") >= 1, + "the tail of this partition was lost: ${tailPartPlan}") + assertEquals(1, countIn(tailPartPlan, "pkTailRanges"), "unexpected tail count: ${tailPartPlan}") + order_qt_part_with_tail """select id, name from lake_pk_part where dt = '20260101'""" + + // --- nothing left in the log --------------------------------------------- + // The lake holds this table in full. Merging costs nothing when there is + // nothing to merge, and only the plan can say so: a reader that built a + // suppression set out of an empty tail would return these same rows while + // opening a log scanner per bucket to do it. + def coldPlan = planOf("""select * from lake_pk_cold""") + assertTrue(coldPlan.contains("unionRead=yes"), "not a merge: ${coldPlan}") + assertTrue(countIn(coldPlan, "lakeSplits") >= 1, "no lake splits: ${coldPlan}") + assertEquals(0, countIn(coldPlan, "suppressedLakeSplits"), + "a table with no tail still wraps splits: ${coldPlan}") + assertEquals(0, countIn(coldPlan, "pkTailRanges"), "a tail was read for nothing: ${coldPlan}") + assertEquals(0, countIn(coldPlan, "pkRanges"), "a bucket was read whole: ${coldPlan}") + order_qt_cold_rows """select id, name from lake_pk_cold""" + compareModes("select id, name from lake_pk_cold order by id") + + // Deletion vectors are not covered here and the reason is in init.sql: fluss + // does create the paimon table with them on, but its tiering service writes + // no deletion vector index, and paimon then reads such a table as empty. A + // fixture that reads as empty asserts nothing. Deletion vectors under the + // suppression filter stay covered by the BE unit tests. + + sql """switch internal""" + sql """drop catalog if exists ${mergeCatalog}""" + sql """drop catalog if exists ${flussOnlyCatalog}""" +} diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_log_table.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_log_table.groovy new file mode 100644 index 00000000000000..070a040bb4e52d --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_log_table.groovy @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Reading fluss log tables end to end: FE planning, the untyped payload it ships, +// the BE java scanner and the partition columns BE materializes from each range. +// Fixtures come from docker/thirdparties/docker-compose/fluss/sql/init.sql and are +// static - this suite never writes, so no polling gate is needed. +// +// Results are recorded into the .out baseline; only what is NOT a result stays as +// an assertion in the code -- the EXPLAIN anchors, which say how the scan was +// planned, and the bounds on range counts, which cannot be exact because a fluss +// writer chooses its own buckets. +suite("test_fluss_log_table", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String catalogName = "test_fluss_log_table" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}" + ); + """ + sql """switch ${catalogName}""" + sql """use fluss_test""" + + // The connector is wired into the v2 file scanner only; the legacy one answers + // "Not supported create reader for table format: fluss". Fuzzy sessions randomize + // this variable, so pinning it is what keeps the suite from failing on half the + // CI runs for a reason that has nothing to do with fluss. + sql """set enable_file_scanner_v2 = true""" + // TIMESTAMP_LTZ renders through the session time zone, and the baseline below + // records what it rendered as. Without pinning it, the recorded value would be + // whatever the machine that generated the baseline happened to be set to. + sql """set time_zone = 'Asia/Shanghai'""" + + def planOf = { String query -> + def planRows = sql("""explain ${query}""") + return planRows.collect { it[0].toString() }.join("\n") + } + def logRangesOf = { String plan -> + def matcher = (plan =~ /logRanges=(\d+)/) + assertTrue(matcher.find(), "plan has no flussScan line: ${plan}") + return matcher.group(1) as int + } + + // --- the whole table ---------------------------------------------------- + order_qt_basic_all """select id, name, price from log_basic""" + + // COUNT(*) projects no column at all: the scanner has to report how many rows it + // read without returning one. A scanner that answered with an empty batch instead + // would make an untouched-looking table out of a populated one. + order_qt_basic_count """select count(*) from log_basic""" + + // --- projection and predicates ----------------------------------------- + order_qt_basic_names """select name from log_basic""" + + // Columns asked for in an order other than the table's: the scanner resolves the + // projection by name, so a positional shortcut anywhere would swap these two. + order_qt_basic_swapped """select name, id from log_basic where id = 2""" + + order_qt_basic_filtered """select id from log_basic where price > 15.00""" + order_qt_basic_count_filtered """select count(*) from log_basic where name = 'bob'""" + + // --- every mapped type survives the round trip -------------------------- + // The three rows are recorded whole: the positive one, the negative one (a sign + // lost in decoding shows up nowhere else) and the all-NULL one (read through a + // null map that is off by one column, every value after it shifts). Recording + // them pins the rendering of map, struct and decimal as well, which is more than + // the decoding this suite is about -- a rendering change will land here and has + // to be re-recorded deliberately. + // + // BINARY and BYTES go through hex(): they map to a Doris string, and a raw byte + // in a recorded baseline is neither readable nor safely round-tripped. + order_qt_types_all """ + select id, f_boolean, f_tinyint, f_smallint, f_int, f_bigint, f_float, f_double, + f_decimal, f_char, f_string, hex(f_binary) as f_binary_hex, + hex(f_bytes) as f_bytes_hex, f_date, f_timestamp, f_timestamp_ltz, + f_array, f_map, f_row + from log_types + """ + + // The nested readers, asked for one element at a time: a struct that decoded into + // the right shape but the wrong field order still renders plausibly above. + order_qt_types_nested """ + select id, array_size(f_array), f_array[1], f_map['k1'], f_map['k2'], + struct_element(f_row, 'r_int'), struct_element(f_row, 'r_string') + from log_types + """ + + // --- partitioned table -------------------------------------------------- + // The partition column is not read by the scanner: FE declares it and BE fills it + // in from each range. Recording it next to the row it belongs to is what catches a + // partition value attached to the wrong split. + order_qt_part_all """select id, name, dt from log_part""" + + // Projecting nothing but the partition column leaves the scanner with an empty + // projection - it still has to report the row count for each range. + order_qt_part_grouped """select dt, count(*) from log_part group by dt""" + + order_qt_part_pruned """select id from log_part where dt = '20260101'""" + + // --- planning is visible in the plan ------------------------------------ + def basicPlan = planOf("""select * from log_basic""") + assertTrue(basicPlan.contains("flussScan: unionRead=no"), + "no fluss scan line: ${basicPlan}") + assertTrue(basicPlan.contains("lakeSplits=0"), "unexpected lake splits: ${basicPlan}") + assertTrue(basicPlan.contains("mode=auto"), "unexpected union read mode: ${basicPlan}") + // One range per bucket that holds data. Which of the three buckets a fluss log row + // lands in is the writer's choice, so the count is bounded rather than fixed. + def basicRanges = logRangesOf(basicPlan) + assertTrue(basicRanges >= 1 && basicRanges <= 3, + "log_basic planned ${basicRanges} ranges over 3 buckets") + + def fullPartPlan = planOf("""select * from log_part""") + assertTrue(fullPartPlan.contains("partition=3/3"), + "all three partitions should be scanned: ${fullPartPlan}") + def fullPartRanges = logRangesOf(fullPartPlan) + assertTrue(fullPartRanges >= 3, "every partition holds data: ${fullPartPlan}") + + def prunedPlan = planOf("""select * from log_part where dt = '20260101'""") + assertTrue(prunedPlan.contains("partition=1/3"), + "partition pruning did not reach the connector: ${prunedPlan}") + // Pruning has to shrink the work, not just the plan line: a partition name rendered + // one way for the listing and another way for the match would prune to nothing here + // while still reporting 1/3. + def prunedRanges = logRangesOf(prunedPlan) + assertTrue(prunedRanges >= 1 && prunedRanges <= 2, + "one partition has 2 buckets, planned ${prunedRanges} ranges: ${prunedPlan}") + + // --- a table that was never written to ---------------------------------- + // Its buckets stop at offset 0, so planning emits no range at all. The empty answer + // has to come from that, not from a scanner opened on an empty bucket. + order_qt_empty_count """select count(*) from log_empty""" + order_qt_empty_rows """select id, name from log_empty""" + def emptyPlan = planOf("""select * from log_empty""") + // An engine that drops a split-less scan altogether is just as correct as one that + // keeps the node, so what is pinned is that no range was planned either way. + def emptyMatcher = (emptyPlan =~ /logRanges=(\d+)/) + assertTrue(!emptyMatcher.find() || emptyMatcher.group(1) == "0", + "a table that was never written to planned ranges: ${emptyPlan}") + + sql """switch internal""" + sql """drop catalog ${catalogName}""" +} diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_misc.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_misc.groovy new file mode 100644 index 00000000000000..e7cd43d91f7135 --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_misc.groovy @@ -0,0 +1,261 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The rest of what a catalog has to answer for: tables with nothing in them, the +// ways a query over one is supposed to fail, and the ordinary SQL a user reaches +// for once the table reads at all. +// +// The suites next to this one each follow one path in depth. This one is the +// breadth: a fluss table has to behave like a table -- joinable, insertable-from, +// a subquery, one side of a union -- and it has to fail in a way that says what to +// do, on the handful of things it genuinely cannot do. +// +// Fixtures: docker/thirdparties/docker-compose/fluss/sql/init.sql. +suite("test_fluss_misc", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String catalogName = "test_fluss_misc" + String requiredCatalog = "test_fluss_misc_required" + String internalDb = "test_fluss_misc_internal" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}" + ); + """ + sql """switch ${catalogName}""" + sql """use fluss_test""" + sql """set enable_file_scanner_v2 = true""" + + def planOf = { String query -> + return sql("""explain ${query}""").collect { it[0].toString() }.join("\n") + } + + // --- tables with nothing in them ------------------------------------------ + // A primary-key table that was never written to is not the same shape as an empty + // log table: its read starts from a kv snapshot, and there is none, so the code that + // has to notice is not the code that reads no log records. + order_qt_pk_empty_count """select count(*) from pk_empty""" + order_qt_pk_empty_rows """select id, name from pk_empty""" + def pkEmptyPlan = planOf("""select * from pk_empty""") + // An engine that drops a split-less scan altogether is as correct as one that keeps + // the node, so what is pinned is that no range was planned either way. + def pkEmptyMatcher = (pkEmptyPlan =~ /pkRanges=(\d+)/) + assertTrue(!pkEmptyMatcher.find() || pkEmptyMatcher.group(1) == "0", + "an empty primary-key table planned ranges: ${pkEmptyPlan}") + + // Aggregates over an empty table, where COUNT and the rest disagree by design: + // count is 0 and every other aggregate is NULL, and a scanner that reported an + // empty batch as "no rows read" rather than "zero rows" gets one of them wrong. + order_qt_empty_aggregates """ + select count(*), count(id), sum(id), min(id), max(id), avg(id) from pk_empty + """ + order_qt_log_empty_aggregates """ + select count(*), count(id), sum(id), min(id), max(id) from log_empty + """ + order_qt_empty_join """ + select count(*) from log_empty a join log_basic b on a.id = b.id + """ + + // A lake table tiering has never committed anything for. Under auto there is + // nothing to read the lake half from, so the fluss-only read is the whole answer -- + // which for a table nobody wrote to is no rows. + def lakeEmptyPlan = planOf("""select * from lake_empty""") + assertTrue(lakeEmptyPlan.contains("unionRead=no"), + "there is no lake snapshot to read: ${lakeEmptyPlan}") + assertTrue(lakeEmptyPlan.contains("lakeSplits=0"), "no lake half: ${lakeEmptyPlan}") + order_qt_lake_empty_count """select count(*) from lake_empty""" + + // Under required the same table is an error: falling back is precisely what that + // mode forbids, and "wait for the tiering service" is the thing its user needs to + // be told rather than left to infer from an empty result. + sql """drop catalog if exists ${requiredCatalog}""" + sql """ + create catalog ${requiredCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "required" + ); + """ + test { + sql """select * from ${requiredCatalog}.fluss_test.lake_empty""" + exception "has no readable lake snapshot yet" + } + + // --- the one fluss type Doris has nowhere to put -------------------------- + // TIME is marked unsupported rather than mapped to a string or to elapsed millis: + // either of those hands back a value whose meaning differs from the source, and + // nothing downstream would ever reveal it. So the column is describable and + // unreadable, and the rest of the table is unaffected -- which is what makes the + // marker useful instead of merely honest. + qt_desc_log_time """desc log_time""" + order_qt_time_other_columns """select id, name from log_time""" + order_qt_time_count """select count(*) from log_time""" + test { + sql """select f_time from log_time""" + exception "UNSUPPORTED" + } + test { + sql """select * from log_time""" + exception "UNSUPPORTED" + } + + // --- things that are simply not there ------------------------------------- + test { + sql """select * from no_such_table""" + exception "no_such_table" + } + test { + sql """select no_such_column from log_basic""" + exception "no_such_column" + } + test { + sql """select * from no_such_db.log_basic""" + exception "no_such_db" + } + + // --- a fluss table is a table --------------------------------------------- + // Two fluss tables joined to each other. The scan of each is the same one the other + // suites make; what is new is two of them in one fragment, which is where a scanner + // holding state on something shared between nodes would show up. + order_qt_join_fluss """ + select b.id, b.name, t.f_string + from log_basic b join log_types t on b.id = t.id + """ + order_qt_join_pk """ + select p.id, p.name, l.name as log_name + from pk_basic p join log_basic l on p.id = l.id + """ + // A join over the seam of a table read as lake plus log: rows from both halves have + // to reach the join operator, not just the half that was planned first. + order_qt_join_lake """ + select l.id, l.name, c.name as cold_name + from lake_log l left join lake_cold c on l.id = c.id + """ + order_qt_join_self """ + select a.id, a.name, b.name + from log_part a join log_part b on a.dt = b.dt and a.id < b.id + """ + + // Subqueries, correlated and not. + order_qt_subquery_in """ + select id, name from log_basic where id in (select id from pk_basic where score > 2.0) + """ + order_qt_subquery_exists """ + select id from log_basic b where exists (select 1 from log_types t where t.id = b.id) + """ + order_qt_subquery_scalar """ + select id, name from log_basic where price > (select min(price) from log_basic) + """ + order_qt_cte """ + with hot as (select id, name from lake_log where id >= 5) + select count(*), min(id) from hot + """ + + // Set operations, where a fluss scan is one arm of several. + order_qt_union_all """ + select id, name from log_basic union all select id, name from lake_cold + """ + order_qt_union_distinct """ + select id from log_basic union select id from lake_cold + """ + order_qt_except """ + select id from lake_log except select id from lake_cold + """ + order_qt_intersect """ + select id from lake_log intersect select id from lake_cold + """ + + // Ordinary aggregation and window shapes over a fluss scan. + order_qt_aggregates """ + select count(*), count(distinct name), min(price), max(price), sum(price), avg(id) + from log_basic + """ + order_qt_group_having """ + select dt, count(*) as c from log_part group by dt having count(*) > 1 + """ + order_qt_order_limit """select id, name from log_basic order by price desc limit 2""" + order_qt_limit_offset """select id from log_basic order by id limit 1 offset 1""" + order_qt_star """select * from log_basic""" + order_qt_expression """select id * 2 as doubled, upper(name) from log_basic where id < 3""" + order_qt_case """ + select id, case when price > 20 then 'high' else 'low' end as band from log_basic + """ + + // --- reading a fluss table into an internal one --------------------------- + // The whole point of the connector for most users, and the one path where the + // scanned rows leave the query that scanned them: they are typed, materialized and + // written. A type that reads correctly but describes itself wrongly fails here and + // nowhere else. + sql """switch internal""" + sql """drop database if exists ${internalDb}""" + sql """create database ${internalDb}""" + sql """use ${internalDb}""" + + sql """ + create table copied ( + id int, + name varchar(64), + price decimalv3(10, 2) + ) distributed by hash(id) buckets 1 properties ("replication_num" = "1") + """ + sql """ + insert into copied select id, name, price from ${catalogName}.fluss_test.log_basic + """ + order_qt_copied """select id, name, price from copied""" + + // CTAS, which takes the column types from the connector rather than from a table + // someone wrote by hand -- so this is where the declared Doris type of every mapped + // fluss type has to be a type Doris can actually create a column of. + sql """ + create table ctas_types properties ("replication_num" = "1") as + select id, f_boolean, f_tinyint, f_smallint, f_int, f_bigint, f_float, f_double, + f_decimal, f_char, f_string, f_date, f_timestamp, f_timestamp_ltz, + f_array, f_map, f_row + from ${catalogName}.fluss_test.log_types + """ + qt_desc_ctas """desc ctas_types""" + order_qt_ctas_count """select count(*), count(f_string), count(f_array) from ctas_types""" + + sql """ + create table ctas_nested properties ("replication_num" = "1") as + select id, f_arr_arr, f_map_row, f_row_deep + from ${catalogName}.fluss_test.log_nested where id = 1 + """ + qt_desc_ctas_nested """desc ctas_nested""" + order_qt_ctas_nested_rows """select id, f_arr_arr, f_map_row, f_row_deep from ctas_nested""" + + // A join between an internal table and a fluss one, which is the shape a user + // actually writes: the two scans are entirely different node types in one plan. + order_qt_join_internal """ + select c.id, c.name, t.f_string + from copied c join ${catalogName}.fluss_test.log_types t on c.id = t.id + """ + + sql """drop database if exists ${internalDb} force""" + sql """switch internal""" + sql """drop catalog if exists ${catalogName}""" + sql """drop catalog if exists ${requiredCatalog}""" +} diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_nested_types.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_nested_types.groovy new file mode 100644 index 00000000000000..57abbac7c7c6c0 --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_nested_types.groovy @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Complex types nested inside complex types, down every path a value can take. +// +// The other type suites cover one column per fluss type, never more than one level +// deep. That is a different question: an element decoder is chosen per level, so a +// reader can be right about MAP and wrong about the MAP> +// beside it, and being right about ARRAY says nothing about ARRAY>. +// Fluss stores these in three unrelated row formats -- arrow for a log table, +// compacted for a primary-key one, parquet through paimon for the lake half -- and +// each has its own nested decoder, so all three are read here. +// +// Fixtures: docker/thirdparties/docker-compose/fluss/sql/init.sql (log_nested, +// pk_nested, lake_nested) and init-lake-tail.sql. +suite("test_fluss_nested_types", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String catalogName = "test_fluss_nested_types" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "required" + ); + """ + sql """switch ${catalogName}""" + sql """use fluss_test""" + + // The connector is wired into the v2 file scanner only, and fuzzy sessions + // randomize the variable that chooses between them. + sql """set enable_file_scanner_v2 = true""" + + def rowsOf = { String query -> sql(query).collect { row -> row.collect { it.toString() } } } + + // --- the mapping, before any row is read --------------------------------- + // Recorded unsorted: which level of nesting each constructor sits at is the whole + // subject, and a type printed one level flat is the bug this suite is about. + qt_desc_log_nested """desc log_nested""" + qt_desc_pk_nested """desc pk_nested""" + + // --- a log table's arrow rows -------------------------------------------- + // All three rows whole: the populated one, the one whose collections are present + // but hold NULL elements, and the one whose columns are NULL outright. The middle + // row is the one worth having -- a decoder that confuses "no element" with "a null + // element" gets the other two right and this one wrong. + order_qt_log_rows """ + select id, f_arr_arr, f_arr_map, f_arr_row, f_map_arr, f_map_row, f_row_deep, + f_arr_arr_arr + from log_nested + """ + + // Element by element, so a value that rendered plausibly but decoded into the wrong + // slot has somewhere to fail. Reaching two levels in at once is the point: the + // outer accessor has to hand the inner one a value it can still take apart. + order_qt_log_elements """ + select id, + array_size(f_arr_arr), f_arr_arr[1], array_size(f_arr_arr[1]), f_arr_arr[1][2], + f_arr_map[1]['a'], + struct_element(f_arr_row[1], 'a'), struct_element(f_arr_row[1], 'b'), + f_map_arr['k1'], f_map_arr['k1'][2], + struct_element(f_map_row['k1'], 'a'), struct_element(f_map_row['k1'], 'b') + from log_nested + """ + + // Three levels of the same constructor, and a struct holding one of each. Nothing + // else here asks a decoder to recurse into itself twice. + order_qt_log_deep """ + select id, + f_arr_arr_arr[1][2][1], + struct_element(f_row_deep, 'r_int'), + struct_element(f_row_deep, 'r_arr')[2], + struct_element(f_row_deep, 'r_map')['m'], + struct_element(struct_element(f_row_deep, 'r_row'), 'x'), + struct_element(struct_element(f_row_deep, 'r_row'), 'y') + from log_nested + """ + + // A predicate on a nested element, which is evaluated after the value has been + // decoded rather than in the scanner: the row it selects is the check. + order_qt_log_nested_predicate """ + select id from log_nested + where f_arr_arr[1][1] = 1 and struct_element(f_map_row['k1'], 'a') = 9 + """ + + // --- a primary-key table's compacted rows --------------------------------- + // Key 1 was written twice, so what comes back must be the SECOND set of nested + // values. A reader that returned the change log rather than the merged view would + // answer with three rows here, two of them the same key. + order_qt_pk_rows """ + select id, f_arr_arr, f_arr_row, f_map_arr, f_map_row, f_row_deep from pk_nested + """ + order_qt_pk_elements """ + select id, + array_size(f_arr_arr), f_arr_arr[1][1], + struct_element(f_arr_row[1], 'b'), + f_map_arr['k9'], f_map_arr['k9'][1], + struct_element(f_map_row['k9'], 'a'), + struct_element(f_row_deep, 'r_arr')[1], + struct_element(struct_element(f_row_deep, 'r_row'), 'y') + from pk_nested + """ + + // --- the same nesting, through paimon ------------------------------------- + // The connector's fluss->Doris mapping has to equal fluss->paimon->Doris, or one + // table has two schemas depending on which door it is read through. For flat types + // that is checked in test_fluss_lake_only; nesting is where the two mappings could + // most easily part company, since each level is converted by its own rule on both + // sides. + qt_desc_lake_nested """desc lake_nested""" + qt_desc_lake_nested_lake """desc lake_nested\$lake""" + def typesOf = { String table -> + def result = [:] + sql("""desc ${table}""").each { row -> result.put(row[0].toString(), row[1].toString()) } + return result + } + def flussTypes = typesOf("lake_nested") + def lakeTypes = typesOf("lake_nested\$lake") + flussTypes.each { column, type -> + assertEquals(type, lakeTypes.get(column), + "column ${column} is ${type} on the fluss table but ${lakeTypes.get(column)} on its lake") + } + // The lake table carries three system columns fluss adds to every one it creates + // (__bucket, __offset, __timestamp) and nothing else, so the count is the other + // half of the parity: a column present on one side only would satisfy the loop. + assertEquals(flussTypes.size() + 3, lakeTypes.size(), + "the lake table should differ by exactly the three system columns") + + // The tiered row read through paimon's own reader. + order_qt_lake_side """ + select id, f_arr_arr, f_arr_row, f_map_arr, f_map_row, f_row_deep from lake_nested\$lake + """ + + // And the front door, where the tiered row arrives through paimon and the all-NULL + // one through fluss: two nested decoders feeding one result set. A level dropped by + // either shows up as a value in the wrong shape next to the same column from the + // other half. + def lakePlan = sql("""explain select * from lake_nested""").collect { it[0].toString() }.join("\n") + assertTrue(lakePlan.contains("unionRead=yes"), "not a union read: ${lakePlan}") + order_qt_lake_union """ + select id, f_arr_arr, f_arr_row, f_map_arr, f_map_row, f_row_deep from lake_nested + """ + order_qt_lake_union_elements """ + select id, f_arr_arr[1][2], struct_element(f_arr_row[1], 'b'), + f_map_arr['k1'][1], struct_element(f_map_row['k1'], 'b'), + struct_element(struct_element(f_row_deep, 'r_row'), 'y') + from lake_nested + """ + + // The union read and a reader that never touches the lake must agree row for row. + // Two entirely different nested decoders over one table: the comparison says + // something no pair of recorded blocks does. + String flussOnlyCatalog = "test_fluss_nested_types_off" + sql """drop catalog if exists ${flussOnlyCatalog}""" + sql """ + create catalog ${flussOnlyCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "disabled" + ); + """ + def unionRows = rowsOf("""select id, f_arr_arr, f_arr_row, f_map_arr, f_map_row, f_row_deep + from lake_nested order by id""") + def flussOnlyRows = rowsOf("""select id, f_arr_arr, f_arr_row, f_map_arr, f_map_row, f_row_deep + from ${flussOnlyCatalog}.fluss_test.lake_nested order by id""") + assertEquals(flussOnlyRows, unionRows, + "paimon and fluss decoded the same nested rows differently\nfluss-only=${flussOnlyRows}\nunion=${unionRows}") + + sql """switch internal""" + sql """drop catalog if exists ${catalogName}""" + sql """drop catalog if exists ${flussOnlyCatalog}""" +} diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_partition_types.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_partition_types.groovy new file mode 100644 index 00000000000000..66b19cb1fe39ac --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_partition_types.groovy @@ -0,0 +1,261 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Partition columns of every type, and the line between the ones that can be read +// and the ones that cannot. +// +// A fluss partition carries its value nowhere but in its own name, and fluss allows +// only ASCII letters, digits, '_' and '-' there -- so a value holding anything else +// is rewritten on the way in and cannot be recovered. Which types survive that is +// not a matter of taste, and every verdict below was established against this +// cluster: STRING, CHAR, BOOLEAN, the integer family, DATE and (as text) BINARY come +// back as written; FLOAT, DOUBLE, TIME and the timestamps do not. +// +// The other suites partition by STRING only, which is also the one type that cannot +// tell a rendering bug from a working one -- a number, a date or a padded CHAR is +// where the two sides of that rendering could disagree. +// +// Fixtures: part_types, part_ts, lake_part_int, lake_pk_part_int in +// docker/thirdparties/docker-compose/fluss/sql/init.sql. +suite("test_fluss_partition_types", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String catalogName = "test_fluss_partition_types" + String varbinaryCatalog = "test_fluss_partition_types_vb" + String unionCatalog = "test_fluss_partition_types_union" + String flussOnlyCatalog = "test_fluss_partition_types_off" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}" + ); + """ + sql """switch ${catalogName}""" + sql """use fluss_test""" + sql """set enable_file_scanner_v2 = true""" + + def planOf = { String query -> + return sql("""explain ${query}""").collect { it[0].toString() }.join("\n") + } + def countIn = { String plan, String field -> + def matcher = (plan =~ /${field}=(\d+)/) + assertTrue(matcher.find(), "plan has no ${field}: ${plan}") + return matcher.group(1) as int + } + def rowsOf = { String query -> sql(query).collect { row -> row.collect { it.toString() } } } + + // --- every readable partition type, in one table -------------------------- + // Recorded unsorted, because the column ORDER is what fe-core zips the partition + // values against positionally: a listing whose values are right but in another + // order assigns each partition someone else's value, silently. + qt_desc_part_types """desc part_types""" + + // The partition names as this connector renders them. Nothing else pins the + // rendering of a boolean, a date or a byte string into a partition name, and both + // the listing and split planning read this same rendering -- so a change here that + // reached only one of them would prune to nothing while still looking planned. + order_qt_part_names """show partitions from part_types""" + + // The values themselves, next to the row they belong to. The partition columns are + // not read by the scanner at all: FE declares them and BE fills each one in from + // the range it came with, so a value on the wrong split shows up only here. + // + // p_bin records as the hex of "0102" rather than of the bytes 0x01 0x02, and that is + // the truth of a partitioned BINARY rather than a defect: what fluss kept is the + // NAME it gave the partition, which is the hex text, and the value in the row is + // that text. A non-partitioned BINARY column, whose bytes are stored in the row + // itself, reads back as the bytes -- covered in the type suites. + order_qt_part_rows """ + select id, name, p_str, p_char, p_bool, p_tiny, p_small, p_int, p_big, p_date, + p_bin, hex(p_bin) as p_bin_hex + from part_types + """ + + // --- pruning, one partition column at a time ------------------------------ + // Each column takes a different value in each of the two partitions, so a predicate + // on any one of them must leave exactly one. Pruning has to shrink the WORK and not + // only the plan line: a name rendered one way for the listing and another for the + // match reports 1/2 here while scanning nothing at all, which is why the recorded + // rows sit beside the plan assertion rather than instead of it. + def prunesToOne = { String column, String predicate -> + def plan = planOf("""select * from part_types where ${predicate}""") + assertTrue(plan.contains("partition=1/2"), + "pruning on ${column} did not reach the connector: ${plan}") + assertTrue(countIn(plan, "logRanges") >= 1, + "pruning on ${column} left no range to read: ${plan}") + } + prunesToOne("p_str", "p_str = 'cn'") + prunesToOne("p_char", "p_char = 'c1'") + prunesToOne("p_bool", "p_bool = true") + prunesToOne("p_tiny", "p_tiny = 1") + prunesToOne("p_small", "p_small = 10") + prunesToOne("p_int", "p_int = 100") + prunesToOne("p_big", "p_big = 1000") + prunesToOne("p_date", "p_date = '2026-01-01'") + + order_qt_pruned_str """select id, name from part_types where p_str = 'cn'""" + order_qt_pruned_int """select id, name from part_types where p_int = 200""" + order_qt_pruned_date """select id, name from part_types where p_date = '2026-01-02'""" + order_qt_pruned_bool """select id, name from part_types where p_bool = false""" + order_qt_pruned_tiny """select id, name from part_types where p_tiny = 2""" + + // Several partition columns at once, which is also the ordinary case for this + // table: they have to agree rather than each prune on its own. + order_qt_pruned_multi """ + select id, name from part_types where p_str = 'cn' and p_int = 100 and p_bool = true + """ + // A combination no partition holds. Pruning to nothing is a correct answer and not + // an error, and it must not be reached by scanning everything and filtering after. + def emptyPlan = planOf("""select * from part_types where p_str = 'cn' and p_int = 200""") + assertTrue(emptyPlan.contains("partition=0/2"), + "an impossible combination should prune to nothing: ${emptyPlan}") + order_qt_pruned_impossible """ + select count(*) from part_types where p_str = 'cn' and p_int = 200 + """ + order_qt_pruned_absent """select count(*) from part_types where p_int = 999""" + + // --- the type whose value fluss cannot store ------------------------------ + // Fluss creates such a table happily and names the partition of + // 2026-01-01 01:02:03.0 as 2026-01-01-01-02-03_0 -- every character it may not + // hold rewritten, many-to-one. Left alone this reaches fe-core's partition parser, + // which fails with the mangled name and nothing else: no column, no type, no fluss. + // The refusal therefore belongs to the connector, and has to name all three. + test { + sql """select * from part_ts""" + exception "its partition column 'p_ts' has fluss type TIMESTAMP(3)" + } + test { + sql """show partitions from part_ts""" + exception "cannot be read back" + } + // DESC still works: the table is describable, it is only unreadable, and the column + // its user has to change is the one this shows. + qt_desc_part_ts """desc part_ts""" + + // --- the type whose verdict the catalog decides --------------------------- + // Fluss names a BINARY partition with the hex text of the bytes. Read as text that + // is exactly what was written; asked for as a VARBINARY it is not a literal of + // anything, so the same table is readable through one catalog and not the other. + sql """drop catalog if exists ${varbinaryCatalog}""" + sql """ + create catalog ${varbinaryCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "enable.mapping.varbinary" = "true" + ); + """ + test { + sql """select id from ${varbinaryCatalog}.fluss_test.part_types""" + exception "enable.mapping.varbinary" + } + + // --- a non-STRING partition under a union read ---------------------------- + // Concatenating a lake with the log written after it needs no partition value + // matched across the halves: each half prunes on its own. So an INT partition works + // here, and this is what says the rule that stops the primary-key merge below was + // not quietly applied to everything partitioned. + sql """drop catalog if exists ${unionCatalog}""" + sql """ + create catalog ${unionCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "required" + ); + """ + sql """drop catalog if exists ${flussOnlyCatalog}""" + sql """ + create catalog ${flussOnlyCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "disabled" + ); + """ + sql """switch ${unionCatalog}""" + sql """use fluss_test""" + sql """set enable_file_scanner_v2 = true""" + + def intPartPlan = planOf("""select * from lake_part_int""") + assertTrue(intPartPlan.contains("unionRead=yes"), + "an INT-partitioned log table should still be read as lake plus log: ${intPartPlan}") + assertTrue(countIn(intPartPlan, "lakeSplits") >= 1, "no lake splits: ${intPartPlan}") + assertEquals(1, countIn(intPartPlan, "logRanges"), + "one partition has a tail and the other does not: ${intPartPlan}") + order_qt_lake_part_int """select id, name, p_int from lake_part_int""" + + // Both halves must land on the same partition: one of these has a tail, the other + // is served entirely from the lake. + def tailPlan = planOf("""select * from lake_part_int where p_int = 1""") + assertTrue(tailPlan.contains("partition=1/2"), "pruning did not reach the connector: ${tailPlan}") + assertEquals(1, countIn(tailPlan, "logRanges"), "the partition with a tail lost it: ${tailPlan}") + order_qt_lake_part_int_tail """select id, name from lake_part_int where p_int = 1""" + + def tieredPlan = planOf("""select * from lake_part_int where p_int = 2""") + assertEquals(0, countIn(tieredPlan, "logRanges"), + "a fully tiered partition still has log ranges: ${tieredPlan}") + order_qt_lake_part_int_tiered """select id, name from lake_part_int where p_int = 2""" + + def unionIntRows = rowsOf("""select id, name, p_int from lake_part_int order by id""") + def flussOnlyIntRows = rowsOf( + """select id, name, p_int from ${flussOnlyCatalog}.fluss_test.lake_part_int order by id""") + assertEquals(flussOnlyIntRows, unionIntRows, + "lake+log and fluss-only disagree\nfluss-only=${flussOnlyIntRows}\nunion=${unionIntRows}") + + // --- a non-STRING partition under a merge BY KEY -------------------------- + // Merging the halves of a primary-key table means deciding which fluss partition a + // paimon split belongs to, by comparing the text each side renders that value as -- + // and only STRING is guaranteed to render alike on both. So this table is not + // merged. Under `required` that is an error naming the reason; the fluss-only read + // it would otherwise fall back to returns every row anyway, so nothing is lost but + // the lake's speed. + test { + sql """select * from lake_pk_part_int""" + exception "cannot be read as its lake plus its change log" + } + + // Under `auto` it falls back instead, and says so in the plan. Without the anchor + // the fallback and a working merge look identical from the rows alone -- which is + // exactly the point of the fallback. + sql """switch ${catalogName}""" + sql """use fluss_test""" + sql """set enable_file_scanner_v2 = true""" + def degradedPlan = planOf("""select * from lake_pk_part_int""") + assertTrue(degradedPlan.contains("unionRead=no"), + "the halves should not have been merged: ${degradedPlan}") + assertTrue(degradedPlan.contains("degraded=partition-type"), + "the plan should say why it fell back: ${degradedPlan}") + assertEquals(0, countIn(degradedPlan, "lakeSplits"), "no lake half is read: ${degradedPlan}") + assertTrue(countIn(degradedPlan, "pkRanges") >= 1, "the whole table comes from fluss: ${degradedPlan}") + + // And the rows are still all of them -- a primary-key table's fluss-only read is the + // whole table, not the part the lake has not taken yet. + order_qt_lake_pk_part_int """select id, name, p_int from lake_pk_part_int""" + order_qt_lake_pk_part_int_pruned """select id, name from lake_pk_part_int where p_int = 1""" + + sql """switch internal""" + sql """drop catalog if exists ${catalogName}""" + sql """drop catalog if exists ${varbinaryCatalog}""" + sql """drop catalog if exists ${unionCatalog}""" + sql """drop catalog if exists ${flussOnlyCatalog}""" +} diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_pk_table.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_pk_table.groovy new file mode 100644 index 00000000000000..af29f1f8a7d79b --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_pk_table.groovy @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Reading fluss primary-key tables end to end. What separates this from the log +// suite is that the answer is not what the fixture wrote but what the writes add +// up to: a fluss primary-key table stores a change log holding every intermediate +// state, and the read has to merge a kv snapshot with the log that followed it. +// Returning the change log instead would show up here as extra rows that each look +// entirely plausible. +// +// The part only this suite can cover is where those snapshot files come from: the +// fluss container writes them and Doris BE reads them from the host filesystem, at +// the same absolute path on both sides. Startup does not report the environment +// ready until every primary-key fixture has a snapshot on disk (see the docker +// README), so these queries take that path rather than replaying the change log. +// +// Fixtures come from docker/thirdparties/docker-compose/fluss/sql/init.sql and are +// static - this suite never writes, so no polling gate is needed. Results are +// recorded into the .out baseline; what stays in the code is what is not a result: +// the EXPLAIN anchors saying how the scan was planned, and range-count bounds, +// which cannot be exact because key-to-bucket hashing is fluss's business. +suite("test_fluss_pk_table", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String catalogName = "test_fluss_pk_table" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}" + ); + """ + sql """switch ${catalogName}""" + sql """use fluss_test""" + + // The connector is wired into the v2 file scanner only; the legacy one answers + // "Not supported create reader for table format: fluss". Fuzzy sessions randomize + // this variable, so pinning it is what keeps the suite from failing on half the + // CI runs for a reason that has nothing to do with fluss. + sql """set enable_file_scanner_v2 = true""" + // TIMESTAMP_LTZ renders through the session time zone, and the baseline below + // records what it rendered as. Without pinning it, the recorded value would be + // whatever the machine that generated the baseline happened to be set to. + sql """set time_zone = 'Asia/Shanghai'""" + + def planOf = { String query -> + def planRows = sql("""explain ${query}""") + return planRows.collect { it[0].toString() }.join("\n") + } + def pkRangesOf = { String plan -> + def matcher = (plan =~ /pkRanges=(\d+)/) + assertTrue(matcher.find(), "plan has no flussScan line: ${plan}") + return matcher.group(1) as int + } + + // --- the merged view ---------------------------------------------------- + // The fixture inserts four rows, updates one and deletes one. Four writes, three + // rows: id 2 carries its later value and id 3 is gone. Reading the change log + // straight through would answer six rows here, all of them real records. + order_qt_basic_all """select id, name, score from pk_basic""" + + // COUNT(*) projects no column at all. On a primary-key table the count still has + // to be the merged one: counting change log records would report six. + order_qt_basic_count """select count(*) from pk_basic""" + + // The deleted key must be absent, not merely superseded; the updated key must + // appear once, not once per version, and never under its old value. + order_qt_basic_deleted_key """select count(*) from pk_basic where id = 3""" + order_qt_basic_updated_key """select count(*) from pk_basic where id = 2""" + order_qt_basic_stale_value """select count(*) from pk_basic where name = 'k2'""" + + // --- projection and predicates ----------------------------------------- + // The merge needs the primary key even when the query does not ask for it, so the + // reader adds it to what it fetches and projects it back out. A leaked key column + // or a fetch-order projection shows up as the wrong values here. + order_qt_basic_names """select name from pk_basic""" + + order_qt_basic_reordered """select score, name from pk_basic where id = 1""" + + order_qt_basic_filtered """select count(*) from pk_basic where score > 2.0""" + + // --- every mapped type, in the kv row format --------------------------- + // Primary-key tables store rows in a different format from a log table's, so this + // repeats the type coverage rather than trusting the log suite for it. Both rows + // are recorded whole -- the populated one and the all-NULL one, where a null map + // off by one column shifts every value after it. + // + // BINARY and BYTES go through hex(): they map to a Doris string, and a raw byte + // in a recorded baseline is neither readable nor safely round-tripped. + order_qt_types_all """ + select id, f_boolean, f_tinyint, f_smallint, f_int, f_bigint, f_float, f_double, + f_decimal, f_char, f_string, hex(f_binary) as f_binary_hex, + hex(f_bytes) as f_bytes_hex, f_date, f_timestamp, f_timestamp_ltz, + f_array, f_map, f_row + from pk_types + """ + + // The nested readers, asked for one element at a time: a struct that decoded into + // the right shape but the wrong field order still renders plausibly above. + order_qt_types_nested """ + select id, array_size(f_array), f_array[1], f_map['k1'], f_map['k2'], + struct_element(f_row, 'r_int'), struct_element(f_row, 'r_string') + from pk_types + """ + + // --- partitioned primary-key table -------------------------------------- + // A partitioned primary-key table is snapshotted per partition, so its snapshots + // have to be asked for per partition too; asking at table level resumes the change + // log at another partition's offset. The partition column itself is not read by the + // scanner - FE declares it and BE fills it in from each range - so recording it + // next to the row it belongs to is what catches a partition value on the wrong split. + order_qt_part_all """select id, name, dt from pk_part""" + + // The delete landed in 20260102 and the update in 20260101: a merge that crossed + // partitions would lose or resurrect one of them. + order_qt_part_grouped """select dt, count(*) from pk_part group by dt""" + + order_qt_part_pruned """select id, name from pk_part where dt = '20260101'""" + + // --- planning is visible in the plan ------------------------------------ + def basicPlan = planOf("""select * from pk_basic""") + assertTrue(basicPlan.contains("flussScan: unionRead=no"), + "no fluss scan line: ${basicPlan}") + assertTrue(basicPlan.contains("lakeSplits=0"), "unexpected lake splits: ${basicPlan}") + assertTrue(basicPlan.contains("mode=auto"), "unexpected union read mode: ${basicPlan}") + // A primary-key table produces primary-key ranges and no log ranges. Counting them + // together would hide a table planned as a raw log read, which returns superseded + // rows and otherwise looks like a working query. + assertTrue(basicPlan.contains("logRanges=0"), + "a primary-key table was planned as a log read: ${basicPlan}") + // One range per bucket that holds anything. Which of the three buckets a key hashes + // into is fluss's choice, so the count is bounded rather than fixed. + def basicRanges = pkRangesOf(basicPlan) + assertTrue(basicRanges >= 1 && basicRanges <= 3, + "pk_basic planned ${basicRanges} ranges over 3 buckets") + + def fullPartPlan = planOf("""select * from pk_part""") + assertTrue(fullPartPlan.contains("partition=2/2"), + "both partitions should be scanned: ${fullPartPlan}") + + def prunedPlan = planOf("""select * from pk_part where dt = '20260101'""") + assertTrue(prunedPlan.contains("partition=1/2"), + "partition pruning did not reach the connector: ${prunedPlan}") + // Pruning has to shrink the work, not just the plan line: a partition name rendered + // one way for the listing and another way for the match would prune to nothing here + // while still reporting 1/2. + def prunedRanges = pkRangesOf(prunedPlan) + assertTrue(prunedRanges >= 1 && prunedRanges <= 2, + "one partition has 2 buckets, planned ${prunedRanges} ranges: ${prunedPlan}") + assertTrue(prunedRanges <= pkRangesOf(fullPartPlan), + "pruning planned more ranges than the full scan: ${prunedPlan}") + + sql """switch internal""" + sql """drop catalog ${catalogName}""" +} diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_predicates.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_predicates.groovy new file mode 100644 index 00000000000000..15d402f4eba328 --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_predicates.groovy @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Predicates that are more than one comparison. +// +// The other suites filter with a single predicate, which is the shape that cannot +// go wrong in an interesting way. Combinations can: a disjunction over a partition +// column has to keep every partition it names rather than the last one; a +// conjunction of a partition predicate and a data predicate must prune on the first +// without dropping rows the second would have kept; a NOT around a partition +// predicate must not prune at all. And for a table read as lake plus log, a +// predicate that reached only one half returns an answer that is short by exactly +// the other half's matching rows -- a plausible number, always. +// +// This suite pins WHAT the answers are; where a predicate is evaluated is not +// asserted, because it is the optimizer's choice and pinning it would fail on the +// day the optimizer gets better rather than worse. The union-read blocks compare +// the two read paths instead, which is the property that has to hold however the +// predicate was routed. +// +// Fixtures: docker/thirdparties/docker-compose/fluss/sql/init.sql. +suite("test_fluss_predicates", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String catalogName = "test_fluss_predicates" + String flussOnlyCatalog = "test_fluss_predicates_off" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "required" + ); + """ + sql """drop catalog if exists ${flussOnlyCatalog}""" + sql """ + create catalog ${flussOnlyCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "disabled" + ); + """ + sql """switch ${catalogName}""" + sql """use fluss_test""" + sql """set enable_file_scanner_v2 = true""" + // The timestamp columns below are compared against literals, and TIMESTAMP_LTZ + // renders through the session time zone. + sql """set time_zone = 'Asia/Shanghai'""" + + def rowsOf = { String query -> sql(query).collect { row -> row.collect { it.toString() } } } + def planOf = { String query -> + return sql("""explain ${query}""").collect { it[0].toString() }.join("\n") + } + def compareModes = { String query -> + def union = rowsOf("""${query}""") + def flussOnly = rowsOf("""${query}""".replace("from ", "from ${flussOnlyCatalog}.fluss_test.")) + assertEquals(flussOnly, union, + "lake+log and fluss-only disagree for: ${query}\nfluss-only=${flussOnly}\nunion=${union}") + } + + // --- conjunction, disjunction, negation on a log table -------------------- + order_qt_and """select id, name from log_basic where id > 1 and price < 25.00""" + order_qt_or """select id, name from log_basic where id = 1 or price > 25.00""" + // The parenthesised mixture: a reader that flattened it into a chain of ANDs or + // ORs answers with everything or nothing, and both look like working code. + order_qt_and_or """ + select id, name from log_basic + where (id = 1 and price < 15.00) or (id = 3 and price > 25.00) + """ + order_qt_or_and """ + select id, name from log_basic + where (id = 1 or id = 3) and price > 25.00 + """ + order_qt_not """select id, name from log_basic where not (id = 1 or price > 25.00)""" + order_qt_not_between """select id from log_basic where id not between 2 and 3""" + order_qt_in """select id, name from log_basic where id in (1, 3) and name <> 'carol'""" + order_qt_not_in """select id from log_basic where id not in (1, 2)""" + order_qt_like """select id from log_basic where name like '%o%' and name not like 'c%'""" + // A predicate that selects nothing, and one that selects everything: the two ends + // of the range, where an inverted condition is easiest to mistake for a working one. + order_qt_always_false """select count(*) from log_basic where id > 0 and id < 0""" + order_qt_always_true """select count(*) from log_basic where id > 0 or name is not null""" + + // --- NULLs, which no comparison is true of -------------------------------- + // The all-NULL row is row 3 of log_types. `f_int <> 3` must not return it, `is null` + // must, and `not (f_int = 3)` must not either -- three-valued logic is exactly what + // a filter pushed down and re-implemented gets wrong. + order_qt_is_null """select id from log_types where f_int is null""" + order_qt_is_not_null """select id from log_types where f_int is not null""" + order_qt_ne_excludes_null """select id from log_types where f_int <> 3""" + order_qt_not_eq_excludes_null """select id from log_types where not (f_int = 3)""" + order_qt_null_or """select id from log_types where f_int is null or f_int > 0""" + order_qt_null_and """select id from log_types where f_string is null and f_boolean is null""" + + // Across types, in one condition. A predicate list that stopped at the first column + // it could not push would return the rows the rest were meant to exclude. + order_qt_mixed_types """ + select id from log_types + where f_boolean = true and f_tinyint > 0 and f_decimal > 0 and f_char = 'char1' + and f_date = '2026-01-01' and f_string like 'string%' + """ + order_qt_mixed_types_or """ + select id from log_types + where f_bigint = 4 or f_double < 0 or (f_float is null and f_smallint is null) + """ + // Reaching into a complex value from a predicate: the element has to be decoded + // before it can be compared, so this cannot have been answered by the scanner. + order_qt_nested_elements """ + select id from log_types + where f_array[1] = 1 and f_map['k1'] = 1 and struct_element(f_row, 'r_int') = 1 + """ + + // --- a partition column in a compound predicate --------------------------- + // Pruning takes the partitions the predicate leaves possible, so a disjunction has + // to keep both of the partitions it names and a negation has to keep every one it + // does not exclude. The plan line is asserted alongside the rows because a + // conservative pruner returning too many partitions still gives the right rows. + def partOrPlan = planOf("""select * from log_part where dt = '20260101' or dt = '20260103'""") + assertTrue(partOrPlan.contains("partition=2/3"), + "a disjunction over partitions should keep both: ${partOrPlan}") + order_qt_part_or """select id, name, dt from log_part where dt = '20260101' or dt = '20260103'""" + + def partInPlan = planOf("""select * from log_part where dt in ('20260101', '20260102')""") + assertTrue(partInPlan.contains("partition=2/3"), "IN over partitions: ${partInPlan}") + order_qt_part_in """select id, dt from log_part where dt in ('20260101', '20260102') and id > 1""" + + def partNotPlan = planOf("""select * from log_part where dt <> '20260101'""") + assertTrue(partNotPlan.contains("partition=2/3"), "a negated partition predicate: ${partNotPlan}") + order_qt_part_ne """select id, dt from log_part where dt <> '20260101'""" + + // A partition predicate and a data predicate together: the first prunes, the second + // must still be applied to what is left rather than assumed satisfied. + def partAndDataPlan = planOf("""select * from log_part where dt = '20260101' and id = 2""") + assertTrue(partAndDataPlan.contains("partition=1/3"), "pruning still applies: ${partAndDataPlan}") + order_qt_part_and_data """select id, name from log_part where dt = '20260101' and id = 2""" + + // A disjunction that mixes the two cannot prune at all: the row it selects lives in + // a partition the partition predicate excludes. + def partOrDataPlan = planOf("""select * from log_part where dt = '20260101' or id = 4""") + assertTrue(partOrDataPlan.contains("partition=3/3"), + "a disjunction with a data predicate must not prune: ${partOrDataPlan}") + order_qt_part_or_data """select id, name, dt from log_part where dt = '20260101' or id = 4""" + + order_qt_part_absent """select count(*) from log_part where dt = '20261231'""" + + // --- a primary-key table, where a predicate is applied after the merge ---- + // The change log holds superseded and deleted rows; a predicate evaluated against + // it rather than against the merged view brings them back. 'k2' is the value key 2 + // used to have, and key 3 was deleted. + order_qt_pk_and """select id, name from pk_basic where id >= 2 and score > 3.0""" + order_qt_pk_or """select id, name from pk_basic where name = 'k1' or score > 20.0""" + order_qt_pk_stale_value """select count(*) from pk_basic where name = 'k2' or id = 3""" + order_qt_pk_not """select id, name from pk_basic where not (id = 1) and name is not null""" + order_qt_pk_in """select id, score from pk_basic where id in (1, 2, 3) and score is not null""" + + // --- a table read as lake plus log ---------------------------------------- + // Whichever half the optimizer pushed the predicate to, both halves have to end up + // filtered by it. The comparison is the check: a predicate that reached only the + // lake returns the log's matching rows unfiltered, and one that reached only the log + // returns the lake's -- either way a plausible count. + order_qt_union_and """select id, name from lake_log where id > 2 and price < 6.00""" + compareModes("""select id, name from lake_log where id > 2 and price < 6.00 order by id""") + + order_qt_union_or """select id, name from lake_log where id = 1 or id = 6""" + compareModes("""select id, name from lake_log where id = 1 or id = 6 order by id""") + + order_qt_union_not """select id from lake_log where not (id between 2 and 5)""" + compareModes("""select id from lake_log where not (id between 2 and 5) order by id""") + + order_qt_union_like """select id, name from lake_log where name like 'hot%' or name like '%1'""" + compareModes("""select id, name from lake_log where name like 'hot%' or name like '%1' order by id""") + + // A predicate that only the lake half can satisfy, and one that only the log half + // can: each is the case where returning the other half unfiltered is invisible in + // the row count of the first. + order_qt_union_lake_only """select id, name from lake_log where id <= 4 and price < 5.00""" + order_qt_union_log_only """select id, name from lake_log where id >= 5""" + order_qt_union_neither """select count(*) from lake_log where id > 100""" + + // The same over a partitioned lake table, where pruning and the seam interact: one + // partition has a tail and the other does not. + order_qt_union_part_or """ + select id, name, dt from lake_part where dt = '20260101' or dt = '20260102' + """ + compareModes(""" + select id, name, dt from lake_part where dt = '20260101' or dt = '20260102' order by id + """) + order_qt_union_part_and """select id from lake_part where dt = '20260101' and id > 2""" + compareModes("""select id from lake_part where dt = '20260101' and id > 2 order by id""") + + // And over a primary-key lake table, where the predicate is applied to the merged + // view of both halves: key 1 was deleted by the tail and key 3 updated by it. + order_qt_union_pk_and """select id, name from lake_pk where id >= 2 and name like '%hot'""" + compareModes("""select id, name from lake_pk where id >= 2 and name like '%hot' order by id""") + order_qt_union_pk_or """select id, name from lake_pk where id = 1 or id = 4""" + compareModes("""select id, name from lake_pk where id = 1 or id = 4 order by id""") + + sql """switch internal""" + sql """drop catalog if exists ${catalogName}""" + sql """drop catalog if exists ${flussOnlyCatalog}""" +} diff --git a/regression-test/suites/external_table_p0/fluss/test_fluss_union_log.groovy b/regression-test/suites/external_table_p0/fluss/test_fluss_union_log.groovy new file mode 100644 index 00000000000000..b5acc6a2802343 --- /dev/null +++ b/regression-test/suites/external_table_p0/fluss/test_fluss_union_log.groovy @@ -0,0 +1,188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Reading a tiered log table as its lake plus the log written after it. +// +// The lake half is planned by the paimon connector on a snapshot the fluss +// coordinator pinned, the log half by fluss from the offsets that snapshot +// stopped at, and both halves end up as ranges of one scan node -- so BE builds +// a paimon reader and a fluss reader for the same query. +// +// The load-bearing assertion is the comparison between the two modes. `required` +// reads lake plus log; `disabled` replays the whole fluss log, which still holds +// everything because tiering copies rather than moves. Two entirely different +// readers over one table: if the seam between the halves is off by a row in +// either direction -- a row counted twice, a row skipped -- the two disagree. +// A single-mode suite could not tell a correct seam from a plausible one. +// +// Fixtures come from docker/thirdparties/docker-compose/fluss/sql/init.sql and +// are frozen: the tiering service is stopped before the log tail is written. +suite("test_fluss_union_log", "p0,external") { + String enabled = context.config.otherConfigs.get("enableFlussTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String coordinatorPort = context.config.otherConfigs.get("fluss_coordinator_port") + String bootstrapServers = "${externalEnvIp}:${coordinatorPort}" + String unionCatalog = "test_fluss_union_log" + String flussOnlyCatalog = "test_fluss_union_log_off" + + // Two catalogs rather than one reconfigured between queries: the read mode is + // a catalog property, and having both live at once is what lets the same query + // be run down both paths and compared. + sql """drop catalog if exists ${unionCatalog}""" + sql """ + create catalog ${unionCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "required" + ); + """ + sql """drop catalog if exists ${flussOnlyCatalog}""" + sql """ + create catalog ${flussOnlyCatalog} properties ( + "type" = "fluss", + "fluss.bootstrap.servers" = "${bootstrapServers}", + "fluss.union_read.mode" = "disabled" + ); + """ + + sql """switch ${unionCatalog}""" + sql """use fluss_test""" + sql """set enable_file_scanner_v2 = true""" + + def rowsOf = { String query -> sql(query).collect { row -> row.collect { it.toString() } } } + def planOf = { String query -> + return sql("""explain ${query}""").collect { it[0].toString() }.join("\n") + } + def countIn = { String plan, String field -> + def matcher = (plan =~ /${field}=(\d+)/) + assertTrue(matcher.find(), "plan has no ${field}: ${plan}") + return matcher.group(1) as int + } + + // --- both halves are planned, and the plan says which is which ----------- + def logPlan = planOf("""select * from lake_log""") + assertTrue(logPlan.contains("flussScan: unionRead=yes"), "not a union read: ${logPlan}") + assertTrue(logPlan.contains("mode=required"), "unexpected mode: ${logPlan}") + assertTrue(countIn(logPlan, "lakeSplits") >= 1, "no lake splits: ${logPlan}") + // Four rows were tiered across three buckets and two written after, so at + // least one bucket has caught up with the lake and must contribute nothing. + // Which buckets the writer chose is its own business, hence a bound. + def logRanges = countIn(logPlan, "logRanges") + assertTrue(logRanges >= 1 && logRanges <= 2, + "lake_log planned ${logRanges} log ranges over 3 buckets: ${logPlan}") + assertEquals(0, countIn(logPlan, "pkRanges")) + + // --- the seam: same rows down both paths --------------------------------- + // The comparison stays in the code even though the results are recorded: it is + // not a hand-written expectation but the load-bearing check of this suite, and + // it says something no pair of recorded blocks does -- that two entirely + // different readers agreed on this table, row for row. + def compareModes = { String query -> + def union = rowsOf("""${query}""") + def flussOnly = rowsOf("""${query}""".replace("from ", "from ${flussOnlyCatalog}.fluss_test.")) + assertEquals(flussOnly, union, + "lake+log and fluss-only disagree for: ${query}\nfluss-only=${flussOnly}\nunion=${union}") + } + + // Four rows out of the lake and two out of the log, in one result. + order_qt_log_rows """select id, name, price from lake_log""" + compareModes("select id, name, price from lake_log order by id") + + // Aggregates run over both halves; a half silently dropped shows up here even + // when the ordered row list is not materialized. + order_qt_log_count """select count(*) from lake_log""" + order_qt_log_sum """select sum(price) from lake_log""" + + // --- a table the lake already holds in full ------------------------------ + // Nothing was written after tiering caught up, so every bucket's log range is + // empty and planning must emit none at all rather than ranges that read + // nothing. + def coldPlan = planOf("""select * from lake_cold""") + assertTrue(coldPlan.contains("unionRead=yes"), "not a union read: ${coldPlan}") + assertTrue(countIn(coldPlan, "lakeSplits") >= 1, "no lake splits: ${coldPlan}") + assertEquals(0, countIn(coldPlan, "logRanges"), "cold table still has log ranges: ${coldPlan}") + order_qt_cold_rows """select id, name from lake_cold""" + compareModes("select id, name from lake_cold order by id") + + // --- every type crosses the seam ----------------------------------------- + // Row 1 comes back through paimon, row 2 (the all-NULL one, written after + // tiering stopped) through the fluss log: two decoders feeding one result set, + // which is what pins the mapping parity on real values rather than on declared + // types. A null map read one column off shifts every value after it, and only + // recording the row column by column notices. + // + // BINARY and BYTES go through hex() as everywhere else. An equality predicate on + // the microsecond TIMESTAMP would match nothing for the lake half -- pushed into + // paimon, it finds no row, which is the paimon connector's own behaviour and not + // something this suite should pin; recording the value sidesteps it. + order_qt_types_all """ + select id, f_boolean, f_tinyint, f_smallint, f_int, f_bigint, f_float, f_double, + f_decimal, f_char, f_string, hex(f_binary) as f_binary_hex, + hex(f_bytes) as f_bytes_hex, f_date, f_timestamp, f_timestamp_ltz, + f_array, f_map, f_row + from lake_types + """ + compareModes("select id from lake_types order by id") + + // --- partitioning, where the two halves prune differently ---------------- + // The fluss half is given the partitions the engine pruned to; the paimon half + // ignores that list and prunes on the pushed-down predicate instead. Both have + // to land on the same partition: one partition here has a log tail, the other is + // served entirely from the lake. + order_qt_part_rows """select id, name, dt from lake_part""" + compareModes("select id, name, dt from lake_part order by id") + + def tieredPartPlan = planOf("""select * from lake_part where dt = '20260102'""") + assertTrue(tieredPartPlan.contains("unionRead=yes"), "not a union read: ${tieredPartPlan}") + assertEquals(0, countIn(tieredPartPlan, "logRanges"), + "a fully tiered partition still has log ranges: ${tieredPartPlan}") + order_qt_part_tiered_only """select id from lake_part where dt = '20260102'""" + + def tailPartPlan = planOf("""select * from lake_part where dt = '20260101'""") + assertEquals(1, countIn(tailPartPlan, "logRanges"), + "the partition with a tail lost its log range: ${tailPartPlan}") + order_qt_part_with_tail """select id from lake_part where dt = '20260101'""" + + // --- a primary-key table is not a log table ------------------------------ + // Concatenating its halves the way this suite's tables are concatenated would + // return superseded and deleted rows, so it is merged BY KEY instead -- a + // different plan out of the same mode, asserted here only so that the two + // shapes cannot quietly become one. What such a table reads as lives in + // test_fluss_lake_pk and test_fluss_lake_pk_merge. + def pkPlan = planOf("""select * from lake_pk""") + assertTrue(pkPlan.contains("unionRead=yes"), "not a union read: ${pkPlan}") + assertEquals(0, countIn(pkPlan, "logRanges"), + "a primary-key table planned a plain log range: ${pkPlan}") + assertTrue(countIn(pkPlan, "pkTailRanges") >= 1, "no tail replayed: ${pkPlan}") + + // --- required does not mean "every table has a lake" --------------------- + // A table with no lake at all is not an error in required mode: there is + // nothing to fall back FROM. Only a lake table whose snapshot cannot be read + // is. + def plainPlan = planOf("""select * from log_basic""") + assertTrue(plainPlan.contains("flussScan: unionRead=no"), "unexpected union read: ${plainPlan}") + assertEquals(0, countIn(plainPlan, "lakeSplits")) + order_qt_plain_count """select count(*) from log_basic""" + + sql """switch internal""" + sql """drop catalog if exists ${unionCatalog}""" + sql """drop catalog if exists ${flussOnlyCatalog}""" +}