From 7491b9a86fbd374ea75f375353cdf3f609547494 Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Wed, 21 Oct 2026 16:53:41 +0800 Subject: [PATCH 1/3] feat(predicate): add full-text match and score projection semantics --- docs/source/api/file_index.rst | 92 +++++ include/paimon/file_index/file_index_reader.h | 5 + include/paimon/predicate/full_text_match.h | 52 +++ include/paimon/predicate/function.h | 3 +- include/paimon/predicate/leaf_predicate.h | 3 +- include/paimon/predicate/predicate_builder.h | 10 + include/paimon/predicate/predicate_utils.h | 3 + include/paimon/read_context.h | 31 +- src/paimon/CMakeLists.txt | 2 + .../common/file_index/file_index_reader.cpp | 5 + .../file_index/file_index_reader_test.cpp | 4 + .../predicate/full_text_match_function.h | 67 ++++ .../predicate/full_text_match_predicate.cpp | 64 ++++ .../predicate/full_text_match_predicate.h | 53 +++ .../full_text_match_predicate_test.cpp | 148 ++++++++ .../common/predicate/predicate_builder.cpp | 16 + .../common/predicate/predicate_utils.cpp | 23 ++ .../common/predicate/predicate_utils_test.cpp | 43 +++ .../predicate/predicate_validator_test.cpp | 43 +++ src/paimon/common/table/special_fields.h | 18 +- .../common/table/special_fields_test.cpp | 13 + src/paimon/core/operation/file_store_scan.cpp | 3 + .../core/operation/file_store_scan_test.cpp | 24 ++ .../core/operation/internal_read_context.cpp | 101 +++++- .../core/operation/internal_read_context.h | 31 +- .../operation/internal_read_context_test.cpp | 329 ++++++++++++++++++ src/paimon/core/operation/read_context.cpp | 18 + .../core/operation/read_context_test.cpp | 37 ++ .../core/table/source/table_read_test.cpp | 52 +++ src/paimon/core/table/table.cpp | 14 + src/paimon/core/utils/field_mapping.cpp | 21 +- src/paimon/core/utils/field_mapping_test.cpp | 43 +++ test/inte/predicate_abi_inte_test.cpp | 15 + 33 files changed, 1361 insertions(+), 25 deletions(-) create mode 100644 include/paimon/predicate/full_text_match.h create mode 100644 src/paimon/common/predicate/full_text_match_function.h create mode 100644 src/paimon/common/predicate/full_text_match_predicate.cpp create mode 100644 src/paimon/common/predicate/full_text_match_predicate.h create mode 100644 src/paimon/common/predicate/full_text_match_predicate_test.cpp diff --git a/docs/source/api/file_index.rst b/docs/source/api/file_index.rst index e95cd26f2..928468f02 100644 --- a/docs/source/api/file_index.rst +++ b/docs/source/api/file_index.rst @@ -21,9 +21,101 @@ File Index .. _cpp-api-file-index: +Full-text predicates and root TopK +================================= + +``PredicateBuilder::FullTextMatch`` creates a ``LeafPredicate`` for a full-text +condition without TopK truncation. It holds a shared ``FullTextMatch`` request, +whose query uses the same JSON DSL as ``FullTextSearch``. Do not modify a request +after passing it to a predicate or read context. + +Compose full-text and scalar predicates with the existing +``PredicateBuilder::And`` and ``PredicateBuilder::Or`` factories. Every field +referenced by this tree must appear in the read schema; field indices refer to +positions in that schema. For example, the following constructs +``body matches "apple" OR price < 10``: + +.. code-block:: cpp + + auto request = std::make_shared( + "body", R"({"match":{"query":"apple"}})"); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr text, + paimon::PredicateBuilder::FullTextMatch(0, request)); + auto price = paimon::PredicateBuilder::LessThan( + 1, "price", paimon::FieldType::INT, paimon::Literal(10)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr filter, + paimon::PredicateBuilder::Or({text, price})); + paimon::ReadContextBuilder builder(table_path); + builder.SetReadFieldNames({"body", "price"}).SetPredicate(filter); + +``SetVectorSearch`` and ``SetFullTextSearch`` configure a separate root TopK +request. They are mutually exclusive and require a positive limit. The intended +semantics are to filter by the complete predicate before ranking; the TopK search +field does not need to be projected. A full-text match inside a predicate never +has a TopK limit. + +Score output is requested only through an explicit ``__INDEX_SCORE`` +projection. ``FullTextMatch``, ``SetVectorSearch`` and ``SetFullTextSearch`` have +no separate score switch. For example, request the score of a full-text +predicate by projecting ``_body_INDEX_SCORE``: + +.. code-block:: cpp + + auto request = std::make_shared( + "body", R"({"match":{"query":"apple"}})"); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr text, + paimon::PredicateBuilder::FullTextMatch(0, request)); + paimon::ReadContextBuilder builder(table_path); + builder.SetReadFieldNames({"body", "_body_INDEX_SCORE"}).SetPredicate(text); + +Score fields have type ``float32`` and preserve the source field's exact name, +including case. Select them with ``SetReadFieldNames`` or ``SetReadSchema``; +they have no persistent field IDs for ``SetReadFieldIds``. Reading all table +fields does not implicitly add score fields. + +Root TopK searches always need scores internally to rank and merge results +across files, regardless of the projection. A full-text predicate without TopK +needs scores only when its score field is projected. + +Each projected score must have exactly one search condition as its source. +The source may be a root TopK request or a full-text predicate. A full-text +predicate whose score is projected cannot have an OR ancestor, and its field +cannot occur in another search condition, including the root TopK request. +An ordinary scalar predicate on the same field is allowed. For example, when +projecting the score of ``match``, ``match AND (p OR q)`` is valid, while +``match OR p`` is rejected. Without that score projection, both OR branches and +repeated full-text search fields are allowed. These checks use the effective +read projection, after resolving the read schema. + +Internally, the read context binds each score output position to a search type +and the original table field name. The binding identifies a logical source, +not a request object's identity. Execution must preserve this source when +copying requests for per-file field mapping or candidate filtering. +``NeedsScores`` derives execution requirements from the root TopK request and +these bindings; it does not add another user configuration. + +This first stage implements predicate construction, validation, field mapping, +and read-context configuration. Executing a full-text predicate, including using +it for scan pruning, returns ``NotImplemented``. Valid per-field score +projections also return ``NotImplemented`` when reading. Existing scalar and root +search execution retain their current capabilities and limitations. + +The next stage will implement exact mixed-predicate execution, per-file index +reader reuse, and selectivity-based AND ordering with candidate propagation. +Missing full-text or vector indexes must fail; scalar conditions may fall back +to exact data scanning. TopK must execute after the complete filter, with scores +and per-file results combined by the read layer. These execution changes are +not part of the predicate interface stage. + Interface ========= +.. doxygenstruct:: paimon::FullTextMatch + :members: + .. doxygenclass:: paimon::FileIndexer :members: :undoc-members: diff --git a/include/paimon/file_index/file_index_reader.h b/include/paimon/file_index/file_index_reader.h index e6cb39c79..6118ebd3c 100644 --- a/include/paimon/file_index/file_index_reader.h +++ b/include/paimon/file_index/file_index_reader.h @@ -31,6 +31,7 @@ #include "paimon/visibility.h" namespace paimon { +struct FullTextMatch; /// Evaluates filter predicates against a file-level index to determine file eligibility. /// /// `FileIndexReader` implements the `FunctionVisitor` interface specialized to produce @@ -79,6 +80,10 @@ class PAIMON_EXPORT FileIndexReader : public FunctionVisitor> VisitFullTextSearch( const std::shared_ptr& full_text_search); + + /// Return every matching file-local physical row position, without TopK truncation. + virtual Result> VisitFullTextMatch( + const std::shared_ptr& full_text_match); }; } // namespace paimon diff --git a/include/paimon/predicate/full_text_match.h b/include/paimon/predicate/full_text_match.h new file mode 100644 index 000000000..f2d8cbcea --- /dev/null +++ b/include/paimon/predicate/full_text_match.h @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/utils/roaring_bitmap64.h" +#include "paimon/visibility.h" + +namespace paimon { + +/// A full-text condition returning every matching row without TopK truncation. +/// Project `__INDEX_SCORE` in the read schema to return its scores. +struct PAIMON_EXPORT FullTextMatch { + FullTextMatch(const std::string& _field_name, const std::string& _query, + std::optional _include_row_ids = std::nullopt) + : field_name(_field_name), query(_query), include_row_ids(std::move(_include_row_ids)) {} + + bool operator==(const FullTextMatch& other) const { + return field_name == other.field_name && query == other.query && + include_row_ids == other.include_row_ids; + } + + /// Table column containing the indexed text. + std::string field_name; + /// JSON DSL query with the same syntax and analyzer semantics as `FullTextSearch::query`. + std::string query; + /// Optional eligible row ids. File readers use file-local physical row positions; + /// global readers use global row ids. An empty set matches no rows. If not + /// set, all rows are searched. + std::optional include_row_ids; +}; + +} // namespace paimon diff --git a/include/paimon/predicate/function.h b/include/paimon/predicate/function.h index f383f7437..5d84f5c1b 100644 --- a/include/paimon/predicate/function.h +++ b/include/paimon/predicate/function.h @@ -43,7 +43,8 @@ class PAIMON_EXPORT Function { STARTS_WITH = 13, ENDS_WITH = 14, CONTAINS = 15, - LIKE = 16 + LIKE = 16, + FULL_TEXT_MATCH = 17 }; virtual ~Function() = default; virtual Type GetType() const = 0; diff --git a/include/paimon/predicate/leaf_predicate.h b/include/paimon/predicate/leaf_predicate.h index 296984875..b57d8173f 100644 --- a/include/paimon/predicate/leaf_predicate.h +++ b/include/paimon/predicate/leaf_predicate.h @@ -31,7 +31,8 @@ class LeafFunction; class Function; enum class FieldType; -/// Leaf node of a `Predicate` tree. Compares a field with literals. +/// Leaf node of a `Predicate` tree describing a condition on one field. +/// Scalar leaves compare the field with literals; full-text leaves hold a search request. class PAIMON_EXPORT LeafPredicate : virtual public Predicate { public: int32_t FieldIndex() const { diff --git a/include/paimon/predicate/predicate_builder.h b/include/paimon/predicate/predicate_builder.h index c8cdf13de..4c829408d 100644 --- a/include/paimon/predicate/predicate_builder.h +++ b/include/paimon/predicate/predicate_builder.h @@ -27,6 +27,7 @@ #include "paimon/visibility.h" namespace paimon { +struct FullTextMatch; class Literal; enum class FieldType; @@ -39,6 +40,15 @@ class PAIMON_EXPORT PredicateBuilder { PredicateBuilder() = delete; ~PredicateBuilder() = delete; + /// Create a full-text predicate returning all matching rows without TopK truncation. + /// The field must have STRING type and a full-text index. + /// Project `__INDEX_SCORE` in the read schema to request score output. + /// @param field_index The index of the field in read schema (0-based). + /// @param match The full-text request, including the field name, JSON DSL query and + /// optional eligible row ids. + static Result> FullTextMatch( + int32_t field_index, const std::shared_ptr& match); + /// Create an equality predicate (field == literal). /// /// @param field_index The index of the field in read schema (0-based). diff --git a/include/paimon/predicate/predicate_utils.h b/include/paimon/predicate/predicate_utils.h index cef3b5fa7..6195188e8 100644 --- a/include/paimon/predicate/predicate_utils.h +++ b/include/paimon/predicate/predicate_utils.h @@ -61,6 +61,9 @@ class PAIMON_EXPORT PredicateUtils { static Status GetAllNames(const std::shared_ptr& predicate, std::set* field_names); + /// Whether the predicate contains a full-text matching condition. + static bool ContainsFullTextMatch(const std::shared_ptr& predicate); + template static Result VisitPredicate(const std::shared_ptr& predicate, const std::shared_ptr>& visitor) { diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index 28305c94c..88fc0a547 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -108,6 +108,7 @@ class PAIMON_EXPORT ReadContext { return full_text_search_; } + /// Whether a root TopK request is configured; full-text predicates are not TopK requests. bool HasVectorOrFullTextSearch() const { return vector_search_ != nullptr || full_text_search_ != nullptr; } @@ -346,24 +347,34 @@ class PAIMON_EXPORT ReadContextBuilder { /// It can significantly improve performance by reducing the amount of data /// that needs to be read and processed. /// + /// All referenced fields, including full-text match fields, must be projected in the read + /// schema. Full-text matches can be combined with scalar predicates using + /// PredicateBuilder::And/Or. Full-text predicate execution is not implemented yet; reading such + /// a predicate returns NotImplemented. + /// /// @param predicate Shared pointer to the predicate for data filtering. /// @return Reference to this builder for method chaining. ReadContextBuilder& SetPredicate(const std::shared_ptr& predicate); - /// Set the vector search request used when reading data. - /// - /// TODO(jinli.zjw): Support combining vector search with predicates and full-text search. Merge - /// per-file candidates to compute read-level top-k results. - /// + /// Configure the root vector TopK request, logically applied after the predicate filter. + /// The search field need not be projected. SetVectorSearch is mutually exclusive with + /// SetFullTextSearch and must have a positive limit. + /// Scores are required internally for TopK. Project `__INDEX_SCORE` to include them + /// in the result. Per-field score output is not implemented yet; reading such a projection + /// returns NotImplemented. + /// TODO(lisizhuo.lsz): Execute combined read predicates and merge TopK across files. /// @param vector_search Vector search request, or nullptr to clear it. /// @return Reference to this builder for method chaining. ReadContextBuilder& SetVectorSearch(const std::shared_ptr& vector_search); - /// Set the full-text search request used when reading data. - /// - /// TODO(jinli.zjw): Support combining full-text search with predicates and vector search. Apply - /// the search limit to merged per-file results at read level. - /// + /// Configure the root full-text TopK request, logically applied after the predicate filter. + /// The search field need not be projected. Use PredicateBuilder::FullTextMatch for a + /// full-text filter without TopK truncation. SetFullTextSearch is mutually exclusive with + /// SetVectorSearch and must have a positive limit. + /// Scores are required internally for TopK. Project `__INDEX_SCORE` to include them + /// in the result. Per-field score output is not implemented yet; reading such a projection + /// returns NotImplemented. + /// TODO(lisizhuo.lsz): Execute combined read predicates and merge TopK across files. /// @param full_text_search Full-text search request, or nullptr to clear it. /// @return Reference to this builder for method chaining. ReadContextBuilder& SetFullTextSearch(const std::shared_ptr& full_text_search); diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index a8a4afe4c..f3f48a96c 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -118,6 +118,7 @@ set(PAIMON_COMMON_SRCS common/predicate/contains.cpp common/predicate/ends_with.cpp common/predicate/equal.cpp + common/predicate/full_text_match_predicate.cpp common/predicate/greater_or_equal.cpp common/predicate/greater_than.cpp common/predicate/in.cpp @@ -669,6 +670,7 @@ if(PAIMON_BUILD_TESTS) common/metrics/histogram_windowing_test.cpp common/options/memory_size_test.cpp common/options/time_duration_test.cpp + common/predicate/full_text_match_predicate_test.cpp common/predicate/full_text_search_test.cpp common/predicate/literal_converter_test.cpp common/predicate/literal_test.cpp diff --git a/src/paimon/common/file_index/file_index_reader.cpp b/src/paimon/common/file_index/file_index_reader.cpp index 9350e23f1..7151c5e5a 100644 --- a/src/paimon/common/file_index/file_index_reader.cpp +++ b/src/paimon/common/file_index/file_index_reader.cpp @@ -111,4 +111,9 @@ Result> FileIndexReader::VisitFullTextSearch( const std::shared_ptr& full_text_search) { return Status::NotImplemented("This file index does not support full text search"); } + +Result> FileIndexReader::VisitFullTextMatch( + const std::shared_ptr& full_text_match) { + return Status::NotImplemented("This file index does not support complete full-text matching"); +} } // namespace paimon diff --git a/src/paimon/common/file_index/file_index_reader_test.cpp b/src/paimon/common/file_index/file_index_reader_test.cpp index 86745c3e4..299eeb2da 100644 --- a/src/paimon/common/file_index/file_index_reader_test.cpp +++ b/src/paimon/common/file_index/file_index_reader_test.cpp @@ -21,6 +21,7 @@ #include #include "gtest/gtest.h" +#include "paimon/predicate/full_text_match.h" #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/vector_search.h" @@ -75,5 +76,8 @@ TEST(FileIndexReaderTest, TestDefaultIndexReader) { std::make_shared("body", R"({"match":{"query":"word"}})", /*limit=*/1); ASSERT_NOK_WITH_MSG(reader.VisitFullTextSearch(full_text_search), "does not support full text search"); + auto full_text_match = std::make_shared("body", R"({"match":{"query":"word"}})"); + ASSERT_NOK_WITH_MSG(reader.VisitFullTextMatch(full_text_match), + "does not support complete full-text matching"); } } // namespace paimon::test diff --git a/src/paimon/common/predicate/full_text_match_function.h b/src/paimon/common/predicate/full_text_match_function.h new file mode 100644 index 000000000..6a36b4ac3 --- /dev/null +++ b/src/paimon/common/predicate/full_text_match_function.h @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "paimon/common/predicate/leaf_function.h" + +namespace paimon { + +/// Identifies an index-dependent predicate without implementing scalar evaluation. +class FullTextMatchFunction : public LeafFunction { + public: + static const FullTextMatchFunction& Instance() { + static const FullTextMatchFunction instance; + return instance; + } + + Result> Test(const arrow::Array& array, const std::vector& literals, + arrow::MemoryPool* pool) const override { + return Status::NotImplemented( + "FullTextMatch does not support scalar Test; a full-text index is required"); + } + + Result Test(const Literal& value, const std::vector& literals) const override { + return Status::NotImplemented( + "FullTextMatch does not support scalar Test; a full-text index is required"); + } + + Result Test(int64_t row_count, const Literal& min_value, const Literal& max_value, + const std::optional& null_count, + const std::vector& literals) const override { + return Status::NotImplemented( + "FullTextMatch does not support statistics Test; a full-text index is required"); + } + + const LeafFunction* Negate() const override { + return nullptr; + } + + Type GetType() const override { + return Type::FULL_TEXT_MATCH; + } + + std::string ToString() const override { + return "FullTextMatch"; + } + + private: + FullTextMatchFunction() = default; +}; + +} // namespace paimon diff --git a/src/paimon/common/predicate/full_text_match_predicate.cpp b/src/paimon/common/predicate/full_text_match_predicate.cpp new file mode 100644 index 000000000..6f481c0c2 --- /dev/null +++ b/src/paimon/common/predicate/full_text_match_predicate.cpp @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/predicate/full_text_match_predicate.h" + +#include "fmt/format.h" +#include "paimon/common/predicate/full_text_match_function.h" +#include "paimon/defs.h" +#include "paimon/predicate/full_text_match.h" + +namespace paimon { + +FullTextMatchPredicate::FullTextMatchPredicate(int32_t field_index, + const std::shared_ptr& match) + : LeafPredicate(FullTextMatchFunction::Instance(), field_index, match->field_name, + FieldType::STRING, {}), + full_text_match_(match) {} + +bool FullTextMatchPredicate::operator==(const Predicate& other) const { + if (this == &other) { + return true; + } + auto text_predicate = dynamic_cast(const_cast(&other)); + if (!text_predicate) { + return false; + } + return LeafPredicate::operator==(other) && + *full_text_match_ == *text_predicate->full_text_match_; +} + +std::string FullTextMatchPredicate::ToString() const { + return fmt::format( + "FullTextMatch({}, {}, include_row_ids={})", FieldName(), full_text_match_->query, + full_text_match_->include_row_ids ? full_text_match_->include_row_ids->ToString() : "null"); +} + +std::shared_ptr FullTextMatchPredicate::NewLeafPredicate( + int32_t new_field_index) const { + return std::make_shared(new_field_index, full_text_match_); +} + +std::shared_ptr FullTextMatchPredicate::NewLeafPredicate( + const std::string& new_field_name) const { + auto request = std::make_shared(*full_text_match_); + request->field_name = new_field_name; + return std::make_shared(FieldIndex(), request); +} + +} // namespace paimon diff --git a/src/paimon/common/predicate/full_text_match_predicate.h b/src/paimon/common/predicate/full_text_match_predicate.h new file mode 100644 index 000000000..61ae13a70 --- /dev/null +++ b/src/paimon/common/predicate/full_text_match_predicate.h @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/predicate/leaf_predicate.h" + +namespace paimon { +struct FullTextMatch; + +/// A full-text leaf evaluated by an index, rather than scalar PredicateFilter::Test. +/// Use PredicateBuilder::FullTextMatch to construct it. +class FullTextMatchPredicate : public LeafPredicate { + public: + FullTextMatchPredicate(int32_t field_index, const std::shared_ptr& match); + + const std::shared_ptr& GetFullTextMatch() const { + return full_text_match_; + } + + bool operator==(const Predicate& other) const override; + std::string ToString() const override; + + /// Rebind the field position while sharing the original request. + std::shared_ptr NewLeafPredicate(int32_t new_field_index) const; + /// Rebind the field name using a request copy, preserving the original request. + std::shared_ptr NewLeafPredicate( + const std::string& new_field_name) const; + + private: + std::shared_ptr full_text_match_; +}; + +} // namespace paimon diff --git a/src/paimon/common/predicate/full_text_match_predicate_test.cpp b/src/paimon/common/predicate/full_text_match_predicate_test.cpp new file mode 100644 index 000000000..cc6ddfc82 --- /dev/null +++ b/src/paimon/common/predicate/full_text_match_predicate_test.cpp @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/predicate/full_text_match_predicate.h" + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/predicate/full_text_match_function.h" +#include "paimon/common/predicate/predicate_filter.h" +#include "paimon/defs.h" +#include "paimon/predicate/compound_predicate.h" +#include "paimon/predicate/full_text_match.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +namespace { + +std::shared_ptr MakeMatch( + const std::string& field_name = "title", + const std::string& query = R"({"match":{"query":"apple"}})") { + return std::make_shared(field_name, query); +} + +} // namespace + +TEST(FullTextMatchPredicateTest, TestBasicAndComposesWithScalarPredicates) { + auto match = MakeMatch(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::FullTextMatch(1, match)); + auto text = std::dynamic_pointer_cast(predicate); + ASSERT_TRUE(text); + ASSERT_EQ(text->GetFullTextMatch(), match); + ASSERT_EQ(text->FieldIndex(), 1); + ASSERT_EQ(text->FieldName(), "title"); + ASSERT_EQ(text->GetFieldType(), FieldType::STRING); + ASSERT_EQ(text->GetFunction().GetType(), Function::Type::FULL_TEXT_MATCH); + ASSERT_TRUE(text->Literals().empty()); + ASSERT_FALSE(std::dynamic_pointer_cast(predicate)); + ASSERT_NE(text->ToString().find(match->query), std::string::npos); + + auto scalar = PredicateBuilder::LessThan(0, "price", FieldType::INT, Literal(10)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr conjunction, + PredicateBuilder::And({scalar, predicate})); + auto compound = std::dynamic_pointer_cast(conjunction); + ASSERT_TRUE(compound); + ASSERT_EQ(compound->Children(), (std::vector>{scalar, predicate})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr disjunction, + PredicateBuilder::Or({scalar, predicate})); + ASSERT_EQ(disjunction->GetFunction().GetType(), Function::Type::OR); + ASSERT_FALSE(text->Negate()); + ASSERT_NOK_WITH_MSG(PredicateBuilder::Not(predicate), "Could not construct A NOT predicate"); + ASSERT_NOK_WITH_MSG(PredicateBuilder::Not(conjunction), "Could not construct A NOT predicate"); +} + +TEST(FullTextMatchPredicateTest, TestRebind) { + auto match = std::make_shared("title", R"({"match":{"query":"apple"}})", + RoaringBitmap64::From({1, 3, 9})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::FullTextMatch(1, match)); + auto text = std::dynamic_pointer_cast(predicate); + ASSERT_TRUE(text); + + auto picked = text->NewLeafPredicate(0); + ASSERT_EQ(picked->FieldIndex(), 0); + ASSERT_EQ(picked->GetFullTextMatch(), match); + ASSERT_EQ(text->FieldIndex(), 1); + + auto renamed = text->NewLeafPredicate("stored_title"); + ASSERT_EQ(renamed->FieldIndex(), 1); + ASSERT_EQ(renamed->FieldName(), "stored_title"); + ASSERT_EQ(renamed->GetFullTextMatch()->field_name, "stored_title"); + ASSERT_EQ(renamed->GetFullTextMatch()->query, match->query); + ASSERT_EQ(renamed->GetFullTextMatch()->include_row_ids, match->include_row_ids); + ASSERT_NE(renamed->GetFullTextMatch(), match); + ASSERT_EQ(match->field_name, "title"); + ASSERT_EQ(text->FieldName(), "title"); +} + +TEST(FullTextMatchPredicateTest, TestEquality) { + auto match = MakeMatch(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::FullTextMatch(1, match)); + auto text = std::dynamic_pointer_cast(predicate); + ASSERT_TRUE(text); + ASSERT_OK_AND_ASSIGN(std::shared_ptr same, + PredicateBuilder::FullTextMatch(1, MakeMatch())); + ASSERT_TRUE(*predicate == *same); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr other_query, + PredicateBuilder::FullTextMatch(1, MakeMatch("title", R"({"match":{"query":"orange"}})"))); + ASSERT_FALSE(*predicate == *other_query); + auto restricted = std::make_shared("title", R"({"match":{"query":"apple"}})", + RoaringBitmap64::From({3, 9})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr candidates, + PredicateBuilder::FullTextMatch(1, restricted)); + ASSERT_FALSE(*predicate == *candidates); + ASSERT_NE(candidates->ToString().find(restricted->include_row_ids->ToString()), + std::string::npos); + ASSERT_OK_AND_ASSIGN(std::shared_ptr other_field, + PredicateBuilder::FullTextMatch(1, MakeMatch("body"))); + ASSERT_FALSE(*predicate == *other_field); + ASSERT_OK_AND_ASSIGN(std::shared_ptr other_index, + PredicateBuilder::FullTextMatch(0, MakeMatch())); + ASSERT_FALSE(*predicate == *other_index); +} + +TEST(FullTextMatchPredicateTest, TestRejectsInvalidRequests) { + ASSERT_NOK_WITH_MSG(PredicateBuilder::FullTextMatch(0, nullptr), + "FullTextMatch request must not be null"); + ASSERT_NOK_WITH_MSG(PredicateBuilder::FullTextMatch(0, MakeMatch("")), + "FullTextMatch field name must not be empty"); + ASSERT_NOK_WITH_MSG(PredicateBuilder::FullTextMatch(0, MakeMatch("title", "")), + "FullTextMatch query must not be empty"); +} + +TEST(FullTextMatchPredicateTest, TestFunctionCannotEvaluateRowsOrStatistics) { + const auto& function = FullTextMatchFunction::Instance(); + arrow::NullArray array(2); + ASSERT_NOK_WITH_MSG( + function.Test(array, {}, arrow::default_memory_pool()), + "FullTextMatch does not support scalar Test; a full-text index is required"); + ASSERT_NOK_WITH_MSG( + function.Test(Literal(1), {}), + "FullTextMatch does not support scalar Test; a full-text index is required"); + ASSERT_NOK_WITH_MSG( + function.Test(2, Literal(1), Literal(2), /*null_count=*/0, {}), + "FullTextMatch does not support statistics Test; a full-text index is required"); + ASSERT_FALSE(function.Negate()); +} + +} // namespace paimon::test diff --git a/src/paimon/common/predicate/predicate_builder.cpp b/src/paimon/common/predicate/predicate_builder.cpp index 529905e65..545fd7f47 100644 --- a/src/paimon/common/predicate/predicate_builder.cpp +++ b/src/paimon/common/predicate/predicate_builder.cpp @@ -25,6 +25,7 @@ #include "paimon/common/predicate/contains.h" #include "paimon/common/predicate/ends_with.h" #include "paimon/common/predicate/equal.h" +#include "paimon/common/predicate/full_text_match_predicate.h" #include "paimon/common/predicate/greater_or_equal.h" #include "paimon/common/predicate/greater_than.h" #include "paimon/common/predicate/in.h" @@ -38,12 +39,27 @@ #include "paimon/common/predicate/not_in.h" #include "paimon/common/predicate/or.h" #include "paimon/common/predicate/starts_with.h" +#include "paimon/predicate/full_text_match.h" #include "paimon/predicate/literal.h" #include "paimon/status.h" namespace paimon { enum class FieldType; +Result> PredicateBuilder::FullTextMatch( + int32_t field_index, const std::shared_ptr& match) { + if (!match) { + return Status::Invalid("FullTextMatch request must not be null"); + } + if (match->field_name.empty()) { + return Status::Invalid("FullTextMatch field name must not be empty"); + } + if (match->query.empty()) { + return Status::Invalid("FullTextMatch query must not be empty"); + } + return std::make_shared(field_index, match); +} + // TODO(xinyu.lxy): predicate field_index use index in read schema now, but java paimon use index // in file schema std::shared_ptr PredicateBuilder::Equal(int32_t field_index, diff --git a/src/paimon/common/predicate/predicate_utils.cpp b/src/paimon/common/predicate/predicate_utils.cpp index ab9c8f4ac..83ad478cb 100644 --- a/src/paimon/common/predicate/predicate_utils.cpp +++ b/src/paimon/common/predicate/predicate_utils.cpp @@ -21,12 +21,28 @@ #include #include "paimon/common/predicate/compound_predicate_impl.h" +#include "paimon/common/predicate/full_text_match_predicate.h" #include "paimon/common/predicate/leaf_predicate_impl.h" #include "paimon/predicate/compound_predicate.h" #include "paimon/predicate/leaf_predicate.h" #include "paimon/predicate/predicate_builder.h" namespace paimon { + +bool PredicateUtils::ContainsFullTextMatch(const std::shared_ptr& predicate) { + if (std::dynamic_pointer_cast(predicate)) { + return true; + } + if (auto compound = std::dynamic_pointer_cast(predicate)) { + for (const auto& child : compound->Children()) { + if (ContainsFullTextMatch(child)) { + return true; + } + } + } + return false; +} + Result PredicateUtils::ContainAnyField(const std::shared_ptr& predicate, const std::set& field_names) { if (auto leaf_predicate = std::dynamic_pointer_cast(predicate)) { @@ -135,6 +151,13 @@ PredicateUtils::ReconstructPredicateWithPickedFields( } return std::optional>( compound_predicate->NewCompoundPredicate(mapped_children)); + } else if (auto full_text = std::dynamic_pointer_cast(predicate)) { + auto iter = picked_field_name_to_idx.find(full_text->FieldName()); + if (iter == picked_field_name_to_idx.end()) { + return std::optional>(); + } + return std::optional>( + full_text->NewLeafPredicate(/*new_field_index=*/iter->second)); } else if (auto leaf_predicate = std::dynamic_pointer_cast(predicate)) { auto iter = picked_field_name_to_idx.find(leaf_predicate->FieldName()); if (iter == picked_field_name_to_idx.end()) { diff --git a/src/paimon/common/predicate/predicate_utils_test.cpp b/src/paimon/common/predicate/predicate_utils_test.cpp index 3d22628a8..8b6a98a81 100644 --- a/src/paimon/common/predicate/predicate_utils_test.cpp +++ b/src/paimon/common/predicate/predicate_utils_test.cpp @@ -21,12 +21,55 @@ #include #include "gtest/gtest.h" +#include "paimon/common/predicate/full_text_match_predicate.h" #include "paimon/defs.h" +#include "paimon/predicate/compound_predicate.h" +#include "paimon/predicate/full_text_match.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { +TEST(PredicateUtilsTest, TestFullTextMatchFieldUtilities) { + auto match = std::make_shared("text", R"({"match":{"query":"apple"}})"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr text, + PredicateBuilder::FullTextMatch(1, match)); + auto scalar = PredicateBuilder::Equal(0, "id", FieldType::INT, Literal(7)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({scalar, text})); + ASSERT_TRUE(PredicateUtils::ContainsFullTextMatch(predicate)); + ASSERT_FALSE(PredicateUtils::ContainsFullTextMatch(scalar)); + ASSERT_FALSE(PredicateUtils::ContainsFullTextMatch(nullptr)); + std::set names; + ASSERT_OK(PredicateUtils::GetAllNames(predicate, &names)); + ASSERT_EQ(names, (std::set{"id", "text"})); + ASSERT_OK_AND_ASSIGN(bool contains, PredicateUtils::ContainAnyField(predicate, {"text"})); + ASSERT_TRUE(contains); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr picked, + PredicateUtils::CreatePickedFieldFilter(predicate, {{"text", 4}})); + auto picked_text = std::dynamic_pointer_cast(picked); + ASSERT_TRUE(picked_text); + ASSERT_EQ(picked_text->FieldIndex(), 4); + ASSERT_EQ(picked_text->GetFullTextMatch(), match); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr disjunction, + PredicateBuilder::Or({scalar, text})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing, + PredicateUtils::CreatePickedFieldFilter(disjunction, {{"text", 4}})); + ASSERT_FALSE(missing); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr remapped, + PredicateUtils::CreatePickedFieldFilter(disjunction, {{"id", 2}, {"text", 4}})); + auto compound = std::dynamic_pointer_cast(remapped); + ASSERT_TRUE(compound); + ASSERT_EQ(compound->GetFunction().GetType(), Function::Type::OR); + picked_text = std::dynamic_pointer_cast(compound->Children()[1]); + ASSERT_TRUE(picked_text); + ASSERT_EQ(picked_text->FieldIndex(), 4); + ASSERT_EQ(picked_text->GetFullTextMatch(), match); +} + TEST(PredicateUtilsTest, TestSplitAnd) { ASSERT_OK_AND_ASSIGN( auto child1, diff --git a/src/paimon/common/predicate/predicate_validator_test.cpp b/src/paimon/common/predicate/predicate_validator_test.cpp index 75ae141c7..89432b727 100644 --- a/src/paimon/common/predicate/predicate_validator_test.cpp +++ b/src/paimon/common/predicate/predicate_validator_test.cpp @@ -22,6 +22,7 @@ #include "gtest/gtest.h" #include "paimon/data/decimal.h" #include "paimon/data/timestamp.h" +#include "paimon/predicate/full_text_match.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/result.h" #include "paimon/testing/utils/testharness.h" @@ -31,6 +32,48 @@ class Schema; } // namespace arrow namespace paimon::test { +TEST(PredicateValidatorTest, TestFullTextMatchRequiresProjectedStringField) { + auto schema = + arrow::schema({arrow::field("id", arrow::int32()), arrow::field("text", arrow::utf8())}); + auto match = std::make_shared("text", R"({"match":{"query":"apple"}})"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr text, + PredicateBuilder::FullTextMatch(1, match)); + ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema, text, true)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr wrong_index, + PredicateBuilder::FullTextMatch(0, match)); + ASSERT_NOK_WITH_MSG( + PredicateValidator::ValidatePredicateWithSchema(*schema, wrong_index, true), + "field text has field idx 1 in input schema, mismatch field idx 0 in predicate"); + ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema, wrong_index, false)); + auto missing = arrow::schema({arrow::field("id", arrow::int32())}); + ASSERT_NOK_WITH_MSG(PredicateValidator::ValidatePredicateWithSchema(*missing, text, true), + "field text does not exist in schema"); + auto numeric = + arrow::schema({arrow::field("id", arrow::int32()), arrow::field("text", arrow::int32())}); + ASSERT_NOK_WITH_MSG(PredicateValidator::ValidatePredicateWithSchema(*numeric, text, true), + "schema type int32 mismatches predicate field type STRING"); +} + +TEST(PredicateValidatorTest, TestFullTextMatchAllowsOrAndRepeatedFields) { + auto schema = + arrow::schema({arrow::field("id", arrow::int32()), arrow::field("text", arrow::utf8())}); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr apple, + PredicateBuilder::FullTextMatch( + 1, std::make_shared("text", R"({"match":{"query":"apple"}})"))); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr orange, + PredicateBuilder::FullTextMatch( + 1, std::make_shared("text", R"({"match":{"query":"orange"}})"))); + auto scalar = PredicateBuilder::Equal(0, "id", FieldType::INT, Literal(7)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr conjunction, + PredicateBuilder::And({apple, orange})); + ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema, conjunction, true)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr disjunction, + PredicateBuilder::Or({conjunction, scalar})); + ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema, disjunction, true)); +} + TEST(PredicateValidatorTest, TestValidateLiterals) { std::string str("apple"); std::shared_ptr schema = arrow::schema(arrow::FieldVector({ diff --git a/src/paimon/common/table/special_fields.h b/src/paimon/common/table/special_fields.h index f3e64fcce..73562140b 100644 --- a/src/paimon/common/table/special_fields.h +++ b/src/paimon/common/table/special_fields.h @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -67,6 +68,20 @@ struct SpecialFields { return data_field; } + static std::string IndexScoreFieldName(const std::string& field_name) { + return "_" + field_name + "_INDEX_SCORE"; + } + + /// Parse a per-field score projection. The global `_INDEX_SCORE` is separate. + static std::optional ParseIndexScoreFieldName(const std::string& name) { + const std::string suffix = "_INDEX_SCORE"; + if (name.size() <= suffix.size() + 1 || !StringUtils::StartsWith(name, "_") || + !StringUtils::EndsWith(name, suffix)) { + return std::nullopt; + } + return name.substr(1, name.size() - suffix.size() - 1); + } + static const DataField& RealtimeOffset() { static const DataField data_field = DataField(SpecialFieldIds::REALTIME_OFFSET, @@ -93,7 +108,8 @@ struct SpecialFields { } return field_name == SequenceNumber().Name() || field_name == ValueKind().Name() || field_name == RowKind().Name() || field_name == RowId().Name() || - field_name == IndexScore().Name(); + field_name == IndexScore().Name() || + ParseIndexScoreFieldName(field_name).has_value(); } // TODO(xinyu.lxy): add a func to complete row-tracking fields diff --git a/src/paimon/common/table/special_fields_test.cpp b/src/paimon/common/table/special_fields_test.cpp index 05c0808e5..3f6f5ad34 100644 --- a/src/paimon/common/table/special_fields_test.cpp +++ b/src/paimon/common/table/special_fields_test.cpp @@ -55,6 +55,19 @@ TEST(SpecialFieldsTest, TestIndexScore) { ASSERT_EQ(SpecialFields::IndexScore().Type()->id(), arrow::Type::FLOAT); } +TEST(SpecialFieldsTest, TestPerFieldIndexScoreNames) { + for (const std::string& field_name : {"F1", "body_text", "_title", "文档"}) { + const auto name = SpecialFields::IndexScoreFieldName(field_name); + ASSERT_EQ(SpecialFields::ParseIndexScoreFieldName(name), field_name); + ASSERT_TRUE(SpecialFields::IsSystemField(name)); + } + ASSERT_EQ(SpecialFields::IndexScoreFieldName("F1"), "_F1_INDEX_SCORE"); + for (const std::string& name : {"_INDEX_SCORE", "__INDEX_SCORE", "F1_INDEX_SCORE", + "_F1_index_score", "_F1_INDEX_SCORE_extra"}) { + ASSERT_FALSE(SpecialFields::ParseIndexScoreFieldName(name)); + } +} + TEST(SpecialFieldsTest, TestRealtimeOffset) { ASSERT_EQ(SpecialFields::RealtimeOffset().Id(), SpecialFieldIds::REALTIME_OFFSET); ASSERT_EQ(SpecialFields::RealtimeOffset().Name(), "_REALTIME_OFFSET"); diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp index e5f132335..4e508bab2 100644 --- a/src/paimon/core/operation/file_store_scan.cpp +++ b/src/paimon/core/operation/file_store_scan.cpp @@ -690,6 +690,9 @@ Result FileStoreScan::FilterNonPartitionManifestEntry(const ManifestEntry& Status FileStoreScan::SplitAndSetFilter(const std::vector& partition_keys, const std::shared_ptr& arrow_schema, const std::shared_ptr& scan_filters) { + if (PredicateUtils::ContainsFullTextMatch(scan_filters->GetPredicate())) { + return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + } if (scan_filters->GetPredicate()) { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr mapping_builder, FieldMappingBuilder::Create(arrow_schema, partition_keys, diff --git a/src/paimon/core/operation/file_store_scan_test.cpp b/src/paimon/core/operation/file_store_scan_test.cpp index ef784f763..5c53f138d 100644 --- a/src/paimon/core/operation/file_store_scan_test.cpp +++ b/src/paimon/core/operation/file_store_scan_test.cpp @@ -29,10 +29,12 @@ #include "paimon/core/manifest/file_source.h" #include "paimon/core/manifest/manifest_entry_serializer.h" #include "paimon/core/manifest/snapshot_live_manifest_entries.h" +#include "paimon/core/operation/append_only_file_store_scan.h" #include "paimon/core/stats/simple_stats.h" #include "paimon/data/timestamp.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_segment.h" +#include "paimon/predicate/full_text_match.h" #include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/testharness.h" @@ -140,6 +142,28 @@ TEST_F(FileStoreScanTest, TestCreatePartitionPredicateWithInvalidPartitionFilter "field invalid does not exist in partition keys"); } +TEST_F(FileStoreScanTest, TestFullTextMatchDoesNotEnterPartitionOrStatsFilters) { + auto match = std::make_shared("f0", R"({"match":{"query":"apple"}})"); + ASSERT_OK_AND_ASSIGN(auto full_text, PredicateBuilder::FullTextMatch(/*field_index=*/0, match)); + auto scalar = PredicateBuilder::Equal(/*field_index=*/1, "f1", FieldType::INT, Literal(1)); + ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::Or({scalar, full_text})); + auto filters = std::make_shared( + predicate, std::vector>{}, std::nullopt); + ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({})); + for (const std::vector& partition_keys : + {std::vector{}, std::vector{"f0"}}) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema_, partition_keys, /*primary_keys=*/{}, {})); + auto result = AppendOnlyFileStoreScan::Create( + /*snapshot_manager=*/nullptr, /*schema_manager=*/nullptr, /*manifest_list=*/nullptr, + /*manifest_file=*/nullptr, table_schema, schema_, filters, options, + CreateDefaultExecutor(), GetDefaultPool()); + ASSERT_NOK_WITH_MSG(result, "FullTextMatch predicate execution is not implemented yet"); + ASSERT_TRUE(result.status().IsNotImplemented()); + } +} + TEST_F(FileStoreScanTest, TestFilterManifestByRowRanges) { class FakeFileStoreScan : public FileStoreScan { public: diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 429302cce..a89eb0aeb 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -18,6 +18,8 @@ #include "paimon/core/operation/internal_read_context.h" +#include +#include #include #include @@ -27,6 +29,7 @@ #include "fmt/format.h" #include "paimon/common/data/variant/variant_access_utils.h" #include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/predicate/full_text_match_predicate.h" #include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" @@ -36,7 +39,9 @@ #include "paimon/core/options/map_storage_layout.h" #include "paimon/core/schema/arrow_schema_validator.h" #include "paimon/core/utils/nested_projection_utils.h" +#include "paimon/predicate/compound_predicate.h" #include "paimon/predicate/full_text_search.h" +#include "paimon/predicate/predicate_utils.h" #include "paimon/predicate/vector_search.h" #include "paimon/status.h" @@ -175,7 +180,12 @@ std::optional InternalReadContext::TryResolveSpecialFieldById( } std::optional InternalReadContext::TryResolveSpecialFieldByName( - const std::string& name, const CoreOptions& core_options, bool has_vector_or_full_text_search) { + const std::string& name, const CoreOptions& core_options, bool has_vector_or_full_text_search, + int32_t output_field_idx) { + if (SpecialFields::ParseIndexScoreFieldName(name)) { + // These virtual columns have read-local negative IDs, not persistent table IDs. + return DataField(-1 - output_field_idx, arrow::field(name, arrow::float32())); + } if (name == SpecialFields::ValueKind().Name()) { return SpecialFields::ValueKind(); } @@ -245,7 +255,8 @@ Result> InternalReadContext::Create( // Align special-field validation with read_field_ids/read_field_names branches. for (const auto& read_field : read_schema->fields()) { if (auto resolved_special_field = TryResolveSpecialFieldByName( - read_field->name(), core_options, has_vector_or_full_text_search)) { + read_field->name(), core_options, has_vector_or_full_text_search, + static_cast(read_data_fields.size()))) { read_data_fields.push_back(*resolved_special_field); continue; } @@ -281,8 +292,9 @@ Result> InternalReadContext::Create( } else if (!context->GetReadFieldNames().empty()) { read_data_fields.reserve(context->GetReadFieldNames().size()); for (const auto& name : context->GetReadFieldNames()) { - if (auto resolved_special_field = TryResolveSpecialFieldByName( - name, core_options, has_vector_or_full_text_search)) { + if (auto resolved_special_field = + TryResolveSpecialFieldByName(name, core_options, has_vector_or_full_text_search, + static_cast(read_data_fields.size()))) { read_data_fields.push_back(*resolved_special_field); continue; } @@ -311,13 +323,85 @@ Result> InternalReadContext::Create( PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( *read_schema, context->GetPredicate(), /*validate_field_idx=*/true)); } - if (!context->GetMemoryPool()) { return Status::Invalid("memory pool is null"); } std::shared_ptr arrow_pool = GetArrowPool(context->GetMemoryPool()); - return std::unique_ptr( + auto internal_context = std::unique_ptr( new InternalReadContext(context, table_schema, read_schema, core_options, arrow_pool)); + PAIMON_RETURN_NOT_OK(internal_context->BindIndexScores()); + return internal_context; +} + +Status InternalReadContext::BindIndexScores() { + std::map projected_scores; + for (int32_t idx = 0; idx < read_schema_->num_fields(); ++idx) { + if (auto field_name = + SpecialFields::ParseIndexScoreFieldName(read_schema_->field(idx)->name())) { + projected_scores.emplace(*field_name, idx); + } + } + if (projected_scores.empty()) { + return Status::OK(); + } + auto bind = [&](const IndexScoreSource& source, bool under_or) -> Status { + auto iter = projected_scores.find(source.field_name); + if (iter == projected_scores.end()) { + return Status::OK(); + } + if (under_or) { + return Status::Invalid("Score-producing FullTextMatch cannot appear under OR"); + } + if (!index_score_bindings_.emplace(iter->second, source).second) { + return Status::Invalid(fmt::format( + "Score-producing search field '{}' occurs more than once", source.field_name)); + } + return Status::OK(); + }; + std::function&, bool)> visit = + [&](const std::shared_ptr& predicate, bool under_or) -> Status { + if (auto full_text = std::dynamic_pointer_cast(predicate)) { + return bind({IndexScoreSource::Type::FULL_TEXT_MATCH, full_text->FieldName()}, + under_or); + } + if (auto compound = std::dynamic_pointer_cast(predicate)) { + const bool child_under_or = + under_or || compound->GetFunction().GetType() == Function::Type::OR; + for (const auto& child : compound->Children()) { + PAIMON_RETURN_NOT_OK(visit(child, child_under_or)); + } + } + return Status::OK(); + }; + PAIMON_RETURN_NOT_OK(visit(GetPredicate(), /*under_or=*/false)); + if (const auto& search = GetVectorSearch()) { + PAIMON_RETURN_NOT_OK( + bind({IndexScoreSource::Type::VECTOR_SEARCH, search->field_name}, /*under_or=*/false)); + } + if (const auto& search = GetFullTextSearch()) { + PAIMON_RETURN_NOT_OK(bind({IndexScoreSource::Type::FULL_TEXT_SEARCH, search->field_name}, + /*under_or=*/false)); + } + for (const auto& [field_name, output_idx] : projected_scores) { + if (index_score_bindings_.find(output_idx) == index_score_bindings_.end()) { + return Status::Invalid(fmt::format("Index score field '{}' has no search source", + read_schema_->field(output_idx)->name())); + } + } + return Status::OK(); +} + +bool InternalReadContext::NeedsScores(const IndexScoreSource& source) const { + if (source.type == IndexScoreSource::Type::VECTOR_SEARCH) { + const auto& search = GetVectorSearch(); + return search && search->field_name == source.field_name; + } + if (source.type == IndexScoreSource::Type::FULL_TEXT_SEARCH) { + const auto& search = GetFullTextSearch(); + return search && search->field_name == source.field_name; + } + return std::any_of(index_score_bindings_.begin(), index_score_bindings_.end(), + [&source](const auto& binding) { return binding.second == source; }); } InternalReadContext::InternalReadContext(const std::shared_ptr& read_context, @@ -334,6 +418,11 @@ InternalReadContext::InternalReadContext(const std::shared_ptr& rea Result> InternalReadContext::CreateWithSchema( const std::shared_ptr& original, const std::shared_ptr& new_read_schema) { + if (original->HasVectorOrFullTextSearch() || + PredicateUtils::ContainsFullTextMatch(original->GetPredicate())) { + return Status::NotImplemented( + "CreateWithSchema does not support search requests or index score projections"); + } // Create a new InternalReadContext sharing all properties except read_schema. // The new read_schema is the minimal column set for COUNT(*). return std::shared_ptr( diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index 8e34a85bd..d3601fdff 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -43,11 +43,31 @@ class RealtimeContext; // internal read context, contains ReadContext, CoreOptions and TableSchema class InternalReadContext { public: + struct IndexScoreSource { + enum class Type { VECTOR_SEARCH, FULL_TEXT_SEARCH, FULL_TEXT_MATCH }; + + bool operator==(const IndexScoreSource& other) const { + return type == other.type && field_name == other.field_name; + } + + Type type; + /// Original table field name, before per-file field mapping. + std::string field_name; + }; + static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, const std::map& options); + /// Maps score output positions in the read schema to their logical search sources. + const std::map& GetIndexScoreBindings() const { + return index_score_bindings_; + } + + /// Root TopK needs scores even without a score projection. + bool NeedsScores(const IndexScoreSource& source) const; + const CoreOptions& GetCoreOptions() const { return options_; } @@ -129,10 +149,8 @@ class InternalReadContext { return read_context_->GetWarmupLevel(); } - /// Create a new InternalReadContext with a different read schema. - /// Useful for creating a context with a minimal column set for specialized reads. - /// All other settings (predicate, options, table_schema, etc.) are inherited - /// from the original context. + /// Create the minimal read schema for PK COUNT(*). Search requests and score + /// projections are unsupported; their bindings cannot be inherited across schemas. static Result> CreateWithSchema( const std::shared_ptr& original, const std::shared_ptr& new_read_schema); @@ -149,16 +167,19 @@ class InternalReadContext { bool has_vector_or_full_text_search); static std::optional TryResolveSpecialFieldByName( const std::string& name, const CoreOptions& core_options, - bool has_vector_or_full_text_search); + bool has_vector_or_full_text_search, int32_t output_field_idx); static Result> AlignReadFieldWithTableFieldIds( const std::shared_ptr& read_field, const std::shared_ptr& table_field); + Status BindIndexScores(); + std::shared_ptr read_context_; std::shared_ptr table_schema_; std::shared_ptr read_schema_; std::shared_ptr arrow_pool_; CoreOptions options_; + std::map index_score_bindings_; }; } // namespace paimon diff --git a/src/paimon/core/operation/internal_read_context_test.cpp b/src/paimon/core/operation/internal_read_context_test.cpp index 086d56d14..ca7b79dc2 100644 --- a/src/paimon/core/operation/internal_read_context_test.cpp +++ b/src/paimon/core/operation/internal_read_context_test.cpp @@ -27,6 +27,7 @@ #include "arrow/c/bridge.h" #include "arrow/type.h" +#include "fmt/format.h" #include "gtest/gtest.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" @@ -34,7 +35,9 @@ #include "paimon/data/shredding/map_shared_shredding_schema_utils.h" #include "paimon/defs.h" #include "paimon/fs/local/local_file_system.h" +#include "paimon/predicate/full_text_match.h" #include "paimon/predicate/full_text_search.h" +#include "paimon/predicate/predicate_builder.h" #include "paimon/predicate/vector_search.h" #include "paimon/status.h" #include "paimon/testing/utils/testharness.h" @@ -278,6 +281,332 @@ TEST(InternalReadContext, TestFileFullTextSearchScoreIsNotSupported) { "does not support score output"); } +TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { + std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; + SchemaManager schema_manager(std::make_shared(), path); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, schema_manager.ReadSchema(0)); + struct TestCase { + int32_t field_index; + std::string field_name; + std::vector projection; + std::string expected_error; + }; + const std::vector cases = { + {0, "f0", {"f0"}, ""}, + {0, "f0", {"f1"}, "field f0 does not exist in schema"}, + {1, + "f0", + {"f0"}, + "field f0 has field idx 0 in input schema, mismatch field idx 1 in predicate"}, + {0, "f1", {"f1"}, "schema type int32 mismatches predicate field type STRING"}}; + for (const auto& test_case : cases) { + auto match = + std::make_shared(test_case.field_name, R"({"match":{"query":"apple"}})"); + ASSERT_OK_AND_ASSIGN(auto predicate, + PredicateBuilder::FullTextMatch(test_case.field_index, match)); + ReadContextBuilder builder(path); + builder.SetReadFieldNames(test_case.projection).SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + if (test_case.expected_error.empty()) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr internal, + InternalReadContext::Create(context, table_schema, table_schema->Options())); + ASSERT_TRUE(internal->GetIndexScoreBindings().empty()); + ASSERT_FALSE(internal->NeedsScores( + {InternalReadContext::IndexScoreSource::Type::FULL_TEXT_MATCH, match->field_name})); + ASSERT_NOK_WITH_MSG( + InternalReadContext::CreateWithSchema(internal, internal->GetReadSchema()), + "CreateWithSchema does not support"); + } else { + ASSERT_NOK_WITH_MSG( + InternalReadContext::Create(context, table_schema, table_schema->Options()), + test_case.expected_error); + } + } +} + +TEST(InternalReadContext, TestPerFieldTopKScoreBindings) { + using SourceType = InternalReadContext::IndexScoreSource::Type; + std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; + SchemaManager schema_manager(std::make_shared(), path); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, schema_manager.ReadSchema(0)); + for (bool vector_search : {false, true}) { + for (bool project_score : {false, true}) { + ReadContextBuilder builder(path); + std::vector projection = {"f1"}; + if (project_score) { + projection.push_back(vector_search ? "_f3_INDEX_SCORE" : "_f0_INDEX_SCORE"); + } + builder.SetReadFieldNames(projection); + const InternalReadContext::IndexScoreSource source = { + vector_search ? SourceType::VECTOR_SEARCH : SourceType::FULL_TEXT_SEARCH, + vector_search ? "f3" : "f0"}; + if (vector_search) { + auto search = std::make_shared( + "f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, std::nullopt, + std::map{}); + builder.SetVectorSearch(search); + } else { + auto search = std::make_shared( + "f0", R"({"match":{"query":"apple"}})", /*limit=*/1); + builder.SetFullTextSearch(search); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr internal, + InternalReadContext::Create(context, table_schema, table_schema->Options())); + ASSERT_EQ(internal->GetReadSchema()->field_names(), projection); + ASSERT_TRUE(internal->NeedsScores(source)); + ASSERT_FALSE(internal->NeedsScores( + {vector_search ? SourceType::FULL_TEXT_SEARCH : SourceType::VECTOR_SEARCH, + source.field_name})); + ASSERT_FALSE(internal->NeedsScores({SourceType::FULL_TEXT_MATCH, source.field_name})); + ASSERT_FALSE(internal->NeedsScores({source.type, "missing"})); + const auto& bindings = internal->GetIndexScoreBindings(); + if (project_score) { + const std::map expected = { + {1, source}}; + ASSERT_EQ(bindings, expected); + ASSERT_EQ(internal->GetReadSchema()->field(1)->type()->id(), arrow::Type::FLOAT); + } else { + ASSERT_TRUE(bindings.empty()); + } + ASSERT_NOK_WITH_MSG( + InternalReadContext::CreateWithSchema(internal, internal->GetReadSchema()), + "CreateWithSchema does not support"); + } + } +} + +class InternalReadContextScoreTest : public testing::Test { + protected: + void SetUp() override { + ASSERT_OK_AND_ASSIGN(table_schema_, + TableSchema::Create(0, + arrow::schema({arrow::field("F1", arrow::utf8()), + arrow::field("F2", arrow::utf8())}), + {}, {}, {})); + } + + Result> Create(ReadContextBuilder* builder) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr context, builder->Finish()); + return InternalReadContext::Create(context, table_schema_, table_schema_->Options()); + } + + std::shared_ptr table_schema_; +}; + +TEST_F(InternalReadContextScoreTest, BindsFullTextScoresByProjection) { + using SourceType = InternalReadContext::IndexScoreSource::Type; + auto first = std::make_shared("F1", R"({"match":{"query":"apple"}})"); + auto second = std::make_shared("F2", first->query); + ASSERT_OK_AND_ASSIGN(std::shared_ptr first_predicate, + PredicateBuilder::FullTextMatch(0, first)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr second_predicate, + PredicateBuilder::FullTextMatch(1, second)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({first_predicate, second_predicate})); + struct TestCase { + int32_t score_count; + bool explicit_schema; + bool reverse_scores; + }; + const std::vector cases = {{0, false, false}, {1, false, false}, {2, false, false}, + {2, false, true}, {2, true, false}, {2, true, true}}; + for (const auto& test_case : cases) { + const int32_t score_count = test_case.score_count; + ReadContextBuilder builder("/tmp/unused-table-path"); + std::vector projection = {"F1", "F2"}; + if (score_count >= 1) { + projection.push_back(test_case.reverse_scores ? "_F1_INDEX_SCORE" : "_F2_INDEX_SCORE"); + } + if (score_count == 2) { + projection.push_back(test_case.reverse_scores ? "_F2_INDEX_SCORE" : "_F1_INDEX_SCORE"); + } + builder.SetPredicate(predicate); + if (test_case.explicit_schema) { + arrow::FieldVector fields = {arrow::field("F1", arrow::utf8()), + arrow::field("F2", arrow::utf8())}; + for (int32_t i = 0; i < score_count; ++i) { + fields.push_back(arrow::field(projection[i + 2], arrow::float32())); + } + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(fields), c_schema.get()).ok()); + builder.SetReadSchema(std::move(c_schema)); + } else if (score_count > 0) { + builder.SetReadFieldNames(projection); + } + ASSERT_OK_AND_ASSIGN(auto internal, Create(&builder)); + ASSERT_EQ(internal->GetReadSchema()->field_names(), projection); + ASSERT_EQ(internal->NeedsScores({SourceType::FULL_TEXT_MATCH, first->field_name}), + score_count == 2); + ASSERT_EQ(internal->NeedsScores({SourceType::FULL_TEXT_MATCH, second->field_name}), + score_count >= 1); + auto copied_match = std::make_shared(*second); + ASSERT_EQ(internal->NeedsScores({SourceType::FULL_TEXT_MATCH, copied_match->field_name}), + score_count >= 1); + ASSERT_FALSE(internal->NeedsScores({SourceType::FULL_TEXT_SEARCH, second->field_name})); + ASSERT_FALSE(internal->NeedsScores({SourceType::FULL_TEXT_MATCH, "missing"})); + const auto& bindings = internal->GetIndexScoreBindings(); + ASSERT_EQ(bindings.size(), score_count); + ASSERT_OK_AND_ASSIGN(std::vector fields, + DataField::ConvertArrowSchemaToDataFields(internal->GetReadSchema())); + for (int32_t i = 0; i < score_count; ++i) { + const int32_t output_idx = i + 2; + auto iter = bindings.find(output_idx); + ASSERT_NE(iter, bindings.end()); + ASSERT_EQ(iter->second.type, SourceType::FULL_TEXT_MATCH); + ASSERT_EQ(iter->second.field_name, + projection[output_idx] == "_F1_INDEX_SCORE" ? "F1" : "F2"); + ASSERT_EQ(fields[output_idx].Type()->id(), arrow::Type::FLOAT); + ASSERT_LT(fields[output_idx].Id(), 0); + ASSERT_NE(fields[output_idx].Id(), SpecialFieldIds::INDEX_SCORE); + for (int32_t previous = 0; previous < output_idx; ++previous) { + ASSERT_NE(fields[output_idx].Id(), fields[previous].Id()); + } + } + } +} + +TEST_F(InternalReadContextScoreTest, BindsRootAndPredicateScores) { + using SourceType = InternalReadContext::IndexScoreSource::Type; + const InternalReadContext::IndexScoreSource search_source = {SourceType::FULL_TEXT_SEARCH, + "F2"}; + const InternalReadContext::IndexScoreSource match_source = {SourceType::FULL_TEXT_MATCH, "F1"}; + auto match = std::make_shared("F1", R"({"match":{"query":"apple"}})"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::FullTextMatch(0, match)); + auto search = std::make_shared("F2", match->query, /*limit=*/10); + ReadContextBuilder builder("/tmp/unused-table-path"); + builder.SetPredicate(predicate).SetFullTextSearch(search).SetReadFieldNames( + {"F1", "_F2_INDEX_SCORE", "_F1_INDEX_SCORE"}); + ASSERT_OK_AND_ASSIGN(auto internal, Create(&builder)); + ASSERT_EQ(internal->GetReadSchema()->GetFieldIndex("F2"), -1); + const auto& bindings = internal->GetIndexScoreBindings(); + const std::map expected = {{1, search_source}, + {2, match_source}}; + ASSERT_EQ(bindings, expected); + ASSERT_TRUE(internal->NeedsScores(search_source)); + ASSERT_TRUE(internal->NeedsScores(match_source)); + + builder.SetPredicate(predicate).SetFullTextSearch(search).SetReadFieldNames( + {"F1", "_F2_INDEX_SCORE", "F2", "_F1_INDEX_SCORE"}); + ASSERT_OK_AND_ASSIGN(internal, Create(&builder)); + const std::map non_contiguous = { + {1, search_source}, {3, match_source}}; + ASSERT_EQ(internal->GetIndexScoreBindings(), non_contiguous); + + auto scalar = PredicateBuilder::IsNotNull(0, "F1", FieldType::STRING); + ASSERT_OK_AND_ASSIGN(predicate, PredicateBuilder::Or({predicate, scalar})); + builder.SetPredicate(predicate).SetFullTextSearch(search).SetReadFieldNames( + {"F1", "_F2_INDEX_SCORE"}); + ASSERT_OK_AND_ASSIGN(internal, Create(&builder)); + ASSERT_TRUE(internal->NeedsScores(search_source)); + ASSERT_FALSE(internal->NeedsScores(match_source)); + ASSERT_EQ(internal->GetIndexScoreBindings().size(), 1); +} + +TEST_F(InternalReadContextScoreTest, RejectsAmbiguousOrConditionalScoreProjection) { + using SourceType = InternalReadContext::IndexScoreSource::Type; + auto match = std::make_shared("F1", R"({"match":{"query":"apple"}})"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr text, + PredicateBuilder::FullTextMatch(0, match)); + auto scalar = PredicateBuilder::IsNotNull(0, "F1", FieldType::STRING); + ASSERT_OK_AND_ASSIGN(std::shared_ptr duplicate, PredicateBuilder::And({text, text})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr conditional, + PredicateBuilder::Or({text, scalar})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr scalar_or, + PredicateBuilder::Or({scalar, scalar})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr valid, + PredicateBuilder::And({text, scalar_or})); + struct TestCase { + std::shared_ptr predicate; + bool root_search; + std::string error; + }; + const std::vector cases = { + {duplicate, false, "Score-producing search field 'F1' occurs more than once"}, + {text, true, "Score-producing search field 'F1' occurs more than once"}, + {conditional, false, "Score-producing FullTextMatch cannot appear under OR"}, + {valid, false, ""}}; + for (const auto& test_case : cases) { + for (bool project_score : {false, true}) { + ReadContextBuilder builder("/tmp/unused-table-path"); + builder.SetPredicate(test_case.predicate) + .SetReadFieldNames(project_score ? std::vector{"F1", "_F1_INDEX_SCORE"} + : std::vector{"F1"}); + if (test_case.root_search) { + builder.SetFullTextSearch( + std::make_shared("F1", match->query, /*limit=*/1)); + } + if (project_score && !test_case.error.empty()) { + ASSERT_NOK_WITH_MSG(Create(&builder), test_case.error); + } else { + ASSERT_OK_AND_ASSIGN(auto internal, Create(&builder)); + ASSERT_EQ(internal->GetIndexScoreBindings().empty(), !project_score); + ASSERT_EQ(internal->NeedsScores({SourceType::FULL_TEXT_SEARCH, "F1"}), + test_case.root_search); + ASSERT_EQ(internal->NeedsScores({SourceType::FULL_TEXT_MATCH, "F1"}), + project_score); + } + } + } +} + +TEST_F(InternalReadContextScoreTest, RejectsMissingScoreSource) { + for (const auto& name : {"_F1_INDEX_SCORE", "_missing_INDEX_SCORE"}) { + ReadContextBuilder builder("/tmp/unused-table-path"); + builder.SetReadFieldNames({name}); + ASSERT_NOK_WITH_MSG(Create(&builder), + fmt::format("Index score field '{}' has no search source", name)); + } +} + +TEST_F(InternalReadContextScoreTest, ResolvesScoresOnlyFromEffectiveProjection) { + const InternalReadContext::IndexScoreSource search_source = { + InternalReadContext::IndexScoreSource::Type::FULL_TEXT_SEARCH, "F2"}; + auto search = std::make_shared("F2", R"({"match":{"query":"apple"}})", 1); + for (bool explicit_schema : {false, true}) { + ReadContextBuilder builder("/tmp/unused-table-path"); + builder.SetFullTextSearch(search) + .SetReadFieldNames({"_missing_INDEX_SCORE"}) + .SetReadFieldIds({0}); + if (explicit_schema) { + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema( + *arrow::schema({arrow::field("_F2_INDEX_SCORE", arrow::float32())}), + c_schema.get()) + .ok()); + builder.SetReadSchema(std::move(c_schema)); + } + ASSERT_OK_AND_ASSIGN(auto internal, Create(&builder)); + ASSERT_EQ(internal->GetReadSchema()->field_names(), + explicit_schema ? std::vector{"_F2_INDEX_SCORE"} + : std::vector{"F1"}); + ASSERT_TRUE(internal->NeedsScores(search_source)); + const auto& bindings = internal->GetIndexScoreBindings(); + ASSERT_EQ(bindings.size(), explicit_schema ? 1 : 0); + if (explicit_schema) { + const std::map expected = { + {0, search_source}}; + ASSERT_EQ(bindings, expected); + } + } +} + +TEST_F(InternalReadContextScoreTest, SchemaReplacementUsesProvidedSchema) { + ReadContextBuilder builder("/tmp/unused-table-path"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr internal, Create(&builder)); + for (const auto& schema : + {arrow::schema({arrow::field("F1", arrow::utf8())}), + arrow::schema({arrow::field("_F1_INDEX_SCORE", arrow::float32())})}) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr replaced, + InternalReadContext::CreateWithSchema(internal, schema)); + ASSERT_TRUE(replaced->GetReadSchema()->Equals(schema)); + ASSERT_TRUE(replaced->GetIndexScoreBindings().empty()); + } +} + TEST(InternalReadContext, TestReadWithValueKindField) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; ReadContextBuilder context_builder(path); diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index 8389ed909..d731c4d3b 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -387,6 +387,24 @@ Result> ReadContextBuilder::Finish() { if (impl_->vector_search_ && impl_->full_text_search_) { return Status::Invalid("VectorSearch and FullTextSearch cannot be configured together"); } + if (impl_->vector_search_) { + const auto& search = impl_->vector_search_; + if (search->limit <= 0) { + return Status::Invalid("VectorSearch TopK limit must be positive"); + } + if (search->field_name.empty() || search->query.empty()) { + return Status::Invalid("VectorSearch field name and query must not be empty"); + } + } + if (impl_->full_text_search_) { + const auto& search = impl_->full_text_search_; + if (search->limit <= 0) { + return Status::Invalid("FullTextSearch TopK limit must be positive"); + } + if (search->field_name.empty() || search->query.empty()) { + return Status::Invalid("FullTextSearch field name and query must not be empty"); + } + } if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { return Status::Invalid("cannot read with null format table"); } diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index 35955807d..d9079e702 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -29,7 +29,9 @@ #include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" +#include "paimon/predicate/full_text_match.h" #include "paimon/predicate/full_text_search.h" +#include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/predicate/vector_search.h" #include "paimon/status.h" @@ -432,4 +434,39 @@ TEST(ReadContextTest, TestRejectConflictingFileIndexSearch) { ASSERT_NOK_WITH_MSG(conflicting.Finish(), "cannot be configured together"); } +TEST(ReadContextTest, TestFullTextPredicateAndRootTopKConfiguration) { + auto match = std::make_shared("body", R"({"match":{"query":"apple"}})"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr text, + PredicateBuilder::FullTextMatch(0, match)); + auto scalar = PredicateBuilder::LessThan(1, "price", FieldType::INT, Literal(10)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr filter, PredicateBuilder::Or({text, scalar})); + auto vector = + std::make_shared("embedding", 3, std::vector{1.0f}, nullptr, nullptr, + std::nullopt, std::map{}); + ReadContextBuilder builder("table_root_path"); + builder.SetReadFieldNames({"body", "price"}).SetPredicate(filter).SetVectorSearch(vector); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_EQ(context->GetPredicate(), filter); + ASSERT_EQ(context->GetVectorSearch(), vector); + + auto ranked_text = std::make_shared("title", match->query, 2); + builder.SetPredicate(filter).SetFullTextSearch(ranked_text); + ASSERT_OK_AND_ASSIGN(context, builder.Finish()); + ASSERT_EQ(context->GetPredicate(), filter); + ASSERT_EQ(context->GetFullTextSearch(), ranked_text); +} + +TEST(ReadContextTest, TestRootTopKValidation) { + auto vector = + std::make_shared("embedding", 0, std::vector{1.0f}, nullptr, nullptr, + std::nullopt, std::map{}); + ReadContextBuilder vector_builder("table_root_path"); + vector_builder.SetVectorSearch(vector); + ASSERT_NOK_WITH_MSG(vector_builder.Finish(), "VectorSearch TopK limit must be positive"); + auto text = std::make_shared("body", R"({"match":{"query":"apple"}})", -1); + ReadContextBuilder text_builder("table_root_path"); + text_builder.SetFullTextSearch(text); + ASSERT_NOK_WITH_MSG(text_builder.Finish(), "FullTextSearch TopK limit must be positive"); +} + } // namespace paimon::test diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 4f5c03b57..31e3f74f0 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -27,20 +27,26 @@ #include #include "gtest/gtest.h" +#include "paimon/catalog/table.h" #include "paimon/common/data/binary_row.h" #include "paimon/core/core_options.h" #include "paimon/core/operation/abstract_split_read.h" #include "paimon/core/operation/split_read.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/key_value_table_read.h" #include "paimon/defs.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/predicate/full_text_match.h" #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" #include "paimon/status.h" +#include "paimon/table/format/format_table.h" #include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/testharness.h" @@ -117,6 +123,52 @@ TEST(TableReadTest, TestReadWithSpecifiedInvalidSchema) { "Get field field_no_exist failed: not exist in table schema"); } +TEST(TableReadTest, TestFullTextMatchExecutionIsNotImplemented) { + auto match = std::make_shared("f0", R"({"match":{"query":"apple"}})"); + ASSERT_OK_AND_ASSIGN(auto full_text, PredicateBuilder::FullTextMatch(/*field_index=*/0, match)); + auto scalar = PredicateBuilder::Equal(/*field_index=*/1, "f1", FieldType::INT, Literal(1)); + ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::Or({scalar, full_text})); + const std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; + auto fs = std::make_shared(); + SchemaManager schema_manager(fs, table_path); + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); + Table table(fs, table_path, schema, Identifier("append_09.db", "append_09")); + for (bool table_entry : {false, true}) { + ReadContextBuilder builder(table_path); + builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); + auto result = + table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + ASSERT_NOK_WITH_MSG(result, "FullTextMatch predicate execution is not implemented yet"); + ASSERT_TRUE(result.status().IsNotImplemented()); + + builder.SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}).SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(context, builder.Finish()); + result = + table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + ASSERT_NOK_WITH_MSG(result, "Score-producing FullTextMatch cannot appear under OR"); + } +} + +TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) { + const std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; + auto search = + std::make_shared("f0", R"({"match":{"query":"apple"}})", /*limit=*/1); + auto fs = std::make_shared(); + SchemaManager schema_manager(fs, path); + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); + Table table(fs, path, schema, Identifier("append_09.db", "append_09")); + for (bool table_entry : {false, true}) { + ReadContextBuilder builder(path); + builder.SetFullTextSearch(search).SetReadFieldNames({"f1", "_f0_INDEX_SCORE"}); + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); + auto result = + table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + ASSERT_NOK_WITH_MSG(result, "Per-field index score output is not implemented yet"); + ASSERT_TRUE(result.status().IsNotImplemented()); + } +} + TEST(TableReadTest, TestCreateKeyValueTableRead) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality/"; diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index bc7085521..aea232116 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -104,6 +104,7 @@ #include "paimon/file_store_write.h" #include "paimon/format/file_format.h" #include "paimon/fs/file_system.h" +#include "paimon/predicate/predicate_utils.h" #include "paimon/read_context.h" #include "paimon/realtime/realtime_context.h" #include "paimon/result.h" @@ -983,6 +984,12 @@ Result> CreateInternalReadContext( Result> CreateTableRead( const std::shared_ptr& internal_context, const std::shared_ptr& memory_pool, const std::shared_ptr& executor) { + if (PredicateUtils::ContainsFullTextMatch(internal_context->GetPredicate())) { + return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + } + if (!internal_context->GetIndexScoreBindings().empty()) { + return Status::NotImplemented("Per-field index score output is not implemented yet"); + } const auto& core_options = internal_context->GetCoreOptions(); const auto& table_schema = internal_context->GetTableSchema(); if (internal_context->GetRealtimeContext() && !core_options.RealtimeEnabled()) { @@ -1046,6 +1053,9 @@ Result> NewDataTableRead( /// refuses by name what a format table cannot honour. Result> NewFormatTableRead(const std::shared_ptr& table, const std::shared_ptr& context) { + if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { + return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, FormatTableRead::Create(table, context)); return std::unique_ptr(std::move(read)); @@ -1081,6 +1091,10 @@ Result> TableRead::Create(std::unique_ptr system_table_path, SystemTableLoader::TryParsePath(context->GetPath())); if (system_table_path) { + if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { + return Status::NotImplemented( + "FullTextMatch predicate execution is not implemented yet"); + } if (context->HasVectorOrFullTextSearch()) { return Status::NotImplemented( "VectorSearch or FullTextSearch is not supported for system tables"); diff --git a/src/paimon/core/utils/field_mapping.cpp b/src/paimon/core/utils/field_mapping.cpp index 70668a5b8..68c97c868 100644 --- a/src/paimon/core/utils/field_mapping.cpp +++ b/src/paimon/core/utils/field_mapping.cpp @@ -25,6 +25,7 @@ #include "arrow/type.h" #include "fmt/format.h" #include "paimon/common/predicate/compound_predicate_impl.h" +#include "paimon/common/predicate/full_text_match_predicate.h" #include "paimon/common/predicate/leaf_predicate_impl.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/field_type_utils.h" @@ -354,7 +355,25 @@ FieldMappingBuilder::ReconstructPredicateWithDataFields( const std::shared_ptr& predicate, const std::map& field_name_to_read_fields, const std::map>& field_id_to_data_fields) { - if (auto leaf_predicate = std::dynamic_pointer_cast(predicate)) { + if (auto full_text = std::dynamic_pointer_cast(predicate)) { + auto read_field = field_name_to_read_fields.find(full_text->FieldName()); + if (read_field == field_name_to_read_fields.end()) { + return Status::Invalid( + fmt::format("FullTextMatch field '{}' is not projected", full_text->FieldName())); + } + auto data_field = field_id_to_data_fields.find(read_field->second.Id()); + if (data_field == field_id_to_data_fields.end()) { + return Status::Invalid(fmt::format( + "FullTextMatch field '{}' does not exist in file schema", full_text->FieldName())); + } + const auto& [field_index, field] = data_field->second; + if (field.Type()->id() != arrow::Type::STRING) { + return Status::Invalid( + fmt::format("FullTextMatch field '{}' must have STRING type", field.Name())); + } + return std::optional>( + full_text->NewLeafPredicate(field.Name())->NewLeafPredicate(field_index)); + } else if (auto leaf_predicate = std::dynamic_pointer_cast(predicate)) { return ReconstructLeafPredicate(leaf_predicate, field_name_to_read_fields, field_id_to_data_fields); } else if (auto compound_predicate = diff --git a/src/paimon/core/utils/field_mapping_test.cpp b/src/paimon/core/utils/field_mapping_test.cpp index dbc1d9326..4f66cd10b 100644 --- a/src/paimon/core/utils/field_mapping_test.cpp +++ b/src/paimon/core/utils/field_mapping_test.cpp @@ -24,10 +24,12 @@ #include "arrow/type_fwd.h" #include "gtest/gtest.h" +#include "paimon/common/predicate/full_text_match_predicate.h" #include "paimon/common/predicate/leaf_predicate_impl.h" #include "paimon/common/predicate/predicate_filter.h" #include "paimon/data/decimal.h" #include "paimon/defs.h" +#include "paimon/predicate/full_text_match.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/status.h" @@ -99,6 +101,47 @@ TEST_F(FieldMappingTest, TestFieldNameMappingUsesIdsAcrossRenameAndReorder) { ASSERT_TRUE(FieldMappingBuilder::CreateFieldNameMapping(table_fields, {}).empty()); } +TEST_F(FieldMappingTest, TestFullTextMatchRenameAndReorder) { + RoaringBitmap64 candidates; + candidates.Add(3); + candidates.Add(9); + auto match = + std::make_shared("text", R"({"match":{"query":"apple"}})", candidates); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::FullTextMatch(1, match)); + std::map read_fields = { + {"text", DataField(7, arrow::field("text", arrow::utf8()))}}; + std::map> data_fields = { + {7, {3, DataField(7, arrow::field("old_text", arrow::utf8()))}}}; + ASSERT_OK_AND_ASSIGN(std::optional> reconstructed, + FieldMappingBuilder::ReconstructPredicateWithDataFields( + predicate, read_fields, data_fields)); + ASSERT_TRUE(reconstructed); + auto full_text = std::dynamic_pointer_cast(reconstructed.value()); + ASSERT_TRUE(full_text); + ASSERT_EQ(full_text->FieldName(), "old_text"); + ASSERT_EQ(full_text->FieldIndex(), 3); + ASSERT_EQ(full_text->GetFullTextMatch()->field_name, "old_text"); + ASSERT_EQ(full_text->GetFullTextMatch()->query, match->query); + ASSERT_EQ(full_text->GetFullTextMatch()->include_row_ids, match->include_row_ids); + ASSERT_NE(full_text->GetFullTextMatch(), match); + ASSERT_EQ(match->field_name, "text"); + auto original = std::dynamic_pointer_cast(predicate); + ASSERT_EQ(original->FieldName(), "text"); + ASSERT_EQ(original->FieldIndex(), 1); + + ASSERT_NOK_WITH_MSG( + FieldMappingBuilder::ReconstructPredicateWithDataFields(predicate, {}, data_fields), + "FullTextMatch field 'text' is not projected"); + ASSERT_NOK_WITH_MSG( + FieldMappingBuilder::ReconstructPredicateWithDataFields(predicate, read_fields, {}), + "FullTextMatch field 'text' does not exist in file schema"); + data_fields = {{7, {0, DataField(7, arrow::field("old_text", arrow::int32()))}}}; + ASSERT_NOK_WITH_MSG(FieldMappingBuilder::ReconstructPredicateWithDataFields( + predicate, read_fields, data_fields), + "FullTextMatch field 'old_text' must have STRING type"); +} + TEST_F(FieldMappingTest, TestEmptyPartitionKeys) { // test no partition keys in schema std::string val("hello"); diff --git a/test/inte/predicate_abi_inte_test.cpp b/test/inte/predicate_abi_inte_test.cpp index 3590eb22f..811307852 100644 --- a/test/inte/predicate_abi_inte_test.cpp +++ b/test/inte/predicate_abi_inte_test.cpp @@ -47,6 +47,8 @@ #include "gtest/gtest.h" #include "paimon/defs.h" #include "paimon/predicate/compound_predicate.h" +#include "paimon/predicate/full_text_match.h" +#include "paimon/predicate/function.h" #include "paimon/predicate/leaf_predicate.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate.h" @@ -74,6 +76,19 @@ TEST(PredicateAbiInteTest, LeafPredicateCastsAcrossModule) { EXPECT_EQ(std::dynamic_pointer_cast(predicate), nullptr); } +TEST(PredicateAbiInteTest, FullTextMatchCastsToLeafAcrossModule) { + auto match = std::make_shared("body", R"({"match":{"query":"apple"}})"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::FullTextMatch(0, match)); + auto leaf = std::dynamic_pointer_cast(predicate); + ASSERT_TRUE(leaf); + ASSERT_EQ(leaf->FieldIndex(), 0); + ASSERT_EQ(leaf->FieldName(), "body"); + ASSERT_EQ(leaf->GetFieldType(), FieldType::STRING); + ASSERT_TRUE(leaf->Literals().empty()); + ASSERT_EQ(leaf->GetFunction().GetType(), Function::Type::FULL_TEXT_MATCH); +} + // A compound predicate built in libpaimon.so must survive dynamic_pointer_cast // to CompoundPredicate, and its children must remain castable to LeafPredicate. TEST(PredicateAbiInteTest, CompoundPredicateCastsAcrossModule) { From abb6771d3c430119560f8e0ef39a66fe6571ac1e Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Thu, 22 Oct 2026 17:32:18 +0800 Subject: [PATCH 2/3] fix comment --- docs/source/api/file_index.rst | 29 ++-------------- include/paimon/file_index/file_index_reader.h | 4 +-- include/paimon/predicate/full_text_match.h | 3 +- include/paimon/predicate/function_visitor.h | 10 ++++++ include/paimon/predicate/predicate_utils.h | 8 +++++ include/paimon/read_context.h | 5 ++- .../file_index/file_index_reader_test.cpp | 34 +++++++++++++++++++ .../common/predicate/predicate_utils.cpp | 9 +++++ .../common/table/special_fields_test.cpp | 6 ++-- src/paimon/core/utils/field_mapping.cpp | 5 ++- test/inte/predicate_abi_inte_test.cpp | 6 ++++ 11 files changed, 81 insertions(+), 38 deletions(-) diff --git a/docs/source/api/file_index.rst b/docs/source/api/file_index.rst index 928468f02..b6a0eb140 100644 --- a/docs/source/api/file_index.rst +++ b/docs/source/api/file_index.rst @@ -56,9 +56,8 @@ semantics are to filter by the complete predicate before ranking; the TopK searc field does not need to be projected. A full-text match inside a predicate never has a TopK limit. -Score output is requested only through an explicit ``__INDEX_SCORE`` -projection. ``FullTextMatch``, ``SetVectorSearch`` and ``SetFullTextSearch`` have -no separate score switch. For example, request the score of a full-text +Score output is requested through an explicit ``__INDEX_SCORE`` +projection. For example, request the score of a full-text predicate by projecting ``_body_INDEX_SCORE``: .. code-block:: cpp @@ -76,10 +75,6 @@ including case. Select them with ``SetReadFieldNames`` or ``SetReadSchema``; they have no persistent field IDs for ``SetReadFieldIds``. Reading all table fields does not implicitly add score fields. -Root TopK searches always need scores internally to rank and merge results -across files, regardless of the projection. A full-text predicate without TopK -needs scores only when its score field is projected. - Each projected score must have exactly one search condition as its source. The source may be a root TopK request or a full-text predicate. A full-text predicate whose score is projected cannot have an OR ancestor, and its field @@ -90,26 +85,6 @@ projecting the score of ``match``, ``match AND (p OR q)`` is valid, while repeated full-text search fields are allowed. These checks use the effective read projection, after resolving the read schema. -Internally, the read context binds each score output position to a search type -and the original table field name. The binding identifies a logical source, -not a request object's identity. Execution must preserve this source when -copying requests for per-file field mapping or candidate filtering. -``NeedsScores`` derives execution requirements from the root TopK request and -these bindings; it does not add another user configuration. - -This first stage implements predicate construction, validation, field mapping, -and read-context configuration. Executing a full-text predicate, including using -it for scan pruning, returns ``NotImplemented``. Valid per-field score -projections also return ``NotImplemented`` when reading. Existing scalar and root -search execution retain their current capabilities and limitations. - -The next stage will implement exact mixed-predicate execution, per-file index -reader reuse, and selectivity-based AND ordering with candidate propagation. -Missing full-text or vector indexes must fail; scalar conditions may fall back -to exact data scanning. TopK must execute after the complete filter, with scores -and per-file results combined by the read layer. These execution changes are -not part of the predicate interface stage. - Interface ========= diff --git a/include/paimon/file_index/file_index_reader.h b/include/paimon/file_index/file_index_reader.h index 6118ebd3c..94f758ea9 100644 --- a/include/paimon/file_index/file_index_reader.h +++ b/include/paimon/file_index/file_index_reader.h @@ -82,8 +82,8 @@ class PAIMON_EXPORT FileIndexReader : public FunctionVisitor& full_text_search); /// Return every matching file-local physical row position, without TopK truncation. - virtual Result> VisitFullTextMatch( - const std::shared_ptr& full_text_match); + Result> VisitFullTextMatch( + const std::shared_ptr& full_text_match) override; }; } // namespace paimon diff --git a/include/paimon/predicate/full_text_match.h b/include/paimon/predicate/full_text_match.h index f2d8cbcea..51c37536a 100644 --- a/include/paimon/predicate/full_text_match.h +++ b/include/paimon/predicate/full_text_match.h @@ -44,8 +44,7 @@ struct PAIMON_EXPORT FullTextMatch { /// JSON DSL query with the same syntax and analyzer semantics as `FullTextSearch::query`. std::string query; /// Optional eligible row ids. File readers use file-local physical row positions; - /// global readers use global row ids. An empty set matches no rows. If not - /// set, all rows are searched. + /// An empty set matches no rows. If not set, all rows are searched. std::optional include_row_ids; }; diff --git a/include/paimon/predicate/function_visitor.h b/include/paimon/predicate/function_visitor.h index 8b759ff27..5e8b54b3e 100644 --- a/include/paimon/predicate/function_visitor.h +++ b/include/paimon/predicate/function_visitor.h @@ -18,15 +18,19 @@ #pragma once +#include #include #include #include "paimon/predicate/leaf_predicate.h" #include "paimon/predicate/literal.h" #include "paimon/result.h" +#include "paimon/status.h" #include "paimon/visibility.h" namespace paimon { +struct FullTextMatch; + /// A visitor interface for evaluating filter predicates on indexed columns. /// @tparam T The result type produced by each visit method (e.g., a file index result or global /// index result). @@ -76,5 +80,11 @@ class PAIMON_EXPORT FunctionVisitor { /// Evaluates whether string values like the given string. virtual Result VisitLike(const Literal& literal) = 0; + + /// Evaluates a full-text condition without TopK truncation. + /// Visitors without full-text support return NotImplemented. + virtual Result VisitFullTextMatch(const std::shared_ptr& full_text_match) { + return Status::NotImplemented("This visitor does not support complete full-text matching"); + } }; } // namespace paimon diff --git a/include/paimon/predicate/predicate_utils.h b/include/paimon/predicate/predicate_utils.h index 6195188e8..24cc47c08 100644 --- a/include/paimon/predicate/predicate_utils.h +++ b/include/paimon/predicate/predicate_utils.h @@ -117,6 +117,11 @@ class PAIMON_EXPORT PredicateUtils { assert(predicate->Literals().size() == 1); return visitor->VisitLike(predicate->Literals()[0]); } + case Function::Type::FULL_TEXT_MATCH: { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr full_text_match, + GetFullTextMatch(predicate)); + return visitor->VisitFullTextMatch(full_text_match); + } default: return Status::Invalid("invalid " + predicate->GetFunction().ToString() + " function in leaf predicate"); @@ -124,6 +129,9 @@ class PAIMON_EXPORT PredicateUtils { } private: + static Result> GetFullTextMatch( + const std::shared_ptr& predicate); + static Result>> ReconstructPredicateWithPickedFields( const std::shared_ptr& predicate, const std::map& picked_field_name_to_idx); diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index 88fc0a547..e62386c65 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -348,7 +348,10 @@ class PAIMON_EXPORT ReadContextBuilder { /// that needs to be read and processed. /// /// All referenced fields, including full-text match fields, must be projected in the read - /// schema. Full-text matches can be combined with scalar predicates using + /// schema. + /// TODO(lisizhuo.lsz): The behavior here will be aligned with Java Paimon in a follow-up + /// change: fields referenced in predicates will no longer be required to appear in the + /// projection. Full-text matches can be combined with scalar predicates using /// PredicateBuilder::And/Or. Full-text predicate execution is not implemented yet; reading such /// a predicate returns NotImplemented. /// diff --git a/src/paimon/common/file_index/file_index_reader_test.cpp b/src/paimon/common/file_index/file_index_reader_test.cpp index 299eeb2da..40a255597 100644 --- a/src/paimon/common/file_index/file_index_reader_test.cpp +++ b/src/paimon/common/file_index/file_index_reader_test.cpp @@ -24,6 +24,8 @@ #include "paimon/predicate/full_text_match.h" #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/predicate/predicate_utils.h" #include "paimon/predicate/vector_search.h" #include "paimon/testing/utils/testharness.h" @@ -80,4 +82,36 @@ TEST(FileIndexReaderTest, TestDefaultIndexReader) { ASSERT_NOK_WITH_MSG(reader.VisitFullTextMatch(full_text_match), "does not support complete full-text matching"); } + +TEST(FileIndexReaderTest, TestFullTextMatchVisitorDispatch) { + class FullTextReader : public FileIndexReader { + public: + Result> VisitFullTextMatch( + const std::shared_ptr& full_text_match) override { + received_match_ = full_text_match; + return FileIndexResult::Skip(); + } + + std::shared_ptr received_match_; + }; + + auto match = std::make_shared("body", R"({"match":{"query":"apple"}})"); + ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::FullTextMatch(0, match)); + auto leaf = std::dynamic_pointer_cast(predicate); + ASSERT_TRUE(leaf); + + auto default_reader = std::make_shared(); + auto result = + PredicateUtils::VisitPredicate>(leaf, default_reader); + ASSERT_NOK_WITH_MSG(result, "This file index does not support complete full-text matching"); + ASSERT_TRUE(result.status().IsNotImplemented()); + + auto reader = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + auto matches, + PredicateUtils::VisitPredicate>(leaf, reader)); + ASSERT_EQ(reader->received_match_, match); + ASSERT_OK_AND_ASSIGN(bool is_remain, matches->IsRemain()); + ASSERT_FALSE(is_remain); +} } // namespace paimon::test diff --git a/src/paimon/common/predicate/predicate_utils.cpp b/src/paimon/common/predicate/predicate_utils.cpp index 83ad478cb..f8dc25005 100644 --- a/src/paimon/common/predicate/predicate_utils.cpp +++ b/src/paimon/common/predicate/predicate_utils.cpp @@ -29,6 +29,15 @@ namespace paimon { +Result> PredicateUtils::GetFullTextMatch( + const std::shared_ptr& predicate) { + auto full_text = std::dynamic_pointer_cast(predicate); + if (!full_text) { + return Status::Invalid("FULL_TEXT_MATCH requires a FullTextMatchPredicate"); + } + return full_text->GetFullTextMatch(); +} + bool PredicateUtils::ContainsFullTextMatch(const std::shared_ptr& predicate) { if (std::dynamic_pointer_cast(predicate)) { return true; diff --git a/src/paimon/common/table/special_fields_test.cpp b/src/paimon/common/table/special_fields_test.cpp index 3f6f5ad34..32cac3960 100644 --- a/src/paimon/common/table/special_fields_test.cpp +++ b/src/paimon/common/table/special_fields_test.cpp @@ -56,14 +56,14 @@ TEST(SpecialFieldsTest, TestIndexScore) { } TEST(SpecialFieldsTest, TestPerFieldIndexScoreNames) { - for (const std::string& field_name : {"F1", "body_text", "_title", "文档"}) { + for (const auto& field_name : {"F1", "body_text", "_title", "文档"}) { const auto name = SpecialFields::IndexScoreFieldName(field_name); ASSERT_EQ(SpecialFields::ParseIndexScoreFieldName(name), field_name); ASSERT_TRUE(SpecialFields::IsSystemField(name)); } ASSERT_EQ(SpecialFields::IndexScoreFieldName("F1"), "_F1_INDEX_SCORE"); - for (const std::string& name : {"_INDEX_SCORE", "__INDEX_SCORE", "F1_INDEX_SCORE", - "_F1_index_score", "_F1_INDEX_SCORE_extra"}) { + for (const auto& name : {"_INDEX_SCORE", "__INDEX_SCORE", "F1_INDEX_SCORE", + "_F1_index_score", "_F1_INDEX_SCORE_extra"}) { ASSERT_FALSE(SpecialFields::ParseIndexScoreFieldName(name)); } } diff --git a/src/paimon/core/utils/field_mapping.cpp b/src/paimon/core/utils/field_mapping.cpp index 68c97c868..7155c16fb 100644 --- a/src/paimon/core/utils/field_mapping.cpp +++ b/src/paimon/core/utils/field_mapping.cpp @@ -345,9 +345,8 @@ Result>> FieldMappingBuilder::Reconstru const auto& [data_field_idx, data_field] = data_field_iter->second; return std::optional>(std::make_shared( - leaf_predicate->GetLeafFunction(), /*data field idx*/ data_field_idx, - /*data field name*/ data_field.Name(), - /*data field type*/ data_type, new_literals)); + leaf_predicate->GetLeafFunction(), /*data_field_idx=*/data_field_idx, + /*data_field_name=*/data_field.Name(), /*data_field_type=*/data_type, new_literals)); } Result>> diff --git a/test/inte/predicate_abi_inte_test.cpp b/test/inte/predicate_abi_inte_test.cpp index 811307852..023400593 100644 --- a/test/inte/predicate_abi_inte_test.cpp +++ b/test/inte/predicate_abi_inte_test.cpp @@ -46,6 +46,7 @@ #include "gtest/gtest.h" #include "paimon/defs.h" +#include "paimon/file_index/file_index_reader.h" #include "paimon/predicate/compound_predicate.h" #include "paimon/predicate/full_text_match.h" #include "paimon/predicate/function.h" @@ -53,6 +54,7 @@ #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate.h" #include "paimon/predicate/predicate_builder.h" +#include "paimon/predicate/predicate_utils.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -87,6 +89,10 @@ TEST(PredicateAbiInteTest, FullTextMatchCastsToLeafAcrossModule) { ASSERT_EQ(leaf->GetFieldType(), FieldType::STRING); ASSERT_TRUE(leaf->Literals().empty()); ASSERT_EQ(leaf->GetFunction().GetType(), Function::Type::FULL_TEXT_MATCH); + auto reader = std::make_shared(); + auto result = PredicateUtils::VisitPredicate>(leaf, reader); + ASSERT_NOK_WITH_MSG(result, "This file index does not support complete full-text matching"); + ASSERT_TRUE(result.status().IsNotImplemented()); } // A compound predicate built in libpaimon.so must survive dynamic_pointer_cast From bdb078f642c1d149ff6cdd961a5401f703b08620 Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Thu, 22 Oct 2026 17:40:13 +0800 Subject: [PATCH 3/3] fix pre-commit --- src/paimon/common/table/special_fields_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/paimon/common/table/special_fields_test.cpp b/src/paimon/common/table/special_fields_test.cpp index 32cac3960..94df21676 100644 --- a/src/paimon/common/table/special_fields_test.cpp +++ b/src/paimon/common/table/special_fields_test.cpp @@ -62,8 +62,8 @@ TEST(SpecialFieldsTest, TestPerFieldIndexScoreNames) { ASSERT_TRUE(SpecialFields::IsSystemField(name)); } ASSERT_EQ(SpecialFields::IndexScoreFieldName("F1"), "_F1_INDEX_SCORE"); - for (const auto& name : {"_INDEX_SCORE", "__INDEX_SCORE", "F1_INDEX_SCORE", - "_F1_index_score", "_F1_INDEX_SCORE_extra"}) { + for (const auto& name : {"_INDEX_SCORE", "__INDEX_SCORE", "F1_INDEX_SCORE", "_F1_index_score", + "_F1_INDEX_SCORE_extra"}) { ASSERT_FALSE(SpecialFields::ParseIndexScoreFieldName(name)); } }