feat(predicate): add full-text match and score projection semantics

docs/source/api/file_index.rst

@@ -21,9 +21,76 @@ File Index
 
 .. _cpp-api-file-index:
 
+Full-text predicates and root TopK
+=================================
+
+``PredicateBuilder::FullTextMatch`` creates a ``LeafPredicate`` for a full-text
+condition without TopK truncation. It holds a shared ``FullTextMatch`` request,
+whose query uses the same JSON DSL as ``FullTextSearch``. Do not modify a request
+after passing it to a predicate or read context.
+
+Compose full-text and scalar predicates with the existing
+``PredicateBuilder::And`` and ``PredicateBuilder::Or`` factories. Every field
+referenced by this tree must appear in the read schema; field indices refer to
+positions in that schema. For example, the following constructs
+``body matches "apple" OR price < 10``:
+
+.. code-block:: cpp
+
+   auto request = std::make_shared<paimon::FullTextMatch>(
+       "body", R"({"match":{"query":"apple"}})");
+   PAIMON_ASSIGN_OR_RAISE(
+       std::shared_ptr<paimon::Predicate> text,
+       paimon::PredicateBuilder::FullTextMatch(0, request));
+   auto price = paimon::PredicateBuilder::LessThan(
+       1, "price", paimon::FieldType::INT, paimon::Literal(10));
+   PAIMON_ASSIGN_OR_RAISE(
+       std::shared_ptr<paimon::Predicate> filter,
+       paimon::PredicateBuilder::Or({text, price}));
+   paimon::ReadContextBuilder builder(table_path);
+   builder.SetReadFieldNames({"body", "price"}).SetPredicate(filter);
+
+``SetVectorSearch`` and ``SetFullTextSearch`` configure a separate root TopK
+request. They are mutually exclusive and require a positive limit. The intended
+semantics are to filter by the complete predicate before ranking; the TopK search
+field does not need to be projected. A full-text match inside a predicate never
+has a TopK limit.
+
+Score output is requested through an explicit ``_<field>_INDEX_SCORE``
+projection. For example, request the score of a full-text
+predicate by projecting ``_body_INDEX_SCORE``:
+
+.. code-block:: cpp
+
+   auto request = std::make_shared<paimon::FullTextMatch>(
+       "body", R"({"match":{"query":"apple"}})");
+   PAIMON_ASSIGN_OR_RAISE(
+       std::shared_ptr<paimon::Predicate> text,
+       paimon::PredicateBuilder::FullTextMatch(0, request));
+   paimon::ReadContextBuilder builder(table_path);
+   builder.SetReadFieldNames({"body", "_body_INDEX_SCORE"}).SetPredicate(text);
+
+Score fields have type ``float32`` and preserve the source field's exact name,
+including case. Select them with ``SetReadFieldNames`` or ``SetReadSchema``;
+they have no persistent field IDs for ``SetReadFieldIds``. Reading all table
+fields does not implicitly add score fields.
+
+Each projected score must have exactly one search condition as its source.
+The source may be a root TopK request or a full-text predicate. A full-text
+predicate whose score is projected cannot have an OR ancestor, and its field
+cannot occur in another search condition, including the root TopK request.
+An ordinary scalar predicate on the same field is allowed. For example, when
+projecting the score of ``match``, ``match AND (p OR q)`` is valid, while
+``match OR p`` is rejected. Without that score projection, both OR branches and
+repeated full-text search fields are allowed. These checks use the effective
+read projection, after resolving the read schema.
+
 Interface
 =========
 
+.. doxygenstruct:: paimon::FullTextMatch
+   :members:
+
 .. doxygenclass:: paimon::FileIndexer
    :members:
    :undoc-members:

include/paimon/file_index/file_index_reader.h

@@ -31,6 +31,7 @@
 #include "paimon/visibility.h"
 
 namespace paimon {
+struct FullTextMatch;
 /// Evaluates filter predicates against a file-level index to determine file eligibility.
 ///
 /// `FileIndexReader` implements the `FunctionVisitor` interface specialized to produce
@@ -79,6 +80,10 @@ class PAIMON_EXPORT FileIndexReader : public FunctionVisitor<std::shared_ptr<Fil
     /// Execute full-text search and return matching file-local physical row positions.
     virtual Result<std::shared_ptr<FileIndexResult>> VisitFullTextSearch(
         const std::shared_ptr<FullTextSearch>& full_text_search);
+
+    /// Return every matching file-local physical row position, without TopK truncation.
+    Result<std::shared_ptr<FileIndexResult>> VisitFullTextMatch(
+        const std::shared_ptr<FullTextMatch>& full_text_match) override;
 };
 
 }  // namespace paimon

include/paimon/predicate/full_text_match.h

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <optional>
+#include <string>
+#include <utility>
+
+#include "paimon/utils/roaring_bitmap64.h"
+#include "paimon/visibility.h"
+
+namespace paimon {
+
+/// A full-text condition returning every matching row without TopK truncation.
+/// Project `_<field>_INDEX_SCORE` in the read schema to return its scores.
+struct PAIMON_EXPORT FullTextMatch {
+    FullTextMatch(const std::string& _field_name, const std::string& _query,
+                  std::optional<RoaringBitmap64> _include_row_ids = std::nullopt)
+        : field_name(_field_name), query(_query), include_row_ids(std::move(_include_row_ids)) {}
+
+    bool operator==(const FullTextMatch& other) const {
+        return field_name == other.field_name && query == other.query &&
+               include_row_ids == other.include_row_ids;
+    }
+
+    /// Table column containing the indexed text.
+    std::string field_name;
+    /// JSON DSL query with the same syntax and analyzer semantics as `FullTextSearch::query`.
+    std::string query;
+    /// Optional eligible row ids. File readers use file-local physical row positions;
+    /// An empty set matches no rows. If not set, all rows are searched.
+    std::optional<RoaringBitmap64> include_row_ids;
+};
+
+}  // namespace paimon

include/paimon/predicate/function.h

@@ -43,7 +43,8 @@ class PAIMON_EXPORT Function {
         STARTS_WITH = 13,
         ENDS_WITH = 14,
         CONTAINS = 15,
-        LIKE = 16
+        LIKE = 16,
+        FULL_TEXT_MATCH = 17
     };
     virtual ~Function() = default;
     virtual Type GetType() const = 0;

include/paimon/predicate/function_visitor.h

@@ -18,15 +18,19 @@
 
 #pragma once
 
+#include <memory>
 #include <string>
 #include <vector>
 
 #include "paimon/predicate/leaf_predicate.h"
 #include "paimon/predicate/literal.h"
 #include "paimon/result.h"
+#include "paimon/status.h"
 #include "paimon/visibility.h"
 
 namespace paimon {
+struct FullTextMatch;
+
 /// A visitor interface for evaluating filter predicates on indexed columns.
 /// @tparam T The result type produced by each visit method (e.g., a file index result or global
 /// index result).
@@ -76,5 +80,11 @@ class PAIMON_EXPORT FunctionVisitor {
 
     /// Evaluates whether string values like the given string.
     virtual Result<T> VisitLike(const Literal& literal) = 0;
+
+    /// Evaluates a full-text condition without TopK truncation.
+    /// Visitors without full-text support return NotImplemented.
+    virtual Result<T> VisitFullTextMatch(const std::shared_ptr<FullTextMatch>& full_text_match) {
+        return Status::NotImplemented("This visitor does not support complete full-text matching");
+    }
 };
 }  // namespace paimon

include/paimon/predicate/leaf_predicate.h

@@ -31,7 +31,8 @@ class LeafFunction;
 class Function;
 enum class FieldType;
 
-/// Leaf node of a `Predicate` tree. Compares a field with literals.
+/// Leaf node of a `Predicate` tree describing a condition on one field.
+/// Scalar leaves compare the field with literals; full-text leaves hold a search request.
 class PAIMON_EXPORT LeafPredicate : virtual public Predicate {
  public:
     int32_t FieldIndex() const {

include/paimon/predicate/predicate_builder.h

@@ -27,6 +27,7 @@
 #include "paimon/visibility.h"
 
 namespace paimon {
+struct FullTextMatch;
 class Literal;
 enum class FieldType;
 
@@ -39,6 +40,15 @@ class PAIMON_EXPORT PredicateBuilder {
     PredicateBuilder() = delete;
     ~PredicateBuilder() = delete;
 
+    /// Create a full-text predicate returning all matching rows without TopK truncation.
+    /// The field must have STRING type and a full-text index.
+    /// Project `_<field>_INDEX_SCORE` in the read schema to request score output.
+    /// @param field_index The index of the field in read schema (0-based).
+    /// @param match The full-text request, including the field name, JSON DSL query and
+    /// optional eligible row ids.
+    static Result<std::shared_ptr<Predicate>> FullTextMatch(
+        int32_t field_index, const std::shared_ptr<FullTextMatch>& match);
+
     /// Create an equality predicate (field == literal).
     ///
     /// @param field_index The index of the field in read schema (0-based).

include/paimon/predicate/predicate_utils.h

@@ -61,6 +61,9 @@ class PAIMON_EXPORT PredicateUtils {
     static Status GetAllNames(const std::shared_ptr<Predicate>& predicate,
                               std::set<std::string>* field_names);
 
+    /// Whether the predicate contains a full-text matching condition.
+    static bool ContainsFullTextMatch(const std::shared_ptr<Predicate>& predicate);
+
     template <typename T>
     static Result<T> VisitPredicate(const std::shared_ptr<LeafPredicate>& predicate,
                                     const std::shared_ptr<FunctionVisitor<T>>& visitor) {
@@ -114,13 +117,21 @@ class PAIMON_EXPORT PredicateUtils {
                 assert(predicate->Literals().size() == 1);
                 return visitor->VisitLike(predicate->Literals()[0]);
             }
+            case Function::Type::FULL_TEXT_MATCH: {
+                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FullTextMatch> full_text_match,
+                                       GetFullTextMatch(predicate));
+                return visitor->VisitFullTextMatch(full_text_match);
+            }
             default:
                 return Status::Invalid("invalid " + predicate->GetFunction().ToString() +
                                        " function in leaf predicate");
         }
     }
 
  private:
+    static Result<std::shared_ptr<FullTextMatch>> GetFullTextMatch(
+        const std::shared_ptr<LeafPredicate>& predicate);
+
     static Result<std::optional<std::shared_ptr<Predicate>>> ReconstructPredicateWithPickedFields(
         const std::shared_ptr<Predicate>& predicate,
         const std::map<std::string, int32_t>& picked_field_name_to_idx);

include/paimon/read_context.h

@@ -108,6 +108,7 @@ class PAIMON_EXPORT ReadContext {
         return full_text_search_;
     }
 
+    /// Whether a root TopK request is configured; full-text predicates are not TopK requests.
     bool HasVectorOrFullTextSearch() const {
         return vector_search_ != nullptr || full_text_search_ != nullptr;
     }
@@ -346,24 +347,37 @@ class PAIMON_EXPORT ReadContextBuilder {
     /// It can significantly improve performance by reducing the amount of data
     /// that needs to be read and processed.
     ///
+    /// All referenced fields, including full-text match fields, must be projected in the read
+    /// schema.
+    /// TODO(lisizhuo.lsz): The behavior here will be aligned with Java Paimon in a follow-up
+    /// change: fields referenced in predicates will no longer be required to appear in the
+    /// projection. Full-text matches can be combined with scalar predicates using
+    /// PredicateBuilder::And/Or. Full-text predicate execution is not implemented yet; reading such
+    /// a predicate returns NotImplemented.
+    ///
     /// @param predicate Shared pointer to the predicate for data filtering.
     /// @return Reference to this builder for method chaining.
     ReadContextBuilder& SetPredicate(const std::shared_ptr<Predicate>& predicate);
 
-    /// Set the vector search request used when reading data.
-    ///
-    /// TODO(jinli.zjw): Support combining vector search with predicates and full-text search. Merge
-    /// per-file candidates to compute read-level top-k results.
-    ///
+    /// Configure the root vector TopK request, logically applied after the predicate filter.
+    /// The search field need not be projected. SetVectorSearch is mutually exclusive with
+    /// SetFullTextSearch and must have a positive limit.
+    /// Scores are required internally for TopK. Project `_<field>_INDEX_SCORE` to include them
+    /// in the result. Per-field score output is not implemented yet; reading such a projection
+    /// returns NotImplemented.
+    /// TODO(lisizhuo.lsz): Execute combined read predicates and merge TopK across files.
     /// @param vector_search Vector search request, or nullptr to clear it.
     /// @return Reference to this builder for method chaining.
     ReadContextBuilder& SetVectorSearch(const std::shared_ptr<VectorSearch>& vector_search);
 
-    /// Set the full-text search request used when reading data.
-    ///
-    /// TODO(jinli.zjw): Support combining full-text search with predicates and vector search. Apply
-    /// the search limit to merged per-file results at read level.
-    ///
+    /// Configure the root full-text TopK request, logically applied after the predicate filter.
+    /// The search field need not be projected. Use PredicateBuilder::FullTextMatch for a
+    /// full-text filter without TopK truncation. SetFullTextSearch is mutually exclusive with
+    /// SetVectorSearch and must have a positive limit.
+    /// Scores are required internally for TopK. Project `_<field>_INDEX_SCORE` to include them
+    /// in the result. Per-field score output is not implemented yet; reading such a projection
+    /// returns NotImplemented.
+    /// TODO(lisizhuo.lsz): Execute combined read predicates and merge TopK across files.
     /// @param full_text_search Full-text search request, or nullptr to clear it.
     /// @return Reference to this builder for method chaining.
     ReadContextBuilder& SetFullTextSearch(const std::shared_ptr<FullTextSearch>& full_text_search);

src/paimon/CMakeLists.txt

@@ -118,6 +118,7 @@ set(PAIMON_COMMON_SRCS
     common/predicate/contains.cpp
     common/predicate/ends_with.cpp
     common/predicate/equal.cpp
+    common/predicate/full_text_match_predicate.cpp
     common/predicate/greater_or_equal.cpp
     common/predicate/greater_than.cpp
     common/predicate/in.cpp
@@ -669,6 +670,7 @@ if(PAIMON_BUILD_TESTS)
                     common/metrics/histogram_windowing_test.cpp
                     common/options/memory_size_test.cpp
                     common/options/time_duration_test.cpp
+                    common/predicate/full_text_match_predicate_test.cpp
                     common/predicate/full_text_search_test.cpp
                     common/predicate/literal_converter_test.cpp
                     common/predicate/literal_test.cpp

src/paimon/common/file_index/file_index_reader.cpp

@@ -111,4 +111,9 @@ Result<std::shared_ptr<FileIndexResult>> FileIndexReader::VisitFullTextSearch(
     const std::shared_ptr<FullTextSearch>& full_text_search) {
     return Status::NotImplemented("This file index does not support full text search");
 }
+
+Result<std::shared_ptr<FileIndexResult>> FileIndexReader::VisitFullTextMatch(
+    const std::shared_ptr<FullTextMatch>& full_text_match) {
+    return Status::NotImplemented("This file index does not support complete full-text matching");
+}
 }  // namespace paimon

src/paimon/common/file_index/file_index_reader_test.cpp

@@ -21,8 +21,11 @@
 #include <utility>
 
 #include "gtest/gtest.h"
+#include "paimon/predicate/full_text_match.h"
 #include "paimon/predicate/full_text_search.h"
 #include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "paimon/predicate/predicate_utils.h"
 #include "paimon/predicate/vector_search.h"
 #include "paimon/testing/utils/testharness.h"
 
@@ -75,5 +78,40 @@ TEST(FileIndexReaderTest, TestDefaultIndexReader) {
         std::make_shared<FullTextSearch>("body", R"({"match":{"query":"word"}})", /*limit=*/1);
     ASSERT_NOK_WITH_MSG(reader.VisitFullTextSearch(full_text_search),
                         "does not support full text search");
+    auto full_text_match = std::make_shared<FullTextMatch>("body", R"({"match":{"query":"word"}})");
+    ASSERT_NOK_WITH_MSG(reader.VisitFullTextMatch(full_text_match),
+                        "does not support complete full-text matching");
+}
+
+TEST(FileIndexReaderTest, TestFullTextMatchVisitorDispatch) {
+    class FullTextReader : public FileIndexReader {
+     public:
+        Result<std::shared_ptr<FileIndexResult>> VisitFullTextMatch(
+            const std::shared_ptr<FullTextMatch>& full_text_match) override {
+            received_match_ = full_text_match;
+            return FileIndexResult::Skip();
+        }
+
+        std::shared_ptr<FullTextMatch> received_match_;
+    };
+
+    auto match = std::make_shared<FullTextMatch>("body", R"({"match":{"query":"apple"}})");
+    ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::FullTextMatch(0, match));
+    auto leaf = std::dynamic_pointer_cast<LeafPredicate>(predicate);
+    ASSERT_TRUE(leaf);
+
+    auto default_reader = std::make_shared<FileIndexReader>();
+    auto result =
+        PredicateUtils::VisitPredicate<std::shared_ptr<FileIndexResult>>(leaf, default_reader);
+    ASSERT_NOK_WITH_MSG(result, "This file index does not support complete full-text matching");
+    ASSERT_TRUE(result.status().IsNotImplemented());
+
+    auto reader = std::make_shared<FullTextReader>();
+    ASSERT_OK_AND_ASSIGN(
+        auto matches,
+        PredicateUtils::VisitPredicate<std::shared_ptr<FileIndexResult>>(leaf, reader));
+    ASSERT_EQ(reader->received_match_, match);
+    ASSERT_OK_AND_ASSIGN(bool is_remain, matches->IsRemain());
+    ASSERT_FALSE(is_remain);
 }
 }  // namespace paimon::test