From 3aa63f1ab691bd9068c15428c6589054fe24e5f8 Mon Sep 17 00:00:00 2001
From: "jinli.zjw" <jinli.zjw@alibaba-inc.com>
Date: Tue, 20 Oct 2026 00:26:20 +0800
Subject: [PATCH 01/10] feat(file-index): support V2 containers and streaming
 writes

---
 include/paimon/defs.h                         |   3 +
 include/paimon/file_index/file_index_format.h |  84 ++---
 include/paimon/file_index/file_index_writer.h |  15 +-
 include/paimon/file_index/file_indexer.h      |   3 +-
 src/paimon/CMakeLists.txt                     |   7 +
 src/paimon/common/defs.cpp                    |   1 +
 .../file_index/bitmap/bitmap_file_index.cpp   |  14 +-
 .../file_index/bitmap/bitmap_file_index.h     |   2 +-
 .../bloomfilter/bloom_filter_file_index.cpp   |   8 +-
 .../bloomfilter/bloom_filter_file_index.h     |   2 +-
 .../bsi/bit_slice_index_bitmap_file_index.cpp |   8 +-
 .../bsi/bit_slice_index_bitmap_file_index.h   |   2 +-
 .../common/file_index/file_index_format.cpp   | 198 ++---------
 .../file_index/file_index_format_test.cpp     | 331 +++++++++++++++++-
 .../file_index/file_index_format_utils.cpp    |  61 ++++
 .../file_index/file_index_format_utils.h      |  51 +++
 .../file_index/file_index_format_v1.cpp       | 196 +++++++++++
 .../common/file_index/file_index_format_v1.h  | 118 +++++++
 .../file_index/file_index_format_v2.cpp       | 146 ++++++++
 .../common/file_index/file_index_format_v2.h  | 110 ++++++
 .../common/file_index/file_index_writer.cpp   |  44 +++
 .../file_index/file_index_writer_test.cpp     | 106 ++++++
 .../rangebitmap/range_bitmap_file_index.cpp   |  10 +-
 .../rangebitmap/range_bitmap_file_index.h     |   4 +-
 src/paimon/core/core_options.cpp              |  10 +
 src/paimon/core/core_options.h                |   1 +
 src/paimon/core/io/data_file_index_writer.cpp |  82 ++---
 src/paimon/core/io/data_file_index_writer.h   |   6 +-
 .../core/io/data_file_index_writer_test.cpp   | 214 ++++++++++-
 src/paimon/core/io/file_index_options.cpp     |  17 +-
 src/paimon/core/io/file_index_options.h       |   7 +-
 .../core/io/file_index_options_test.cpp       |  16 +
 .../core/io/spillable_index_output_stream.cpp |  97 +++++
 .../core/io/spillable_index_output_stream.h   |  68 ++++
 .../io/spillable_index_output_stream_test.cpp | 119 +++++++
 .../indexer/lumina/lumina_file_index.cpp      |  65 +++-
 src/paimon/indexer/lumina/lumina_file_index.h |   3 +-
 .../indexer/lumina/lumina_file_index_test.cpp |  92 ++++-
 .../testing/mock/mock_search_file_index.cpp   |  17 +-
 .../testing/mock/mock_search_file_index.h     |   2 +-
 40 files changed, 2007 insertions(+), 333 deletions(-)
 create mode 100644 src/paimon/common/file_index/file_index_format_utils.cpp
 create mode 100644 src/paimon/common/file_index/file_index_format_utils.h
 create mode 100644 src/paimon/common/file_index/file_index_format_v1.cpp
 create mode 100644 src/paimon/common/file_index/file_index_format_v1.h
 create mode 100644 src/paimon/common/file_index/file_index_format_v2.cpp
 create mode 100644 src/paimon/common/file_index/file_index_format_v2.h
 create mode 100644 src/paimon/common/file_index/file_index_writer.cpp
 create mode 100644 src/paimon/common/file_index/file_index_writer_test.cpp
 create mode 100644 src/paimon/core/io/spillable_index_output_stream.cpp
 create mode 100644 src/paimon/core/io/spillable_index_output_stream.h
 create mode 100644 src/paimon/core/io/spillable_index_output_stream_test.cpp

diff --git a/include/paimon/defs.h b/include/paimon/defs.h
index 8c88c3de6..e4223f12b 100644
--- a/include/paimon/defs.h
+++ b/include/paimon/defs.h
@@ -469,6 +469,9 @@ struct PAIMON_EXPORT Options {
     /// manifest. Default value is 500B.
     static const char FILE_INDEX_IN_MANIFEST_THRESHOLD[];
 
+    /// "file-index.format-version" - File index container write version, 1 or 2. Defaults to 1.
+    static const char FILE_INDEX_FORMAT_VERSION[];
+
     /// "data-file.external-paths" - The external paths where the data of this table will be
     /// written, multiple elements separated by commas.
     static const char DATA_FILE_EXTERNAL_PATHS[];
diff --git a/include/paimon/file_index/file_index_format.h b/include/paimon/file_index/file_index_format.h
index b02f581ce..d69b6eef7 100644
--- a/include/paimon/file_index/file_index_format.h
+++ b/include/paimon/file_index/file_index_format.h
@@ -19,6 +19,7 @@
 #pragma once
 
 #include <cstdint>
+#include <functional>
 #include <map>
 #include <memory>
 #include <string>
@@ -37,57 +38,10 @@ class Bytes;
 class OutputStream;
 
 /// Defines the on-disk format and versioning for Paimon file-level indexes.
-/// File index file format. Put all column and offset in the header.
-///
-/// <pre>
-///  _______________________________________    _____________________
-/// |     magic    | version | head length |
-/// |--------------------------------------|
-/// |            column number             |
-/// |--------------------------------------|
-/// |   column 1        |  index number    |
-/// |--------------------------------------|
-/// |  index name 1 | start pos | length   |
-/// |--------------------------------------|
-/// |  index name 2 | start pos | length   |
-/// |--------------------------------------|
-/// |  index name 3 | start pos | length   |
-/// |--------------------------------------|            HEAD
-/// |   column 2        |  index number    |
-/// |--------------------------------------|
-/// |  index name 1 | start pos | length   |
-/// |--------------------------------------|
-/// |  index name 2 | start pos | length   |
-/// |--------------------------------------|
-/// |  index name 3 | start pos | length   |
-/// |--------------------------------------|
-/// |                 ...                  |
-/// |--------------------------------------|
-/// |                 ...                  |
-/// |--------------------------------------|
-/// |  redundant length | redundant bytes  |
-/// |--------------------------------------|    ---------------------
-/// |                BODY                  |
-/// |                BODY                  |
-/// |                BODY                  |             BODY
-/// |                BODY                  |
-/// |______________________________________|    _____________________
-///
-/// magic:               8 bytes long
-/// version:             4 bytes int
-/// head length:         4 bytes int
-/// column number:       4 bytes int
-/// column x:            var bytes utf (length + bytes)
-/// index number:        4 bytes int (how many column items below)
-/// index name x:        var bytes utf
-/// start pos:           4 bytes int
-/// length:              4 bytes int
-/// redundant length:    4 bytes int (for compatibility with later versions, in this version,
-///                      content is zero)
-/// redundant bytes:     var bytes (for compatibility with later version, in this version, is empty)
-/// BODY:                column index bytes + column index bytes + column index bytes + .......
-/// </pre>
-///
+/// Supports V1 and V2 containers. V2 uses int64 offsets and lengths and supports streaming index
+/// payloads directly to the output; avoiding full payload buffering requires a streaming WriteTo()
+/// implementation in the index plugin.
+/// See file_index_format_v1.h and file_index_format_v2.h for the serialized layouts.
 class PAIMON_EXPORT FileIndexFormat {
  public:
     class Reader;
@@ -109,35 +63,47 @@ class PAIMON_EXPORT FileIndexFormat {
         const std::shared_ptr<InputStream>& input_stream, const std::shared_ptr<MemoryPool>& pool,
         const std::map<std::string, std::string>& options);
 
-    /// Creates a `Writer` which serializes a complete V1 file index container.
+    /// Creates a `Writer` for file index container.
     ///
     /// @param output_stream Destination stream for serialized index data.
     /// @param pool Memory pool for writer-side allocations.
+    /// @param version Container format version; defaults to V1.
     /// @return A unique pointer to a `Writer` on success.
     static Result<std::unique_ptr<Writer>> CreateWriter(
-        const std::shared_ptr<OutputStream>& output_stream,
-        const std::shared_ptr<MemoryPool>& pool);
+        const std::shared_ptr<OutputStream>& output_stream, const std::shared_ptr<MemoryPool>& pool,
+        int32_t version = V_1);
 
  public:
     static const int64_t MAGIC;
     static const int32_t EMPTY_INDEX_FLAG;
     static const int32_t V_1;
+    static const int32_t V_2;
 };
 
 /// Writer for file index file.
-class FileIndexFormat::Writer {
+class PAIMON_EXPORT FileIndexFormat::Writer {
  public:
+    /// Writes one payload without closing the stream. Returns false for an absent index
+    /// (without writing bytes), or true after writing a non-empty payload.
+    using PayloadWriter = std::function<Result<bool>(const std::shared_ptr<OutputStream>&)>;
     virtual ~Writer() = default;
 
-    /// Writes all column indexes. This is a terminal, one-shot operation.
-    virtual Status WriteColumnIndexes(const FileIndexFormat::ColumnIndexes& indexes) = 0;
-
     /// Flushes and closes the output stream supplied to `CreateWriter()`.
     virtual Status Close() = 0;
+
+    /// Writes one index. V1 buffers payloads; V2 writes them directly to the destination.
+    /// An empty callback or a false result denotes an empty index (-1, 0).
+    /// A true result indicates that a non-empty payload was written.
+    /// Call Finish() after the last index. Payload failures are terminal.
+    virtual Status WriteIndex(const std::string& column_name, const std::string& index_type,
+                              const PayloadWriter& payload) = 0;
+
+    /// Completes the container: writes V1's header and buffered bodies, or V2's footer and trailer.
+    virtual Status Finish() = 0;
 };
 
 /// Reader for file index file.
-class FileIndexFormat::Reader {
+class PAIMON_EXPORT FileIndexFormat::Reader {
  public:
     virtual ~Reader() = default;
     /// Reads index data for a specific column from the index file.
diff --git a/include/paimon/file_index/file_index_writer.h b/include/paimon/file_index/file_index_writer.h
index 4fc388bb9..7f2aed7db 100644
--- a/include/paimon/file_index/file_index_writer.h
+++ b/include/paimon/file_index/file_index_writer.h
@@ -29,6 +29,8 @@
 struct ArrowArray;
 
 namespace paimon {
+class OutputStream;
+
 /// Interface for writing file-level index data from Arrow batches.
 class PAIMON_EXPORT FileIndexWriter {
  public:
@@ -48,9 +50,20 @@ class PAIMON_EXPORT FileIndexWriter {
     /// have been added. It can be called only once and typically assumes no further calls to
     /// `AddBatch()` will occur afterward.
     ///
+    /// Streaming-only writers may override WriteTo() and use the default NotImplemented result
+    /// from this method.
+    ///
     /// @return A unique pointer to a byte array containing the serialized index data,
     ///         or an error if serialization fails.
-    virtual Result<PAIMON_UNIQUE_PTR<Bytes>> SerializedBytes() const = 0;
+    virtual Result<PAIMON_UNIQUE_PTR<Bytes>> SerializedBytes() const;
+
+    /// Writes one index payload to the caller-owned stream without closing it.
+    /// The default implementation writes SerializedBytes(); streaming writers can override this
+    /// method to avoid materializing the complete payload. Write failures must be propagated.
+    /// @return true if a non-empty index payload was produced, false if there is no index, or an
+    /// error. Returning false must not write any bytes. The default implementation treats null or
+    /// zero-length SerializedBytes() results as no index.
+    virtual Result<bool> WriteTo(const std::shared_ptr<OutputStream>& output) const;
 };
 
 }  // namespace paimon
diff --git a/include/paimon/file_index/file_indexer.h b/include/paimon/file_index/file_indexer.h
index 47af01c00..2f2fdd9a2 100644
--- a/include/paimon/file_index/file_indexer.h
+++ b/include/paimon/file_index/file_indexer.h
@@ -44,6 +44,7 @@ class PAIMON_EXPORT FileIndexer {
     }
 
     /// Create `FileIndexReader` with input stream.
+    /// @note Plugin implementations should validate their own payload size limits.
     ///
     /// @param arrow_schema ArrowSchema derived from arrow schema or struct type with
     /// specified indexed field.
@@ -53,7 +54,7 @@ class PAIMON_EXPORT FileIndexer {
     /// @param pool Memory pool for memory allocation.
     /// @return A `FileIndexReader` to read index.
     virtual Result<std::shared_ptr<FileIndexReader>> CreateReader(
-        ::ArrowSchema* arrow_schema, int32_t start, int32_t length,
+        ::ArrowSchema* arrow_schema, int64_t start, int64_t length,
         const std::shared_ptr<InputStream>& input_stream,
         const std::shared_ptr<MemoryPool>& pool) const = 0;
 
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index ba89aa825..ef7ecec15 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -71,7 +71,11 @@ set(PAIMON_COMMON_SRCS
     common/file_index/bitmap/bitmap_index_result.cpp
     common/file_index/file_indexer_factory.cpp
     common/file_index/file_index_format.cpp
+    common/file_index/file_index_format_utils.cpp
+    common/file_index/file_index_format_v1.cpp
+    common/file_index/file_index_format_v2.cpp
     common/file_index/file_index_reader.cpp
+    common/file_index/file_index_writer.cpp
     common/file_index/file_index_result.cpp
     common/file_index/scored_file_index_result.cpp
     common/format/column_stats.cpp
@@ -294,6 +298,7 @@ set(PAIMON_CORE_SRCS
     core/io/data_file_meta_write_cols_legacy_serializer.cpp
     core/io/data_file_path_factory.cpp
     core/io/data_file_index_writer.cpp
+    core/io/spillable_index_output_stream.cpp
     core/io/file_index_options.cpp
     core/io/append_data_file_writer_factory.cpp
     core/io/blob_data_file_writer_factory.cpp
@@ -597,6 +602,7 @@ if(PAIMON_BUILD_TESTS)
                     common/file_index/file_indexer_factory_test.cpp
                     common/file_index/file_index_result_test.cpp
                     common/file_index/file_index_reader_test.cpp
+                    common/file_index/file_index_writer_test.cpp
                     common/file_index/scored_file_index_result_test.cpp
                     common/file_index/file_index_format_test.cpp
                     common/file_index/empty/empty_file_index_reader_test.cpp
@@ -829,6 +835,7 @@ if(PAIMON_BUILD_TESTS)
                     core/io/data_file_meta_test.cpp
                     core/io/data_file_writer_base_test.cpp
                     core/io/data_file_index_writer_test.cpp
+                    core/io/spillable_index_output_stream_test.cpp
                     core/io/file_index_options_test.cpp
                     core/io/file_index_evaluator_test.cpp
                     core/io/single_file_writer_test.cpp
diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp
index aa59ed5b1..571cbbb93 100644
--- a/src/paimon/common/defs.cpp
+++ b/src/paimon/common/defs.cpp
@@ -118,6 +118,7 @@ const char Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP[] =
 const char Options::SCAN_FALLBACK_BRANCH[] = "scan.fallback-branch";
 const char Options::BRANCH[] = "branch";
 const char Options::FILE_INDEX_READ_ENABLED[] = "file-index.read.enabled";
+const char Options::FILE_INDEX_FORMAT_VERSION[] = "file-index.format-version";
 const char Options::FILE_INDEX_IN_MANIFEST_THRESHOLD[] = "file-index.in-manifest-threshold";
 const char Options::DATA_FILE_EXTERNAL_PATHS[] = "data-file.external-paths";
 const char Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY[] = "data-file.external-paths.strategy";
diff --git a/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp b/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp
index 43bb71b0e..4bd1deb9f 100644
--- a/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp
+++ b/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp
@@ -18,6 +18,7 @@
 
 #include "paimon/common/file_index/bitmap/bitmap_file_index.h"
 
+#include <limits>
 #include <utility>
 
 #include "arrow/c/bridge.h"
@@ -25,6 +26,7 @@
 #include "paimon/common/file_index/bitmap/bitmap_file_index_meta.h"
 #include "paimon/common/file_index/bitmap/bitmap_file_index_meta_v1.h"
 #include "paimon/common/file_index/bitmap/bitmap_file_index_meta_v2.h"
+#include "paimon/common/io/offset_input_stream.h"
 #include "paimon/common/memory/memory_segment_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
@@ -77,9 +79,15 @@ FieldType BitmapFileIndex::ConvertType(const FieldType& data_type) {
 }
 
 Result<std::shared_ptr<FileIndexReader>> BitmapFileIndex::CreateReader(
-    ::ArrowSchema* c_arrow_schema, int32_t start, int32_t length,
+    ::ArrowSchema* c_arrow_schema, int64_t start, int64_t length,
     const std::shared_ptr<InputStream>& input_stream,
     const std::shared_ptr<MemoryPool>& pool) const {
+    if (length > std::numeric_limits<int32_t>::max()) {
+        return Status::NotImplemented(
+            "Bitmap file index does not support payloads larger than INT32_MAX");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OffsetInputStream> bounded_stream,
+                           OffsetInputStream::Create(input_stream, length, start));
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> arrow_schema,
                                       arrow::ImportSchema(c_arrow_schema));
     if (arrow_schema->num_fields() != 1) {
@@ -90,8 +98,8 @@ Result<std::shared_ptr<FileIndexReader>> BitmapFileIndex::CreateReader(
     auto arrow_type = arrow_schema->field(0)->type();
     PAIMON_ASSIGN_OR_RAISE(FieldType data_type,
                            FieldTypeUtils::ConvertToFieldType(arrow_type->id()));
-    return std::make_shared<BitmapFileIndexReader>(arrow_type, data_type, start, length,
-                                                   input_stream, pool);
+    return std::make_shared<BitmapFileIndexReader>(
+        arrow_type, data_type, 0, static_cast<int32_t>(length), bounded_stream, pool);
 }
 
 Result<std::shared_ptr<FileIndexWriter>> BitmapFileIndex::CreateWriter(
diff --git a/src/paimon/common/file_index/bitmap/bitmap_file_index.h b/src/paimon/common/file_index/bitmap/bitmap_file_index.h
index 6aa0f4169..6c6e41a5e 100644
--- a/src/paimon/common/file_index/bitmap/bitmap_file_index.h
+++ b/src/paimon/common/file_index/bitmap/bitmap_file_index.h
@@ -46,7 +46,7 @@ class PAIMON_EXPORT BitmapFileIndex : public FileIndexer {
     ~BitmapFileIndex() override = default;
 
     Result<std::shared_ptr<FileIndexReader>> CreateReader(
-        ::ArrowSchema* arrow_schema, int32_t start, int32_t length,
+        ::ArrowSchema* arrow_schema, int64_t start, int64_t length,
         const std::shared_ptr<InputStream>& input_stream,
         const std::shared_ptr<MemoryPool>& pool) const override;
 
diff --git a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp
index 86d07d63b..713302120 100644
--- a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp
+++ b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp
@@ -27,6 +27,7 @@
 
 #include "arrow/c/bridge.h"
 #include "fmt/format.h"
+#include "paimon/common/io/offset_input_stream.h"
 #include "paimon/common/predicate/literal_converter.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
@@ -43,9 +44,11 @@ class MemoryPool;
 BloomFilterFileIndex::BloomFilterFileIndex(const std::map<std::string, std::string>& options)
     : options_(options) {}
 Result<std::shared_ptr<FileIndexReader>> BloomFilterFileIndex::CreateReader(
-    ::ArrowSchema* c_arrow_schema, int32_t start, int32_t length,
+    ::ArrowSchema* c_arrow_schema, int64_t start, int64_t length,
     const std::shared_ptr<InputStream>& input_stream,
     const std::shared_ptr<MemoryPool>& pool) const {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OffsetInputStream> bounded_stream,
+                           OffsetInputStream::Create(input_stream, length, start));
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> arrow_schema,
                                       arrow::ImportSchema(c_arrow_schema));
     if (arrow_schema->num_fields() != 1) {
@@ -55,10 +58,9 @@ Result<std::shared_ptr<FileIndexReader>> BloomFilterFileIndex::CreateReader(
     }
     auto arrow_type = arrow_schema->field(0)->type();
 
-    PAIMON_RETURN_NOT_OK(input_stream->Seek(start, SeekOrigin::FS_SEEK_SET));
     auto bytes = std::make_shared<Bytes>(length, pool.get());
     PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_len,
-                           input_stream->Read(bytes->data(), bytes->size()));
+                           bounded_stream->Read(bytes->data(), bytes->size()));
     if (static_cast<size_t>(actual_read_len) != bytes->size()) {
         return Status::Invalid(
             fmt::format("create reader for BloomFilterFileIndex failed, expected read len "
diff --git a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.h b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.h
index 62d082c76..4ab6a94f8 100644
--- a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.h
+++ b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.h
@@ -50,7 +50,7 @@ class BloomFilterFileIndex : public FileIndexer {
     ~BloomFilterFileIndex() override = default;
 
     Result<std::shared_ptr<FileIndexReader>> CreateReader(
-        ::ArrowSchema* arrow_schema, int32_t start, int32_t length,
+        ::ArrowSchema* arrow_schema, int64_t start, int64_t length,
         const std::shared_ptr<InputStream>& input_stream,
         const std::shared_ptr<MemoryPool>& pool) const override;
 
diff --git a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp
index 558dec502..37266e494 100644
--- a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp
+++ b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp
@@ -32,6 +32,7 @@
 #include "fmt/format.h"
 #include "paimon/common/file_index/bsi/bit_slice_index_roaring_bitmap.h"
 #include "paimon/common/io/memory_segment_output_stream.h"
+#include "paimon/common/io/offset_input_stream.h"
 #include "paimon/common/memory/memory_segment_utils.h"
 #include "paimon/common/predicate/literal_converter.h"
 #include "paimon/common/utils/arrow/status_utils.h"
@@ -160,9 +161,11 @@ Result<PAIMON_UNIQUE_PTR<Bytes>> BitSliceIndexBitmapFileIndexWriter::SerializedB
 }
 
 Result<std::shared_ptr<FileIndexReader>> BitSliceIndexBitmapFileIndex::CreateReader(
-    ::ArrowSchema* c_arrow_schema, int32_t start, int32_t length,
+    ::ArrowSchema* c_arrow_schema, int64_t start, int64_t length,
     const std::shared_ptr<InputStream>& input_stream,
     const std::shared_ptr<MemoryPool>& pool) const {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OffsetInputStream> bounded_stream,
+                           OffsetInputStream::Create(input_stream, length, start));
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> arrow_schema,
                                       arrow::ImportSchema(c_arrow_schema));
     if (arrow_schema->num_fields() != 1) {
@@ -172,10 +175,9 @@ Result<std::shared_ptr<FileIndexReader>> BitSliceIndexBitmapFileIndex::CreateRea
     }
     auto arrow_type = arrow_schema->field(0)->type();
 
-    PAIMON_RETURN_NOT_OK(input_stream->Seek(start, SeekOrigin::FS_SEEK_SET));
     auto bytes = std::make_unique<Bytes>(length, pool.get());
     PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_len,
-                           input_stream->Read(bytes->data(), bytes->size()));
+                           bounded_stream->Read(bytes->data(), bytes->size()));
     if (static_cast<size_t>(actual_read_len) != bytes->size()) {
         return Status::Invalid(
             fmt::format("create reader for BitSliceIndexBitmapFileIndex failed, expected read len "
diff --git a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h
index d7dfe092f..69f1b65a2 100644
--- a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h
+++ b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h
@@ -49,7 +49,7 @@ class BitSliceIndexBitmapFileIndex : public FileIndexer {
     ~BitSliceIndexBitmapFileIndex() override = default;
 
     Result<std::shared_ptr<FileIndexReader>> CreateReader(
-        ::ArrowSchema* arrow_schema, int32_t start, int32_t length,
+        ::ArrowSchema* arrow_schema, int64_t start, int64_t length,
         const std::shared_ptr<InputStream>& input_stream,
         const std::shared_ptr<MemoryPool>& pool) const override;
 
diff --git a/src/paimon/common/file_index/file_index_format.cpp b/src/paimon/common/file_index/file_index_format.cpp
index 0b5bf4cde..4e8af8200 100644
--- a/src/paimon/common/file_index/file_index_format.cpp
+++ b/src/paimon/common/file_index/file_index_format.cpp
@@ -19,21 +19,21 @@
 #include "paimon/file_index/file_index_format.h"
 
 #include <cassert>
-#include <map>
-#include <unordered_map>
 #include <utility>
 
 #include "arrow/c/bridge.h"
+#include "arrow/c/helpers.h"
 #include "arrow/type.h"
 #include "fmt/format.h"
 #include "paimon/common/file_index/empty/empty_file_index_reader.h"
-#include "paimon/common/io/data_output_stream.h"
+#include "paimon/common/file_index/file_index_format_utils.h"
+#include "paimon/common/file_index/file_index_format_v1.h"
+#include "paimon/common/file_index/file_index_format_v2.h"
 #include "paimon/common/utils/arrow/status_utils.h"
-#include "paimon/common/utils/math.h"
+#include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/io/file_index_options.h"
 #include "paimon/file_index/file_indexer.h"
 #include "paimon/file_index/file_indexer_factory.h"
-#include "paimon/io/byte_array_input_stream.h"
 #include "paimon/io/data_input_stream.h"
 #include "paimon/memory/bytes.h"
 #include "paimon/status.h"
@@ -42,134 +42,8 @@ namespace paimon {
 class InputStream;
 class MemoryPool;
 
-class FileIndexFormatWriterImpl : public FileIndexFormat::Writer {
- public:
-    explicit FileIndexFormatWriterImpl(const std::shared_ptr<OutputStream>& output_stream)
-        : output_stream_(output_stream) {
-        assert(output_stream_);
-    }
-
-    Status WriteColumnIndexes(const FileIndexFormat::ColumnIndexes& indexes) override {
-        if (written_) {
-            return Status::Invalid("File index column indexes have already been written");
-        }
-
-        PAIMON_RETURN_NOT_OK(WriteHead(indexes));
-        // Write body.
-        DataOutputStream data_output(output_stream_);
-        for (const auto& [column_name, column_indexes] : indexes) {
-            for (const auto& [index_type, bytes] : column_indexes) {
-                if (bytes) {
-                    PAIMON_RETURN_NOT_OK(data_output.WriteBytes(bytes));
-                }
-            }
-        }
-        written_ = true;
-        return Status::OK();
-    }
-
-    Status Close() override {
-        if (closed_) {
-            return Status::OK();
-        }
-        closed_ = true;
-        PAIMON_RETURN_NOT_OK(output_stream_->Flush());
-        return output_stream_->Close();
-    }
-
- private:
-    static constexpr int32_t kRedundantLength = 0;
-
-    static Result<int32_t> CalculateHeadLength(const FileIndexFormat::ColumnIndexes& indexes) {
-        // magic(8), version(4), header length(4), and column count(4).
-        int64_t head_length = 8 + 4 + 4 + 4;
-        int64_t body_length = 0;
-        PAIMON_RETURN_NOT_OK(
-            ValidateValueInRange<int32_t>(indexes.size(), "file index column count"));
-        for (const auto& [column_name, column_indexes] : indexes) {
-            PAIMON_RETURN_NOT_OK(ValidateValueInRange<uint16_t>(column_name.size(),
-                                                                "file index column name length"));
-            PAIMON_RETURN_NOT_OK(
-                ValidateValueInRange<int32_t>(column_indexes.size(), "column index count"));
-            // column name(2 + N) + index count(4)
-            head_length += 2 + static_cast<int64_t>(column_name.size()) + 4;
-            for (const auto& [index_type, bytes] : column_indexes) {
-                PAIMON_RETURN_NOT_OK(ValidateValueInRange<uint16_t>(index_type.size(),
-                                                                    "file index type name length"));
-                // index type(2 + N) + body offset(4) + body length(4)
-                head_length += 2 + static_cast<int64_t>(index_type.size()) + 4 + 4;
-                if (bytes) {
-                    PAIMON_RETURN_NOT_OK(AddChecked(bytes->size(), "index body", &body_length));
-                }
-            }
-        }
-
-        head_length += 4;  // The trailing redundant-length field(4).
-        PAIMON_RETURN_NOT_OK(
-            ValidateValueInRange<int32_t>(head_length, "file index header length"));
-        int64_t container_length = head_length + body_length;
-        PAIMON_RETURN_NOT_OK(ValidateValueInRange<int32_t>(container_length, "file index size"));
-        return static_cast<int32_t>(head_length);
-    }
-
-    Status WriteHead(const FileIndexFormat::ColumnIndexes& indexes) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t head_length, CalculateHeadLength(indexes));
-        DataOutputStream data_output(output_stream_);
-        // Write magic.
-        PAIMON_RETURN_NOT_OK(data_output.WriteValue<int64_t>(FileIndexFormat::MAGIC));
-        // Write version.
-        PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(FileIndexFormat::V_1));
-        // Write head length.
-        PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(head_length));
-        // Write column count.
-        PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(static_cast<int32_t>(indexes.size())));
-
-        int64_t body_offset = head_length;
-        for (const auto& [column_name, column_indexes] : indexes) {
-            // Write column name.
-            PAIMON_RETURN_NOT_OK(data_output.WriteString(column_name));
-            // Write index count for the column.
-            PAIMON_RETURN_NOT_OK(
-                data_output.WriteValue<int32_t>(static_cast<int32_t>(column_indexes.size())));
-            for (const auto& [index_type, bytes] : column_indexes) {
-                // Write index type.
-                PAIMON_RETURN_NOT_OK(data_output.WriteString(index_type));
-                // Write body offset and length.
-                if (bytes) {
-                    PAIMON_RETURN_NOT_OK(
-                        data_output.WriteValue<int32_t>(static_cast<int32_t>(body_offset)));
-                    PAIMON_RETURN_NOT_OK(
-                        data_output.WriteValue<int32_t>(static_cast<int32_t>(bytes->size())));
-                    body_offset += static_cast<int64_t>(bytes->size());
-                } else {
-                    PAIMON_RETURN_NOT_OK(
-                        data_output.WriteValue<int32_t>(FileIndexFormat::EMPTY_INDEX_FLAG));
-                    PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(0));
-                }
-            }
-        }
-        // Write redundant length for future format extensions.
-        return data_output.WriteValue<int32_t>(kRedundantLength);
-    }
-
-    template <typename T>
-    static Status AddChecked(T value, const char* name, int64_t* total) {
-        PAIMON_RETURN_NOT_OK(ValidateValueInRange<int32_t>(value, name));
-        *total += static_cast<int64_t>(value);
-        return ValidateValueInRange<int32_t>(*total, name);
-    }
-
-    std::shared_ptr<OutputStream> output_stream_;
-    bool written_ = false;
-    bool closed_ = false;
-};
-
 class FileIndexFormatReaderImpl : public FileIndexFormat::Reader {
  public:
-    using HeaderType =
-        std::unordered_map<std::string,
-                           std::unordered_map<std::string, std::pair<int32_t, int32_t>>>;
-
     static Result<std::unique_ptr<FileIndexFormatReaderImpl>> Create(
         const std::shared_ptr<InputStream>& input_stream, const std::shared_ptr<MemoryPool>& pool,
         const std::map<std::string, std::string>& options) {
@@ -181,36 +55,15 @@ class FileIndexFormatReaderImpl : public FileIndexFormat::Reader {
             return Status::Invalid("This file is not file index file.");
         }
         PAIMON_ASSIGN_OR_RAISE(int32_t version, data_input_stream.ReadValue<int32_t>());
-        if (version != FileIndexFormat::V_1) {
-            return Status::Invalid(
-                fmt::format("This index file is version of {}, not in supported version list [{}]",
-                            version, FileIndexFormat::V_1));
-        }
-        PAIMON_ASSIGN_OR_RAISE(int32_t head_length, data_input_stream.ReadValue<int32_t>());
-        auto head_bytes = std::make_unique<Bytes>(head_length - 8 - 4 - 4, pool.get());
-        PAIMON_RETURN_NOT_OK(data_input_stream.ReadBytes(head_bytes.get()));
-
-        auto byte_array_input_stream =
-            std::make_shared<ByteArrayInputStream>(head_bytes->data(), head_bytes->size());
-        DataInputStream inner_data_input_stream(byte_array_input_stream);
-        PAIMON_ASSIGN_OR_RAISE(int32_t column_size, inner_data_input_stream.ReadValue<int32_t>());
-
-        HeaderType header;
-        for (int32_t i = 0; i < column_size; i++) {
-            PAIMON_ASSIGN_OR_RAISE(std::string column_name, inner_data_input_stream.ReadString());
-            PAIMON_ASSIGN_OR_RAISE(int32_t index_size,
-                                   inner_data_input_stream.ReadValue<int32_t>());
-            auto& index_map = header[column_name];
-            for (int32_t j = 0; j < index_size; j++) {
-                PAIMON_ASSIGN_OR_RAISE(std::string index_type,
-                                       inner_data_input_stream.ReadString());
-                PAIMON_ASSIGN_OR_RAISE(int32_t offset,
-                                       inner_data_input_stream.ReadValue<int32_t>());
-                PAIMON_ASSIGN_OR_RAISE(int32_t length,
-                                       inner_data_input_stream.ReadValue<int32_t>());
-                index_map[index_type] = std::make_pair(offset, length);
-            }
+        if (version != FileIndexFormat::V_1 && version != FileIndexFormat::V_2) {
+            return Status::Invalid(fmt::format("Unsupported file index version {}", version));
         }
+        PAIMON_ASSIGN_OR_RAISE(int64_t file_length, input_stream->Length());
+        PAIMON_ASSIGN_OR_RAISE(
+            FileIndexEntries header,
+            version == FileIndexFormat::V_1
+                ? FileIndexFormatV1::ReadIndexEntries(input_stream, pool, file_length)
+                : FileIndexFormatV2::ReadIndexEntries(input_stream, pool, file_length));
         return std::unique_ptr<FileIndexFormatReaderImpl>(new FileIndexFormatReaderImpl(
             input_stream, std::move(header), pool, std::move(file_index_options)));
     }
@@ -242,8 +95,9 @@ class FileIndexFormatReaderImpl : public FileIndexFormat::Reader {
     }
 
  private:
-    FileIndexFormatReaderImpl(const std::shared_ptr<InputStream>& input_stream, HeaderType&& header,
-                              const std::shared_ptr<MemoryPool>& pool, FileIndexOptions&& options)
+    FileIndexFormatReaderImpl(const std::shared_ptr<InputStream>& input_stream,
+                              FileIndexEntries&& header, const std::shared_ptr<MemoryPool>& pool,
+                              FileIndexOptions&& options)
         : input_stream_(input_stream),
           pool_(pool),
           header_(std::move(header)),
@@ -253,7 +107,7 @@ class FileIndexFormatReaderImpl : public FileIndexFormat::Reader {
 
     Result<std::shared_ptr<FileIndexReader>> GetFileIndexReader(
         const std::string& column_name, const std::shared_ptr<arrow::Schema>& arrow_schema,
-        const std::string& index_type, const std::pair<int32_t, int32_t>& offset_and_length) const {
+        const std::string& index_type, const std::pair<int64_t, int64_t>& offset_and_length) const {
         if (offset_and_length.first == FileIndexFormat::EMPTY_INDEX_FLAG) {
             return std::make_shared<EmptyFileIndexReader>();
         }
@@ -266,6 +120,8 @@ class FileIndexFormatReaderImpl : public FileIndexFormat::Reader {
             return std::shared_ptr<FileIndexReader>();
         }
         ArrowSchema c_arrow_schema;
+        ArrowSchemaMarkReleased(&c_arrow_schema);
+        ScopeGuard schema_guard([&c_arrow_schema]() { ArrowSchemaRelease(&c_arrow_schema); });
         PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema));
         return file_indexer->CreateReader(&c_arrow_schema, offset_and_length.first,
                                           offset_and_length.second, input_stream_, pool_);
@@ -276,13 +132,14 @@ class FileIndexFormatReaderImpl : public FileIndexFormat::Reader {
     std::shared_ptr<MemoryPool> pool_;
     // get header and cache it.
     // [column_name : [index_type : {offset, length}]]
-    HeaderType header_;
+    FileIndexEntries header_;
     FileIndexOptions options_;
 };
 
 const int64_t FileIndexFormat::MAGIC = 1493475289347502LL;
 const int32_t FileIndexFormat::EMPTY_INDEX_FLAG = -1;
 const int32_t FileIndexFormat::V_1 = 1;
+const int32_t FileIndexFormat::V_2 = 2;
 
 Result<std::unique_ptr<FileIndexFormat::Reader>> FileIndexFormat::CreateReader(
     const std::shared_ptr<InputStream>& input_stream, const std::shared_ptr<MemoryPool>& pool,
@@ -291,13 +148,22 @@ Result<std::unique_ptr<FileIndexFormat::Reader>> FileIndexFormat::CreateReader(
 }
 
 Result<std::unique_ptr<FileIndexFormat::Writer>> FileIndexFormat::CreateWriter(
-    const std::shared_ptr<OutputStream>& output_stream, const std::shared_ptr<MemoryPool>& pool) {
+    const std::shared_ptr<OutputStream>& output_stream, const std::shared_ptr<MemoryPool>& pool,
+    int32_t version) {
     if (!output_stream) {
         return Status::Invalid("File index output stream cannot be null");
     }
     if (!pool) {
         return Status::Invalid("File index memory pool cannot be null");
     }
-    return std::make_unique<FileIndexFormatWriterImpl>(output_stream);
+    if (version == V_1) {
+        return std::make_unique<FileIndexFormatV1::Writer>(output_stream, pool);
+    }
+    if (version == V_2) {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<Writer> writer,
+                               FileIndexFormatV2::Writer::Create(output_stream));
+        return writer;
+    }
+    return Status::Invalid(fmt::format("Unsupported file index version {}", version));
 }
 }  // namespace paimon
diff --git a/src/paimon/common/file_index/file_index_format_test.cpp b/src/paimon/common/file_index/file_index_format_test.cpp
index c54701762..f19296696 100644
--- a/src/paimon/common/file_index/file_index_format_test.cpp
+++ b/src/paimon/common/file_index/file_index_format_test.cpp
@@ -17,31 +17,60 @@
  */
 #include "paimon/file_index/file_index_format.h"
 
+#include <algorithm>
+#include <array>
+#include <cstring>
+#include <limits>
+#include <map>
 #include <utility>
 
+#include "arrow/api.h"
 #include "gtest/gtest.h"
 #include "paimon/common/file_index/bitmap/bitmap_file_index.h"
 #include "paimon/common/file_index/bloomfilter/bloom_filter_file_index.h"
 #include "paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h"
 #include "paimon/common/file_index/empty/empty_file_index_reader.h"
+#include "paimon/common/file_index/file_index_format_utils.h"
+#include "paimon/common/file_index/rangebitmap/range_bitmap_file_index.h"
 #include "paimon/common/io/byte_array_output_stream.h"
+#include "paimon/common/io/data_output_stream.h"
 #include "paimon/data/timestamp.h"
 #include "paimon/defs.h"
 #include "paimon/file_index/file_index_result.h"
 #include "paimon/file_index/scored_file_index_result.h"
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/io/byte_array_input_stream.h"
+#include "paimon/io/data_input_stream.h"
 #include "paimon/memory/bytes.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/predicate/literal.h"
 #include "paimon/predicate/vector_search.h"
 #include "paimon/status.h"
+#include "paimon/testing/mock/mock_file_system.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
 
 class FileIndexFormatTest : public ::testing::Test {
  public:
+    Status WriteIndexes(FileIndexFormat::Writer* writer,
+                        const FileIndexFormat::ColumnIndexes& indexes) const {
+        for (const auto& [column, entries] : indexes) {
+            for (const auto& [type, bytes] : entries) {
+                FileIndexFormat::Writer::PayloadWriter payload =
+                    [bytes](const std::shared_ptr<OutputStream>& output) -> Result<bool> {
+                    if (!bytes || bytes->size() == 0) {
+                        return false;
+                    }
+                    PAIMON_RETURN_NOT_OK(DataOutputStream(output).WriteBytes(bytes));
+                    return true;
+                };
+                PAIMON_RETURN_NOT_OK(writer->WriteIndex(column, type, payload));
+            }
+        }
+        return writer->Finish();
+    }
+
     void SetUp() override {
         pool_ = GetDefaultPool();
     }
@@ -72,7 +101,7 @@ TEST_F(FileIndexFormatTest, TestWriteAndReadEmptyIndexGoldenBytes) {
     auto output = std::make_shared<ByteArrayOutputStream>(std::move(segment_output));
 
     ASSERT_OK_AND_ASSIGN(auto writer, FileIndexFormat::CreateWriter(output, pool_));
-    ASSERT_OK(writer->WriteColumnIndexes(indexes));
+    ASSERT_OK(WriteIndexes(writer.get(), indexes));
     ASSERT_OK(writer->Close());
     ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> actual, output->Finish(pool_.get()));
 
@@ -95,7 +124,7 @@ TEST_F(FileIndexFormatTest, TestEmptyVectorIndexCanBeSearched) {
         MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_);
     auto output = std::make_shared<ByteArrayOutputStream>(std::move(segment_output));
     ASSERT_OK_AND_ASSIGN(auto writer, FileIndexFormat::CreateWriter(output, pool_));
-    ASSERT_OK(writer->WriteColumnIndexes(indexes));
+    ASSERT_OK(WriteIndexes(writer.get(), indexes));
     ASSERT_OK(writer->Close());
     ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> bytes, output->Finish(pool_.get()));
 
@@ -865,4 +894,302 @@ TEST_F(FileIndexFormatTest, TestBitmapIndexWithTimestamp) {
     check_nano("ts_tz_nano");
 }
 
+namespace {
+
+std::shared_ptr<ByteArrayOutputStream> NewIndexOutput(const std::shared_ptr<MemoryPool>& pool) {
+    return std::make_shared<ByteArrayOutputStream>(std::make_unique<MemorySegmentOutputStream>(
+        MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool));
+}
+
+// Generated with FileIndexFormatV2.Writer from Java Paimon.
+// Contains an empty index, a two-byte unknown index, and a present zero-byte index.
+const std::vector<uint8_t> kJavaV2Golden = {
+    0,   5,   78,  78,  208, 26, 53, 174, 0,  0,  0,   2,   42,  43,  0,   0,   0,   1,
+    0,   2,   99,  49,  0,   0,  0,  2,   0,  5,  101, 109, 112, 116, 121, 255, 255, 255,
+    255, 255, 255, 255, 255, 0,  0,  0,   0,  0,  0,   0,   0,   0,   7,   117, 110, 107,
+    110, 111, 119, 110, 0,   0,  0,  0,   0,  0,  0,   12,  0,   0,   0,   0,   0,   0,
+    0,   2,   0,   0,   0,   60, 80, 65,  70, 73, 68,  88,  48,  50,
+};
+
+struct SparseIndexState {
+    std::map<int64_t, std::string> segments;
+    int64_t length = 0;
+    bool discard = false;
+};
+
+class SparseIndexOutput : public MockOutputStream {
+ public:
+    explicit SparseIndexOutput(const std::shared_ptr<SparseIndexState>& state) : state_(state) {}
+    Result<int64_t> Write(const char* buffer, int64_t size) override {
+        if (!state_->discard) {
+            state_->segments[state_->length] = std::string(buffer, size);
+        }
+        state_->length += size;
+        return size;
+    }
+    Result<int64_t> GetPos() const override {
+        return state_->length;
+    }
+
+ private:
+    std::shared_ptr<SparseIndexState> state_;
+};
+
+class SparseIndexInput : public MockInputStream {
+ public:
+    explicit SparseIndexInput(const std::shared_ptr<SparseIndexState>& state) : state_(state) {}
+    Status Seek(int64_t offset, SeekOrigin origin) override {
+        if (origin == FS_SEEK_CUR) {
+            offset += position_;
+        }
+        if (origin == FS_SEEK_END) {
+            offset += state_->length;
+        }
+        if (offset < 0 || offset > state_->length) {
+            return Status::Invalid("Invalid seek");
+        }
+        position_ = offset;
+        return Status::OK();
+    }
+    Result<int64_t> GetPos() const override {
+        return position_;
+    }
+    Result<int64_t> Length() const override {
+        return state_->length;
+    }
+    Result<int64_t> Read(char* buffer, int64_t size) override {
+        PAIMON_ASSIGN_OR_RAISE(int64_t read, Read(buffer, size, position_));
+        position_ += read;
+        return read;
+    }
+    Result<int64_t> Read(char* buffer, int64_t size, int64_t offset) override {
+        if (offset < 0 || size < 0 || offset > state_->length) {
+            return Status::Invalid("Invalid read");
+        }
+        size = std::min(size, state_->length - offset);
+        std::memset(buffer, 0, size);
+        for (const auto& [start, bytes] : state_->segments) {
+            int64_t end = start + bytes.size();
+            int64_t from = std::max(start, offset);
+            int64_t to = std::min(end, offset + size);
+            if (from < to) {
+                std::memcpy(buffer + from - offset, bytes.data() + from - start, to - from);
+            }
+        }
+        return size;
+    }
+
+ private:
+    std::shared_ptr<SparseIndexState> state_;
+    int64_t position_ = 0;
+};
+}  // namespace
+
+TEST_F(FileIndexFormatTest, TestV2JavaGoldenBytes) {
+    FileIndexFormat::ColumnIndexes indexes;
+    indexes["c1"]["empty"] = nullptr;
+    indexes["c1"]["unknown"] = std::make_shared<Bytes>(std::string("*+"), pool_.get());
+    auto output = NewIndexOutput(pool_);
+    ASSERT_OK_AND_ASSIGN(auto writer,
+                         FileIndexFormat::CreateWriter(output, pool_, FileIndexFormat::V_2));
+    ASSERT_OK(WriteIndexes(writer.get(), indexes));
+    ASSERT_OK(writer->Close());
+    ASSERT_OK_AND_ASSIGN(auto bytes, output->Finish(pool_.get()));
+    ASSERT_EQ(kJavaV2Golden, std::vector<uint8_t>(bytes->data(), bytes->data() + bytes->size()));
+
+    // Read the Java golden bytes, including empty-index handling and
+    // skipping unknown backend types without opening their payloads.
+    auto input = std::make_shared<ByteArrayInputStream>(
+        reinterpret_cast<const char*>(kJavaV2Golden.data()), kJavaV2Golden.size());
+    ASSERT_OK_AND_ASSIGN(auto reader, FileIndexFormat::CreateReader(input, pool_, /*options=*/{}));
+    auto schema = arrow::schema({arrow::field("c1", arrow::int32())});
+    ASSERT_OK_AND_ASSIGN(auto readers,
+                         reader->ReadColumnIndex("c1", CreateArrowSchema(schema).get()));
+    ASSERT_EQ(1, readers.size());
+    ASSERT_TRUE(dynamic_cast<EmptyFileIndexReader*>(readers[0].get()));
+}
+
+TEST_F(FileIndexFormatTest, TestV2InvalidContainers) {
+    auto check = [this](const std::vector<uint8_t>& bytes) {
+        auto input = std::make_shared<ByteArrayInputStream>(
+            bytes.empty() ? "" : reinterpret_cast<const char*>(bytes.data()), bytes.size());
+        ASSERT_NOK(FileIndexFormat::CreateReader(input, pool_, /*options=*/{}));
+    };
+    for (size_t size : {size_t(0), size_t(11), size_t(24), kJavaV2Golden.size() - 1}) {
+        check(std::vector<uint8_t>(kJavaV2Golden.begin(), kJavaV2Golden.begin() + size));
+    }
+    for (const auto& [offset, value] : std::vector<std::pair<size_t, uint8_t>>{
+             {11, 3},
+             {kJavaV2Golden.size() - 1, 0},
+             {kJavaV2Golden.size() - 12, 0x7f},     // Impossible footer length.
+             {kJavaV2Golden.size() - 12, 0xff}}) {  // Negative footer length.
+        auto bytes = kJavaV2Golden;
+        bytes[offset] = value;
+        check(bytes);
+    }
+}
+
+TEST_F(FileIndexFormatTest, TestV2StreamingLargeContainer) {
+    auto state = std::make_shared<SparseIndexState>();
+    auto output = std::make_shared<SparseIndexOutput>(state);
+    ASSERT_OK_AND_ASSIGN(auto writer,
+                         FileIndexFormat::CreateWriter(output, pool_, FileIndexFormat::V_2));
+    // Stream a virtual >2 GiB payload in bounded chunks; the mock discards only this gap.
+    const int64_t gap_length = int64_t{1} << 31;
+    ASSERT_OK(writer->WriteIndex(
+        "gap", "unknown",
+        [state, gap_length](const std::shared_ptr<OutputStream>& stream) -> Result<bool> {
+            state->discard = true;
+            std::array<char, 65536> zeros{};
+            for (int64_t i = 0; i < gap_length; i += zeros.size()) {
+                PAIMON_ASSIGN_OR_RAISE(int64_t written, stream->Write(zeros.data(), zeros.size()));
+                if (written != static_cast<int64_t>(zeros.size())) {
+                    return Status::IOError("short write");
+                }
+            }
+            state->discard = false;
+            return true;
+        }));
+    auto schema = arrow::schema({arrow::field("f0", arrow::int32())});
+    std::vector<std::pair<std::string, std::shared_ptr<FileIndexer>>> indexers = {
+        {"bitmap", std::make_shared<BitmapFileIndex>(std::map<std::string, std::string>{})},
+        {"bsi",
+         std::make_shared<BitSliceIndexBitmapFileIndex>(std::map<std::string, std::string>{})},
+        {"bloom-filter",
+         std::make_shared<BloomFilterFileIndex>(std::map<std::string, std::string>{})},
+        {"range-bitmap",
+         std::make_shared<RangeBitmapFileIndex>(std::map<std::string, std::string>{})}};
+    arrow::Int32Builder builder;
+    ASSERT_TRUE(builder.AppendValues({1, 2, 1}).ok());
+    auto values_result = builder.Finish();
+    ASSERT_TRUE(values_result.ok());
+    std::shared_ptr<arrow::Array> values = values_result.ValueOrDie();
+    auto batch_result = arrow::StructArray::Make({values}, schema->fields());
+    ASSERT_TRUE(batch_result.ok());
+    auto batch = batch_result.ValueOrDie();
+    for (const auto& [type, indexer] : indexers) {
+        ASSERT_OK_AND_ASSIGN(auto index_writer,
+                             indexer->CreateWriter(CreateArrowSchema(schema).get(), pool_));
+        ArrowArray c_batch;
+        ASSERT_TRUE(arrow::ExportArray(*batch, &c_batch).ok());
+        ASSERT_OK(index_writer->AddBatch(&c_batch));
+        ASSERT_OK_AND_ASSIGN(auto payload, index_writer->SerializedBytes());
+        ASSERT_OK(writer->WriteIndex(
+            type, type, [&payload](const std::shared_ptr<OutputStream>& stream) -> Result<bool> {
+                PAIMON_ASSIGN_OR_RAISE(int64_t written,
+                                       stream->Write(payload->data(), payload->size()));
+                if (written != static_cast<int64_t>(payload->size())) {
+                    return Status::IOError("short write");
+                }
+                return true;
+            }));
+    }
+    ASSERT_OK(writer->Finish());
+    ASSERT_OK(writer->Close());
+    ASSERT_GT(state->length, gap_length);
+    auto input = std::make_shared<SparseIndexInput>(state);
+    ASSERT_OK_AND_ASSIGN(auto reader, FileIndexFormat::CreateReader(input, pool_, /*options=*/{}));
+    for (const auto& [type, indexer] : indexers) {
+        ASSERT_OK_AND_ASSIGN(
+            auto readers,
+            reader->ReadColumnIndex(
+                type,
+                CreateArrowSchema(arrow::schema({arrow::field(type, arrow::int32())})).get()));
+        ASSERT_EQ(1, readers.size()) << type;
+        ASSERT_OK_AND_ASSIGN(auto matches, readers[0]->VisitEqual(Literal(1)));
+        if (type == "bloom-filter") {
+            ASSERT_OK_AND_ASSIGN(bool remains, matches->IsRemain());
+            ASSERT_TRUE(remains);
+        } else {
+            ASSERT_EQ("{0,2}", matches->ToString()) << type;
+        }
+    }
+}
+
+TEST_F(FileIndexFormatTest, TestPluginRangeLimits) {
+    auto schema = arrow::schema({arrow::field("f0", arrow::int32())});
+    auto state = std::make_shared<SparseIndexState>();
+    state->length = int64_t{1} << 32;
+    auto input = std::make_shared<SparseIndexInput>(state);
+    BitmapFileIndex bitmap({});
+    RangeBitmapFileIndex range({});
+    const int64_t oversized = int64_t{1} << 31;
+    for (FileIndexer* indexer : std::vector<FileIndexer*>{&bitmap, &range}) {
+        auto c_schema = CreateArrowSchema(schema);
+        ASSERT_NOK_WITH_MSG(indexer->CreateReader(c_schema.get(), 0, oversized, input, pool_),
+                            "INT32_MAX");
+        if (c_schema->release) {
+            c_schema->release(c_schema.get());
+        }
+    }
+}
+
+TEST_F(FileIndexFormatTest, TestV2EmptyContainer) {
+    auto output = NewIndexOutput(pool_);
+    ASSERT_OK_AND_ASSIGN(auto writer,
+                         FileIndexFormat::CreateWriter(output, pool_, FileIndexFormat::V_2));
+    ASSERT_OK_AND_ASSIGN(int64_t prefix_length, output->GetPos());
+    ASSERT_EQ(12, prefix_length);
+    ASSERT_OK(writer->Finish());
+    ASSERT_OK(writer->Close());
+    ASSERT_OK_AND_ASSIGN(auto bytes, output->Finish(pool_.get()));
+    ASSERT_EQ(28, bytes->size());
+    auto input = std::make_shared<ByteArrayInputStream>(bytes->data(), bytes->size());
+    ASSERT_OK_AND_ASSIGN(auto reader, FileIndexFormat::CreateReader(input, pool_, /*options=*/{}));
+}
+
+TEST_F(FileIndexFormatTest, TestV1BuffersPayloadsUntilFinish) {
+    FileIndexFormat::ColumnIndexes indexes = {
+        {"f0", {{"bitmap", std::make_shared<Bytes>("payload", pool_.get())}, {"empty", nullptr}}}};
+    auto output = NewIndexOutput(pool_);
+    ASSERT_OK_AND_ASSIGN(auto writer, FileIndexFormat::CreateWriter(output, pool_));
+    for (const auto& [column, entries] : indexes) {
+        for (const auto& [type, bytes] : entries) {
+            FileIndexFormat::Writer::PayloadWriter payload;
+            if (bytes) {
+                payload =
+                    [bytes](const std::shared_ptr<OutputStream>& destination) -> Result<bool> {
+                    PAIMON_RETURN_NOT_OK(DataOutputStream(destination).WriteBytes(bytes));
+                    return true;
+                };
+            }
+            ASSERT_OK(writer->WriteIndex(column, type, payload));
+        }
+    }
+    ASSERT_OK_AND_ASSIGN(int64_t before_finish, output->GetPos());
+    ASSERT_EQ(0, before_finish);
+    ASSERT_OK(writer->Finish());
+    ASSERT_OK(writer->Close());
+    ASSERT_OK_AND_ASSIGN(auto actual, output->Finish(pool_.get()));
+    auto input = std::make_shared<ByteArrayInputStream>(actual->data(), actual->size());
+    DataInputStream in(input);
+    ASSERT_OK(in.Seek(12));
+    ASSERT_OK_AND_ASSIGN(int32_t head_length, in.ReadValue<int32_t>());
+    ASSERT_OK_AND_ASSIGN(FileIndexEntries entries,
+                         FileIndexFormatUtils::ReadFileIndexEntries(in, FileIndexFormat::V_1));
+    ASSERT_EQ(std::make_pair(int64_t{head_length}, int64_t{7}), entries.at("f0").at("bitmap"));
+    ASSERT_EQ(std::make_pair(int64_t{-1}, int64_t{0}), entries.at("f0").at("empty"));
+    ASSERT_EQ("payload", std::string(actual->data() + head_length, 7));
+}
+
+TEST_F(FileIndexFormatTest, TestAbsentPayloadMustNotWriteBytes) {
+    for (int32_t version : {FileIndexFormat::V_1, FileIndexFormat::V_2}) {
+        SCOPED_TRACE(version);
+        ASSERT_OK_AND_ASSIGN(auto writer,
+                             FileIndexFormat::CreateWriter(NewIndexOutput(pool_), pool_, version));
+        ASSERT_NOK_WITH_MSG(
+            writer->WriteIndex("f0", "bad",
+                               [](const std::shared_ptr<OutputStream>& output) -> Result<bool> {
+                                   PAIMON_ASSIGN_OR_RAISE(int64_t written, output->Write("x", 1));
+                                   if (written != 1) {
+                                       return Status::IOError("short write");
+                                   }
+                                   return false;
+                               }),
+            "Absent file index payload must not write bytes");
+        ASSERT_NOK(writer->Finish());
+        ASSERT_OK(writer->Close());
+    }
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/common/file_index/file_index_format_utils.cpp b/src/paimon/common/file_index/file_index_format_utils.cpp
new file mode 100644
index 000000000..892fa3a69
--- /dev/null
+++ b/src/paimon/common/file_index/file_index_format_utils.cpp
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/common/file_index/file_index_format_utils.h"
+
+#include "paimon/common/io/offset_input_stream.h"
+#include "paimon/file_index/file_index_format.h"
+#include "paimon/io/buffered_input_stream.h"
+#include "paimon/io/data_input_stream.h"
+
+namespace paimon {
+Result<std::shared_ptr<InputStream>> FileIndexFormatUtils::CreateFileIndexDirectory(
+    const std::shared_ptr<InputStream>& input, const std::shared_ptr<MemoryPool>& pool,
+    int64_t start, int64_t length, int64_t file_length) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OffsetInputStream> bounded,
+                           OffsetInputStream::Create(input, length, start, file_length));
+    return std::make_shared<BufferedInputStream>(bounded, BufferedInputStream::DEFAULT_BUFFER_SIZE,
+                                                 pool.get());
+}
+
+Result<FileIndexEntries> FileIndexFormatUtils::ReadFileIndexEntries(const DataInputStream& in,
+                                                                    int32_t version) {
+    PAIMON_ASSIGN_OR_RAISE(int32_t column_size, in.ReadValue<int32_t>());
+    FileIndexEntries header;
+    for (int32_t i = 0; i < column_size; ++i) {
+        PAIMON_ASSIGN_OR_RAISE(std::string column, in.ReadString());
+        PAIMON_ASSIGN_OR_RAISE(int32_t index_size, in.ReadValue<int32_t>());
+        auto& entries = header[column];
+        for (int32_t j = 0; j < index_size; ++j) {
+            PAIMON_ASSIGN_OR_RAISE(std::string type, in.ReadString());
+            int64_t offset;
+            int64_t length;
+            if (version == FileIndexFormat::V_1) {
+                PAIMON_ASSIGN_OR_RAISE(offset, in.ReadValue<int32_t>());
+                PAIMON_ASSIGN_OR_RAISE(length, in.ReadValue<int32_t>());
+            } else {
+                PAIMON_ASSIGN_OR_RAISE(offset, in.ReadValue<int64_t>());
+                PAIMON_ASSIGN_OR_RAISE(length, in.ReadValue<int64_t>());
+            }
+            entries[type] = {offset, length};
+        }
+    }
+    return header;
+}
+
+}  // namespace paimon
diff --git a/src/paimon/common/file_index/file_index_format_utils.h b/src/paimon/common/file_index/file_index_format_utils.h
new file mode 100644
index 000000000..4d9c2cbf6
--- /dev/null
+++ b/src/paimon/common/file_index/file_index_format_utils.h
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <utility>
+
+#include "paimon/result.h"
+
+namespace paimon {
+class DataInputStream;
+class InputStream;
+class MemoryPool;
+
+using FileIndexEntries =
+    std::unordered_map<std::string, std::unordered_map<std::string, std::pair<int64_t, int64_t>>>;
+
+class FileIndexFormatUtils {
+ public:
+    FileIndexFormatUtils() = delete;
+    ~FileIndexFormatUtils() = delete;
+
+    /// Creates a bounded, buffered view of the container directory.
+    static Result<std::shared_ptr<InputStream>> CreateFileIndexDirectory(
+        const std::shared_ptr<InputStream>& input, const std::shared_ptr<MemoryPool>& pool,
+        int64_t start, int64_t length, int64_t file_length);
+
+    /// Reads shared column/type entries using the offsets and lengths of the given version.
+    static Result<FileIndexEntries> ReadFileIndexEntries(const DataInputStream& in,
+                                                         int32_t version);
+};
+}  // namespace paimon
diff --git a/src/paimon/common/file_index/file_index_format_v1.cpp b/src/paimon/common/file_index/file_index_format_v1.cpp
new file mode 100644
index 000000000..55554cc43
--- /dev/null
+++ b/src/paimon/common/file_index/file_index_format_v1.cpp
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/common/file_index/file_index_format_v1.h"
+
+#include <cassert>
+#include <utility>
+
+#include "paimon/common/io/byte_array_output_stream.h"
+#include "paimon/common/io/data_output_stream.h"
+#include "paimon/common/utils/math.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/io/data_input_stream.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/status.h"
+
+namespace paimon {
+namespace {
+template <typename T>
+Status AddChecked(T value, const char* name, int64_t* total) {
+    PAIMON_RETURN_NOT_OK(ValidateValueInRange<int32_t>(value, name));
+    *total += static_cast<int64_t>(value);
+    return ValidateValueInRange<int32_t>(*total, name);
+}
+}  // namespace
+
+FileIndexFormatV1::Writer::Writer(const std::shared_ptr<OutputStream>& output,
+                                  const std::shared_ptr<MemoryPool>& pool)
+    : output_stream_(output), pool_(pool) {
+    assert(output_stream_);
+}
+
+Status FileIndexFormatV1::Writer::WriteIndex(const std::string& column, const std::string& type,
+                                             const PayloadWriter& payload) {
+    if (finished_ || closed_ || failed_) {
+        return Status::Invalid("File index writer is not writable");
+    }
+    if (buffered_indexes_[column].count(type)) {
+        return Status::Invalid("Duplicate file index entry");
+    }
+    std::shared_ptr<Bytes> bytes;
+    if (payload) {
+        auto output =
+            std::make_shared<ByteArrayOutputStream>(std::make_unique<MemorySegmentOutputStream>(
+                MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_));
+        failed_ = true;
+        PAIMON_ASSIGN_OR_RAISE(bool written, payload(output));
+        PAIMON_ASSIGN_OR_RAISE(int64_t length, output->GetPos());
+        if (!written && length != 0) {
+            return Status::Invalid("Absent file index payload must not write bytes");
+        }
+        if (written) {
+            PAIMON_ASSIGN_OR_RAISE(bytes, output->Finish(pool_.get()));
+        }
+        failed_ = false;
+    }
+    buffered_indexes_[column][type] = std::move(bytes);
+    return Status::OK();
+}
+
+Status FileIndexFormatV1::Writer::Finish() {
+    if (finished_ || closed_ || failed_) {
+        return Status::Invalid("File index writer is not writable");
+    }
+    Status status = WriteIndexes(buffered_indexes_);
+    failed_ = !status.ok();
+    if (status.ok()) {
+        buffered_indexes_.clear();
+    }
+    return status;
+}
+
+Status FileIndexFormatV1::Writer::WriteIndexes(const FileIndexFormat::ColumnIndexes& indexes) {
+    PAIMON_RETURN_NOT_OK(WriteHead(indexes));
+    // Write body.
+    DataOutputStream data_output(output_stream_);
+    for (const auto& [column_name, column_indexes] : indexes) {
+        for (const auto& [index_type, bytes] : column_indexes) {
+            if (bytes) {
+                PAIMON_RETURN_NOT_OK(data_output.WriteBytes(bytes));
+            }
+        }
+    }
+    finished_ = true;
+    return Status::OK();
+}
+
+Status FileIndexFormatV1::Writer::Close() {
+    if (closed_) {
+        return Status::OK();
+    }
+    closed_ = true;
+    PAIMON_RETURN_NOT_OK(output_stream_->Flush());
+    return output_stream_->Close();
+}
+
+Result<int32_t> FileIndexFormatV1::Writer::CalculateHeadLength(
+    const FileIndexFormat::ColumnIndexes& indexes) {
+    // magic(8), version(4), header length(4), and column count(4).
+    int64_t head_length = 8 + 4 + 4 + 4;
+    int64_t body_length = 0;
+    PAIMON_RETURN_NOT_OK(ValidateValueInRange<int32_t>(indexes.size(), "file index column count"));
+    for (const auto& [column_name, column_indexes] : indexes) {
+        PAIMON_RETURN_NOT_OK(
+            ValidateValueInRange<uint16_t>(column_name.size(), "file index column name length"));
+        PAIMON_RETURN_NOT_OK(
+            ValidateValueInRange<int32_t>(column_indexes.size(), "column index count"));
+        // column name(2 + N) + index count(4)
+        head_length += 2 + static_cast<int64_t>(column_name.size()) + 4;
+        for (const auto& [index_type, bytes] : column_indexes) {
+            PAIMON_RETURN_NOT_OK(
+                ValidateValueInRange<uint16_t>(index_type.size(), "file index type name length"));
+            // index type(2 + N) + body offset(4) + body length(4)
+            head_length += 2 + static_cast<int64_t>(index_type.size()) + 4 + 4;
+            if (bytes) {
+                PAIMON_RETURN_NOT_OK(AddChecked(bytes->size(), "index body", &body_length));
+            }
+        }
+    }
+
+    head_length += 4;  // The trailing redundant-length field(4).
+    PAIMON_RETURN_NOT_OK(ValidateValueInRange<int32_t>(head_length, "file index header length"));
+    int64_t container_length = head_length + body_length;
+    PAIMON_RETURN_NOT_OK(ValidateValueInRange<int32_t>(container_length, "file index size"));
+    return static_cast<int32_t>(head_length);
+}
+
+Status FileIndexFormatV1::Writer::WriteHead(const FileIndexFormat::ColumnIndexes& indexes) {
+    PAIMON_ASSIGN_OR_RAISE(int32_t head_length, CalculateHeadLength(indexes));
+    DataOutputStream data_output(output_stream_);
+    // Write magic.
+    PAIMON_RETURN_NOT_OK(data_output.WriteValue<int64_t>(FileIndexFormat::MAGIC));
+    // Write version.
+    PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(FileIndexFormat::V_1));
+    // Write head length.
+    PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(head_length));
+    // Write column count.
+    PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(static_cast<int32_t>(indexes.size())));
+
+    int64_t body_offset = head_length;
+    for (const auto& [column_name, column_indexes] : indexes) {
+        // Write column name.
+        PAIMON_RETURN_NOT_OK(data_output.WriteString(column_name));
+        // Write index count for the column.
+        PAIMON_RETURN_NOT_OK(
+            data_output.WriteValue<int32_t>(static_cast<int32_t>(column_indexes.size())));
+        for (const auto& [index_type, bytes] : column_indexes) {
+            // Write index type.
+            PAIMON_RETURN_NOT_OK(data_output.WriteString(index_type));
+            // Write body offset and length.
+            if (bytes) {
+                PAIMON_RETURN_NOT_OK(
+                    data_output.WriteValue<int32_t>(static_cast<int32_t>(body_offset)));
+                PAIMON_RETURN_NOT_OK(
+                    data_output.WriteValue<int32_t>(static_cast<int32_t>(bytes->size())));
+                body_offset += static_cast<int64_t>(bytes->size());
+            } else {
+                PAIMON_RETURN_NOT_OK(
+                    data_output.WriteValue<int32_t>(FileIndexFormat::EMPTY_INDEX_FLAG));
+                PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(0));
+            }
+        }
+    }
+    // Write redundant length for future format extensions.
+    return data_output.WriteValue<int32_t>(kRedundantLength);
+}
+
+Result<FileIndexEntries> FileIndexFormatV1::ReadIndexEntries(
+    const std::shared_ptr<InputStream>& input, const std::shared_ptr<MemoryPool>& pool,
+    int64_t file_length) {
+    DataInputStream data_input(input);
+    PAIMON_ASSIGN_OR_RAISE(int32_t head_length, data_input.ReadValue<int32_t>());
+    constexpr int64_t kDirectoryStart = 16;
+    int64_t directory_length = head_length - kDirectoryStart;
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> directory,
+                           FileIndexFormatUtils::CreateFileIndexDirectory(
+                               input, pool, kDirectoryStart, directory_length, file_length));
+    DataInputStream in(directory);
+    return FileIndexFormatUtils::ReadFileIndexEntries(in, FileIndexFormat::V_1);
+}
+}  // namespace paimon
diff --git a/src/paimon/common/file_index/file_index_format_v1.h b/src/paimon/common/file_index/file_index_format_v1.h
new file mode 100644
index 000000000..ef1c40dcb
--- /dev/null
+++ b/src/paimon/common/file_index/file_index_format_v1.h
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+
+#include "paimon/common/file_index/file_index_format_utils.h"
+#include "paimon/file_index/file_index_format.h"
+
+namespace paimon {
+/// V1 container layout and writer implementation.
+/// - All numeric fields use big-endian byte order.
+/// - Column metadata precedes the index payloads.
+/// - Payload offsets and lengths are int32; an empty index uses (-1, 0).
+/// - Incremental writes buffer payloads until Finish() can write the header and bodies.
+/// - The complete container must fit within INT32_MAX bytes.
+///
+/// <pre>
+///  _______________________________________    _____________________
+/// |     magic    | version | head length |
+/// |--------------------------------------|
+/// |            column number             |
+/// |--------------------------------------|
+/// |   column 1        |  index number    |
+/// |--------------------------------------|
+/// |  index name 1 | start pos | length   |
+/// |--------------------------------------|
+/// |  index name 2 | start pos | length   |
+/// |--------------------------------------|
+/// |  index name 3 | start pos | length   |
+/// |--------------------------------------|            HEAD
+/// |   column 2        |  index number    |
+/// |--------------------------------------|
+/// |  index name 1 | start pos | length   |
+/// |--------------------------------------|
+/// |  index name 2 | start pos | length   |
+/// |--------------------------------------|
+/// |  index name 3 | start pos | length   |
+/// |--------------------------------------|
+/// |                 ...                  |
+/// |--------------------------------------|
+/// |                 ...                  |
+/// |--------------------------------------|
+/// |  redundant length | redundant bytes  |
+/// |--------------------------------------|    ---------------------
+/// |                BODY                  |
+/// |                BODY                  |
+/// |                BODY                  |             BODY
+/// |                BODY                  |
+/// |______________________________________|    _____________________
+///
+/// magic:               8 bytes long
+/// version:             4 bytes int
+/// head length:         4 bytes int
+/// column number:       4 bytes int
+/// column x:            var bytes utf (length + bytes)
+/// index number:        4 bytes int (how many column items below)
+/// index name x:        var bytes utf
+/// start pos:           4 bytes int
+/// length:              4 bytes int
+/// redundant length:    4 bytes int (for compatibility with later versions, in this version,
+///                      content is zero)
+/// redundant bytes:     var bytes (for compatibility with later version, in this version, is empty)
+/// BODY:                column index bytes + column index bytes + column index bytes + .......
+/// </pre>
+///
+class FileIndexFormatV1 {
+ public:
+    FileIndexFormatV1() = delete;
+    ~FileIndexFormatV1() = delete;
+
+    class Writer;
+
+    /// Reads the directory after the magic/version prefix has been consumed.
+    static Result<FileIndexEntries> ReadIndexEntries(const std::shared_ptr<InputStream>& input,
+                                                     const std::shared_ptr<MemoryPool>& pool,
+                                                     int64_t file_length);
+};
+
+class FileIndexFormatV1::Writer : public FileIndexFormat::Writer {
+ public:
+    Writer(const std::shared_ptr<OutputStream>& output, const std::shared_ptr<MemoryPool>& pool);
+    Status WriteIndex(const std::string& column, const std::string& type,
+                      const PayloadWriter& payload) override;
+    Status Finish() override;
+    Status Close() override;
+
+ private:
+    static constexpr int32_t kRedundantLength = 0;
+    static Result<int32_t> CalculateHeadLength(const FileIndexFormat::ColumnIndexes& indexes);
+    Status WriteIndexes(const FileIndexFormat::ColumnIndexes& indexes);
+    Status WriteHead(const FileIndexFormat::ColumnIndexes& indexes);
+
+    std::shared_ptr<OutputStream> output_stream_;
+    std::shared_ptr<MemoryPool> pool_;
+    FileIndexFormat::ColumnIndexes buffered_indexes_;
+    bool failed_ = false;
+    bool finished_ = false;
+    bool closed_ = false;
+};
+}  // namespace paimon
diff --git a/src/paimon/common/file_index/file_index_format_v2.cpp b/src/paimon/common/file_index/file_index_format_v2.cpp
new file mode 100644
index 000000000..cdfc2358e
--- /dev/null
+++ b/src/paimon/common/file_index/file_index_format_v2.cpp
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/common/file_index/file_index_format_v2.h"
+
+#include <utility>
+
+#include "paimon/common/io/data_output_stream.h"
+#include "paimon/common/utils/math.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/io/data_input_stream.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/status.h"
+
+namespace paimon {
+namespace {
+constexpr int64_t kV2PrefixLength = 12;
+constexpr int64_t kV2TrailerLength = 12;
+constexpr int64_t kV2TailMagic = 0x5041464944583032LL;
+
+}  // namespace
+
+FileIndexFormatV2::Writer::Writer(const std::shared_ptr<OutputStream>& output) : output_(output) {}
+
+Result<std::unique_ptr<FileIndexFormatV2::Writer>> FileIndexFormatV2::Writer::Create(
+    const std::shared_ptr<OutputStream>& output) {
+    auto writer = std::unique_ptr<Writer>(new Writer(output));
+    DataOutputStream data_output(output);
+    PAIMON_RETURN_NOT_OK(data_output.WriteValue<int64_t>(FileIndexFormat::MAGIC));
+    PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(FileIndexFormat::V_2));
+    return writer;
+}
+
+Status FileIndexFormatV2::Writer::WriteIndex(const std::string& column, const std::string& type,
+                                             const PayloadWriter& payload) {
+    if (failed_ || finished_ || closed_) {
+        return Status::Invalid("File index writer is not writable");
+    }
+    PAIMON_RETURN_NOT_OK(ValidateValueInRange<uint16_t>(column.size(), "column name length"));
+    PAIMON_RETURN_NOT_OK(ValidateValueInRange<uint16_t>(type.size(), "index type length"));
+    auto column_iter = entries_.find(column);
+    if (column_iter != entries_.end() && column_iter->second.count(type)) {
+        return Status::Invalid("Duplicate file index entry");
+    }
+    failed_ = true;
+    int64_t start = FileIndexFormat::EMPTY_INDEX_FLAG;
+    int64_t length = 0;
+    if (payload) {
+        PAIMON_ASSIGN_OR_RAISE(start, output_->GetPos());
+        PAIMON_ASSIGN_OR_RAISE(bool written, payload(output_));
+        PAIMON_ASSIGN_OR_RAISE(int64_t end, output_->GetPos());
+        length = end - start;
+        if (!written) {
+            if (length != 0) {
+                return Status::Invalid("Absent file index payload must not write bytes");
+            }
+            start = FileIndexFormat::EMPTY_INDEX_FLAG;
+        }
+    }
+    entries_[column][type] = {start, length};
+    failed_ = false;
+    return Status::OK();
+}
+
+Status FileIndexFormatV2::Writer::Finish() {
+    if (failed_ || finished_ || closed_) {
+        return Status::Invalid("File index writer is not writable");
+    }
+    failed_ = true;
+    // Validate the directory size before emitting it; the Java trailer uses int32.
+    int64_t footer_length = 4;
+    for (const auto& [column, entries] : entries_) {
+        footer_length += 2 + static_cast<int64_t>(column.size()) + 4;
+        for (const auto& [type, range] : entries) {
+            footer_length += 2 + static_cast<int64_t>(type.size()) + 16;
+        }
+        PAIMON_RETURN_NOT_OK(ValidateValueInRange<int32_t>(footer_length, "footer length"));
+    }
+    DataOutputStream data_output(output_);
+    PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(static_cast<int32_t>(entries_.size())));
+    for (const auto& [column, entries] : entries_) {
+        PAIMON_RETURN_NOT_OK(data_output.WriteString(column));
+        PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(static_cast<int32_t>(entries.size())));
+        for (const auto& [type, range] : entries) {
+            PAIMON_RETURN_NOT_OK(data_output.WriteString(type));
+            PAIMON_RETURN_NOT_OK(data_output.WriteValue<int64_t>(range.first));
+            PAIMON_RETURN_NOT_OK(data_output.WriteValue<int64_t>(range.second));
+        }
+    }
+    PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(static_cast<int32_t>(footer_length)));
+    PAIMON_RETURN_NOT_OK(data_output.WriteValue<int64_t>(kV2TailMagic));
+    finished_ = true;
+    failed_ = false;
+    return Status::OK();
+}
+
+Status FileIndexFormatV2::Writer::Close() {
+    if (closed_) {
+        return Status::OK();
+    }
+    closed_ = true;
+    Status flush_status = output_->Flush();
+    Status close_status = output_->Close();
+    PAIMON_RETURN_NOT_OK(flush_status);
+    return close_status;
+}
+
+Result<FileIndexEntries> FileIndexFormatV2::ReadIndexEntries(
+    const std::shared_ptr<InputStream>& input, const std::shared_ptr<MemoryPool>& pool,
+    int64_t file_length) {
+    if (file_length < kV2PrefixLength + 4 + kV2TrailerLength) {
+        return Status::Invalid("Truncated V2 file index");
+    }
+    DataInputStream data_input(input);
+    PAIMON_RETURN_NOT_OK(data_input.Seek(file_length - kV2TrailerLength));
+    PAIMON_ASSIGN_OR_RAISE(int32_t footer_length, data_input.ReadValue<int32_t>());
+    PAIMON_ASSIGN_OR_RAISE(int64_t tail_magic, data_input.ReadValue<int64_t>());
+    if (tail_magic != kV2TailMagic) {
+        return Status::Invalid("Invalid file index tail magic");
+    }
+    if (footer_length < 4 || footer_length > file_length - kV2TrailerLength - kV2PrefixLength) {
+        return Status::Invalid("Invalid file index footer length");
+    }
+    int64_t footer_start = file_length - kV2TrailerLength - footer_length;
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> directory,
+                           FileIndexFormatUtils::CreateFileIndexDirectory(
+                               input, pool, footer_start, footer_length, file_length));
+    DataInputStream in(directory);
+    return FileIndexFormatUtils::ReadFileIndexEntries(in, FileIndexFormat::V_2);
+}
+}  // namespace paimon
diff --git a/src/paimon/common/file_index/file_index_format_v2.h b/src/paimon/common/file_index/file_index_format_v2.h
new file mode 100644
index 000000000..71504c8a9
--- /dev/null
+++ b/src/paimon/common/file_index/file_index_format_v2.h
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "paimon/common/file_index/file_index_format_utils.h"
+#include "paimon/file_index/file_index_format.h"
+
+namespace paimon {
+/// V2 container layout and writer implementation.
+/// - All numeric fields use big-endian byte order.
+/// - Column metadata follows the index payloads.
+/// - Payload offsets and lengths are int64; an empty index uses (-1, 0).
+/// - Incremental writes stream payloads directly; Finish() writes the footer and trailer.
+/// - The trailer locates the footer, footer_length must fit within INT32_MAX bytes.
+///
+/// <pre>
+///  _______________________________________    _____________________
+/// |          magic          | version    |            PREFIX
+/// |--------------------------------------|    ---------------------
+/// |            index payload 1           |
+/// |--------------------------------------|
+/// |            index payload 2           |             BODY
+/// |--------------------------------------|
+/// |                 ...                  |
+/// |--------------------------------------|    ---------------------
+/// |            column number             |
+/// |--------------------------------------|
+/// |   column 1        |  index number    |
+/// |--------------------------------------|
+/// |  index name 1 | start pos | length   |
+/// |--------------------------------------|
+/// |  index name 2 | start pos | length   |
+/// |--------------------------------------|
+/// |                 ...                  |            FOOTER
+/// |--------------------------------------|
+/// |   column 2        |  index number    |
+/// |--------------------------------------|
+/// |  index name 1 | start pos | length   |
+/// |--------------------------------------|
+/// |                 ...                  |
+/// |--------------------------------------|    ---------------------
+/// |     footer length     | tail magic   |            TRAILER
+/// |______________________________________|    _____________________
+///
+/// magic:               8 bytes long
+/// version:             4 bytes int (2)
+/// column number:       4 bytes int
+/// column x:            var bytes utf (length + bytes)
+/// index number:        4 bytes int (number of indexes for the column)
+/// index name x:        var bytes utf (index type)
+/// start pos:           8 bytes long (absolute payload offset)
+/// length:              8 bytes long (payload length)
+/// footer length:       4 bytes int (footer bytes only, excludes trailer)
+/// tail magic:          8 bytes long (0x5041464944583032, "PAFIDX02")
+/// BODY:                serialized index payloads
+/// </pre>
+///
+class FileIndexFormatV2 {
+ public:
+    FileIndexFormatV2() = delete;
+    ~FileIndexFormatV2() = delete;
+
+    class Writer;
+
+    /// Reads the directory after the magic/version prefix has been consumed.
+    static Result<FileIndexEntries> ReadIndexEntries(const std::shared_ptr<InputStream>& input,
+                                                     const std::shared_ptr<MemoryPool>& pool,
+                                                     int64_t file_length);
+};
+
+class FileIndexFormatV2::Writer : public FileIndexFormat::Writer {
+ public:
+    static Result<std::unique_ptr<Writer>> Create(const std::shared_ptr<OutputStream>& output);
+    Status Close() override;
+    Status WriteIndex(const std::string& column, const std::string& type,
+                      const PayloadWriter& payload) override;
+    Status Finish() override;
+
+ private:
+    explicit Writer(const std::shared_ptr<OutputStream>& output);
+
+    std::shared_ptr<OutputStream> output_;
+    std::map<std::string, std::map<std::string, std::pair<int64_t, int64_t>>> entries_;
+    bool failed_ = false;
+    bool finished_ = false;
+    bool closed_ = false;
+};
+}  // namespace paimon
diff --git a/src/paimon/common/file_index/file_index_writer.cpp b/src/paimon/common/file_index/file_index_writer.cpp
new file mode 100644
index 000000000..7a8e2916e
--- /dev/null
+++ b/src/paimon/common/file_index/file_index_writer.cpp
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/file_index/file_index_writer.h"
+
+#include "paimon/fs/file_system.h"
+#include "paimon/status.h"
+
+namespace paimon {
+Result<PAIMON_UNIQUE_PTR<Bytes>> FileIndexWriter::SerializedBytes() const {
+    return Status::NotImplemented("This file index writer requires streaming output");
+}
+
+Result<bool> FileIndexWriter::WriteTo(const std::shared_ptr<OutputStream>& output) const {
+    if (!output) {
+        return Status::Invalid("File index output stream cannot be null");
+    }
+    PAIMON_ASSIGN_OR_RAISE(PAIMON_UNIQUE_PTR<Bytes> bytes, SerializedBytes());
+    if (!bytes || bytes->size() == 0) {
+        return false;
+    }
+    auto length = static_cast<int64_t>(bytes->size());
+    PAIMON_ASSIGN_OR_RAISE(int64_t written, output->Write(bytes->data(), length));
+    if (written != length) {
+        return Status::IOError("Short write for file index payload");
+    }
+    return true;
+}
+}  // namespace paimon
diff --git a/src/paimon/common/file_index/file_index_writer_test.cpp b/src/paimon/common/file_index/file_index_writer_test.cpp
new file mode 100644
index 000000000..8988b4c20
--- /dev/null
+++ b/src/paimon/common/file_index/file_index_writer_test.cpp
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/file_index/file_index_writer.h"
+
+#include <string>
+
+#include "gtest/gtest.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/mock/mock_file_system.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+namespace {
+class RecordingIndexOutput : public MockOutputStream {
+ public:
+    Result<int64_t> Write(const char* buffer, int64_t size) override {
+        if (short_write) {
+            return size - 1;
+        }
+        data.append(buffer, size);
+        return size;
+    }
+    Status Flush() override {
+        ++flush_count;
+        return Status::OK();
+    }
+    Status Close() override {
+        ++close_count;
+        return Status::OK();
+    }
+
+    std::string data;
+    bool short_write = false;
+    int32_t flush_count = 0;
+    int32_t close_count = 0;
+};
+
+class LegacyIndexWriter : public FileIndexWriter {
+ public:
+    Status AddBatch(::ArrowArray*) override {
+        return Status::OK();
+    }
+    Result<PAIMON_UNIQUE_PTR<Bytes>> SerializedBytes() const override {
+        if (absent) {
+            return PAIMON_UNIQUE_PTR<Bytes>();
+        }
+        if (zero_length) {
+            return Bytes::AllocateBytes("", GetDefaultPool().get());
+        }
+        return Bytes::AllocateBytes("legacy payload", GetDefaultPool().get());
+    }
+    bool absent = false;
+    bool zero_length = false;
+};
+
+}  // namespace
+
+TEST(FileIndexWriterTest, TestLegacyDefaultWriteTo) {
+    LegacyIndexWriter writer;
+    auto output = std::make_shared<RecordingIndexOutput>();
+    ASSERT_OK_AND_ASSIGN(bool written, writer.WriteTo(output));
+    ASSERT_TRUE(written);
+    ASSERT_EQ("legacy payload", output->data);
+    ASSERT_EQ(0, output->flush_count);
+    ASSERT_EQ(0, output->close_count);
+    ASSERT_NOK_WITH_MSG(writer.WriteTo(nullptr), "cannot be null");
+}
+
+TEST(FileIndexWriterTest, TestLegacyWriteToRejectsShortWrite) {
+    LegacyIndexWriter writer;
+    auto output = std::make_shared<RecordingIndexOutput>();
+    output->short_write = true;
+    ASSERT_NOK_WITH_MSG(writer.WriteTo(output), "Short write");
+}
+
+TEST(FileIndexWriterTest, TestNullAndZeroLengthPayloadsProduceNoIndex) {
+    LegacyIndexWriter writer;
+    auto output = std::make_shared<RecordingIndexOutput>();
+    writer.absent = true;
+    ASSERT_OK_AND_ASSIGN(bool written, writer.WriteTo(output));
+    ASSERT_FALSE(written);
+    ASSERT_TRUE(output->data.empty());
+    writer.absent = false;
+    writer.zero_length = true;
+    ASSERT_OK_AND_ASSIGN(written, writer.WriteTo(output));
+    ASSERT_FALSE(written);
+    ASSERT_TRUE(output->data.empty());
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp
index 8384e7df0..862271bad 100644
--- a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp
+++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp
@@ -21,6 +21,8 @@
 #include <arrow/c/bridge.h>
 #include <arrow/type.h>
 
+#include <limits>
+
 #include "paimon/common/file_index/rangebitmap/range_bitmap.h"
 #include "paimon/common/file_index/rangebitmap/range_bitmap_type_adapter.h"
 #include "paimon/common/io/offset_input_stream.h"
@@ -39,7 +41,7 @@ RangeBitmapFileIndex::RangeBitmapFileIndex(const std::map<std::string, std::stri
     : options_(options) {}
 
 Result<std::shared_ptr<FileIndexReader>> RangeBitmapFileIndex::CreateReader(
-    ::ArrowSchema* arrow_schema, const int32_t start, const int32_t length,
+    ::ArrowSchema* arrow_schema, const int64_t start, const int64_t length,
     const std::shared_ptr<InputStream>& input_stream,
     const std::shared_ptr<MemoryPool>& pool) const {
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> arrow_schema_ptr,
@@ -113,8 +115,12 @@ RangeBitmapFileIndexWriter::RangeBitmapFileIndexWriter(
       appender_(std::move(appender)) {}
 
 Result<std::shared_ptr<RangeBitmapFileIndexReader>> RangeBitmapFileIndexReader::Create(
-    const std::shared_ptr<arrow::DataType>& arrow_type, const int32_t start, const int32_t length,
+    const std::shared_ptr<arrow::DataType>& arrow_type, const int64_t start, const int64_t length,
     const std::shared_ptr<InputStream>& input_stream, const std::shared_ptr<MemoryPool>& pool) {
+    if (length > std::numeric_limits<int32_t>::max()) {
+        return Status::NotImplemented(
+            "Range bitmap file index does not support payloads larger than INT32_MAX");
+    }
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RangeBitmapTypeAdapter> type_adapter,
                            RangeBitmapTypeAdapter::Create(arrow_type));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OffsetInputStream> bounded_stream,
diff --git a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.h b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.h
index 3e7d511e0..36b2bd675 100644
--- a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.h
+++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.h
@@ -45,7 +45,7 @@ class PAIMON_EXPORT RangeBitmapFileIndex final : public FileIndexer {
     ~RangeBitmapFileIndex() override = default;
 
     Result<std::shared_ptr<FileIndexReader>> CreateReader(
-        ::ArrowSchema* arrow_schema, int32_t start, int32_t length,
+        ::ArrowSchema* arrow_schema, int64_t start, int64_t length,
         const std::shared_ptr<InputStream>& input_stream,
         const std::shared_ptr<MemoryPool>& pool) const override;
 
@@ -86,7 +86,7 @@ class RangeBitmapFileIndexReader final
       public std::enable_shared_from_this<RangeBitmapFileIndexReader> {
  public:
     static Result<std::shared_ptr<RangeBitmapFileIndexReader>> Create(
-        const std::shared_ptr<arrow::DataType>& arrow_type, int32_t start, int32_t length,
+        const std::shared_ptr<arrow::DataType>& arrow_type, int64_t start, int64_t length,
         const std::shared_ptr<InputStream>& input_stream, const std::shared_ptr<MemoryPool>& pool);
 
     ~RangeBitmapFileIndexReader() override;
diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp
index 28af19140..ee71f42cb 100644
--- a/src/paimon/core/core_options.cpp
+++ b/src/paimon/core/core_options.cpp
@@ -341,6 +341,7 @@ struct CoreOptions::Impl {
     int64_t deletion_vector_target_file_size = 2 * 1024 * 1024;
     int64_t manifest_full_compaction_file_size = 16 * 1024 * 1024;
     int64_t file_index_in_manifest_threshold = 500;
+    int32_t file_index_format_version = 1;
     int64_t write_buffer_size = 256 * 1024 * 1024;
     int64_t commit_timeout = std::numeric_limits<int64_t>::max();
     int64_t commit_min_retry_wait = 10;
@@ -869,6 +870,11 @@ struct CoreOptions::Impl {
 
     // Parse index-related configurations: file index, global index.
     Status ParseIndexOptions(const ConfigParser& parser) {
+        PAIMON_RETURN_NOT_OK(
+            parser.Parse<int32_t>(Options::FILE_INDEX_FORMAT_VERSION, &file_index_format_version));
+        if (file_index_format_version != 1 && file_index_format_version != 2) {
+            return Status::Invalid("file-index.format-version must be 1 or 2");
+        }
         // Parse file-index.in-manifest-threshold - max inline file index size, default 500B
         PAIMON_RETURN_NOT_OK(parser.ParseMemorySize(Options::FILE_INDEX_IN_MANIFEST_THRESHOLD,
                                                     &file_index_in_manifest_threshold));
@@ -1749,6 +1755,10 @@ bool CoreOptions::FileIndexReadEnabled() const {
     return impl_->file_index_read_enabled;
 }
 
+int32_t CoreOptions::FileIndexFormatVersion() const {
+    return impl_->file_index_format_version;
+}
+
 int64_t CoreOptions::FileIndexInManifestThreshold() const {
     return impl_->file_index_in_manifest_threshold;
 }
diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h
index a16f75e4c..6af866753 100644
--- a/src/paimon/core/core_options.h
+++ b/src/paimon/core/core_options.h
@@ -222,6 +222,7 @@ class PAIMON_EXPORT CoreOptions {
     bool PrepareCommitWaitCompaction() const;
     bool FileIndexReadEnabled() const;
     int64_t FileIndexInManifestThreshold() const;
+    int32_t FileIndexFormatVersion() const;
 
     std::map<std::string, std::string> GetFieldsSequenceGroups() const;
     bool PartialUpdateRemoveRecordOnDelete() const;
diff --git a/src/paimon/core/io/data_file_index_writer.cpp b/src/paimon/core/io/data_file_index_writer.cpp
index f95b4bdd9..8af0ad6b2 100644
--- a/src/paimon/core/io/data_file_index_writer.cpp
+++ b/src/paimon/core/io/data_file_index_writer.cpp
@@ -28,15 +28,15 @@
 #include "arrow/c/helpers.h"
 #include "arrow/compute/cast.h"
 #include "fmt/format.h"
-#include "paimon/common/io/byte_array_output_stream.h"
-#include "paimon/common/io/memory_segment_output_stream.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/math.h"
 #include "paimon/common/utils/path_util.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/casting/casting_utils.h"
 #include "paimon/core/io/data_file_path_factory.h"
+#include "paimon/core/io/spillable_index_output_stream.h"
 #include "paimon/file_index/file_index_format.h"
 #include "paimon/file_index/file_index_writer.h"
 #include "paimon/file_index/file_indexer.h"
@@ -92,17 +92,19 @@ Result<std::unique_ptr<DataFileIndexWriter>> DataFileIndexWriter::Create(
         writers.push_back(
             {definition.column_name, definition.index_type, field_index, field, std::move(writer)});
     }
-    return std::unique_ptr<DataFileIndexWriter>(new DataFileIndexWriter(
-        std::move(writers), options.InManifestThreshold(), file_system, path_factory, pool));
+    return std::unique_ptr<DataFileIndexWriter>(
+        new DataFileIndexWriter(std::move(writers), options.InManifestThreshold(),
+                                options.FormatVersion(), file_system, path_factory, pool));
 }
 
 DataFileIndexWriter::DataFileIndexWriter(std::vector<IndexWriterEntry>&& writers,
-                                         int64_t in_manifest_threshold,
+                                         int64_t in_manifest_threshold, int32_t format_version,
                                          const std::shared_ptr<FileSystem>& file_system,
                                          const std::shared_ptr<DataFilePathFactory>& path_factory,
                                          const std::shared_ptr<MemoryPool>& pool)
     : writers_(std::move(writers)),
       in_manifest_threshold_(in_manifest_threshold),
+      format_version_(format_version),
       file_system_(file_system),
       path_factory_(path_factory),
       pool_(pool) {}
@@ -149,62 +151,36 @@ Status DataFileIndexWriter::AddBatch(const std::shared_ptr<arrow::StructArray>&
     return Status::OK();
 }
 
-Result<std::shared_ptr<Bytes>> DataFileIndexWriter::SerializeContainer() {
-    FileIndexFormat::ColumnIndexes column_indexes;
-    for (const IndexWriterEntry& entry : writers_) {
-        PAIMON_ASSIGN_OR_RAISE(column_indexes[entry.column_name][entry.index_type],
-                               entry.writer->SerializedBytes());
-    }
-
-    auto segment_output = std::make_unique<MemorySegmentOutputStream>(
-        MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_);
-    std::shared_ptr<ByteArrayOutputStream> output =
-        std::make_shared<ByteArrayOutputStream>(std::move(segment_output));
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileIndexFormat::Writer> format_writer,
-                           FileIndexFormat::CreateWriter(output, pool_));
-    PAIMON_RETURN_NOT_OK(format_writer->WriteColumnIndexes(column_indexes));
-    PAIMON_RETURN_NOT_OK(format_writer->Close());
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Bytes> bytes, output->Finish(pool_.get()));
-    return bytes;
-}
-
 Result<FileIndexWriteResult> DataFileIndexWriter::Finish(const std::string& data_file_path) {
     if (finished_) {
         return Status::Invalid("Data file index writer has already finished");
     }
     finished_ = true;
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Bytes> bytes, SerializeContainer());
-    if (static_cast<int64_t>(bytes->size()) <= in_manifest_threshold_) {
-        return FileIndexWriteResult{bytes, {}};
+    PAIMON_RETURN_NOT_OK(
+        ValidateValueInRange<int32_t>(in_manifest_threshold_, "file index inline threshold"));
+    std::string index_path = path_factory_->ToFileIndexPath(data_file_path);
+    auto output = std::make_shared<SpillableIndexOutputStream>(file_system_, index_path,
+                                                               in_manifest_threshold_, pool_);
+    ScopeGuard guard([&output]() { [[maybe_unused]] Status ignored = output->Abort(); });
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileIndexFormat::Writer> writer,
+                           FileIndexFormat::CreateWriter(output, pool_, format_version_));
+    for (const IndexWriterEntry& entry : writers_) {
+        FileIndexFormat::Writer::PayloadWriter payload =
+            [index_writer = entry.writer](const std::shared_ptr<OutputStream>& destination) {
+                return index_writer->WriteTo(destination);
+            };
+        PAIMON_RETURN_NOT_OK(writer->WriteIndex(entry.column_name, entry.index_type, payload));
     }
-
-    external_index_path_ = path_factory_->ToFileIndexPath(data_file_path);
-    PAIMON_RETURN_NOT_OK(WriteExternal(external_index_path_.value(), bytes));
-    return FileIndexWriteResult{nullptr, {PathUtil::GetName(external_index_path_.value())}};
-}
-
-Status DataFileIndexWriter::WriteExternal(const std::string& path,
-                                          const std::shared_ptr<Bytes>& bytes) {
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OutputStream> output,
-                           file_system_->Create(path, /*overwrite=*/false));
-    ScopeGuard guard([this, &output]() {
-        if (output) {
-            [[maybe_unused]] Status _ = output->Close();
-        }
-        Abort();
-    });
-    PAIMON_ASSIGN_OR_RAISE(int64_t written,
-                           output->Write(bytes->data(), static_cast<int64_t>(bytes->size())));
-    if (written != static_cast<int64_t>(bytes->size())) {
-        return Status::IOError(fmt::format("Short write for file index {}: expected {}, wrote {}",
-                                           path, bytes->size(), written));
+    PAIMON_RETURN_NOT_OK(writer->Finish());
+    PAIMON_RETURN_NOT_OK(writer->Close());
+    if (output->Spilled()) {
+        external_index_path_ = index_path;
+        guard.Release();
+        return FileIndexWriteResult{nullptr, {PathUtil::GetName(index_path)}};
     }
-    PAIMON_RETURN_NOT_OK(output->Flush());
-    Status close_status = output->Close();
-    output.reset();
-    PAIMON_RETURN_NOT_OK(close_status);
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Bytes> embedded, output->EmbeddedBytes());
     guard.Release();
-    return Status::OK();
+    return FileIndexWriteResult{embedded, {}};
 }
 
 void DataFileIndexWriter::Abort() {
diff --git a/src/paimon/core/io/data_file_index_writer.h b/src/paimon/core/io/data_file_index_writer.h
index 322e70ff5..feb8327e7 100644
--- a/src/paimon/core/io/data_file_index_writer.h
+++ b/src/paimon/core/io/data_file_index_writer.h
@@ -82,13 +82,10 @@ class DataFileIndexWriter {
     };
 
     DataFileIndexWriter(std::vector<IndexWriterEntry>&& writers, int64_t in_manifest_threshold,
-                        const std::shared_ptr<FileSystem>& file_system,
+                        int32_t format_version, const std::shared_ptr<FileSystem>& file_system,
                         const std::shared_ptr<DataFilePathFactory>& path_factory,
                         const std::shared_ptr<MemoryPool>& pool);
 
-    Result<std::shared_ptr<Bytes>> SerializeContainer();
-    Status WriteExternal(const std::string& path, const std::shared_ptr<Bytes>& bytes);
-
     /// Decodes the dictionary-encoded columns the parquet passthrough forwards, and nothing else.
     /// Built on first use, since most batches decode nothing.
     ///
@@ -98,6 +95,7 @@ class DataFileIndexWriter {
     std::shared_ptr<arrow::MemoryPool> arrow_pool_;
     std::vector<IndexWriterEntry> writers_;
     int64_t in_manifest_threshold_;
+    int32_t format_version_;
     std::shared_ptr<FileSystem> file_system_;
     std::shared_ptr<DataFilePathFactory> path_factory_;
     std::shared_ptr<MemoryPool> pool_;
diff --git a/src/paimon/core/io/data_file_index_writer_test.cpp b/src/paimon/core/io/data_file_index_writer_test.cpp
index b6f25f3c4..0e929f81d 100644
--- a/src/paimon/core/io/data_file_index_writer_test.cpp
+++ b/src/paimon/core/io/data_file_index_writer_test.cpp
@@ -30,7 +30,9 @@
 #include "arrow/c/bridge.h"
 #include "arrow/ipc/json_simple.h"
 #include "arrow/type.h"
+#include "fmt/format.h"
 #include "gtest/gtest.h"
+#include "paimon/common/file_index/empty/empty_file_index_reader.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/checked_cast.h"
@@ -38,9 +40,14 @@
 #include "paimon/core/io/data_file_path_factory.h"
 #include "paimon/core/io/file_index_options.h"
 #include "paimon/defs.h"
+#include "paimon/factories/factory.h"
 #include "paimon/file_index/file_index_format.h"
+#include "paimon/file_index/file_index_writer.h"
+#include "paimon/file_index/file_indexer.h"
+#include "paimon/file_index/file_indexer_factory.h"
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/io/byte_array_input_stream.h"
+#include "paimon/io/data_input_stream.h"
 #include "paimon/memory/bytes.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/predicate/literal.h"
@@ -50,6 +57,83 @@
 namespace paimon::test {
 namespace {
 
+// Deliberately implements only WriteTo(), so the V2 integration cannot fall back to a buffer.
+class StreamingTestIndexWriter : public FileIndexWriter {
+ public:
+    StreamingTestIndexWriter(bool fail, bool empty) : fail_(fail), empty_(empty) {}
+    Status AddBatch(::ArrowArray* batch) override {
+        batch->release(batch);
+        return Status::OK();
+    }
+    Result<bool> WriteTo(const std::shared_ptr<OutputStream>& output) const override {
+        if (empty_) {
+            return false;
+        }
+        for (const std::string& chunk : {"stream", "ed"}) {
+            PAIMON_ASSIGN_OR_RAISE(int64_t written, output->Write(chunk.data(), chunk.size()));
+            if (written != static_cast<int64_t>(chunk.size())) {
+                return Status::IOError("short streamed payload");
+            }
+            if (fail_) {
+                return Status::IOError("streamed payload failed");
+            }
+        }
+        return true;
+    }
+
+ private:
+    bool fail_;
+    bool empty_;
+};
+
+class StreamingTestIndexer : public FileIndexer {
+ public:
+    explicit StreamingTestIndexer(const std::map<std::string, std::string>& options)
+        : fail_(options.count("fail")), empty_(options.count("empty")) {}
+    Result<std::shared_ptr<FileIndexWriter>> CreateWriter(
+        ::ArrowSchema* c_schema, const std::shared_ptr<MemoryPool>&) const override {
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> schema,
+                                          arrow::ImportSchema(c_schema));
+        if (schema->num_fields() != 1) {
+            return Status::Invalid("expected one field");
+        }
+        return std::make_shared<StreamingTestIndexWriter>(fail_, empty_);
+    }
+    Result<std::shared_ptr<FileIndexReader>> CreateReader(
+        ::ArrowSchema* c_schema, int64_t start, int64_t length,
+        const std::shared_ptr<InputStream>& input,
+        const std::shared_ptr<MemoryPool>&) const override {
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> schema,
+                                          arrow::ImportSchema(c_schema));
+        if (schema->num_fields() != 1 || length != 8) {
+            return Status::Invalid("invalid streamed payload metadata");
+        }
+        PAIMON_RETURN_NOT_OK(input->Seek(start, SeekOrigin::FS_SEEK_SET));
+        char bytes[8];
+        PAIMON_ASSIGN_OR_RAISE(int64_t read, input->Read(bytes, length));
+        if (read != length || std::string(bytes, sizeof(bytes)) != "streamed") {
+            return Status::Invalid("invalid streamed payload bytes");
+        }
+        return std::make_shared<FileIndexReader>();
+    }
+
+ private:
+    bool fail_;
+    bool empty_;
+};
+
+class StreamingTestIndexFactory : public FileIndexerFactory {
+ public:
+    const char* Identifier() const override {
+        return "test-streaming-index";
+    }
+    Result<std::unique_ptr<FileIndexer>> Create(
+        const std::map<std::string, std::string>& options) const override {
+        return std::make_unique<StreamingTestIndexer>(options);
+    }
+};
+REGISTER_PAIMON_FACTORY(StreamingTestIndexFactory);
+
 struct CloseFailingState {
     int32_t close_count = 0;
     int32_t delete_count = 0;
@@ -95,6 +179,10 @@ class CloseFailingFileSystem : public MockFileSystem {
 
 class DataFileIndexWriterTest : public ::testing::Test {
  public:
+    virtual int32_t FormatVersion() const {
+        return FileIndexFormat::V_1;
+    }
+
     void SetUp() override {
         pool_ = GetDefaultPool();
         file_system_ = std::make_shared<LocalFileSystem>();
@@ -108,13 +196,20 @@ class DataFileIndexWriterTest : public ::testing::Test {
 
     Result<std::unique_ptr<DataFileIndexWriter>> CreateWriter(
         const std::map<std::string, std::string>& index_options) const {
-        PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options,
-                               CoreOptions::FromMap(index_options, file_system_));
+        PAIMON_ASSIGN_OR_RAISE(
+            CoreOptions core_options,
+            CoreOptions::FromMap(WithFormatVersion(index_options), file_system_));
         PAIMON_ASSIGN_OR_RAISE(FileIndexOptions parsed,
                                FileIndexOptions::FromCoreOptions(core_options));
         return DataFileIndexWriter::Create(schema_, parsed, file_system_, path_factory_, pool_);
     }
 
+    std::map<std::string, std::string> WithFormatVersion(
+        std::map<std::string, std::string> options) const {
+        options[Options::FILE_INDEX_FORMAT_VERSION] = std::to_string(FormatVersion());
+        return options;
+    }
+
     std::shared_ptr<arrow::StructArray> CreateBatch(const std::string& json) const {
         std::shared_ptr<arrow::Array> array =
             arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema_->fields()), json)
@@ -143,7 +238,60 @@ class DataFileIndexWriterTest : public ::testing::Test {
     std::shared_ptr<arrow::Schema> schema_;
 };
 
-TEST_F(DataFileIndexWriterTest, TestMultipleIndexesOnSameColumnEmbeddedRoundTrip) {
+class VersionedDataFileIndexWriterTest : public DataFileIndexWriterTest,
+                                         public ::testing::WithParamInterface<int32_t> {
+ public:
+    int32_t FormatVersion() const override {
+        return GetParam();
+    }
+};
+
+class DataFileIndexWriterV2Test : public DataFileIndexWriterTest {
+ public:
+    int32_t FormatVersion() const override {
+        return FileIndexFormat::V_2;
+    }
+};
+
+TEST_P(VersionedDataFileIndexWriterTest, TestStreamingOnlyPlugin) {
+    ASSERT_OK_AND_ASSIGN(auto writer,
+                         CreateWriter({{"file-index.test-streaming-index.columns", "f0"}}));
+    ASSERT_OK(writer->AddBatch(CreateBatch(R"([{"f0": 1, "f1": 2}])")));
+    ASSERT_OK_AND_ASSIGN(FileIndexWriteResult result, writer->Finish("stream.orc"));
+    ASSERT_TRUE(result.embedded_index);
+    ASSERT_TRUE(result.extra_files.empty());
+    ASSERT_OK_AND_ASSIGN(auto reader, CreateReader(result.embedded_index));
+    ASSERT_OK_AND_ASSIGN(auto indexes, ReadColumn(reader.get(), "f0"));
+    ASSERT_EQ(1, indexes.size());
+}
+
+TEST_F(DataFileIndexWriterV2Test, TestStreamingFailureRemovesPartialFile) {
+    ASSERT_OK_AND_ASSIGN(auto writer,
+                         CreateWriter({{"file-index.test-streaming-index.columns", "f0"},
+                                       {"file-index.test-streaming-index.f0.fail", "true"},
+                                       {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1B"}}));
+    ASSERT_OK(writer->AddBatch(CreateBatch(R"([{"f0": 1, "f1": 2}])")));
+    const std::string data_path = fmt::format("{}/partial.orc", directory_->Str());
+    ASSERT_NOK_WITH_MSG(writer->Finish(data_path), "streamed payload failed");
+    ASSERT_OK_AND_ASSIGN(bool exists,
+                         file_system_->Exists(path_factory_->ToFileIndexPath(data_path)));
+    ASSERT_FALSE(exists);
+}
+
+TEST_P(VersionedDataFileIndexWriterTest, TestEmptyStreamingIndex) {
+    ASSERT_OK_AND_ASSIGN(auto writer,
+                         CreateWriter({{"file-index.test-streaming-index.columns", "f0"},
+                                       {"file-index.test-streaming-index.f0.empty", "true"}}));
+    ASSERT_OK(writer->AddBatch(CreateBatch(R"([{"f0": 1, "f1": 2}])")));
+    ASSERT_OK_AND_ASSIGN(FileIndexWriteResult result, writer->Finish("empty.orc"));
+    ASSERT_TRUE(result.embedded_index);
+    ASSERT_OK_AND_ASSIGN(auto reader, CreateReader(result.embedded_index));
+    ASSERT_OK_AND_ASSIGN(auto indexes, ReadColumn(reader.get(), "f0"));
+    ASSERT_EQ(1, indexes.size());
+    ASSERT_TRUE(dynamic_cast<EmptyFileIndexReader*>(indexes[0].get()));
+}
+
+TEST_P(VersionedDataFileIndexWriterTest, TestMultipleIndexesOnSameColumnEmbeddedRoundTrip) {
     ASSERT_OK_AND_ASSIGN(auto writer,
                          CreateWriter({{"file-index.bitmap.columns", "f0"},
                                        {"file-index.bsi.columns", "f0"},
@@ -158,6 +306,13 @@ TEST_F(DataFileIndexWriterTest, TestMultipleIndexesOnSameColumnEmbeddedRoundTrip
     ASSERT_OK_AND_ASSIGN(FileIndexWriteResult result, writer->Finish("unused.orc"));
     ASSERT_TRUE(result.embedded_index);
     ASSERT_TRUE(result.extra_files.empty());
+    auto input = std::make_shared<ByteArrayInputStream>(result.embedded_index->data(),
+                                                        result.embedded_index->size());
+    DataInputStream data_input(input);
+    ASSERT_OK_AND_ASSIGN(int64_t magic, data_input.ReadValue<int64_t>());
+    ASSERT_EQ(FileIndexFormat::MAGIC, magic);
+    ASSERT_OK_AND_ASSIGN(int32_t version, data_input.ReadValue<int32_t>());
+    ASSERT_EQ(FormatVersion(), version);
     ASSERT_OK_AND_ASSIGN(auto reader, CreateReader(result.embedded_index));
 
     ASSERT_OK_AND_ASSIGN(auto f0_readers, ReadColumn(reader.get(), "f0"));
@@ -175,7 +330,7 @@ TEST_F(DataFileIndexWriterTest, TestMultipleIndexesOnSameColumnEmbeddedRoundTrip
     ASSERT_EQ("{2,3}", greater_result->ToString());
 }
 
-TEST_F(DataFileIndexWriterTest, TestDictionaryEncodedIndexedColumnRoundTrip) {
+TEST_P(VersionedDataFileIndexWriterTest, TestDictionaryEncodedIndexedColumnRoundTrip) {
     // The parquet dictionary passthrough hands compaction batches over still encoded, and the
     // bitmap index only sees the right values if the indexed column is decoded first.
     schema_ =
@@ -227,7 +382,7 @@ TEST_F(DataFileIndexWriterTest, TestDictionaryEncodedIndexedColumnRoundTrip) {
     ASSERT_EQ("{4}", second_batch_result->ToString());
 }
 
-TEST_F(DataFileIndexWriterTest, TestExternalIndexAndAbortCleanup) {
+TEST_P(VersionedDataFileIndexWriterTest, TestExternalIndexAndAbortCleanup) {
     ASSERT_OK_AND_ASSIGN(auto writer,
                          CreateWriter({{"file-index.bitmap.columns", "f0"},
                                        {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1B"}}));
@@ -255,7 +410,7 @@ TEST_F(DataFileIndexWriterTest, TestExternalIndexAndAbortCleanup) {
     ASSERT_FALSE(exists);
 }
 
-TEST_F(DataFileIndexWriterTest, TestBsiAndBloomFilterEmbeddedRoundTrip) {
+TEST_P(VersionedDataFileIndexWriterTest, TestBsiAndBloomFilterEmbeddedRoundTrip) {
     ASSERT_OK_AND_ASSIGN(auto writer,
                          CreateWriter({{"file-index.bsi.columns", "f0"},
                                        {"file-index.bloom-filter.columns", "f1"},
@@ -317,10 +472,12 @@ TEST_F(DataFileIndexWriterTest, TestFinishIsOneShot) {
                         "already finished");
 }
 
-TEST_F(DataFileIndexWriterTest, TestCloseFailureClosesExternalStreamOnce) {
+TEST_P(VersionedDataFileIndexWriterTest, TestCloseFailureClosesExternalStreamOnce) {
     ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
                          CoreOptions::FromMap({{"file-index.bitmap.columns", "f0"},
-                                               {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1B"}},
+                                               {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1B"},
+                                               {Options::FILE_INDEX_FORMAT_VERSION,
+                                                std::to_string(FormatVersion())}},
                                               file_system_));
     ASSERT_OK_AND_ASSIGN(FileIndexOptions options, FileIndexOptions::FromCoreOptions(core_options));
     auto state = std::make_shared<CloseFailingState>();
@@ -335,4 +492,45 @@ TEST_F(DataFileIndexWriterTest, TestCloseFailureClosesExternalStreamOnce) {
     ASSERT_EQ(1, state->delete_count);
 }
 
+TEST_F(DataFileIndexWriterV2Test, TestThresholdIncludesFooterAndTrailer) {
+    auto batch = CreateBatch(R"([{"f0": 1, "f1": 10}])");
+    ASSERT_OK_AND_ASSIGN(auto sizing_writer,
+                         CreateWriter({{"file-index.bitmap.columns", "f0"},
+                                       {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1MB"}}));
+    ASSERT_OK(sizing_writer->AddBatch(batch));
+    ASSERT_OK_AND_ASSIGN(auto sized, sizing_writer->Finish("unused.orc"));
+    ASSERT_TRUE(sized.embedded_index);
+    int64_t container_size = sized.embedded_index->size();
+    {
+        // Keep the index embedded when the threshold equals the complete container size.
+        ASSERT_OK_AND_ASSIGN(auto writer, CreateWriter({{"file-index.bitmap.columns", "f0"},
+                                                        {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD,
+                                                         fmt::format("{}B", container_size)}}));
+        ASSERT_OK(writer->AddBatch(batch));
+        std::string path = path_factory_->NewPath();
+        ASSERT_OK_AND_ASSIGN(auto result, writer->Finish(path));
+        ASSERT_TRUE(result.embedded_index);
+        ASSERT_EQ(container_size, result.embedded_index->size());
+        ASSERT_TRUE(result.extra_files.empty());
+    }
+    {
+        // Spill to an external file when the threshold is one byte below the complete container
+        // size.
+        ASSERT_OK_AND_ASSIGN(auto writer, CreateWriter({{"file-index.bitmap.columns", "f0"},
+                                                        {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD,
+                                                         fmt::format("{}B", container_size - 1)}}));
+        ASSERT_OK(writer->AddBatch(batch));
+        std::string path = path_factory_->NewPath();
+        ASSERT_OK_AND_ASSIGN(auto result, writer->Finish(path));
+        ASSERT_FALSE(result.embedded_index);
+        ASSERT_EQ(1, result.extra_files.size());
+        ASSERT_OK_AND_ASSIGN(auto file_status,
+                             file_system_->GetFileStatus(path_factory_->ToFileIndexPath(path)));
+        ASSERT_EQ(container_size, file_status.GetLen());
+    }
+}
+
+INSTANTIATE_TEST_SUITE_P(ContainerVersions, VersionedDataFileIndexWriterTest,
+                         ::testing::Values(1, 2));
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/io/file_index_options.cpp b/src/paimon/core/io/file_index_options.cpp
index 0d9dff9ab..568ea97c9 100644
--- a/src/paimon/core/io/file_index_options.cpp
+++ b/src/paimon/core/io/file_index_options.cpp
@@ -25,6 +25,7 @@
 
 #include "fmt/format.h"
 #include "paimon/common/options/memory_size.h"
+#include "paimon/common/utils/options_utils.h"
 #include "paimon/common/utils/string_utils.h"
 #include "paimon/core/core_options.h"
 #include "paimon/defs.h"
@@ -41,7 +42,8 @@ constexpr size_t kColumnsSuffixLength = sizeof(kColumnsSuffix) - 1;
 }  // namespace
 
 Result<FileIndexOptions> FileIndexOptions::FromCoreOptions(const CoreOptions& options) {
-    return Parse(options.ToMap(), options.FileIndexInManifestThreshold());
+    return Parse(options.ToMap(), options.FileIndexInManifestThreshold(),
+                 options.FileIndexFormatVersion());
 }
 
 Result<FileIndexOptions> FileIndexOptions::FromMap(
@@ -51,13 +53,22 @@ Result<FileIndexOptions> FileIndexOptions::FromMap(
     if (iter != raw_options.end()) {
         PAIMON_ASSIGN_OR_RAISE(in_manifest_threshold, MemorySize::ParseBytes(iter->second));
     }
-    return Parse(raw_options, in_manifest_threshold);
+    PAIMON_ASSIGN_OR_RAISE(
+        int32_t format_version,
+        OptionsUtils::GetValueFromMap<int32_t>(raw_options, Options::FILE_INDEX_FORMAT_VERSION,
+                                               CoreOptions().FileIndexFormatVersion()));
+    if (format_version != 1 && format_version != 2) {
+        return Status::Invalid("file-index.format-version must be 1 or 2");
+    }
+    return Parse(raw_options, in_manifest_threshold, format_version);
 }
 
 Result<FileIndexOptions> FileIndexOptions::Parse(
-    const std::map<std::string, std::string>& raw_options, int64_t in_manifest_threshold) {
+    const std::map<std::string, std::string>& raw_options, int64_t in_manifest_threshold,
+    int32_t format_version) {
     FileIndexOptions result;
     result.in_manifest_threshold_ = in_manifest_threshold;
+    result.format_version_ = format_version;
 
     std::set<std::pair<std::string, std::string>> declared;
     for (const auto& [key, value] : raw_options) {
diff --git a/src/paimon/core/io/file_index_options.h b/src/paimon/core/io/file_index_options.h
index 27c85b178..85c760235 100644
--- a/src/paimon/core/io/file_index_options.h
+++ b/src/paimon/core/io/file_index_options.h
@@ -57,6 +57,10 @@ class FileIndexOptions {
         return in_manifest_threshold_;
     }
 
+    int32_t FormatVersion() const {
+        return format_version_;
+    }
+
     bool Empty() const {
         return definitions_.empty();
     }
@@ -68,10 +72,11 @@ class FileIndexOptions {
     FileIndexOptions() = default;
 
     static Result<FileIndexOptions> Parse(const std::map<std::string, std::string>& raw_options,
-                                          int64_t in_manifest_threshold);
+                                          int64_t in_manifest_threshold, int32_t format_version);
 
     std::vector<FileIndexDefinition> definitions_;
     int64_t in_manifest_threshold_ = 0;
+    int32_t format_version_ = 1;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/io/file_index_options_test.cpp b/src/paimon/core/io/file_index_options_test.cpp
index 3f5b12532..0c59590ca 100644
--- a/src/paimon/core/io/file_index_options_test.cpp
+++ b/src/paimon/core/io/file_index_options_test.cpp
@@ -145,4 +145,20 @@ TEST(FileIndexOptionsTest, TestRewriteFieldNamesRejectsUndeclaredColumnOptions)
         "column 'vec' is not declared");
 }
 
+TEST(FileIndexOptionsTest, TestFormatVersion) {
+    ASSERT_OK_AND_ASSIGN(auto defaults, ParseOptions({}));
+    ASSERT_EQ(1, defaults.FormatVersion());
+    for (const std::string& version : {"1", "2"}) {
+        ASSERT_OK_AND_ASSIGN(auto options, ParseOptions({{"file-index.format-version", version}}));
+        ASSERT_EQ(std::stoi(version), options.FormatVersion());
+        ASSERT_OK_AND_ASSIGN(FileIndexOptions from_map,
+                             FileIndexOptions::FromMap({{"file-index.format-version", version}}));
+        ASSERT_EQ(std::stoi(version), from_map.FormatVersion());
+    }
+    for (const std::string& version : {"0", "3", "-1", "bad"}) {
+        ASSERT_NOK(ParseOptions({{"file-index.format-version", version}}));
+        ASSERT_NOK(FileIndexOptions::FromMap({{"file-index.format-version", version}}));
+    }
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/io/spillable_index_output_stream.cpp b/src/paimon/core/io/spillable_index_output_stream.cpp
new file mode 100644
index 000000000..31128ef99
--- /dev/null
+++ b/src/paimon/core/io/spillable_index_output_stream.cpp
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/io/spillable_index_output_stream.h"
+
+#include <limits>
+
+#include "paimon/common/io/byte_array_output_stream.h"
+#include "paimon/common/io/memory_segment_output_stream.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/memory/memory_pool.h"
+
+namespace paimon {
+SpillableIndexOutputStream::SpillableIndexOutputStream(
+    const std::shared_ptr<FileSystem>& file_system, const std::string& path, int64_t threshold,
+    const std::shared_ptr<MemoryPool>& pool)
+    : threshold_(threshold),
+      pool_(pool),
+      file_system_(file_system),
+      path_(path),
+      memory_(std::make_shared<ByteArrayOutputStream>(std::make_unique<MemorySegmentOutputStream>(
+          MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool))) {}
+
+Result<int64_t> SpillableIndexOutputStream::Write(const char* buffer, int64_t size) {
+    if (size < 0 || size > std::numeric_limits<int64_t>::max() - position_) {
+        return Status::Invalid("File index size overflow");
+    }
+    if (!external_ && position_ + size > threshold_) {
+        PAIMON_ASSIGN_OR_RAISE(external_, file_system_->Create(path_, /*overwrite=*/false));
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Bytes> prefix, memory_->Finish(pool_.get()));
+        PAIMON_ASSIGN_OR_RAISE(int64_t written, external_->Write(prefix->data(), prefix->size()));
+        if (written != static_cast<int64_t>(prefix->size())) {
+            return Status::IOError("Short write for file index");
+        }
+        memory_.reset();
+    }
+    PAIMON_ASSIGN_OR_RAISE(int64_t written,
+                           (external_ ? external_.get() : memory_.get())->Write(buffer, size));
+    if (written != size) {
+        return Status::IOError("Short write for file index");
+    }
+    position_ += written;
+    return written;
+}
+
+Result<int64_t> SpillableIndexOutputStream::GetPos() const {
+    return position_;
+}
+
+Result<std::string> SpillableIndexOutputStream::GetUri() const {
+    return path_;
+}
+
+Status SpillableIndexOutputStream::Flush() {
+    return external_ ? external_->Flush() : memory_->Flush();
+}
+
+Status SpillableIndexOutputStream::Close() {
+    if (closed_) {
+        return Status::OK();
+    }
+    closed_ = true;
+    return external_ ? external_->Close() : memory_->Close();
+}
+
+Result<std::shared_ptr<Bytes>> SpillableIndexOutputStream::EmbeddedBytes() {
+    if (external_) {
+        return std::shared_ptr<Bytes>();
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Bytes> bytes, memory_->Finish(pool_.get()));
+    return bytes;
+}
+
+Status SpillableIndexOutputStream::Abort() {
+    Status close_status = Close();
+    if (Spilled()) {
+        PAIMON_RETURN_NOT_OK(file_system_->Delete(path_, /*recursive=*/false));
+    }
+    return close_status;
+}
+}  // namespace paimon
diff --git a/src/paimon/core/io/spillable_index_output_stream.h b/src/paimon/core/io/spillable_index_output_stream.h
new file mode 100644
index 000000000..0373c1934
--- /dev/null
+++ b/src/paimon/core/io/spillable_index_output_stream.h
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <string>
+
+#include "paimon/fs/file_system.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+
+namespace paimon {
+class ByteArrayOutputStream;
+class Bytes;
+class MemoryPool;
+
+/// Keeps a small file index in memory and spills to an external file above the threshold.
+class SpillableIndexOutputStream : public OutputStream {
+ public:
+    SpillableIndexOutputStream(const std::shared_ptr<FileSystem>& file_system,
+                               const std::string& path, int64_t threshold,
+                               const std::shared_ptr<MemoryPool>& pool);
+
+    Result<int64_t> Write(const char* buffer, int64_t size) override;
+    Result<int64_t> GetPos() const override;
+    Result<std::string> GetUri() const override;
+    Status Flush() override;
+    Status Close() override;
+
+    bool Spilled() const {
+        return external_ != nullptr;
+    }
+
+    /// Closes the stream and deletes any unpublished external index, even if Close() fails.
+    Status Abort();
+
+    /// Returns the buffered index, or null if it has spilled to an external file.
+    Result<std::shared_ptr<Bytes>> EmbeddedBytes();
+
+ private:
+    int64_t threshold_;
+    std::shared_ptr<MemoryPool> pool_;
+    std::shared_ptr<FileSystem> file_system_;
+    std::string path_;
+    std::shared_ptr<ByteArrayOutputStream> memory_;
+    std::shared_ptr<OutputStream> external_;
+    int64_t position_ = 0;
+    bool closed_ = false;
+};
+}  // namespace paimon
diff --git a/src/paimon/core/io/spillable_index_output_stream_test.cpp b/src/paimon/core/io/spillable_index_output_stream_test.cpp
new file mode 100644
index 000000000..ca6bee73f
--- /dev/null
+++ b/src/paimon/core/io/spillable_index_output_stream_test.cpp
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/io/spillable_index_output_stream.h"
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "paimon/fs/local/local_file_system.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+class SpillableIndexOutputStreamTest : public ::testing::Test {
+ protected:
+    void SetUp() override {
+        directory_ = UniqueTestDirectory::Create();
+        ASSERT_TRUE(directory_);
+        path_ = directory_->Str() + "/test.index";
+    }
+
+    std::unique_ptr<SpillableIndexOutputStream> CreateStream(int64_t threshold) {
+        return std::make_unique<SpillableIndexOutputStream>(file_system_, path_, threshold, pool_);
+    }
+
+    void Write(SpillableIndexOutputStream* stream, const std::string& data) {
+        ASSERT_OK_AND_ASSIGN(int64_t written, stream->Write(data.data(), data.size()));
+        ASSERT_EQ(data.size(), written);
+    }
+
+    std::shared_ptr<MemoryPool> pool_ = GetDefaultPool();
+    std::shared_ptr<FileSystem> file_system_ = std::make_shared<LocalFileSystem>();
+    std::unique_ptr<UniqueTestDirectory> directory_;
+    std::string path_;
+};
+
+TEST_F(SpillableIndexOutputStreamTest, KeepsPayloadAtThresholdInMemory) {
+    auto stream = CreateStream(4);
+    ASSERT_NO_FATAL_FAILURE(Write(stream.get(), "ab"));
+    ASSERT_NO_FATAL_FAILURE(Write(stream.get(), "cd"));
+    ASSERT_OK_AND_ASSIGN(int64_t position, stream->GetPos());
+    ASSERT_EQ(4, position);
+    ASSERT_OK(stream->Flush());
+    ASSERT_OK(stream->Close());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Bytes> bytes, stream->EmbeddedBytes());
+    ASSERT_TRUE(bytes);
+    ASSERT_EQ("abcd", std::string(bytes->data(), bytes->size()));
+    ASSERT_FALSE(stream->Spilled());
+    ASSERT_OK_AND_ASSIGN(bool exists, file_system_->Exists(path_));
+    ASSERT_FALSE(exists);
+}
+
+TEST_F(SpillableIndexOutputStreamTest, SpillsAndPreservesBufferedPrefix) {
+    for (const auto& [threshold, prefix] :
+         std::vector<std::pair<int64_t, std::string>>{{4, ""}, {4, "abcd"}}) {
+        auto stream = CreateStream(threshold);
+        if (!prefix.empty()) {
+            ASSERT_NO_FATAL_FAILURE(Write(stream.get(), prefix));
+        }
+        ASSERT_FALSE(stream->Spilled());
+        ASSERT_NO_FATAL_FAILURE(Write(stream.get(), "efghi"));
+        ASSERT_NO_FATAL_FAILURE(Write(stream.get(), "j"));
+        ASSERT_TRUE(stream->Spilled());
+        ASSERT_OK_AND_ASSIGN(int64_t position, stream->GetPos());
+        ASSERT_EQ(prefix.size() + 6, position);
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<Bytes> bytes, stream->EmbeddedBytes());
+        ASSERT_FALSE(bytes);
+        ASSERT_OK(stream->Flush());
+        ASSERT_OK(stream->Close());
+        std::string content;
+        ASSERT_OK(file_system_->ReadFile(path_, &content));
+        ASSERT_EQ(prefix + "efghij", content);
+        ASSERT_OK(file_system_->Delete(path_));
+    }
+}
+
+TEST_F(SpillableIndexOutputStreamTest, AbortRespectsFileOwnership) {
+    for (bool spilled : {false, true}) {
+        if (!spilled) {
+            ASSERT_OK_AND_ASSIGN(std::unique_ptr<OutputStream> existing,
+                                 file_system_->Create(path_, /*overwrite=*/false));
+            ASSERT_OK_AND_ASSIGN(int64_t written, existing->Write("original", 8));
+            ASSERT_EQ(8, written);
+            ASSERT_OK(existing->Close());
+        }
+        auto stream = CreateStream(spilled ? 0 : 4);
+        ASSERT_NO_FATAL_FAILURE(Write(stream.get(), "ab"));
+        ASSERT_OK(stream->Abort());
+        ASSERT_OK_AND_ASSIGN(bool exists, file_system_->Exists(path_));
+        ASSERT_EQ(!spilled, exists);
+        if (!spilled) {
+            std::string content;
+            ASSERT_OK(file_system_->ReadFile(path_, &content));
+            ASSERT_EQ("original", content);
+            ASSERT_OK(file_system_->Delete(path_));
+        }
+    }
+}
+}  // namespace paimon::test
diff --git a/src/paimon/indexer/lumina/lumina_file_index.cpp b/src/paimon/indexer/lumina/lumina_file_index.cpp
index e30a5838c..4fcf209f2 100644
--- a/src/paimon/indexer/lumina/lumina_file_index.cpp
+++ b/src/paimon/indexer/lumina/lumina_file_index.cpp
@@ -39,6 +39,42 @@
 namespace paimon::lumina {
 namespace {
 
+/// Reports positions relative to the wrapped stream's position at creation.
+/// Flush forwards to the wrapped stream; Close leaves it open so the container can continue
+/// writing.
+class OffsetOutputStream : public OutputStream {
+ public:
+    static Result<std::unique_ptr<OffsetOutputStream>> Create(
+        const std::shared_ptr<OutputStream>& output) {
+        PAIMON_ASSIGN_OR_RAISE(int64_t start, output->GetPos());
+        return std::unique_ptr<OffsetOutputStream>(new OffsetOutputStream(output, start));
+    }
+
+    Result<int64_t> Write(const char* buffer, int64_t size) override {
+        return output_->Write(buffer, size);
+    }
+    Result<int64_t> GetPos() const override {
+        PAIMON_ASSIGN_OR_RAISE(int64_t position, output_->GetPos());
+        return position - start_;
+    }
+    Result<std::string> GetUri() const override {
+        return output_->GetUri();
+    }
+    Status Flush() override {
+        return output_->Flush();
+    }
+    Status Close() override {
+        return Status::OK();
+    }
+
+ private:
+    OffsetOutputStream(const std::shared_ptr<OutputStream>& output, int64_t start)
+        : output_(output), start_(start) {}
+
+    std::shared_ptr<OutputStream> output_;
+    int64_t start_;
+};
+
 Result<std::shared_ptr<arrow::Schema>> ImportVectorSchema(::ArrowSchema* c_schema,
                                                           const std::string& owner) {
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> schema,
@@ -103,28 +139,39 @@ Status LuminaFileIndexWriter::AddBatch(::ArrowArray* batch) {
 }
 
 Result<PAIMON_UNIQUE_PTR<Bytes>> LuminaFileIndexWriter::SerializedBytes() const {
+    auto segment_output = std::make_unique<MemorySegmentOutputStream>(
+        MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_->GetPaimonPool());
+    auto output = std::make_shared<ByteArrayOutputStream>(std::move(segment_output));
+    PAIMON_ASSIGN_OR_RAISE(bool written, WriteTo(output));
+    if (!written) {
+        return PAIMON_UNIQUE_PTR<Bytes>();
+    }
+    return output->Finish(pool_->GetPaimonPool().get());
+}
+
+Result<bool> LuminaFileIndexWriter::WriteTo(const std::shared_ptr<OutputStream>& output) const {
+    if (!output) {
+        return Status::Invalid("Lumina File Index output stream cannot be null");
+    }
     if (serialized_) {
         return Status::Invalid("Lumina File Index has already been serialized");
     }
     serialized_ = true;
     if (accumulator_.IndexedCount() == 0) {
-        return PAIMON_UNIQUE_PTR<Bytes>();
+        return false;
     }
-
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OffsetOutputStream> offset_output,
+                           OffsetOutputStream::Create(output));
     std::unique_ptr<::lumina::extensions::experimental::CkptManager> checkpoint_manager;
     PAIMON_ASSIGN_OR_RAISE(
         std::unique_ptr<LuminaIndexBuildContext> build_context,
         accumulator_.Build(builder_options_, index_info_.dimension, !tag_fields_.empty(),
                            std::move(checkpoint_manager), pool_.get()));
 
-    auto segment_output = std::make_unique<MemorySegmentOutputStream>(
-        MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_->GetPaimonPool());
-    std::shared_ptr<ByteArrayOutputStream> output =
-        std::make_shared<ByteArrayOutputStream>(std::move(segment_output));
-    auto file_writer = std::make_unique<LuminaFileWriter>(output);
+    auto file_writer = std::make_unique<LuminaFileWriter>(offset_output);
     PAIMON_RETURN_NOT_OK_FROM_LUMINA(
         build_context->builder.Dump(std::move(file_writer), ::lumina::api::IOOptions()));
-    return output->Finish(pool_->GetPaimonPool().get());
+    return true;
 }
 
 LuminaFileIndexReader::LuminaFileIndexReader(std::unique_ptr<LuminaIndexSearcher>&& searcher)
@@ -181,7 +228,7 @@ Result<std::shared_ptr<FileIndexWriter>> LuminaFileIndexer::CreateWriter(
 }
 
 Result<std::shared_ptr<FileIndexReader>> LuminaFileIndexer::CreateReader(
-    ::ArrowSchema* c_schema, int32_t start, int32_t length,
+    ::ArrowSchema* c_schema, int64_t start, int64_t length,
     const std::shared_ptr<InputStream>& input_stream,
     const std::shared_ptr<MemoryPool>& pool) const {
     if (!input_stream || !pool) {
diff --git a/src/paimon/indexer/lumina/lumina_file_index.h b/src/paimon/indexer/lumina/lumina_file_index.h
index 9a7c5f790..750ca4209 100644
--- a/src/paimon/indexer/lumina/lumina_file_index.h
+++ b/src/paimon/indexer/lumina/lumina_file_index.h
@@ -44,7 +44,7 @@ class LuminaFileIndexer final : public FileIndexer {
     Result<std::optional<std::vector<std::string>>> GetExtraFieldNames() const override;
 
     Result<std::shared_ptr<FileIndexReader>> CreateReader(
-        ::ArrowSchema* arrow_schema, int32_t start, int32_t length,
+        ::ArrowSchema* arrow_schema, int64_t start, int64_t length,
         const std::shared_ptr<InputStream>& input_stream,
         const std::shared_ptr<MemoryPool>& pool) const override;
 
@@ -66,6 +66,7 @@ class LuminaFileIndexWriter final : public FileIndexWriter {
     Status AddBatch(::ArrowArray* batch) override;
 
     Result<PAIMON_UNIQUE_PTR<Bytes>> SerializedBytes() const override;
+    Result<bool> WriteTo(const std::shared_ptr<OutputStream>& output) const override;
 
  private:
     std::string field_name_;
diff --git a/src/paimon/indexer/lumina/lumina_file_index_test.cpp b/src/paimon/indexer/lumina/lumina_file_index_test.cpp
index d78e1f137..986c5cda2 100644
--- a/src/paimon/indexer/lumina/lumina_file_index_test.cpp
+++ b/src/paimon/indexer/lumina/lumina_file_index_test.cpp
@@ -28,6 +28,7 @@
 #include "arrow/c/bridge.h"
 #include "arrow/ipc/json_simple.h"
 #include "gtest/gtest.h"
+#include "paimon/common/io/byte_array_output_stream.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/io/byte_array_input_stream.h"
 #include "paimon/memory/memory_pool.h"
@@ -103,7 +104,7 @@ TEST(LuminaFileIndexTest, RoundTripUsesFileLocalRowPositions) {
     ASSERT_TRUE(arrow::ExportSchema(*schema, &reader_schema).ok());
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexReader> reader,
                          indexer.CreateReader(&reader_schema, /*start=*/6,
-                                              static_cast<int32_t>(artifact->size()), input, pool));
+                                              static_cast<int64_t>(artifact->size()), input, pool));
 
     std::shared_ptr<VectorSearch> search = std::make_shared<VectorSearch>(
         "embedding", /*limit=*/1, std::vector<float>{1.0f, 1.0f, 1.0f, 1.0f},
@@ -133,7 +134,7 @@ TEST(LuminaFileIndexTest, RoundTripUsesFileLocalRowPositions) {
                         "lumina index was not built with tag");
 }
 
-TEST(LuminaFileIndexTest, AllNullVectorsProduceEmptyArtifact) {
+TEST(LuminaFileIndexTest, WriteToUsesRelativeOffsetsAndKeepsStreamOpen) {
     const std::map<std::string, std::string> options = {
         {"index.dimension", "4"},
         {"index.type", "bruteforce"},
@@ -141,11 +142,10 @@ TEST(LuminaFileIndexTest, AllNullVectorsProduceEmptyArtifact) {
         {"encoding.type", "rawf32"},
     };
     std::shared_ptr<MemoryPool> pool = GetDefaultPool();
-    std::shared_ptr<arrow::StructArray> batch = CreateAllNullVectors();
+    std::shared_ptr<arrow::StructArray> batch = CreateVectors();
     std::shared_ptr<arrow::Schema> schema =
         arrow::schema({arrow::field("embedding", arrow::list(arrow::float32()))});
     LuminaFileIndexer indexer(options);
-
     ::ArrowSchema writer_schema;
     ASSERT_TRUE(arrow::ExportSchema(*schema, &writer_schema).ok());
     ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexWriter> writer,
@@ -153,8 +153,88 @@ TEST(LuminaFileIndexTest, AllNullVectorsProduceEmptyArtifact) {
     ::ArrowArray c_batch;
     ASSERT_TRUE(arrow::ExportArray(*batch, &c_batch).ok());
     ASSERT_OK(writer->AddBatch(&c_batch));
-    ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> artifact, writer->SerializedBytes());
-    ASSERT_FALSE(artifact);
+
+    // Write after a prefix so the payload starts at a nonzero stream position.
+    // Lumina must record its internal offsets relative to this payload start.
+    auto output =
+        std::make_shared<ByteArrayOutputStream>(std::make_unique<MemorySegmentOutputStream>(
+            MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool));
+    ASSERT_OK_AND_ASSIGN(int64_t prefix_length, output->Write("prefix", 6));
+    ASSERT_EQ(6, prefix_length);
+    ASSERT_OK_AND_ASSIGN(bool written, writer->WriteTo(output));
+    ASSERT_TRUE(written);
+    ASSERT_OK_AND_ASSIGN(int64_t end, output->GetPos());
+    int64_t length = end - prefix_length;
+    ASSERT_GT(length, 0);
+
+    // Appending a suffix verifies that WriteTo() leaves the caller's stream open.
+    // Check that the surrounding bytes remain intact when retrieving the container.
+    ASSERT_OK_AND_ASSIGN(int64_t suffix_length, output->Write("suffix", 6));
+    ASSERT_EQ(6, suffix_length);
+    ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> container, output->Finish(pool.get()));
+    ASSERT_EQ("prefix", std::string(container->data(), prefix_length));
+    ASSERT_EQ("suffix", std::string(container->data() + end, suffix_length));
+
+    // Reopen only the payload range, excluding the prefix and suffix.
+    // This checks that the reader can locate the index inside a larger stream.
+    auto input = std::make_shared<ByteArrayInputStream>(container->data(), container->size());
+    ::ArrowSchema reader_schema;
+    ASSERT_TRUE(arrow::ExportSchema(*schema, &reader_schema).ok());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexReader> reader,
+                         indexer.CreateReader(&reader_schema, prefix_length, length, input, pool));
+
+    // Query the reopened index to exercise the offsets stored inside the payload.
+    // The exact matching vector must still resolve to file row 2.
+    auto search = std::make_shared<VectorSearch>(
+        "embedding", /*limit=*/1, std::vector<float>{1.0f, 1.0f, 1.0f, 1.0f},
+        /*pre_filter=*/nullptr, /*predicate=*/nullptr, VectorSearch::DistanceType::EUCLIDEAN,
+        std::map<std::string, std::string>{});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<ScoredFileIndexResult> result,
+                         reader->VisitVectorSearch(search));
+    ASSERT_EQ(RoaringBitmap32::From({2}), result->GetRowPositions());
+}
+
+TEST(LuminaFileIndexTest, AllNullVectorsProduceEmptyArtifact) {
+    const std::map<std::string, std::string> options = {
+        {"index.dimension", "4"},
+        {"index.type", "bruteforce"},
+        {"distance.metric", "l2"},
+        {"encoding.type", "rawf32"},
+    };
+    std::shared_ptr<MemoryPool> pool = GetDefaultPool();
+    std::shared_ptr<arrow::StructArray> batch = CreateAllNullVectors();
+    std::shared_ptr<arrow::Schema> schema =
+        arrow::schema({arrow::field("embedding", arrow::list(arrow::float32()))});
+    LuminaFileIndexer indexer(options);
+    {
+        // WriteTo() reports no index and leaves the output empty.
+        ::ArrowSchema writer_schema;
+        ASSERT_TRUE(arrow::ExportSchema(*schema, &writer_schema).ok());
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexWriter> writer,
+                             indexer.CreateWriter(&writer_schema, pool));
+        ::ArrowArray c_batch;
+        ASSERT_TRUE(arrow::ExportArray(*batch, &c_batch).ok());
+        ASSERT_OK(writer->AddBatch(&c_batch));
+        auto output =
+            std::make_shared<ByteArrayOutputStream>(std::make_unique<MemorySegmentOutputStream>(
+                MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool));
+        ASSERT_OK_AND_ASSIGN(bool written, writer->WriteTo(output));
+        ASSERT_FALSE(written);
+        ASSERT_OK_AND_ASSIGN(int64_t position, output->GetPos());
+        ASSERT_EQ(0, position);
+    }
+    {
+        // SerializedBytes() reports no index by returning a null artifact.
+        ::ArrowSchema writer_schema;
+        ASSERT_TRUE(arrow::ExportSchema(*schema, &writer_schema).ok());
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexWriter> writer,
+                             indexer.CreateWriter(&writer_schema, pool));
+        ::ArrowArray c_batch;
+        ASSERT_TRUE(arrow::ExportArray(*batch, &c_batch).ok());
+        ASSERT_OK(writer->AddBatch(&c_batch));
+        ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> artifact, writer->SerializedBytes());
+        ASSERT_FALSE(artifact);
+    }
 }
 
 TEST(LuminaFileIndexTest, RoundTripWithTagPredicate) {
diff --git a/src/paimon/testing/mock/mock_search_file_index.cpp b/src/paimon/testing/mock/mock_search_file_index.cpp
index 3e8e39a1c..26a329064 100644
--- a/src/paimon/testing/mock/mock_search_file_index.cpp
+++ b/src/paimon/testing/mock/mock_search_file_index.cpp
@@ -91,7 +91,7 @@ MockSearchFileIndexer::MockSearchFileIndexer(SearchIndexType index_type)
     : index_type_(index_type) {}
 
 Result<std::shared_ptr<FileIndexReader>> MockSearchFileIndexer::CreateReader(
-    ::ArrowSchema* c_schema, int32_t start, int32_t length,
+    ::ArrowSchema* c_schema, int64_t start, int64_t length,
     const std::shared_ptr<InputStream>& input, const std::shared_ptr<MemoryPool>&) const {
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> schema,
                                       arrow::ImportSchema(c_schema));
@@ -174,7 +174,20 @@ Result<std::shared_ptr<Bytes>> MockSearchFileIndex::Serialize(
     auto output = MakeOutput(pool);
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileIndexFormat::Writer> writer,
                            FileIndexFormat::CreateWriter(output, pool));
-    PAIMON_RETURN_NOT_OK(writer->WriteColumnIndexes(indexes));
+    for (const auto& [column, entries] : indexes) {
+        for (const auto& [type, bytes] : entries) {
+            FileIndexFormat::Writer::PayloadWriter payload;
+            if (bytes && bytes->size() > 0) {
+                payload =
+                    [bytes](const std::shared_ptr<OutputStream>& destination) -> Result<bool> {
+                    PAIMON_RETURN_NOT_OK(DataOutputStream(destination).WriteBytes(bytes));
+                    return true;
+                };
+            }
+            PAIMON_RETURN_NOT_OK(writer->WriteIndex(column, type, payload));
+        }
+    }
+    PAIMON_RETURN_NOT_OK(writer->Finish());
     PAIMON_RETURN_NOT_OK(writer->Close());
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Bytes> bytes, output->Finish(pool.get()));
     return bytes;
diff --git a/src/paimon/testing/mock/mock_search_file_index.h b/src/paimon/testing/mock/mock_search_file_index.h
index b0980fe0b..8def6b0cb 100644
--- a/src/paimon/testing/mock/mock_search_file_index.h
+++ b/src/paimon/testing/mock/mock_search_file_index.h
@@ -99,7 +99,7 @@ class MockSearchFileIndexer final : public FileIndexer {
     explicit MockSearchFileIndexer(SearchIndexType index_type);
 
     Result<std::shared_ptr<FileIndexReader>> CreateReader(
-        ::ArrowSchema* c_schema, int32_t start, int32_t length,
+        ::ArrowSchema* c_schema, int64_t start, int64_t length,
         const std::shared_ptr<InputStream>& input,
         const std::shared_ptr<MemoryPool>& pool) const override;
 

From fd7df9fc7218232a5157d672419e34a468f89c87 Mon Sep 17 00:00:00 2001
From: "jinli.zjw" <jinli.zjw@alibaba-inc.com>
Date: Tue, 20 Oct 2026 09:56:42 +0800
Subject: [PATCH 02/10] fix(file-index): keep streaming tests compatible with
 C++17

---
 src/paimon/common/file_index/file_index_format_test.cpp | 6 +++---
 src/paimon/core/io/data_file_index_writer_test.cpp      | 2 +-
 src/paimon/core/io/file_index_options_test.cpp          | 4 ++--
 src/paimon/testing/mock/mock_search_file_index.cpp      | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/paimon/common/file_index/file_index_format_test.cpp b/src/paimon/common/file_index/file_index_format_test.cpp
index f19296696..a5e691c70 100644
--- a/src/paimon/common/file_index/file_index_format_test.cpp
+++ b/src/paimon/common/file_index/file_index_format_test.cpp
@@ -58,7 +58,7 @@ class FileIndexFormatTest : public ::testing::Test {
         for (const auto& [column, entries] : indexes) {
             for (const auto& [type, bytes] : entries) {
                 FileIndexFormat::Writer::PayloadWriter payload =
-                    [bytes](const std::shared_ptr<OutputStream>& output) -> Result<bool> {
+                    [bytes = bytes](const std::shared_ptr<OutputStream>& output) -> Result<bool> {
                     if (!bytes || bytes->size() == 0) {
                         return false;
                     }
@@ -1147,8 +1147,8 @@ TEST_F(FileIndexFormatTest, TestV1BuffersPayloadsUntilFinish) {
         for (const auto& [type, bytes] : entries) {
             FileIndexFormat::Writer::PayloadWriter payload;
             if (bytes) {
-                payload =
-                    [bytes](const std::shared_ptr<OutputStream>& destination) -> Result<bool> {
+                payload = [bytes = bytes](
+                              const std::shared_ptr<OutputStream>& destination) -> Result<bool> {
                     PAIMON_RETURN_NOT_OK(DataOutputStream(destination).WriteBytes(bytes));
                     return true;
                 };
diff --git a/src/paimon/core/io/data_file_index_writer_test.cpp b/src/paimon/core/io/data_file_index_writer_test.cpp
index 0e929f81d..cc2f0ba7d 100644
--- a/src/paimon/core/io/data_file_index_writer_test.cpp
+++ b/src/paimon/core/io/data_file_index_writer_test.cpp
@@ -69,7 +69,7 @@ class StreamingTestIndexWriter : public FileIndexWriter {
         if (empty_) {
             return false;
         }
-        for (const std::string& chunk : {"stream", "ed"}) {
+        for (const std::string chunk : {"stream", "ed"}) {
             PAIMON_ASSIGN_OR_RAISE(int64_t written, output->Write(chunk.data(), chunk.size()));
             if (written != static_cast<int64_t>(chunk.size())) {
                 return Status::IOError("short streamed payload");
diff --git a/src/paimon/core/io/file_index_options_test.cpp b/src/paimon/core/io/file_index_options_test.cpp
index 0c59590ca..28cd27ab8 100644
--- a/src/paimon/core/io/file_index_options_test.cpp
+++ b/src/paimon/core/io/file_index_options_test.cpp
@@ -148,14 +148,14 @@ TEST(FileIndexOptionsTest, TestRewriteFieldNamesRejectsUndeclaredColumnOptions)
 TEST(FileIndexOptionsTest, TestFormatVersion) {
     ASSERT_OK_AND_ASSIGN(auto defaults, ParseOptions({}));
     ASSERT_EQ(1, defaults.FormatVersion());
-    for (const std::string& version : {"1", "2"}) {
+    for (const std::string version : {"1", "2"}) {
         ASSERT_OK_AND_ASSIGN(auto options, ParseOptions({{"file-index.format-version", version}}));
         ASSERT_EQ(std::stoi(version), options.FormatVersion());
         ASSERT_OK_AND_ASSIGN(FileIndexOptions from_map,
                              FileIndexOptions::FromMap({{"file-index.format-version", version}}));
         ASSERT_EQ(std::stoi(version), from_map.FormatVersion());
     }
-    for (const std::string& version : {"0", "3", "-1", "bad"}) {
+    for (const std::string version : {"0", "3", "-1", "bad"}) {
         ASSERT_NOK(ParseOptions({{"file-index.format-version", version}}));
         ASSERT_NOK(FileIndexOptions::FromMap({{"file-index.format-version", version}}));
     }
diff --git a/src/paimon/testing/mock/mock_search_file_index.cpp b/src/paimon/testing/mock/mock_search_file_index.cpp
index 26a329064..83f72f461 100644
--- a/src/paimon/testing/mock/mock_search_file_index.cpp
+++ b/src/paimon/testing/mock/mock_search_file_index.cpp
@@ -178,8 +178,8 @@ Result<std::shared_ptr<Bytes>> MockSearchFileIndex::Serialize(
         for (const auto& [type, bytes] : entries) {
             FileIndexFormat::Writer::PayloadWriter payload;
             if (bytes && bytes->size() > 0) {
-                payload =
-                    [bytes](const std::shared_ptr<OutputStream>& destination) -> Result<bool> {
+                payload = [bytes = bytes](
+                              const std::shared_ptr<OutputStream>& destination) -> Result<bool> {
                     PAIMON_RETURN_NOT_OK(DataOutputStream(destination).WriteBytes(bytes));
                     return true;
                 };

From 9ceeae1f47a95263e4e16916b173b162b090d2a2 Mon Sep 17 00:00:00 2001
From: "jinli.zjw" <jinli.zjw@alibaba-inc.com>
Date: Tue, 20 Oct 2026 10:29:45 +0800
Subject: [PATCH 03/10] fix(file-index): remove unnecessary constant lambda
 capture

---
 src/paimon/common/file_index/file_index_format_test.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/paimon/common/file_index/file_index_format_test.cpp b/src/paimon/common/file_index/file_index_format_test.cpp
index a5e691c70..d2c2a9e99 100644
--- a/src/paimon/common/file_index/file_index_format_test.cpp
+++ b/src/paimon/common/file_index/file_index_format_test.cpp
@@ -1037,8 +1037,7 @@ TEST_F(FileIndexFormatTest, TestV2StreamingLargeContainer) {
     // Stream a virtual >2 GiB payload in bounded chunks; the mock discards only this gap.
     const int64_t gap_length = int64_t{1} << 31;
     ASSERT_OK(writer->WriteIndex(
-        "gap", "unknown",
-        [state, gap_length](const std::shared_ptr<OutputStream>& stream) -> Result<bool> {
+        "gap", "unknown", [state](const std::shared_ptr<OutputStream>& stream) -> Result<bool> {
             state->discard = true;
             std::array<char, 65536> zeros{};
             for (int64_t i = 0; i < gap_length; i += zeros.size()) {

From 53e47ac6c02cc643533503592047dde7ff431ca1 Mon Sep 17 00:00:00 2001
From: "jinli.zjw" <jinli.zjw@alibaba-inc.com>
Date: Tue, 20 Oct 2026 11:48:50 +0800
Subject: [PATCH 04/10] fix(file-index): use explicit casts in V2 container
 tests

---
 src/paimon/common/file_index/file_index_format_test.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/paimon/common/file_index/file_index_format_test.cpp b/src/paimon/common/file_index/file_index_format_test.cpp
index d2c2a9e99..05b5f2db6 100644
--- a/src/paimon/common/file_index/file_index_format_test.cpp
+++ b/src/paimon/common/file_index/file_index_format_test.cpp
@@ -1015,7 +1015,8 @@ TEST_F(FileIndexFormatTest, TestV2InvalidContainers) {
             bytes.empty() ? "" : reinterpret_cast<const char*>(bytes.data()), bytes.size());
         ASSERT_NOK(FileIndexFormat::CreateReader(input, pool_, /*options=*/{}));
     };
-    for (size_t size : {size_t(0), size_t(11), size_t(24), kJavaV2Golden.size() - 1}) {
+    for (size_t size : {static_cast<size_t>(0), static_cast<size_t>(11), static_cast<size_t>(24),
+                        kJavaV2Golden.size() - 1}) {
         check(std::vector<uint8_t>(kJavaV2Golden.begin(), kJavaV2Golden.begin() + size));
     }
     for (const auto& [offset, value] : std::vector<std::pair<size_t, uint8_t>>{

From cf803343936fdd317c030a4a7ea925d321006852 Mon Sep 17 00:00:00 2001
From: "jinli.zjw" <jinli.zjw@alibaba-inc.com>
Date: Thu, 22 Oct 2026 16:47:16 +0800
Subject: [PATCH 05/10] fix(file-index): align V2 configuration and streaming
 APIs

Align the format version option with Java and simplify option parsing and metadata helpers.

Clarify borrowed output stream ownership and update writer APIs and callers.

Cover Java reads and C++ V1/V2 write-read paths across file formats, prefetch settings, and embedded or external indexes.
---
 include/paimon/defs.h                         |    2 +-
 include/paimon/file_index/file_index_format.h |    8 +-
 include/paimon/file_index/file_index_writer.h |    3 +-
 src/paimon/common/defs.cpp                    |    2 +-
 .../file_index/bitmap/bitmap_file_index.cpp   |   22 +-
 .../file_index/bitmap/bitmap_file_index.h     |    4 +-
 .../common/file_index/file_index_format.cpp   |   13 +-
 .../file_index/file_index_format_test.cpp     |  121 +-
 .../file_index/file_index_format_utils.cpp    |   10 +-
 .../file_index/file_index_format_utils.h      |   18 +-
 .../file_index/file_index_format_v1.cpp       |   16 +-
 .../common/file_index/file_index_format_v1.h  |    8 +-
 .../file_index/file_index_format_v2.cpp       |   14 +-
 .../common/file_index/file_index_format_v2.h  |   11 +-
 .../common/file_index/file_index_writer.cpp   |    2 +-
 .../file_index/file_index_writer_test.cpp     |   14 +-
 src/paimon/core/core_options.cpp              |    5 +-
 src/paimon/core/core_options_test.cpp         |   10 +
 src/paimon/core/io/data_file_index_writer.cpp |    7 +-
 .../core/io/data_file_index_writer_test.cpp   |    2 +-
 src/paimon/core/io/file_index_options.cpp     |   31 +-
 src/paimon/core/io/file_index_options.h       |    3 -
 .../core/io/file_index_options_test.cpp       |   19 +-
 .../core/io/spillable_index_output_stream.cpp |    2 +-
 .../core/io/spillable_index_output_stream.h   |    2 +-
 .../io/spillable_index_output_stream_test.cpp |    3 +-
 .../indexer/lumina/lumina_file_index.cpp      |   12 +-
 src/paimon/indexer/lumina/lumina_file_index.h |    2 +-
 .../indexer/lumina/lumina_file_index_test.cpp |    4 +-
 .../testing/mock/mock_search_file_index.cpp   |    5 +-
 test/inte/read_inte_with_index_test.cpp       | 1600 ++++++++++-------
 31 files changed, 1131 insertions(+), 844 deletions(-)

diff --git a/include/paimon/defs.h b/include/paimon/defs.h
index e4223f12b..565c90fe5 100644
--- a/include/paimon/defs.h
+++ b/include/paimon/defs.h
@@ -469,7 +469,7 @@ struct PAIMON_EXPORT Options {
     /// manifest. Default value is 500B.
     static const char FILE_INDEX_IN_MANIFEST_THRESHOLD[];
 
-    /// "file-index.format-version" - File index container write version, 1 or 2. Defaults to 1.
+    /// "file-index.format.version" - File index container write version, 1 or 2. Defaults to 1.
     static const char FILE_INDEX_FORMAT_VERSION[];
 
     /// "data-file.external-paths" - The external paths where the data of this table will be
diff --git a/include/paimon/file_index/file_index_format.h b/include/paimon/file_index/file_index_format.h
index d69b6eef7..ec4e3f00d 100644
--- a/include/paimon/file_index/file_index_format.h
+++ b/include/paimon/file_index/file_index_format.h
@@ -66,12 +66,12 @@ class PAIMON_EXPORT FileIndexFormat {
     /// Creates a `Writer` for file index container.
     ///
     /// @param output_stream Destination stream for serialized index data.
+    /// @param version Container format version (V1 or V2).
     /// @param pool Memory pool for writer-side allocations.
-    /// @param version Container format version; defaults to V1.
     /// @return A unique pointer to a `Writer` on success.
     static Result<std::unique_ptr<Writer>> CreateWriter(
-        const std::shared_ptr<OutputStream>& output_stream, const std::shared_ptr<MemoryPool>& pool,
-        int32_t version = V_1);
+        const std::shared_ptr<OutputStream>& output_stream, int32_t version,
+        const std::shared_ptr<MemoryPool>& pool);
 
  public:
     static const int64_t MAGIC;
@@ -85,7 +85,7 @@ class PAIMON_EXPORT FileIndexFormat::Writer {
  public:
     /// Writes one payload without closing the stream. Returns false for an absent index
     /// (without writing bytes), or true after writing a non-empty payload.
-    using PayloadWriter = std::function<Result<bool>(const std::shared_ptr<OutputStream>&)>;
+    using PayloadWriter = std::function<Result<bool>(OutputStream*)>;
     virtual ~Writer() = default;
 
     /// Flushes and closes the output stream supplied to `CreateWriter()`.
diff --git a/include/paimon/file_index/file_index_writer.h b/include/paimon/file_index/file_index_writer.h
index 7f2aed7db..43028bf40 100644
--- a/include/paimon/file_index/file_index_writer.h
+++ b/include/paimon/file_index/file_index_writer.h
@@ -58,12 +58,13 @@ class PAIMON_EXPORT FileIndexWriter {
     virtual Result<PAIMON_UNIQUE_PTR<Bytes>> SerializedBytes() const;
 
     /// Writes one index payload to the caller-owned stream without closing it.
+    /// The stream is borrowed for this call and must not be retained by the writer.
     /// The default implementation writes SerializedBytes(); streaming writers can override this
     /// method to avoid materializing the complete payload. Write failures must be propagated.
     /// @return true if a non-empty index payload was produced, false if there is no index, or an
     /// error. Returning false must not write any bytes. The default implementation treats null or
     /// zero-length SerializedBytes() results as no index.
-    virtual Result<bool> WriteTo(const std::shared_ptr<OutputStream>& output) const;
+    virtual Result<bool> WriteTo(OutputStream* output) const;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp
index 571cbbb93..1088b75eb 100644
--- a/src/paimon/common/defs.cpp
+++ b/src/paimon/common/defs.cpp
@@ -118,7 +118,7 @@ const char Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP[] =
 const char Options::SCAN_FALLBACK_BRANCH[] = "scan.fallback-branch";
 const char Options::BRANCH[] = "branch";
 const char Options::FILE_INDEX_READ_ENABLED[] = "file-index.read.enabled";
-const char Options::FILE_INDEX_FORMAT_VERSION[] = "file-index.format-version";
+const char Options::FILE_INDEX_FORMAT_VERSION[] = "file-index.format.version";
 const char Options::FILE_INDEX_IN_MANIFEST_THRESHOLD[] = "file-index.in-manifest-threshold";
 const char Options::DATA_FILE_EXTERNAL_PATHS[] = "data-file.external-paths";
 const char Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY[] = "data-file.external-paths.strategy";
diff --git a/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp b/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp
index 4bd1deb9f..f883224b9 100644
--- a/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp
+++ b/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp
@@ -98,8 +98,7 @@ Result<std::shared_ptr<FileIndexReader>> BitmapFileIndex::CreateReader(
     auto arrow_type = arrow_schema->field(0)->type();
     PAIMON_ASSIGN_OR_RAISE(FieldType data_type,
                            FieldTypeUtils::ConvertToFieldType(arrow_type->id()));
-    return std::make_shared<BitmapFileIndexReader>(
-        arrow_type, data_type, 0, static_cast<int32_t>(length), bounded_stream, pool);
+    return std::make_shared<BitmapFileIndexReader>(arrow_type, data_type, bounded_stream, pool);
 }
 
 Result<std::shared_ptr<FileIndexWriter>> BitmapFileIndex::CreateWriter(
@@ -236,16 +235,10 @@ Result<PAIMON_UNIQUE_PTR<Bytes>> BitmapFileIndexWriter::SerializedBytes() const
 }
 
 BitmapFileIndexReader::BitmapFileIndexReader(const std::shared_ptr<arrow::DataType>& arrow_type,
-                                             const FieldType& data_type, int32_t start,
-                                             int32_t length,
+                                             const FieldType& data_type,
                                              const std::shared_ptr<InputStream>& input_stream,
                                              const std::shared_ptr<MemoryPool>& pool)
-    : head_start_(start),
-      length_(length),
-      data_type_(data_type),
-      arrow_type_(arrow_type),
-      pool_(pool),
-      input_stream_(input_stream) {}
+    : data_type_(data_type), arrow_type_(arrow_type), pool_(pool), input_stream_(input_stream) {}
 
 Result<std::shared_ptr<FileIndexResult>> BitmapFileIndexReader::VisitEqual(const Literal& literal) {
     return VisitIn({literal});
@@ -337,16 +330,17 @@ Result<RoaringBitmap32> BitmapFileIndexReader::ReadBitmap(const Literal& literal
 
 Status BitmapFileIndexReader::ReadInternalMeta() {
     if (!bitmap_file_index_meta_) {
-        PAIMON_RETURN_NOT_OK(input_stream_->Seek(head_start_, SeekOrigin::FS_SEEK_SET));
+        PAIMON_RETURN_NOT_OK(input_stream_->Seek(0, SeekOrigin::FS_SEEK_SET));
+        PAIMON_ASSIGN_OR_RAISE(int64_t length, input_stream_->Length());
         DataInputStream data_input_stream(input_stream_);
         PAIMON_ASSIGN_OR_RAISE(int8_t version, data_input_stream.ReadValue<int8_t>());
         FieldType converted_type = BitmapFileIndex::ConvertType(data_type_);
         if (version == BitmapFileIndex::VERSION_1) {
             bitmap_file_index_meta_ = std::make_shared<BitmapFileIndexMetaV1>(
-                converted_type, head_start_, length_, pool_);
+                converted_type, /*start=*/0, static_cast<int32_t>(length), pool_);
         } else if (version == BitmapFileIndex::VERSION_2) {
-            bitmap_file_index_meta_ =
-                std::make_shared<BitmapFileIndexMetaV2>(converted_type, length_, pool_);
+            bitmap_file_index_meta_ = std::make_shared<BitmapFileIndexMetaV2>(
+                converted_type, static_cast<int32_t>(length), pool_);
         } else {
             return Status::Invalid(fmt::format("unknown bitmap file index version {}", version));
         }
diff --git a/src/paimon/common/file_index/bitmap/bitmap_file_index.h b/src/paimon/common/file_index/bitmap/bitmap_file_index.h
index 6c6e41a5e..9f8272f52 100644
--- a/src/paimon/common/file_index/bitmap/bitmap_file_index.h
+++ b/src/paimon/common/file_index/bitmap/bitmap_file_index.h
@@ -101,7 +101,7 @@ class BitmapFileIndexReader : public FileIndexReader,
                               public std::enable_shared_from_this<BitmapFileIndexReader> {
  public:
     BitmapFileIndexReader(const std::shared_ptr<arrow::DataType>& arrow_type,
-                          const FieldType& data_type, int32_t start, int32_t length,
+                          const FieldType& data_type,
                           const std::shared_ptr<InputStream>& input_stream,
                           const std::shared_ptr<MemoryPool>& pool);
 
@@ -123,8 +123,6 @@ class BitmapFileIndexReader : public FileIndexReader,
     Status ReadInternalMeta();
 
  private:
-    int32_t head_start_;
-    int32_t length_;
     FieldType data_type_;
     std::shared_ptr<arrow::DataType> arrow_type_;
     std::shared_ptr<MemoryPool> pool_;
diff --git a/src/paimon/common/file_index/file_index_format.cpp b/src/paimon/common/file_index/file_index_format.cpp
index 4e8af8200..97c3b772e 100644
--- a/src/paimon/common/file_index/file_index_format.cpp
+++ b/src/paimon/common/file_index/file_index_format.cpp
@@ -60,7 +60,7 @@ class FileIndexFormatReaderImpl : public FileIndexFormat::Reader {
         }
         PAIMON_ASSIGN_OR_RAISE(int64_t file_length, input_stream->Length());
         PAIMON_ASSIGN_OR_RAISE(
-            FileIndexEntries header,
+            FileIndexFormatUtils::FileIndexEntries header,
             version == FileIndexFormat::V_1
                 ? FileIndexFormatV1::ReadIndexEntries(input_stream, pool, file_length)
                 : FileIndexFormatV2::ReadIndexEntries(input_stream, pool, file_length));
@@ -96,8 +96,8 @@ class FileIndexFormatReaderImpl : public FileIndexFormat::Reader {
 
  private:
     FileIndexFormatReaderImpl(const std::shared_ptr<InputStream>& input_stream,
-                              FileIndexEntries&& header, const std::shared_ptr<MemoryPool>& pool,
-                              FileIndexOptions&& options)
+                              FileIndexFormatUtils::FileIndexEntries&& header,
+                              const std::shared_ptr<MemoryPool>& pool, FileIndexOptions&& options)
         : input_stream_(input_stream),
           pool_(pool),
           header_(std::move(header)),
@@ -131,8 +131,7 @@ class FileIndexFormatReaderImpl : public FileIndexFormat::Reader {
     std::shared_ptr<InputStream> input_stream_;
     std::shared_ptr<MemoryPool> pool_;
     // get header and cache it.
-    // [column_name : [index_type : {offset, length}]]
-    FileIndexEntries header_;
+    FileIndexFormatUtils::FileIndexEntries header_;
     FileIndexOptions options_;
 };
 
@@ -148,8 +147,8 @@ Result<std::unique_ptr<FileIndexFormat::Reader>> FileIndexFormat::CreateReader(
 }
 
 Result<std::unique_ptr<FileIndexFormat::Writer>> FileIndexFormat::CreateWriter(
-    const std::shared_ptr<OutputStream>& output_stream, const std::shared_ptr<MemoryPool>& pool,
-    int32_t version) {
+    const std::shared_ptr<OutputStream>& output_stream, int32_t version,
+    const std::shared_ptr<MemoryPool>& pool) {
     if (!output_stream) {
         return Status::Invalid("File index output stream cannot be null");
     }
diff --git a/src/paimon/common/file_index/file_index_format_test.cpp b/src/paimon/common/file_index/file_index_format_test.cpp
index 05b5f2db6..fbd82fbb3 100644
--- a/src/paimon/common/file_index/file_index_format_test.cpp
+++ b/src/paimon/common/file_index/file_index_format_test.cpp
@@ -25,6 +25,7 @@
 #include <utility>
 
 #include "arrow/api.h"
+#include "arrow/ipc/json_simple.h"
 #include "gtest/gtest.h"
 #include "paimon/common/file_index/bitmap/bitmap_file_index.h"
 #include "paimon/common/file_index/bloomfilter/bloom_filter_file_index.h"
@@ -34,6 +35,7 @@
 #include "paimon/common/file_index/rangebitmap/range_bitmap_file_index.h"
 #include "paimon/common/io/byte_array_output_stream.h"
 #include "paimon/common/io/data_output_stream.h"
+#include "paimon/common/utils/checked_cast.h"
 #include "paimon/data/timestamp.h"
 #include "paimon/defs.h"
 #include "paimon/file_index/file_index_result.h"
@@ -53,12 +55,12 @@ namespace paimon::test {
 
 class FileIndexFormatTest : public ::testing::Test {
  public:
-    Status WriteIndexes(FileIndexFormat::Writer* writer,
-                        const FileIndexFormat::ColumnIndexes& indexes) const {
+    Status WriteIndexes(const FileIndexFormat::ColumnIndexes& indexes,
+                        FileIndexFormat::Writer* writer) const {
         for (const auto& [column, entries] : indexes) {
             for (const auto& [type, bytes] : entries) {
                 FileIndexFormat::Writer::PayloadWriter payload =
-                    [bytes = bytes](const std::shared_ptr<OutputStream>& output) -> Result<bool> {
+                    [bytes = bytes](OutputStream* output) -> Result<bool> {
                     if (!bytes || bytes->size() == 0) {
                         return false;
                     }
@@ -68,7 +70,7 @@ class FileIndexFormatTest : public ::testing::Test {
                 PAIMON_RETURN_NOT_OK(writer->WriteIndex(column, type, payload));
             }
         }
-        return writer->Finish();
+        return Status::OK();
     }
 
     void SetUp() override {
@@ -100,8 +102,10 @@ TEST_F(FileIndexFormatTest, TestWriteAndReadEmptyIndexGoldenBytes) {
         MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_);
     auto output = std::make_shared<ByteArrayOutputStream>(std::move(segment_output));
 
-    ASSERT_OK_AND_ASSIGN(auto writer, FileIndexFormat::CreateWriter(output, pool_));
-    ASSERT_OK(WriteIndexes(writer.get(), indexes));
+    ASSERT_OK_AND_ASSIGN(auto writer,
+                         FileIndexFormat::CreateWriter(output, FileIndexFormat::V_1, pool_));
+    ASSERT_OK(WriteIndexes(indexes, writer.get()));
+    ASSERT_OK(writer->Finish());
     ASSERT_OK(writer->Close());
     ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> actual, output->Finish(pool_.get()));
 
@@ -123,8 +127,10 @@ TEST_F(FileIndexFormatTest, TestEmptyVectorIndexCanBeSearched) {
     auto segment_output = std::make_unique<MemorySegmentOutputStream>(
         MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_);
     auto output = std::make_shared<ByteArrayOutputStream>(std::move(segment_output));
-    ASSERT_OK_AND_ASSIGN(auto writer, FileIndexFormat::CreateWriter(output, pool_));
-    ASSERT_OK(WriteIndexes(writer.get(), indexes));
+    ASSERT_OK_AND_ASSIGN(auto writer,
+                         FileIndexFormat::CreateWriter(output, FileIndexFormat::V_1, pool_));
+    ASSERT_OK(WriteIndexes(indexes, writer.get()));
+    ASSERT_OK(writer->Finish());
     ASSERT_OK(writer->Close());
     ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> bytes, output->Finish(pool_.get()));
 
@@ -991,8 +997,9 @@ TEST_F(FileIndexFormatTest, TestV2JavaGoldenBytes) {
     indexes["c1"]["unknown"] = std::make_shared<Bytes>(std::string("*+"), pool_.get());
     auto output = NewIndexOutput(pool_);
     ASSERT_OK_AND_ASSIGN(auto writer,
-                         FileIndexFormat::CreateWriter(output, pool_, FileIndexFormat::V_2));
-    ASSERT_OK(WriteIndexes(writer.get(), indexes));
+                         FileIndexFormat::CreateWriter(output, FileIndexFormat::V_2, pool_));
+    ASSERT_OK(WriteIndexes(indexes, writer.get()));
+    ASSERT_OK(writer->Finish());
     ASSERT_OK(writer->Close());
     ASSERT_OK_AND_ASSIGN(auto bytes, output->Finish(pool_.get()));
     ASSERT_EQ(kJavaV2Golden, std::vector<uint8_t>(bytes->data(), bytes->data() + bytes->size()));
@@ -1034,22 +1041,21 @@ TEST_F(FileIndexFormatTest, TestV2StreamingLargeContainer) {
     auto state = std::make_shared<SparseIndexState>();
     auto output = std::make_shared<SparseIndexOutput>(state);
     ASSERT_OK_AND_ASSIGN(auto writer,
-                         FileIndexFormat::CreateWriter(output, pool_, FileIndexFormat::V_2));
+                         FileIndexFormat::CreateWriter(output, FileIndexFormat::V_2, pool_));
     // Stream a virtual >2 GiB payload in bounded chunks; the mock discards only this gap.
     const int64_t gap_length = int64_t{1} << 31;
-    ASSERT_OK(writer->WriteIndex(
-        "gap", "unknown", [state](const std::shared_ptr<OutputStream>& stream) -> Result<bool> {
-            state->discard = true;
-            std::array<char, 65536> zeros{};
-            for (int64_t i = 0; i < gap_length; i += zeros.size()) {
-                PAIMON_ASSIGN_OR_RAISE(int64_t written, stream->Write(zeros.data(), zeros.size()));
-                if (written != static_cast<int64_t>(zeros.size())) {
-                    return Status::IOError("short write");
-                }
+    ASSERT_OK(writer->WriteIndex("gap", "unknown", [state](OutputStream* stream) -> Result<bool> {
+        state->discard = true;
+        std::array<char, 65536> zeros{};
+        for (int64_t i = 0; i < gap_length; i += zeros.size()) {
+            PAIMON_ASSIGN_OR_RAISE(int64_t written, stream->Write(zeros.data(), zeros.size()));
+            if (written != static_cast<int64_t>(zeros.size())) {
+                return Status::IOError("short write");
             }
-            state->discard = false;
-            return true;
-        }));
+        }
+        state->discard = false;
+        return true;
+    }));
     auto schema = arrow::schema({arrow::field("f0", arrow::int32())});
     std::vector<std::pair<std::string, std::shared_ptr<FileIndexer>>> indexers = {
         {"bitmap", std::make_shared<BitmapFileIndex>(std::map<std::string, std::string>{})},
@@ -1059,30 +1065,20 @@ TEST_F(FileIndexFormatTest, TestV2StreamingLargeContainer) {
          std::make_shared<BloomFilterFileIndex>(std::map<std::string, std::string>{})},
         {"range-bitmap",
          std::make_shared<RangeBitmapFileIndex>(std::map<std::string, std::string>{})}};
-    arrow::Int32Builder builder;
-    ASSERT_TRUE(builder.AppendValues({1, 2, 1}).ok());
-    auto values_result = builder.Finish();
-    ASSERT_TRUE(values_result.ok());
-    std::shared_ptr<arrow::Array> values = values_result.ValueOrDie();
-    auto batch_result = arrow::StructArray::Make({values}, schema->fields());
-    ASSERT_TRUE(batch_result.ok());
-    auto batch = batch_result.ValueOrDie();
+    std::shared_ptr<arrow::Array> array =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()),
+                                                  R"([[1], [2], [1]])")
+            .ValueOrDie();
+    auto batch = checked_pointer_cast<arrow::StructArray>(array);
     for (const auto& [type, indexer] : indexers) {
         ASSERT_OK_AND_ASSIGN(auto index_writer,
                              indexer->CreateWriter(CreateArrowSchema(schema).get(), pool_));
         ArrowArray c_batch;
         ASSERT_TRUE(arrow::ExportArray(*batch, &c_batch).ok());
         ASSERT_OK(index_writer->AddBatch(&c_batch));
-        ASSERT_OK_AND_ASSIGN(auto payload, index_writer->SerializedBytes());
-        ASSERT_OK(writer->WriteIndex(
-            type, type, [&payload](const std::shared_ptr<OutputStream>& stream) -> Result<bool> {
-                PAIMON_ASSIGN_OR_RAISE(int64_t written,
-                                       stream->Write(payload->data(), payload->size()));
-                if (written != static_cast<int64_t>(payload->size())) {
-                    return Status::IOError("short write");
-                }
-                return true;
-            }));
+        ASSERT_OK(writer->WriteIndex(type, type, [index_writer](OutputStream* stream) {
+            return index_writer->WriteTo(stream);
+        }));
     }
     ASSERT_OK(writer->Finish());
     ASSERT_OK(writer->Close());
@@ -1127,7 +1123,7 @@ TEST_F(FileIndexFormatTest, TestPluginRangeLimits) {
 TEST_F(FileIndexFormatTest, TestV2EmptyContainer) {
     auto output = NewIndexOutput(pool_);
     ASSERT_OK_AND_ASSIGN(auto writer,
-                         FileIndexFormat::CreateWriter(output, pool_, FileIndexFormat::V_2));
+                         FileIndexFormat::CreateWriter(output, FileIndexFormat::V_2, pool_));
     ASSERT_OK_AND_ASSIGN(int64_t prefix_length, output->GetPos());
     ASSERT_EQ(12, prefix_length);
     ASSERT_OK(writer->Finish());
@@ -1142,20 +1138,9 @@ TEST_F(FileIndexFormatTest, TestV1BuffersPayloadsUntilFinish) {
     FileIndexFormat::ColumnIndexes indexes = {
         {"f0", {{"bitmap", std::make_shared<Bytes>("payload", pool_.get())}, {"empty", nullptr}}}};
     auto output = NewIndexOutput(pool_);
-    ASSERT_OK_AND_ASSIGN(auto writer, FileIndexFormat::CreateWriter(output, pool_));
-    for (const auto& [column, entries] : indexes) {
-        for (const auto& [type, bytes] : entries) {
-            FileIndexFormat::Writer::PayloadWriter payload;
-            if (bytes) {
-                payload = [bytes = bytes](
-                              const std::shared_ptr<OutputStream>& destination) -> Result<bool> {
-                    PAIMON_RETURN_NOT_OK(DataOutputStream(destination).WriteBytes(bytes));
-                    return true;
-                };
-            }
-            ASSERT_OK(writer->WriteIndex(column, type, payload));
-        }
-    }
+    ASSERT_OK_AND_ASSIGN(auto writer,
+                         FileIndexFormat::CreateWriter(output, FileIndexFormat::V_1, pool_));
+    ASSERT_OK(WriteIndexes(indexes, writer.get()));
     ASSERT_OK_AND_ASSIGN(int64_t before_finish, output->GetPos());
     ASSERT_EQ(0, before_finish);
     ASSERT_OK(writer->Finish());
@@ -1165,7 +1150,7 @@ TEST_F(FileIndexFormatTest, TestV1BuffersPayloadsUntilFinish) {
     DataInputStream in(input);
     ASSERT_OK(in.Seek(12));
     ASSERT_OK_AND_ASSIGN(int32_t head_length, in.ReadValue<int32_t>());
-    ASSERT_OK_AND_ASSIGN(FileIndexEntries entries,
+    ASSERT_OK_AND_ASSIGN(FileIndexFormatUtils::FileIndexEntries entries,
                          FileIndexFormatUtils::ReadFileIndexEntries(in, FileIndexFormat::V_1));
     ASSERT_EQ(std::make_pair(int64_t{head_length}, int64_t{7}), entries.at("f0").at("bitmap"));
     ASSERT_EQ(std::make_pair(int64_t{-1}, int64_t{0}), entries.at("f0").at("empty"));
@@ -1176,17 +1161,17 @@ TEST_F(FileIndexFormatTest, TestAbsentPayloadMustNotWriteBytes) {
     for (int32_t version : {FileIndexFormat::V_1, FileIndexFormat::V_2}) {
         SCOPED_TRACE(version);
         ASSERT_OK_AND_ASSIGN(auto writer,
-                             FileIndexFormat::CreateWriter(NewIndexOutput(pool_), pool_, version));
-        ASSERT_NOK_WITH_MSG(
-            writer->WriteIndex("f0", "bad",
-                               [](const std::shared_ptr<OutputStream>& output) -> Result<bool> {
-                                   PAIMON_ASSIGN_OR_RAISE(int64_t written, output->Write("x", 1));
-                                   if (written != 1) {
-                                       return Status::IOError("short write");
-                                   }
-                                   return false;
-                               }),
-            "Absent file index payload must not write bytes");
+                             FileIndexFormat::CreateWriter(NewIndexOutput(pool_), version, pool_));
+        ASSERT_NOK_WITH_MSG(writer->WriteIndex("f0", "bad",
+                                               [](OutputStream* output) -> Result<bool> {
+                                                   PAIMON_ASSIGN_OR_RAISE(int64_t written,
+                                                                          output->Write("x", 1));
+                                                   if (written != 1) {
+                                                       return Status::IOError("short write");
+                                                   }
+                                                   return false;
+                                               }),
+                            "Absent file index payload must not write bytes");
         ASSERT_NOK(writer->Finish());
         ASSERT_OK(writer->Close());
     }
diff --git a/src/paimon/common/file_index/file_index_format_utils.cpp b/src/paimon/common/file_index/file_index_format_utils.cpp
index 892fa3a69..c1565d698 100644
--- a/src/paimon/common/file_index/file_index_format_utils.cpp
+++ b/src/paimon/common/file_index/file_index_format_utils.cpp
@@ -24,17 +24,17 @@
 #include "paimon/io/data_input_stream.h"
 
 namespace paimon {
-Result<std::shared_ptr<InputStream>> FileIndexFormatUtils::CreateFileIndexDirectory(
-    const std::shared_ptr<InputStream>& input, const std::shared_ptr<MemoryPool>& pool,
-    int64_t start, int64_t length, int64_t file_length) {
+Result<std::shared_ptr<InputStream>> FileIndexFormatUtils::CreateMetadataInputStream(
+    const std::shared_ptr<InputStream>& input, int64_t start, int64_t length, int64_t file_length,
+    const std::shared_ptr<MemoryPool>& pool) {
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OffsetInputStream> bounded,
                            OffsetInputStream::Create(input, length, start, file_length));
     return std::make_shared<BufferedInputStream>(bounded, BufferedInputStream::DEFAULT_BUFFER_SIZE,
                                                  pool.get());
 }
 
-Result<FileIndexEntries> FileIndexFormatUtils::ReadFileIndexEntries(const DataInputStream& in,
-                                                                    int32_t version) {
+Result<FileIndexFormatUtils::FileIndexEntries> FileIndexFormatUtils::ReadFileIndexEntries(
+    const DataInputStream& in, int32_t version) {
     PAIMON_ASSIGN_OR_RAISE(int32_t column_size, in.ReadValue<int32_t>());
     FileIndexEntries header;
     for (int32_t i = 0; i < column_size; ++i) {
diff --git a/src/paimon/common/file_index/file_index_format_utils.h b/src/paimon/common/file_index/file_index_format_utils.h
index 4d9c2cbf6..5ab3b6c78 100644
--- a/src/paimon/common/file_index/file_index_format_utils.h
+++ b/src/paimon/common/file_index/file_index_format_utils.h
@@ -19,9 +19,9 @@
 #pragma once
 
 #include <cstdint>
+#include <map>
 #include <memory>
 #include <string>
-#include <unordered_map>
 #include <utility>
 
 #include "paimon/result.h"
@@ -31,18 +31,20 @@ class DataInputStream;
 class InputStream;
 class MemoryPool;
 
-using FileIndexEntries =
-    std::unordered_map<std::string, std::unordered_map<std::string, std::pair<int64_t, int64_t>>>;
-
 class FileIndexFormatUtils {
  public:
+    /// Column name -> index type -> (payload offset, payload length), ordered by name.
+    /// Empty indexes use (-1, 0).
+    using FileIndexEntries =
+        std::map<std::string, std::map<std::string, std::pair<int64_t, int64_t>>>;
+
     FileIndexFormatUtils() = delete;
     ~FileIndexFormatUtils() = delete;
 
-    /// Creates a bounded, buffered view of the container directory.
-    static Result<std::shared_ptr<InputStream>> CreateFileIndexDirectory(
-        const std::shared_ptr<InputStream>& input, const std::shared_ptr<MemoryPool>& pool,
-        int64_t start, int64_t length, int64_t file_length);
+    /// Creates a bounded, buffered input stream for the container's index metadata.
+    static Result<std::shared_ptr<InputStream>> CreateMetadataInputStream(
+        const std::shared_ptr<InputStream>& input, int64_t start, int64_t length,
+        int64_t file_length, const std::shared_ptr<MemoryPool>& pool);
 
     /// Reads shared column/type entries using the offsets and lengths of the given version.
     static Result<FileIndexEntries> ReadFileIndexEntries(const DataInputStream& in,
diff --git a/src/paimon/common/file_index/file_index_format_v1.cpp b/src/paimon/common/file_index/file_index_format_v1.cpp
index 55554cc43..25ce9be5d 100644
--- a/src/paimon/common/file_index/file_index_format_v1.cpp
+++ b/src/paimon/common/file_index/file_index_format_v1.cpp
@@ -59,7 +59,7 @@ Status FileIndexFormatV1::Writer::WriteIndex(const std::string& column, const st
             std::make_shared<ByteArrayOutputStream>(std::make_unique<MemorySegmentOutputStream>(
                 MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_));
         failed_ = true;
-        PAIMON_ASSIGN_OR_RAISE(bool written, payload(output));
+        PAIMON_ASSIGN_OR_RAISE(bool written, payload(output.get()));
         PAIMON_ASSIGN_OR_RAISE(int64_t length, output->GetPos());
         if (!written && length != 0) {
             return Status::Invalid("Absent file index payload must not write bytes");
@@ -180,17 +180,17 @@ Status FileIndexFormatV1::Writer::WriteHead(const FileIndexFormat::ColumnIndexes
     return data_output.WriteValue<int32_t>(kRedundantLength);
 }
 
-Result<FileIndexEntries> FileIndexFormatV1::ReadIndexEntries(
+Result<FileIndexFormatUtils::FileIndexEntries> FileIndexFormatV1::ReadIndexEntries(
     const std::shared_ptr<InputStream>& input, const std::shared_ptr<MemoryPool>& pool,
     int64_t file_length) {
     DataInputStream data_input(input);
     PAIMON_ASSIGN_OR_RAISE(int32_t head_length, data_input.ReadValue<int32_t>());
-    constexpr int64_t kDirectoryStart = 16;
-    int64_t directory_length = head_length - kDirectoryStart;
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> directory,
-                           FileIndexFormatUtils::CreateFileIndexDirectory(
-                               input, pool, kDirectoryStart, directory_length, file_length));
-    DataInputStream in(directory);
+    constexpr int64_t kMetadataStart = 16;
+    int64_t metadata_length = head_length - kMetadataStart;
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> metadata_input,
+                           FileIndexFormatUtils::CreateMetadataInputStream(
+                               input, kMetadataStart, metadata_length, file_length, pool));
+    DataInputStream in(metadata_input);
     return FileIndexFormatUtils::ReadFileIndexEntries(in, FileIndexFormat::V_1);
 }
 }  // namespace paimon
diff --git a/src/paimon/common/file_index/file_index_format_v1.h b/src/paimon/common/file_index/file_index_format_v1.h
index ef1c40dcb..96af5f68c 100644
--- a/src/paimon/common/file_index/file_index_format_v1.h
+++ b/src/paimon/common/file_index/file_index_format_v1.h
@@ -88,10 +88,10 @@ class FileIndexFormatV1 {
 
     class Writer;
 
-    /// Reads the directory after the magic/version prefix has been consumed.
-    static Result<FileIndexEntries> ReadIndexEntries(const std::shared_ptr<InputStream>& input,
-                                                     const std::shared_ptr<MemoryPool>& pool,
-                                                     int64_t file_length);
+    /// Reads index metadata from the header after the magic/version prefix has been consumed.
+    static Result<FileIndexFormatUtils::FileIndexEntries> ReadIndexEntries(
+        const std::shared_ptr<InputStream>& input, const std::shared_ptr<MemoryPool>& pool,
+        int64_t file_length);
 };
 
 class FileIndexFormatV1::Writer : public FileIndexFormat::Writer {
diff --git a/src/paimon/common/file_index/file_index_format_v2.cpp b/src/paimon/common/file_index/file_index_format_v2.cpp
index cdfc2358e..eb3a9fa29 100644
--- a/src/paimon/common/file_index/file_index_format_v2.cpp
+++ b/src/paimon/common/file_index/file_index_format_v2.cpp
@@ -62,7 +62,7 @@ Status FileIndexFormatV2::Writer::WriteIndex(const std::string& column, const st
     int64_t length = 0;
     if (payload) {
         PAIMON_ASSIGN_OR_RAISE(start, output_->GetPos());
-        PAIMON_ASSIGN_OR_RAISE(bool written, payload(output_));
+        PAIMON_ASSIGN_OR_RAISE(bool written, payload(output_.get()));
         PAIMON_ASSIGN_OR_RAISE(int64_t end, output_->GetPos());
         length = end - start;
         if (!written) {
@@ -82,7 +82,7 @@ Status FileIndexFormatV2::Writer::Finish() {
         return Status::Invalid("File index writer is not writable");
     }
     failed_ = true;
-    // Validate the directory size before emitting it; the Java trailer uses int32.
+    // Validate the footer size before emitting it; the Java trailer uses int32.
     int64_t footer_length = 4;
     for (const auto& [column, entries] : entries_) {
         footer_length += 2 + static_cast<int64_t>(column.size()) + 4;
@@ -120,7 +120,7 @@ Status FileIndexFormatV2::Writer::Close() {
     return close_status;
 }
 
-Result<FileIndexEntries> FileIndexFormatV2::ReadIndexEntries(
+Result<FileIndexFormatUtils::FileIndexEntries> FileIndexFormatV2::ReadIndexEntries(
     const std::shared_ptr<InputStream>& input, const std::shared_ptr<MemoryPool>& pool,
     int64_t file_length) {
     if (file_length < kV2PrefixLength + 4 + kV2TrailerLength) {
@@ -137,10 +137,10 @@ Result<FileIndexEntries> FileIndexFormatV2::ReadIndexEntries(
         return Status::Invalid("Invalid file index footer length");
     }
     int64_t footer_start = file_length - kV2TrailerLength - footer_length;
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> directory,
-                           FileIndexFormatUtils::CreateFileIndexDirectory(
-                               input, pool, footer_start, footer_length, file_length));
-    DataInputStream in(directory);
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> metadata_input,
+                           FileIndexFormatUtils::CreateMetadataInputStream(
+                               input, footer_start, footer_length, file_length, pool));
+    DataInputStream in(metadata_input);
     return FileIndexFormatUtils::ReadFileIndexEntries(in, FileIndexFormat::V_2);
 }
 }  // namespace paimon
diff --git a/src/paimon/common/file_index/file_index_format_v2.h b/src/paimon/common/file_index/file_index_format_v2.h
index 71504c8a9..11e621839 100644
--- a/src/paimon/common/file_index/file_index_format_v2.h
+++ b/src/paimon/common/file_index/file_index_format_v2.h
@@ -19,7 +19,6 @@
 #pragma once
 
 #include <cstdint>
-#include <map>
 #include <memory>
 #include <string>
 #include <utility>
@@ -84,10 +83,10 @@ class FileIndexFormatV2 {
 
     class Writer;
 
-    /// Reads the directory after the magic/version prefix has been consumed.
-    static Result<FileIndexEntries> ReadIndexEntries(const std::shared_ptr<InputStream>& input,
-                                                     const std::shared_ptr<MemoryPool>& pool,
-                                                     int64_t file_length);
+    /// Reads index metadata from the footer after the magic/version prefix has been consumed.
+    static Result<FileIndexFormatUtils::FileIndexEntries> ReadIndexEntries(
+        const std::shared_ptr<InputStream>& input, const std::shared_ptr<MemoryPool>& pool,
+        int64_t file_length);
 };
 
 class FileIndexFormatV2::Writer : public FileIndexFormat::Writer {
@@ -102,7 +101,7 @@ class FileIndexFormatV2::Writer : public FileIndexFormat::Writer {
     explicit Writer(const std::shared_ptr<OutputStream>& output);
 
     std::shared_ptr<OutputStream> output_;
-    std::map<std::string, std::map<std::string, std::pair<int64_t, int64_t>>> entries_;
+    FileIndexFormatUtils::FileIndexEntries entries_;
     bool failed_ = false;
     bool finished_ = false;
     bool closed_ = false;
diff --git a/src/paimon/common/file_index/file_index_writer.cpp b/src/paimon/common/file_index/file_index_writer.cpp
index 7a8e2916e..f4525c49f 100644
--- a/src/paimon/common/file_index/file_index_writer.cpp
+++ b/src/paimon/common/file_index/file_index_writer.cpp
@@ -26,7 +26,7 @@ Result<PAIMON_UNIQUE_PTR<Bytes>> FileIndexWriter::SerializedBytes() const {
     return Status::NotImplemented("This file index writer requires streaming output");
 }
 
-Result<bool> FileIndexWriter::WriteTo(const std::shared_ptr<OutputStream>& output) const {
+Result<bool> FileIndexWriter::WriteTo(OutputStream* output) const {
     if (!output) {
         return Status::Invalid("File index output stream cannot be null");
     }
diff --git a/src/paimon/common/file_index/file_index_writer_test.cpp b/src/paimon/common/file_index/file_index_writer_test.cpp
index 8988b4c20..55a7281bc 100644
--- a/src/paimon/common/file_index/file_index_writer_test.cpp
+++ b/src/paimon/common/file_index/file_index_writer_test.cpp
@@ -36,10 +36,6 @@ class RecordingIndexOutput : public MockOutputStream {
         data.append(buffer, size);
         return size;
     }
-    Status Flush() override {
-        ++flush_count;
-        return Status::OK();
-    }
     Status Close() override {
         ++close_count;
         return Status::OK();
@@ -47,7 +43,6 @@ class RecordingIndexOutput : public MockOutputStream {
 
     std::string data;
     bool short_write = false;
-    int32_t flush_count = 0;
     int32_t close_count = 0;
 };
 
@@ -74,10 +69,9 @@ class LegacyIndexWriter : public FileIndexWriter {
 TEST(FileIndexWriterTest, TestLegacyDefaultWriteTo) {
     LegacyIndexWriter writer;
     auto output = std::make_shared<RecordingIndexOutput>();
-    ASSERT_OK_AND_ASSIGN(bool written, writer.WriteTo(output));
+    ASSERT_OK_AND_ASSIGN(bool written, writer.WriteTo(output.get()));
     ASSERT_TRUE(written);
     ASSERT_EQ("legacy payload", output->data);
-    ASSERT_EQ(0, output->flush_count);
     ASSERT_EQ(0, output->close_count);
     ASSERT_NOK_WITH_MSG(writer.WriteTo(nullptr), "cannot be null");
 }
@@ -86,19 +80,19 @@ TEST(FileIndexWriterTest, TestLegacyWriteToRejectsShortWrite) {
     LegacyIndexWriter writer;
     auto output = std::make_shared<RecordingIndexOutput>();
     output->short_write = true;
-    ASSERT_NOK_WITH_MSG(writer.WriteTo(output), "Short write");
+    ASSERT_NOK_WITH_MSG(writer.WriteTo(output.get()), "Short write");
 }
 
 TEST(FileIndexWriterTest, TestNullAndZeroLengthPayloadsProduceNoIndex) {
     LegacyIndexWriter writer;
     auto output = std::make_shared<RecordingIndexOutput>();
     writer.absent = true;
-    ASSERT_OK_AND_ASSIGN(bool written, writer.WriteTo(output));
+    ASSERT_OK_AND_ASSIGN(bool written, writer.WriteTo(output.get()));
     ASSERT_FALSE(written);
     ASSERT_TRUE(output->data.empty());
     writer.absent = false;
     writer.zero_length = true;
-    ASSERT_OK_AND_ASSIGN(written, writer.WriteTo(output));
+    ASSERT_OK_AND_ASSIGN(written, writer.WriteTo(output.get()));
     ASSERT_FALSE(written);
     ASSERT_TRUE(output->data.empty());
 }
diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp
index ee71f42cb..ec52eac14 100644
--- a/src/paimon/core/core_options.cpp
+++ b/src/paimon/core/core_options.cpp
@@ -29,6 +29,7 @@
 #include "paimon/common/fs/resolving_file_system.h"
 #include "paimon/common/options/memory_size.h"
 #include "paimon/common/options/time_duration.h"
+#include "paimon/common/utils/math.h"
 #include "paimon/common/utils/options_utils.h"
 #include "paimon/common/utils/path_util.h"
 #include "paimon/common/utils/string_utils.h"
@@ -873,11 +874,13 @@ struct CoreOptions::Impl {
         PAIMON_RETURN_NOT_OK(
             parser.Parse<int32_t>(Options::FILE_INDEX_FORMAT_VERSION, &file_index_format_version));
         if (file_index_format_version != 1 && file_index_format_version != 2) {
-            return Status::Invalid("file-index.format-version must be 1 or 2");
+            return Status::Invalid("file-index.format.version must be 1 or 2");
         }
         // Parse file-index.in-manifest-threshold - max inline file index size, default 500B
         PAIMON_RETURN_NOT_OK(parser.ParseMemorySize(Options::FILE_INDEX_IN_MANIFEST_THRESHOLD,
                                                     &file_index_in_manifest_threshold));
+        PAIMON_RETURN_NOT_OK(ValidateValueInRange<int32_t>(file_index_in_manifest_threshold,
+                                                           "file index inline threshold"));
         // Parse file-index.read.enabled - whether to enable reading file index, default true
         PAIMON_RETURN_NOT_OK(
             parser.Parse<bool>(Options::FILE_INDEX_READ_ENABLED, &file_index_read_enabled));
diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp
index 0d8ab2960..c533de71a 100644
--- a/src/paimon/core/core_options_test.cpp
+++ b/src/paimon/core/core_options_test.cpp
@@ -159,6 +159,7 @@ TEST(CoreOptionsTest, TestDefaultValue) {
     ASSERT_EQ("main", core_options.GetBranch());
     ASSERT_TRUE(core_options.FileIndexReadEnabled());
     ASSERT_EQ(500, core_options.FileIndexInManifestThreshold());
+    ASSERT_EQ(1, core_options.FileIndexFormatVersion());
     ASSERT_EQ(std::nullopt, core_options.GetDataFileExternalPaths());
     ASSERT_EQ(ExternalPathStrategy::NONE, core_options.GetExternalPathStrategy());
     ASSERT_TRUE(core_options.EnableAdaptivePrefetchStrategy());
@@ -283,6 +284,7 @@ TEST(CoreOptionsTest, TestFromMap) {
         {Options::BRANCH, "rt"},
         {Options::FILE_INDEX_READ_ENABLED, "false"},
         {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "2KB"},
+        {Options::FILE_INDEX_FORMAT_VERSION, "2"},
         {Options::DATA_FILE_EXTERNAL_PATHS, "FILE:///tmp/index"},
         {Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY, "round-robin"},
         {Options::FILE_COMPRESSION, "snappy"},
@@ -445,6 +447,7 @@ TEST(CoreOptionsTest, TestFromMap) {
     ASSERT_EQ(core_options.GetBranch(), "rt");
     ASSERT_FALSE(core_options.FileIndexReadEnabled());
     ASSERT_EQ(2 * 1024, core_options.FileIndexInManifestThreshold());
+    ASSERT_EQ(2, core_options.FileIndexFormatVersion());
     ASSERT_EQ(core_options.GetDataFileExternalPaths(),
               std::optional<std::string>("FILE:///tmp/index"));
     ASSERT_EQ(core_options.GetExternalPathStrategy(), ExternalPathStrategy::ROUND_ROBIN);
@@ -523,6 +526,13 @@ TEST(CoreOptionsTest, TestFromMap) {
 }
 
 TEST(CoreOptionsTest, TestInvalidCase) {
+    ASSERT_NOK_WITH_MSG(
+        CoreOptions::FromMap({{Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "2147483648B"}}),
+        "file index inline threshold");
+    for (const std::string version : {"0", "3", "-1", "invalid"}) {
+        ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"file-index.format.version", version}}),
+                            "file-index.format.version must be 1 or 2");
+    }
     ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{Options::TARGET_FILE_ROW_NUM, "0"}}),
                         "target-file-row-num should be at least 1");
     ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{Options::BUCKET, "3.5"}}),
diff --git a/src/paimon/core/io/data_file_index_writer.cpp b/src/paimon/core/io/data_file_index_writer.cpp
index 8af0ad6b2..5faecf6c4 100644
--- a/src/paimon/core/io/data_file_index_writer.cpp
+++ b/src/paimon/core/io/data_file_index_writer.cpp
@@ -31,7 +31,6 @@
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
-#include "paimon/common/utils/math.h"
 #include "paimon/common/utils/path_util.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/casting/casting_utils.h"
@@ -156,17 +155,15 @@ Result<FileIndexWriteResult> DataFileIndexWriter::Finish(const std::string& data
         return Status::Invalid("Data file index writer has already finished");
     }
     finished_ = true;
-    PAIMON_RETURN_NOT_OK(
-        ValidateValueInRange<int32_t>(in_manifest_threshold_, "file index inline threshold"));
     std::string index_path = path_factory_->ToFileIndexPath(data_file_path);
     auto output = std::make_shared<SpillableIndexOutputStream>(file_system_, index_path,
                                                                in_manifest_threshold_, pool_);
     ScopeGuard guard([&output]() { [[maybe_unused]] Status ignored = output->Abort(); });
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileIndexFormat::Writer> writer,
-                           FileIndexFormat::CreateWriter(output, pool_, format_version_));
+                           FileIndexFormat::CreateWriter(output, format_version_, pool_));
     for (const IndexWriterEntry& entry : writers_) {
         FileIndexFormat::Writer::PayloadWriter payload =
-            [index_writer = entry.writer](const std::shared_ptr<OutputStream>& destination) {
+            [index_writer = entry.writer](OutputStream* destination) {
                 return index_writer->WriteTo(destination);
             };
         PAIMON_RETURN_NOT_OK(writer->WriteIndex(entry.column_name, entry.index_type, payload));
diff --git a/src/paimon/core/io/data_file_index_writer_test.cpp b/src/paimon/core/io/data_file_index_writer_test.cpp
index cc2f0ba7d..042e94b83 100644
--- a/src/paimon/core/io/data_file_index_writer_test.cpp
+++ b/src/paimon/core/io/data_file_index_writer_test.cpp
@@ -65,7 +65,7 @@ class StreamingTestIndexWriter : public FileIndexWriter {
         batch->release(batch);
         return Status::OK();
     }
-    Result<bool> WriteTo(const std::shared_ptr<OutputStream>& output) const override {
+    Result<bool> WriteTo(OutputStream* output) const override {
         if (empty_) {
             return false;
         }
diff --git a/src/paimon/core/io/file_index_options.cpp b/src/paimon/core/io/file_index_options.cpp
index 568ea97c9..3acf85a53 100644
--- a/src/paimon/core/io/file_index_options.cpp
+++ b/src/paimon/core/io/file_index_options.cpp
@@ -24,8 +24,6 @@
 #include <utility>
 
 #include "fmt/format.h"
-#include "paimon/common/options/memory_size.h"
-#include "paimon/common/utils/options_utils.h"
 #include "paimon/common/utils/string_utils.h"
 #include "paimon/core/core_options.h"
 #include "paimon/defs.h"
@@ -41,34 +39,17 @@ constexpr size_t kColumnsSuffixLength = sizeof(kColumnsSuffix) - 1;
 
 }  // namespace
 
-Result<FileIndexOptions> FileIndexOptions::FromCoreOptions(const CoreOptions& options) {
-    return Parse(options.ToMap(), options.FileIndexInManifestThreshold(),
-                 options.FileIndexFormatVersion());
-}
-
 Result<FileIndexOptions> FileIndexOptions::FromMap(
     const std::map<std::string, std::string>& raw_options) {
-    int64_t in_manifest_threshold = CoreOptions().FileIndexInManifestThreshold();
-    auto iter = raw_options.find(Options::FILE_INDEX_IN_MANIFEST_THRESHOLD);
-    if (iter != raw_options.end()) {
-        PAIMON_ASSIGN_OR_RAISE(in_manifest_threshold, MemorySize::ParseBytes(iter->second));
-    }
-    PAIMON_ASSIGN_OR_RAISE(
-        int32_t format_version,
-        OptionsUtils::GetValueFromMap<int32_t>(raw_options, Options::FILE_INDEX_FORMAT_VERSION,
-                                               CoreOptions().FileIndexFormatVersion()));
-    if (format_version != 1 && format_version != 2) {
-        return Status::Invalid("file-index.format-version must be 1 or 2");
-    }
-    return Parse(raw_options, in_manifest_threshold, format_version);
+    PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(raw_options));
+    return FromCoreOptions(options);
 }
 
-Result<FileIndexOptions> FileIndexOptions::Parse(
-    const std::map<std::string, std::string>& raw_options, int64_t in_manifest_threshold,
-    int32_t format_version) {
+Result<FileIndexOptions> FileIndexOptions::FromCoreOptions(const CoreOptions& options) {
+    const std::map<std::string, std::string>& raw_options = options.ToMap();
     FileIndexOptions result;
-    result.in_manifest_threshold_ = in_manifest_threshold;
-    result.format_version_ = format_version;
+    result.in_manifest_threshold_ = options.FileIndexInManifestThreshold();
+    result.format_version_ = options.FileIndexFormatVersion();
 
     std::set<std::pair<std::string, std::string>> declared;
     for (const auto& [key, value] : raw_options) {
diff --git a/src/paimon/core/io/file_index_options.h b/src/paimon/core/io/file_index_options.h
index 85c760235..aabd06c8d 100644
--- a/src/paimon/core/io/file_index_options.h
+++ b/src/paimon/core/io/file_index_options.h
@@ -71,9 +71,6 @@ class FileIndexOptions {
  private:
     FileIndexOptions() = default;
 
-    static Result<FileIndexOptions> Parse(const std::map<std::string, std::string>& raw_options,
-                                          int64_t in_manifest_threshold, int32_t format_version);
-
     std::vector<FileIndexDefinition> definitions_;
     int64_t in_manifest_threshold_ = 0;
     int32_t format_version_ = 1;
diff --git a/src/paimon/core/io/file_index_options_test.cpp b/src/paimon/core/io/file_index_options_test.cpp
index 28cd27ab8..3fa188c05 100644
--- a/src/paimon/core/io/file_index_options_test.cpp
+++ b/src/paimon/core/io/file_index_options_test.cpp
@@ -68,22 +68,15 @@ TEST(FileIndexOptionsTest, TestPreserveDottedBackendOptionName) {
     EXPECT_TRUE(options.GetIndexerOptions("missing", "lumina").empty());
 }
 
-TEST(FileIndexOptionsTest, TestFromMapOnlyParsesFileIndexOptions) {
+TEST(FileIndexOptionsTest, TestFromMap) {
     ASSERT_OK_AND_ASSIGN(
         FileIndexOptions options,
-        FileIndexOptions::FromMap({{Options::FILE_FORMAT, "missing-format"},
-                                   {"manifest.format", "missing-format"},
-                                   {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1KB"},
+        FileIndexOptions::FromMap({{Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1KB"},
                                    {"file-index.bitmap.columns", "f0"}}));
     ASSERT_EQ(1, options.Definitions().size());
     EXPECT_EQ("f0", options.Definitions()[0].column_name);
     EXPECT_EQ("bitmap", options.Definitions()[0].index_type);
     EXPECT_EQ(1024, options.InManifestThreshold());
-
-    ASSERT_OK_AND_ASSIGN(FileIndexOptions default_options, FileIndexOptions::FromMap({}));
-    EXPECT_EQ(CoreOptions().FileIndexInManifestThreshold(), default_options.InManifestThreshold());
-    ASSERT_NOK(
-        FileIndexOptions::FromMap({{Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "invalid-size"}}));
 }
 
 TEST(FileIndexOptionsTest, TestRewriteFieldNamesPreservesOptions) {
@@ -149,15 +142,15 @@ TEST(FileIndexOptionsTest, TestFormatVersion) {
     ASSERT_OK_AND_ASSIGN(auto defaults, ParseOptions({}));
     ASSERT_EQ(1, defaults.FormatVersion());
     for (const std::string version : {"1", "2"}) {
-        ASSERT_OK_AND_ASSIGN(auto options, ParseOptions({{"file-index.format-version", version}}));
+        ASSERT_OK_AND_ASSIGN(auto options, ParseOptions({{"file-index.format.version", version}}));
         ASSERT_EQ(std::stoi(version), options.FormatVersion());
         ASSERT_OK_AND_ASSIGN(FileIndexOptions from_map,
-                             FileIndexOptions::FromMap({{"file-index.format-version", version}}));
+                             FileIndexOptions::FromMap({{"file-index.format.version", version}}));
         ASSERT_EQ(std::stoi(version), from_map.FormatVersion());
     }
     for (const std::string version : {"0", "3", "-1", "bad"}) {
-        ASSERT_NOK(ParseOptions({{"file-index.format-version", version}}));
-        ASSERT_NOK(FileIndexOptions::FromMap({{"file-index.format-version", version}}));
+        ASSERT_NOK(ParseOptions({{"file-index.format.version", version}}));
+        ASSERT_NOK(FileIndexOptions::FromMap({{"file-index.format.version", version}}));
     }
 }
 
diff --git a/src/paimon/core/io/spillable_index_output_stream.cpp b/src/paimon/core/io/spillable_index_output_stream.cpp
index 31128ef99..402dcdcf6 100644
--- a/src/paimon/core/io/spillable_index_output_stream.cpp
+++ b/src/paimon/core/io/spillable_index_output_stream.cpp
@@ -81,7 +81,7 @@ Status SpillableIndexOutputStream::Close() {
 
 Result<std::shared_ptr<Bytes>> SpillableIndexOutputStream::EmbeddedBytes() {
     if (external_) {
-        return std::shared_ptr<Bytes>();
+        return Status::Invalid("File index has spilled to an external file");
     }
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Bytes> bytes, memory_->Finish(pool_.get()));
     return bytes;
diff --git a/src/paimon/core/io/spillable_index_output_stream.h b/src/paimon/core/io/spillable_index_output_stream.h
index 0373c1934..68d7e1a58 100644
--- a/src/paimon/core/io/spillable_index_output_stream.h
+++ b/src/paimon/core/io/spillable_index_output_stream.h
@@ -52,7 +52,7 @@ class SpillableIndexOutputStream : public OutputStream {
     /// Closes the stream and deletes any unpublished external index, even if Close() fails.
     Status Abort();
 
-    /// Returns the buffered index, or null if it has spilled to an external file.
+    /// Returns the buffered index, or an error if it has spilled to an external file.
     Result<std::shared_ptr<Bytes>> EmbeddedBytes();
 
  private:
diff --git a/src/paimon/core/io/spillable_index_output_stream_test.cpp b/src/paimon/core/io/spillable_index_output_stream_test.cpp
index ca6bee73f..69afdd1a6 100644
--- a/src/paimon/core/io/spillable_index_output_stream_test.cpp
+++ b/src/paimon/core/io/spillable_index_output_stream_test.cpp
@@ -83,8 +83,7 @@ TEST_F(SpillableIndexOutputStreamTest, SpillsAndPreservesBufferedPrefix) {
         ASSERT_TRUE(stream->Spilled());
         ASSERT_OK_AND_ASSIGN(int64_t position, stream->GetPos());
         ASSERT_EQ(prefix.size() + 6, position);
-        ASSERT_OK_AND_ASSIGN(std::shared_ptr<Bytes> bytes, stream->EmbeddedBytes());
-        ASSERT_FALSE(bytes);
+        ASSERT_NOK_WITH_MSG(stream->EmbeddedBytes(), "spilled to an external file");
         ASSERT_OK(stream->Flush());
         ASSERT_OK(stream->Close());
         std::string content;
diff --git a/src/paimon/indexer/lumina/lumina_file_index.cpp b/src/paimon/indexer/lumina/lumina_file_index.cpp
index 4fcf209f2..7ad70102c 100644
--- a/src/paimon/indexer/lumina/lumina_file_index.cpp
+++ b/src/paimon/indexer/lumina/lumina_file_index.cpp
@@ -44,8 +44,7 @@ namespace {
 /// writing.
 class OffsetOutputStream : public OutputStream {
  public:
-    static Result<std::unique_ptr<OffsetOutputStream>> Create(
-        const std::shared_ptr<OutputStream>& output) {
+    static Result<std::unique_ptr<OffsetOutputStream>> Create(OutputStream* output) {
         PAIMON_ASSIGN_OR_RAISE(int64_t start, output->GetPos());
         return std::unique_ptr<OffsetOutputStream>(new OffsetOutputStream(output, start));
     }
@@ -68,10 +67,9 @@ class OffsetOutputStream : public OutputStream {
     }
 
  private:
-    OffsetOutputStream(const std::shared_ptr<OutputStream>& output, int64_t start)
-        : output_(output), start_(start) {}
+    OffsetOutputStream(OutputStream* output, int64_t start) : output_(output), start_(start) {}
 
-    std::shared_ptr<OutputStream> output_;
+    OutputStream* output_;
     int64_t start_;
 };
 
@@ -142,14 +140,14 @@ Result<PAIMON_UNIQUE_PTR<Bytes>> LuminaFileIndexWriter::SerializedBytes() const
     auto segment_output = std::make_unique<MemorySegmentOutputStream>(
         MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_->GetPaimonPool());
     auto output = std::make_shared<ByteArrayOutputStream>(std::move(segment_output));
-    PAIMON_ASSIGN_OR_RAISE(bool written, WriteTo(output));
+    PAIMON_ASSIGN_OR_RAISE(bool written, WriteTo(output.get()));
     if (!written) {
         return PAIMON_UNIQUE_PTR<Bytes>();
     }
     return output->Finish(pool_->GetPaimonPool().get());
 }
 
-Result<bool> LuminaFileIndexWriter::WriteTo(const std::shared_ptr<OutputStream>& output) const {
+Result<bool> LuminaFileIndexWriter::WriteTo(OutputStream* output) const {
     if (!output) {
         return Status::Invalid("Lumina File Index output stream cannot be null");
     }
diff --git a/src/paimon/indexer/lumina/lumina_file_index.h b/src/paimon/indexer/lumina/lumina_file_index.h
index 750ca4209..9021756e2 100644
--- a/src/paimon/indexer/lumina/lumina_file_index.h
+++ b/src/paimon/indexer/lumina/lumina_file_index.h
@@ -66,7 +66,7 @@ class LuminaFileIndexWriter final : public FileIndexWriter {
     Status AddBatch(::ArrowArray* batch) override;
 
     Result<PAIMON_UNIQUE_PTR<Bytes>> SerializedBytes() const override;
-    Result<bool> WriteTo(const std::shared_ptr<OutputStream>& output) const override;
+    Result<bool> WriteTo(OutputStream* output) const override;
 
  private:
     std::string field_name_;
diff --git a/src/paimon/indexer/lumina/lumina_file_index_test.cpp b/src/paimon/indexer/lumina/lumina_file_index_test.cpp
index 986c5cda2..fe4558db2 100644
--- a/src/paimon/indexer/lumina/lumina_file_index_test.cpp
+++ b/src/paimon/indexer/lumina/lumina_file_index_test.cpp
@@ -161,7 +161,7 @@ TEST(LuminaFileIndexTest, WriteToUsesRelativeOffsetsAndKeepsStreamOpen) {
             MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool));
     ASSERT_OK_AND_ASSIGN(int64_t prefix_length, output->Write("prefix", 6));
     ASSERT_EQ(6, prefix_length);
-    ASSERT_OK_AND_ASSIGN(bool written, writer->WriteTo(output));
+    ASSERT_OK_AND_ASSIGN(bool written, writer->WriteTo(output.get()));
     ASSERT_TRUE(written);
     ASSERT_OK_AND_ASSIGN(int64_t end, output->GetPos());
     int64_t length = end - prefix_length;
@@ -218,7 +218,7 @@ TEST(LuminaFileIndexTest, AllNullVectorsProduceEmptyArtifact) {
         auto output =
             std::make_shared<ByteArrayOutputStream>(std::make_unique<MemorySegmentOutputStream>(
                 MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool));
-        ASSERT_OK_AND_ASSIGN(bool written, writer->WriteTo(output));
+        ASSERT_OK_AND_ASSIGN(bool written, writer->WriteTo(output.get()));
         ASSERT_FALSE(written);
         ASSERT_OK_AND_ASSIGN(int64_t position, output->GetPos());
         ASSERT_EQ(0, position);
diff --git a/src/paimon/testing/mock/mock_search_file_index.cpp b/src/paimon/testing/mock/mock_search_file_index.cpp
index 83f72f461..27576e4b7 100644
--- a/src/paimon/testing/mock/mock_search_file_index.cpp
+++ b/src/paimon/testing/mock/mock_search_file_index.cpp
@@ -173,13 +173,12 @@ Result<std::shared_ptr<Bytes>> MockSearchFileIndex::Serialize(
     const FileIndexFormat::ColumnIndexes& indexes, const std::shared_ptr<MemoryPool>& pool) {
     auto output = MakeOutput(pool);
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileIndexFormat::Writer> writer,
-                           FileIndexFormat::CreateWriter(output, pool));
+                           FileIndexFormat::CreateWriter(output, FileIndexFormat::V_1, pool));
     for (const auto& [column, entries] : indexes) {
         for (const auto& [type, bytes] : entries) {
             FileIndexFormat::Writer::PayloadWriter payload;
             if (bytes && bytes->size() > 0) {
-                payload = [bytes = bytes](
-                              const std::shared_ptr<OutputStream>& destination) -> Result<bool> {
+                payload = [bytes = bytes](OutputStream* destination) -> Result<bool> {
                     PAIMON_RETURN_NOT_OK(DataOutputStream(destination).WriteBytes(bytes));
                     return true;
                 };
diff --git a/test/inte/read_inte_with_index_test.cpp b/test/inte/read_inte_with_index_test.cpp
index 515b812fc..8a7a2b3fd 100644
--- a/test/inte/read_inte_with_index_test.cpp
+++ b/test/inte/read_inte_with_index_test.cpp
@@ -16,18 +16,22 @@
  * limitations under the License.
  */
 
+#include <algorithm>
 #include <cstdint>
 #include <cstring>
 #include <iostream>
+#include <map>
 #include <memory>
 #include <optional>
 #include <string>
+#include <tuple>
 #include <utility>
 #include <vector>
 
 #include "arrow/api.h"
 #include "arrow/array/array_base.h"
 #include "arrow/c/abi.h"
+#include "arrow/c/bridge.h"
 #include "arrow/ipc/json_simple.h"
 #include "arrow/util/string_builder.h"
 #include "gtest/gtest.h"
@@ -38,7 +42,12 @@
 #include "paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_factory.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
+#include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/deletionvectors/bitmap_deletion_vector.h"
+#include "paimon/core/deletionvectors/deletion_vector_index_file_writer.h"
+#include "paimon/core/index/index_file_meta.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/manifest/file_source.h"
 #include "paimon/core/stats/simple_stats.h"
@@ -55,11 +64,17 @@
 #include "paimon/predicate/predicate_builder.h"
 #include "paimon/read_context.h"
 #include "paimon/reader/batch_reader.h"
+#include "paimon/record_batch.h"
 #include "paimon/result.h"
+#include "paimon/scan_context.h"
 #include "paimon/status.h"
+#include "paimon/table/source/plan.h"
 #include "paimon/table/source/table_read.h"
+#include "paimon/table/source/table_scan.h"
+#include "paimon/testing/mock/mock_index_path_factory.h"
 #include "paimon/testing/utils/io_exception_helper.h"
 #include "paimon/testing/utils/read_result_collector.h"
+#include "paimon/testing/utils/test_helper.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon {
@@ -68,11 +83,49 @@ class Predicate;
 }  // namespace paimon
 
 namespace paimon::test {
-class ReadInteWithIndexTest : public testing::Test,
-                              public ::testing::WithParamInterface<std::pair<std::string, bool>> {
+enum class IndexDataSource { CppV1, CppV2, Java };
+
+constexpr char kBitmapData[] = R"([
+    ["Alice", 10, 1, 11.1], ["Bob", 10, 1, 12.1],
+    ["Emily", 10, 0, 13.1], ["Tony", 10, 0, 14.1],
+    ["Lucy", 20, 1, 15.1], ["Bob", 10, 1, 16.1],
+    ["Tony", 20, 0, 17.1], ["Alice", 20, null, 18.1]
+])";
+
+constexpr char kNumericData[] = R"([
+    ["Alice", 100, -2, 11.1, 1745542802000123000],
+    ["Bob", 200, -3, 12.1, 1745542902000123000],
+    ["Emily", 300, 1, 13.1, 1745542602000123000],
+    ["Tony", 50, 1, 14.1, -1744877000],
+    ["Lucy", 500, -1, 15.1, -1764877000],
+    ["Bob", 100, 2, 16.1, null],
+    ["Tony", null, -2, 17.1, 1745542802000123001],
+    ["Alice", 20, null, 18.1, -1724877000]
+])";
+
+constexpr char kRangeBitmapData[] = R"([
+    [17, 100, 1.1, 11.11, 19739, "row0"],
+    [3, 200, 2.2, 22.22, 19725, "row1"],
+    [5, 300, 3.3, 33.33, 19727, "row2"],
+    [7, 400, 4.4, 44.44, 19729, "row3"],
+    [9, 500, 5.5, 55.55, 19731, "row4"],
+    [null, null, null, null, null, null],
+    [null, null, null, null, null, "null_row"],
+    [10, 600, 6.6, 66.66, 19732, "row7"]
+])";
+
+class ReadInteWithIndexTest
+    : public testing::Test,
+      public ::testing::WithParamInterface<std::tuple<std::string, bool, IndexDataSource>> {
  public:
     void SetUp() override {
         pool_ = GetDefaultPool();
+        std::tie(file_format_, enable_prefetch_, data_source_) = GetParam();
+        if (data_source_ != IndexDataSource::Java) {
+            dir_ = UniqueTestDirectory::Create();
+            embedded_index_ = RandomNumber(0, 1) == 0;
+            RecordProperty("index_storage", embedded_index_ ? "embedded" : "external");
+        }
     }
     void TearDown() override {}
 
@@ -80,13 +133,12 @@ class ReadInteWithIndexTest : public testing::Test,
                      const std::vector<std::shared_ptr<Split>> splits,
                      const std::shared_ptr<Predicate>& predicate,
                      const std::shared_ptr<arrow::ChunkedArray>& expected_array) const {
-        auto [file_format, enable_prefetch] = GetParam();
         ReadContextBuilder context_builder(table_path);
         context_builder.AddOption("read.batch-size", "2")
             .AddOption("test.enable-adaptive-prefetch-strategy", "false")
             .SetPredicate(predicate)
             .EnableLateMaterializing(false);
-        if (enable_prefetch) {
+        if (enable_prefetch_) {
             context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3);
         }
         ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
@@ -898,33 +950,177 @@ class ReadInteWithIndexTest : public testing::Test,
         }
     }
 
- private:
+    std::map<std::string, std::string> CppOptions(
+        std::map<std::string, std::string> options) const {
+        options[Options::FILE_FORMAT] = file_format_;
+        options.emplace(Options::BUCKET, "-1");
+        options[Options::FILE_INDEX_FORMAT_VERSION] =
+            data_source_ == IndexDataSource::CppV1 ? "1" : "2";
+        options[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = embedded_index_ ? "100MB" : "1B";
+        return options;
+    }
+
+    void CheckIndexStorage(const std::shared_ptr<DataSplitImpl>& split) const {
+        SCOPED_TRACE(embedded_index_ ? "embedded index" : "external index");
+        for (const auto& file : split->DataFiles()) {
+            ASSERT_EQ(embedded_index_, file->embedded_index != nullptr);
+            ASSERT_EQ(embedded_index_ ? 0 : 1, file->extra_files.size());
+        }
+    }
+
+    void ScanCppTable(TestHelper* helper, std::shared_ptr<DataSplitImpl>* split) const {
+        ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<Split>> splits,
+                             helper->NewScan(StartupMode::LatestFull(), std::nullopt, false));
+        ASSERT_EQ(1, splits.size());
+        *split = checked_pointer_cast<DataSplitImpl>(splits[0]);
+        ASSERT_NO_FATAL_FAILURE(CheckIndexStorage(*split));
+    }
+
+    void WriteCppTable(const std::vector<DataField>& read_fields, const std::string& json,
+                       const std::map<std::string, std::string>& index_options, std::string* path,
+                       std::shared_ptr<DataSplitImpl>* split, bool single_row_batches = false,
+                       const std::vector<std::string>& primary_keys = {}) const {
+        auto options = CppOptions(index_options);
+        arrow::FieldVector fields;
+        for (size_t i = 1; i < read_fields.size(); ++i) {
+            auto field = read_fields[i].ArrowField();
+            if (std::find(primary_keys.begin(), primary_keys.end(), field->name()) !=
+                primary_keys.end()) {
+                field = field->WithNullable(false);
+            }
+            fields.push_back(field);
+        }
+        ASSERT_OK_AND_ASSIGN(
+            std::unique_ptr<TestHelper> helper,
+            TestHelper::Create(dir_->Str(), arrow::schema(fields), {}, primary_keys, options,
+                               /*is_streaming_mode=*/false, /*ignore_if_exists=*/false,
+                               /*temp_directory=*/dir_->Str() + "/tmp"));
+        if (single_row_batches) {
+            // Flush each record separately so a tiny stripe/row-group size produces eight parts.
+            auto array_result =
+                arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), json);
+            ASSERT_TRUE(array_result.ok()) << array_result.status().ToString();
+            auto array = array_result.ValueOrDie();
+            std::vector<std::unique_ptr<RecordBatch>> batches;
+            for (int64_t i = 0; i < array->length(); ++i) {
+                ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> row,
+                                     ArrowUtils::NormalizeArrayOffsets(
+                                         array->Slice(i, 1), arrow::default_memory_pool()));
+                ::ArrowArray c_array;
+                ASSERT_TRUE(arrow::ExportArray(*row, &c_array).ok());
+                RecordBatchBuilder builder(&c_array);
+                ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                                     builder.SetBucket(0).Finish());
+                batches.push_back(std::move(batch));
+            }
+            ASSERT_OK(helper->WriteAndCommit(std::move(batches), 0, std::nullopt));
+        } else {
+            ASSERT_OK_AND_ASSIGN(
+                std::unique_ptr<RecordBatch> batch,
+                TestHelper::MakeRecordBatch(arrow::struct_(fields), json, {}, 0, {}));
+            ASSERT_OK(helper->WriteAndCommit(std::move(batch), 0, std::nullopt));
+        }
+        *path = dir_->Str() + "/foo.db/bar";
+        ASSERT_NO_FATAL_FAILURE(ScanCppTable(helper.get(), split));
+    }
+
+    void WriteCppTableWithDv(const std::vector<DataField>& read_fields, std::string* path,
+                             std::shared_ptr<DataSplitImpl>* split) const {
+        ASSERT_NO_FATAL_FAILURE(WriteCppTable(read_fields, R"([
+            [0, "Alice", 10, 1, 11.1], [1, "Bob", 10, 1, 12.1],
+            [2, "Emily", 10, 0, 13.1], [3, "Tony", 10, 0, 14.1],
+            [4, "Lucy", 20, 1, 15.1], [5, "Bob", 10, 1, 16.1],
+            [6, "Tony", 20, 0, 17.1], [7, "Alice", 20, null, 18.1]
+        ])",
+                                              {{"file-index.bitmap.columns", "f0,f1,f2"},
+                                               {Options::BUCKET, "1"},
+                                               {"deletion-vectors.enabled", "true"}},
+                                              path, split, /*single_row_batches=*/false,
+                                              /*primary_keys=*/{"key"}));
+        ASSERT_EQ(1, (*split)->DataFiles().size());
+        auto deletion_vector = std::make_shared<BitmapDeletionVector>(RoaringBitmap32());
+        ASSERT_OK(deletion_vector->Delete(0));
+        ASSERT_OK(deletion_vector->Delete(4));
+        auto index_paths = std::make_shared<MockIndexPathFactory>(dir_->Str());
+        DeletionVectorIndexFileWriter dv_writer(dir_->GetFileSystem(), index_paths, pool_);
+        const auto& file_name = (*split)->DataFiles()[0]->file_name;
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<IndexFileMeta> dv_meta,
+                             dv_writer.WriteSingleFile({{file_name, deletion_vector}}));
+        const auto& range = dv_meta->DvRanges().value().find(file_name)->second;
+        DeletionFile deletion_file(index_paths->ToPath(dv_meta), range.GetOffset(),
+                                   range.GetLength(), range.GetCardinality());
+        DataSplitImpl::Builder builder((*split)->Partition(), (*split)->Bucket(),
+                                       (*split)->BucketPath(), {(*split)->DataFiles()[0]});
+        ASSERT_OK_AND_ASSIGN(*split, builder.WithSnapshot((*split)->SnapshotId())
+                                         .WithDataDeletionFiles({deletion_file})
+                                         .IsStreaming(false)
+                                         .RawConvertible(true)
+                                         .Build());
+    }
+
+    void WriteCppTableWithAlter(const std::vector<DataField>& read_fields, std::string* path,
+                                std::shared_ptr<DataSplitImpl>* split) const {
+        std::vector<DataField> original_fields = {
+            SpecialFields::ValueKind(), DataField(0, arrow::field("f0", arrow::utf8())),
+            DataField(1, arrow::field("f1", arrow::int32())),
+            DataField(2, arrow::field("f2", arrow::int32())),
+            DataField(3, arrow::field("f3", arrow::float64()))};
+        ASSERT_NO_FATAL_FAILURE(WriteCppTable(original_fields, kBitmapData,
+                                              {{"file-index.bitmap.columns", "f0,f1,f2"}}, path,
+                                              split));
+        auto options = CppOptions({{"file-index.bitmap.columns", "f1,f4,f5"}});
+        std::vector<DataField> current_fields(read_fields.begin() + 1, read_fields.end());
+        ASSERT_OK(TestHelper::WriteNextSchema(dir_->GetFileSystem(), *path, current_fields,
+                                              /*highest_field_id=*/4, options));
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<TestHelper> helper,
+                             TestHelper::Create(*path, options, /*is_streaming_mode=*/false));
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+                             TestHelper::MakeRecordBatch(
+                                 DataField::ConvertDataFieldsToArrowStructType(current_fields), R"([
+            [30, "Alice", 21.1, 100], [20, "Emily", 22.1, 101],
+            [10, "Bob", 23.1, 100], [30, "David", 24.1, null]
+        ])",
+                                 {}, 0, {}));
+        ASSERT_OK(helper->WriteAndCommit(std::move(batch), 1, std::nullopt));
+        ASSERT_NO_FATAL_FAILURE(ScanCppTable(helper.get(), split));
+        ASSERT_EQ(2, (*split)->DataFiles().size());
+        ASSERT_EQ(0, (*split)->DataFiles()[0]->schema_id);
+        ASSERT_EQ(1, (*split)->DataFiles()[1]->schema_id);
+    }
+
+ protected:
+    std::string file_format_;
+    bool enable_prefetch_ = false;
+    bool embedded_index_ = false;
+    IndexDataSource data_source_ = IndexDataSource::Java;
     std::shared_ptr<MemoryPool> pool_;
+    std::unique_ptr<UniqueTestDirectory> dir_;
 };
 
-std::vector<std::pair<std::string, bool>> GetTestValuesForReadInteWithIndexTest() {
-    std::vector<std::pair<std::string, bool>> values = {{"parquet", false}, {"parquet", true}};
+std::vector<std::string> GetFileFormatsForReadInteWithIndexTest() {
+    std::vector<std::string> values = {"parquet"};
 #ifdef PAIMON_ENABLE_ORC
-    values.emplace_back("orc", false);
-    values.emplace_back("orc", true);
+    values.emplace_back("orc");
 #endif
     return values;
 }
 
-INSTANTIATE_TEST_SUITE_P(FileFormatAndEnablePaimonPrefetch, ReadInteWithIndexTest,
-                         ::testing::ValuesIn(std::vector<std::pair<std::string, bool>>(
-                             GetTestValuesForReadInteWithIndexTest())));
+INSTANTIATE_TEST_SUITE_P(
+    FileFormatAndPrefetchAndDataSource, ReadInteWithIndexTest,
+    ::testing::Combine(::testing::ValuesIn(GetFileFormatsForReadInteWithIndexTest()),
+                       ::testing::Bool(),
+                       ::testing::Values(IndexDataSource::CppV1, IndexDataSource::CppV2,
+                                         IndexDataSource::Java)),
+    [](const ::testing::TestParamInfo<ReadInteWithIndexTest::ParamType>& info) {
+        IndexDataSource source = std::get<2>(info.param);
+        std::string source_name = source == IndexDataSource::Java
+                                      ? "Java"
+                                      : (source == IndexDataSource::CppV1 ? "CppV1" : "CppV2");
+        return std::get<0>(info.param) + (std::get<1>(info.param) ? "Prefetch" : "NoPrefetch") +
+               source_name;
+    });
 
 TEST_P(ReadInteWithIndexTest, TestSimple) {
-    auto [file_format, enable_prefetch] = GetParam();
-    std::string path = GetDataDir() + "/" + file_format +
-                       "/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/";
-    std::string file_name;
-    if (file_format == "orc") {
-        file_name = "data-414509f5-e40c-4245-b992-bbf486778ac9-0.orc";
-    } else if (file_format == "parquet") {
-        file_name = "data-783929b2-49d4-4006-a898-194a62e3278d-0.parquet";
-    }
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),
@@ -932,25 +1128,41 @@ TEST_P(ReadInteWithIndexTest, TestSimple) {
                                           DataField(3, arrow::field("f3", arrow::float64()))};
     std::shared_ptr<arrow::DataType> arrow_data_type =
         DataField::ConvertDataFieldsToArrowStructType(read_fields);
+    std::string path;
+    std::shared_ptr<DataSplitImpl> split;
+    if (data_source_ != IndexDataSource::Java) {
+        ASSERT_NO_FATAL_FAILURE(WriteCppTable(
+            read_fields, kBitmapData, {{"file-index.bitmap.columns", "f0,f1,f2"}}, &path, &split));
+    } else {
+        path = GetDataDir() + "/" + file_format_ +
+               "/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/";
+        std::string file_name;
+        if (file_format_ == "orc") {
+            file_name = "data-414509f5-e40c-4245-b992-bbf486778ac9-0.orc";
+        } else if (file_format_ == "parquet") {
+            file_name = "data-783929b2-49d4-4006-a898-194a62e3278d-0.parquet";
+        }
 
-    auto data_file_meta = std::make_shared<DataFileMeta>(
-        file_name, /*file_size=*/689,
-        /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
-        /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
-        /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
-        /*max_sequence_number=*/7, /*schema_id=*/0,
-        /*level=*/0,
-        /*extra_files=*/
-        std::vector<std::optional<std::string>>({file_name + ".index"}),
-        /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
-        /*embedded_index=*/nullptr, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
-        /*column_max_sequence_numbers=*/std::nullopt);
-    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
-                                   /*bucket_path=*/path + "bucket-0/", {data_file_meta});
-    ASSERT_OK_AND_ASSIGN(auto split,
-                         builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+        auto data_file_meta = std::make_shared<DataFileMeta>(
+            file_name, /*file_size=*/689,
+            /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
+            /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+            /*max_sequence_number=*/7, /*schema_id=*/0,
+            /*level=*/0,
+            /*extra_files=*/
+            std::vector<std::optional<std::string>>({file_name + ".index"}),
+            /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+            /*embedded_index=*/nullptr, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
+        DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                       /*bucket_path=*/path + "bucket-0/", {data_file_meta});
+        ASSERT_OK_AND_ASSIGN(
+            split, builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    }
 
     auto predicate =
         PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING,
@@ -968,7 +1180,7 @@ TEST_P(ReadInteWithIndexTest, TestSimple) {
         .AddOption("test.enable-adaptive-prefetch-strategy", "false")
         .AddOption("orc.read.enable-metrics", "true")
         .SetPredicate(predicate);
-    if (enable_prefetch) {
+    if (enable_prefetch_) {
         context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3);
     }
     ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
@@ -980,7 +1192,7 @@ TEST_P(ReadInteWithIndexTest, TestSimple) {
     ASSERT_TRUE(result_array->Equals(*expected_array));
 
     // test metrics
-    if (file_format == "orc") {
+    if (file_format_ == "orc") {
         ASSERT_OK_AND_ASSIGN(uint64_t io_count, read_metrics->GetCounter("orc.read.io.count"));
         ASSERT_GT(io_count, 0);
         ASSERT_OK_AND_ASSIGN(uint64_t latency,
@@ -990,16 +1202,6 @@ TEST_P(ReadInteWithIndexTest, TestSimple) {
 }
 
 TEST_P(ReadInteWithIndexTest, TestReadWithLimits) {
-    auto [file_format, enable_prefetch] = GetParam();
-    std::string path = GetDataDir() + "/" + file_format +
-                       "/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/";
-    std::string file_name;
-    if (file_format == "orc") {
-        file_name = "data-414509f5-e40c-4245-b992-bbf486778ac9-0.orc";
-    } else if (file_format == "parquet") {
-        file_name = "data-783929b2-49d4-4006-a898-194a62e3278d-0.parquet";
-    }
-
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),
@@ -1007,24 +1209,41 @@ TEST_P(ReadInteWithIndexTest, TestReadWithLimits) {
                                           DataField(3, arrow::field("f3", arrow::float64()))};
     std::shared_ptr<arrow::DataType> arrow_data_type =
         DataField::ConvertDataFieldsToArrowStructType(read_fields);
-    auto data_file_meta = std::make_shared<DataFileMeta>(
-        file_name, /*file_size=*/689,
-        /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
-        /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
-        /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
-        /*max_sequence_number=*/7, /*schema_id=*/0,
-        /*level=*/0,
-        /*extra_files=*/
-        std::vector<std::optional<std::string>>({file_name + ".index"}),
-        /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
-        /*embedded_index=*/nullptr, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
-        /*column_max_sequence_numbers=*/std::nullopt);
-    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
-                                   /*bucket_path=*/path + "bucket-0/", {data_file_meta});
-    ASSERT_OK_AND_ASSIGN(auto split,
-                         builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    std::string path;
+    std::shared_ptr<DataSplitImpl> split;
+    if (data_source_ != IndexDataSource::Java) {
+        ASSERT_NO_FATAL_FAILURE(WriteCppTable(
+            read_fields, kBitmapData, {{"file-index.bitmap.columns", "f0,f1,f2"}}, &path, &split));
+    } else {
+        path = GetDataDir() + "/" + file_format_ +
+               "/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/";
+        std::string file_name;
+        if (file_format_ == "orc") {
+            file_name = "data-414509f5-e40c-4245-b992-bbf486778ac9-0.orc";
+        } else if (file_format_ == "parquet") {
+            file_name = "data-783929b2-49d4-4006-a898-194a62e3278d-0.parquet";
+        }
+
+        auto data_file_meta = std::make_shared<DataFileMeta>(
+            file_name, /*file_size=*/689,
+            /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
+            /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+            /*max_sequence_number=*/7, /*schema_id=*/0,
+            /*level=*/0,
+            /*extra_files=*/
+            std::vector<std::optional<std::string>>({file_name + ".index"}),
+            /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+            /*embedded_index=*/nullptr, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
+        DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                       /*bucket_path=*/path + "bucket-0/", {data_file_meta});
+        ASSERT_OK_AND_ASSIGN(
+            split, builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    }
 
     auto predicate =
         PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING,
@@ -1035,7 +1254,7 @@ TEST_P(ReadInteWithIndexTest, TestReadWithLimits) {
         .AddOption(Options::READ_BATCH_SIZE, "1")
         .AddOption("orc.read.enable-metrics", "true")
         .SetPredicate(predicate);
-    if (enable_prefetch) {
+    if (enable_prefetch_) {
         context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3);
     }
     ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
@@ -1059,7 +1278,7 @@ TEST_P(ReadInteWithIndexTest, TestReadWithLimits) {
     }
 
     // test metrics
-    if (file_format == "orc") {
+    if (file_format_ == "orc") {
         ASSERT_TRUE(read_metrics);
         ASSERT_OK_AND_ASSIGN(uint64_t io_count, read_metrics->GetCounter("orc.read.io.count"));
         ASSERT_GT(io_count, 0);
@@ -1070,16 +1289,6 @@ TEST_P(ReadInteWithIndexTest, TestReadWithLimits) {
 }
 
 TEST_P(ReadInteWithIndexTest, TestEmbeddingBitmapIndex) {
-    auto [file_format, enable_prefetch] = GetParam();
-    std::string path =
-        GetDataDir() + "/" + file_format + "/append_with_bitmap.db/append_with_bitmap/";
-    std::string file_name;
-    if (file_format == "orc") {
-        file_name = "data-62feb610-c83f-4217-9b50-bbad9cd08eb4-0.orc";
-    } else if (file_format == "parquet") {
-        file_name = "data-54b396b6-df2d-4c6a-a0ae-b6a5afb612d7-0.parquet";
-    }
-
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),
@@ -1087,69 +1296,74 @@ TEST_P(ReadInteWithIndexTest, TestEmbeddingBitmapIndex) {
                                           DataField(3, arrow::field("f3", arrow::float64()))};
     std::shared_ptr<arrow::DataType> arrow_data_type =
         DataField::ConvertDataFieldsToArrowStructType(read_fields);
+    std::string path;
+    std::shared_ptr<DataSplitImpl> split;
+    if (data_source_ != IndexDataSource::Java) {
+        ASSERT_NO_FATAL_FAILURE(WriteCppTable(
+            read_fields, kBitmapData, {{"file-index.bitmap.columns", "f0,f1,f2"}}, &path, &split));
+    } else {
+        path = GetDataDir() + "/" + file_format_ + "/append_with_bitmap.db/append_with_bitmap/";
+        std::string file_name;
+        if (file_format_ == "orc") {
+            file_name = "data-62feb610-c83f-4217-9b50-bbad9cd08eb4-0.orc";
+        } else if (file_format_ == "parquet") {
+            file_name = "data-54b396b6-df2d-4c6a-a0ae-b6a5afb612d7-0.parquet";
+        }
 
-    std::vector<uint8_t> embedded_bytes = {
-        0,  5,   78,  78,  208, 26,  53,  174, 0,   0,   0,   1,   0,   0,   0,   96,  0,   0,
-        0,  3,   0,   2,   102, 48,  0,   0,   0,   1,   0,   6,   98,  105, 116, 109, 97,  112,
-        0,  0,   0,   96,  0,   0,   0,   176, 0,   2,   102, 49,  0,   0,   0,   1,   0,   6,
-        98, 105, 116, 109, 97,  112, 0,   0,   1,   16,  0,   0,   0,   102, 0,   2,   102, 50,
-        0,  0,   0,   1,   0,   6,   98,  105, 116, 109, 97,  112, 0,   0,   1,   118, 0,   0,
-        0,  108, 0,   0,   0,   0,   2,   0,   0,   0,   8,   0,   0,   0,   5,   0,   0,   0,
-        0,  1,   0,   0,   0,   5,   65,  108, 105, 99,  101, 0,   0,   0,   0,   0,   0,   0,
-        85, 0,   0,   0,   5,   0,   0,   0,   5,   65,  108, 105, 99,  101, 0,   0,   0,   0,
-        0,  0,   0,   20,  0,   0,   0,   3,   66,  111, 98,  0,   0,   0,   20,  0,   0,   0,
-        20, 0,   0,   0,   5,   69,  109, 105, 108, 121, 255, 255, 255, 253, 255, 255, 255, 255,
-        0,  0,   0,   4,   76,  117, 99,  121, 255, 255, 255, 251, 255, 255, 255, 255, 0,   0,
-        0,  4,   84,  111, 110, 121, 0,   0,   0,   40,  0,   0,   0,   20,  58,  48,  0,   0,
-        1,  0,   0,   0,   0,   0,   1,   0,   16,  0,   0,   0,   0,   0,   7,   0,   58,  48,
-        0,  0,   1,   0,   0,   0,   0,   0,   1,   0,   16,  0,   0,   0,   1,   0,   5,   0,
-        58, 48,  0,   0,   1,   0,   0,   0,   0,   0,   1,   0,   16,  0,   0,   0,   3,   0,
-        6,  0,   2,   0,   0,   0,   8,   0,   0,   0,   2,   0,   0,   0,   0,   1,   0,   0,
-        0,  10,  0,   0,   0,   0,   0,   0,   0,   28,  0,   0,   0,   2,   0,   0,   0,   10,
-        0,  0,   0,   22,  0,   0,   0,   26,  0,   0,   0,   20,  0,   0,   0,   0,   0,   0,
-        0,  22,  58,  48,  0,   0,   1,   0,   0,   0,   0,   0,   2,   0,   16,  0,   0,   0,
-        4,  0,   6,   0,   7,   0,   58,  48,  0,   0,   1,   0,   0,   0,   0,   0,   4,   0,
-        16, 0,   0,   0,   0,   0,   1,   0,   2,   0,   3,   0,   5,   0,   2,   0,   0,   0,
-        8,  0,   0,   0,   2,   1,   255, 255, 255, 248, 0,   0,   0,   18,  0,   0,   0,   1,
-        0,  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   28,  0,   0,   0,   2,   0,   0,
-        0,  0,   0,   0,   0,   0,   0,   0,   0,   22,  0,   0,   0,   1,   0,   0,   0,   22,
-        0,  0,   0,   24,  58,  48,  0,   0,   1,   0,   0,   0,   0,   0,   2,   0,   16,  0,
-        0,  0,   2,   0,   3,   0,   6,   0,   58,  48,  0,   0,   1,   0,   0,   0,   0,   0,
-        3,  0,   16,  0,   0,   0,   0,   0,   1,   0,   4,   0,   5,   0};
-    auto embedded_index = std::make_shared<Bytes>(embedded_bytes.size(), pool_.get());
-    memcpy(embedded_index->data(), reinterpret_cast<const void*>(embedded_bytes.data()),
-           embedded_bytes.size());
-    auto data_file_meta = std::make_shared<DataFileMeta>(
-        file_name, /*file_size=*/689,
-        /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
-        /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
-        /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
-        /*max_sequence_number=*/7, /*schema_id=*/0,
-        /*level=*/0,
-        /*extra_files=*/std::vector<std::optional<std::string>>(),
-        /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
-        /*embedded_index=*/embedded_index, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
-        /*column_max_sequence_numbers=*/std::nullopt);
-    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
-                                   /*bucket_path=*/path + "bucket-0/", {data_file_meta});
-    ASSERT_OK_AND_ASSIGN(auto split,
-                         builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+        std::vector<uint8_t> embedded_bytes = {
+            0,  5,   78,  78,  208, 26,  53,  174, 0,   0,   0,   1,   0,   0,   0,   96,  0,   0,
+            0,  3,   0,   2,   102, 48,  0,   0,   0,   1,   0,   6,   98,  105, 116, 109, 97,  112,
+            0,  0,   0,   96,  0,   0,   0,   176, 0,   2,   102, 49,  0,   0,   0,   1,   0,   6,
+            98, 105, 116, 109, 97,  112, 0,   0,   1,   16,  0,   0,   0,   102, 0,   2,   102, 50,
+            0,  0,   0,   1,   0,   6,   98,  105, 116, 109, 97,  112, 0,   0,   1,   118, 0,   0,
+            0,  108, 0,   0,   0,   0,   2,   0,   0,   0,   8,   0,   0,   0,   5,   0,   0,   0,
+            0,  1,   0,   0,   0,   5,   65,  108, 105, 99,  101, 0,   0,   0,   0,   0,   0,   0,
+            85, 0,   0,   0,   5,   0,   0,   0,   5,   65,  108, 105, 99,  101, 0,   0,   0,   0,
+            0,  0,   0,   20,  0,   0,   0,   3,   66,  111, 98,  0,   0,   0,   20,  0,   0,   0,
+            20, 0,   0,   0,   5,   69,  109, 105, 108, 121, 255, 255, 255, 253, 255, 255, 255, 255,
+            0,  0,   0,   4,   76,  117, 99,  121, 255, 255, 255, 251, 255, 255, 255, 255, 0,   0,
+            0,  4,   84,  111, 110, 121, 0,   0,   0,   40,  0,   0,   0,   20,  58,  48,  0,   0,
+            1,  0,   0,   0,   0,   0,   1,   0,   16,  0,   0,   0,   0,   0,   7,   0,   58,  48,
+            0,  0,   1,   0,   0,   0,   0,   0,   1,   0,   16,  0,   0,   0,   1,   0,   5,   0,
+            58, 48,  0,   0,   1,   0,   0,   0,   0,   0,   1,   0,   16,  0,   0,   0,   3,   0,
+            6,  0,   2,   0,   0,   0,   8,   0,   0,   0,   2,   0,   0,   0,   0,   1,   0,   0,
+            0,  10,  0,   0,   0,   0,   0,   0,   0,   28,  0,   0,   0,   2,   0,   0,   0,   10,
+            0,  0,   0,   22,  0,   0,   0,   26,  0,   0,   0,   20,  0,   0,   0,   0,   0,   0,
+            0,  22,  58,  48,  0,   0,   1,   0,   0,   0,   0,   0,   2,   0,   16,  0,   0,   0,
+            4,  0,   6,   0,   7,   0,   58,  48,  0,   0,   1,   0,   0,   0,   0,   0,   4,   0,
+            16, 0,   0,   0,   0,   0,   1,   0,   2,   0,   3,   0,   5,   0,   2,   0,   0,   0,
+            8,  0,   0,   0,   2,   1,   255, 255, 255, 248, 0,   0,   0,   18,  0,   0,   0,   1,
+            0,  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   28,  0,   0,   0,   2,   0,   0,
+            0,  0,   0,   0,   0,   0,   0,   0,   0,   22,  0,   0,   0,   1,   0,   0,   0,   22,
+            0,  0,   0,   24,  58,  48,  0,   0,   1,   0,   0,   0,   0,   0,   2,   0,   16,  0,
+            0,  0,   2,   0,   3,   0,   6,   0,   58,  48,  0,   0,   1,   0,   0,   0,   0,   0,
+            3,  0,   16,  0,   0,   0,   0,   0,   1,   0,   4,   0,   5,   0};
+        auto embedded_index = std::make_shared<Bytes>(embedded_bytes.size(), pool_.get());
+        memcpy(embedded_index->data(), reinterpret_cast<const void*>(embedded_bytes.data()),
+               embedded_bytes.size());
+        auto data_file_meta = std::make_shared<DataFileMeta>(
+            file_name, /*file_size=*/689,
+            /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
+            /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+            /*max_sequence_number=*/7, /*schema_id=*/0,
+            /*level=*/0,
+            /*extra_files=*/std::vector<std::optional<std::string>>(),
+            /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+            /*embedded_index=*/embedded_index, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
+        DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                       /*bucket_path=*/path + "bucket-0/", {data_file_meta});
+        ASSERT_OK_AND_ASSIGN(
+            split, builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    }
     CheckResultForBitmapWithSingleRowGroup(path, arrow_data_type, split);
 }
 
 TEST_P(ReadInteWithIndexTest, TestBitmapWithV1) {
-    auto [file_format, enable_prefetch] = GetParam();
-    std::string path =
-        GetDataDir() + "/" + file_format + "/append_with_bitmap_v1.db/append_with_bitmap_v1/";
-    std::string file_name;
-    if (file_format == "orc") {
-        file_name = "data-c29cf741-80ce-4b74-9cf3-e42efef557b3-0.orc";
-    } else if (file_format == "parquet") {
-        file_name = "data-6b38a150-c99c-4add-a449-a48a5d34e36c-0.parquet";
-    }
-
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),
@@ -1157,63 +1371,73 @@ TEST_P(ReadInteWithIndexTest, TestBitmapWithV1) {
                                           DataField(3, arrow::field("f3", arrow::float64()))};
     std::shared_ptr<arrow::DataType> arrow_data_type =
         DataField::ConvertDataFieldsToArrowStructType(read_fields);
+    std::string path;
+    std::shared_ptr<DataSplitImpl> split;
+    if (data_source_ != IndexDataSource::Java) {
+        ASSERT_NO_FATAL_FAILURE(WriteCppTable(read_fields, kBitmapData,
+                                              {{"file-index.bitmap.columns", "f0,f1,f2"},
+                                               {"file-index.bitmap.f0.version", "1"},
+                                               {"file-index.bitmap.f1.version", "1"},
+                                               {"file-index.bitmap.f2.version", "1"}},
+                                              &path, &split));
+    } else {
+        path =
+            GetDataDir() + "/" + file_format_ + "/append_with_bitmap_v1.db/append_with_bitmap_v1/";
+        std::string file_name;
+        if (file_format_ == "orc") {
+            file_name = "data-c29cf741-80ce-4b74-9cf3-e42efef557b3-0.orc";
+        } else if (file_format_ == "parquet") {
+            file_name = "data-6b38a150-c99c-4add-a449-a48a5d34e36c-0.parquet";
+        }
 
-    std::vector<uint8_t> embedded_bytes = {
-        0,   5,   78,  78,  208, 26,  53,  174, 0,   0,   0,   1,   0,  0,   0,   96,  0,   0,
-        0,   3,   0,   2,   102, 48,  0,   0,   0,   1,   0,   6,   98, 105, 116, 109, 97,  112,
-        0,   0,   0,   96,  0,   0,   0,   131, 0,   2,   102, 49,  0,  0,   0,   1,   0,   6,
-        98,  105, 116, 109, 97,  112, 0,   0,   0,   227, 0,   0,   0,  74,  0,   2,   102, 50,
-        0,   0,   0,   1,   0,   6,   98,  105, 116, 109, 97,  112, 0,  0,   1,   45,  0,   0,
-        0,   76,  0,   0,   0,   0,   1,   0,   0,   0,   8,   0,   0,  0,   5,   0,   0,   0,
-        0,   5,   65,  108, 105, 99,  101, 0,   0,   0,   0,   0,   0,  0,   4,   76,  117, 99,
-        121, 255, 255, 255, 251, 0,   0,   0,   3,   66,  111, 98,  0,  0,   0,   20,  0,   0,
-        0,   5,   69,  109, 105, 108, 121, 255, 255, 255, 253, 0,   0,  0,   4,   84,  111, 110,
-        121, 0,   0,   0,   40,  58,  48,  0,   0,   1,   0,   0,   0,  0,   0,   1,   0,   16,
-        0,   0,   0,   0,   0,   7,   0,   58,  48,  0,   0,   1,   0,  0,   0,   0,   0,   1,
-        0,   16,  0,   0,   0,   1,   0,   5,   0,   58,  48,  0,   0,  1,   0,   0,   0,   0,
-        0,   1,   0,   16,  0,   0,   0,   3,   0,   6,   0,   1,   0,  0,   0,   8,   0,   0,
-        0,   2,   0,   0,   0,   0,   20,  0,   0,   0,   0,   0,   0,  0,   10,  0,   0,   0,
-        22,  58,  48,  0,   0,   1,   0,   0,   0,   0,   0,   2,   0,  16,  0,   0,   0,   4,
-        0,   6,   0,   7,   0,   58,  48,  0,   0,   1,   0,   0,   0,  0,   0,   4,   0,   16,
-        0,   0,   0,   0,   0,   1,   0,   2,   0,   3,   0,   5,   0,  1,   0,   0,   0,   8,
-        0,   0,   0,   2,   1,   255, 255, 255, 248, 0,   0,   0,   0,  0,   0,   0,   0,   0,
-        0,   0,   1,   0,   0,   0,   22,  58,  48,  0,   0,   1,   0,  0,   0,   0,   0,   2,
-        0,   16,  0,   0,   0,   2,   0,   3,   0,   6,   0,   58,  48, 0,   0,   1,   0,   0,
-        0,   0,   0,   3,   0,   16,  0,   0,   0,   0,   0,   1,   0,  4,   0,   5,   0};
-    auto embedded_index = std::make_shared<Bytes>(embedded_bytes.size(), pool_.get());
-    memcpy(embedded_index->data(), reinterpret_cast<const void*>(embedded_bytes.data()),
-           embedded_bytes.size());
-    auto data_file_meta = std::make_shared<DataFileMeta>(
-        file_name, /*file_size=*/689,
-        /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
-        /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
-        /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
-        /*max_sequence_number=*/7, /*schema_id=*/0,
-        /*level=*/0,
-        /*extra_files=*/std::vector<std::optional<std::string>>(),
-        /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
-        /*embedded_index=*/embedded_index, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
-        /*column_max_sequence_numbers=*/std::nullopt);
-    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
-                                   /*bucket_path=*/path + "bucket-0/", {data_file_meta});
-    ASSERT_OK_AND_ASSIGN(auto split,
-                         builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+        std::vector<uint8_t> embedded_bytes = {
+            0,   5,   78,  78,  208, 26,  53,  174, 0,   0,   0,   1,   0,  0,   0,   96,  0,   0,
+            0,   3,   0,   2,   102, 48,  0,   0,   0,   1,   0,   6,   98, 105, 116, 109, 97,  112,
+            0,   0,   0,   96,  0,   0,   0,   131, 0,   2,   102, 49,  0,  0,   0,   1,   0,   6,
+            98,  105, 116, 109, 97,  112, 0,   0,   0,   227, 0,   0,   0,  74,  0,   2,   102, 50,
+            0,   0,   0,   1,   0,   6,   98,  105, 116, 109, 97,  112, 0,  0,   1,   45,  0,   0,
+            0,   76,  0,   0,   0,   0,   1,   0,   0,   0,   8,   0,   0,  0,   5,   0,   0,   0,
+            0,   5,   65,  108, 105, 99,  101, 0,   0,   0,   0,   0,   0,  0,   4,   76,  117, 99,
+            121, 255, 255, 255, 251, 0,   0,   0,   3,   66,  111, 98,  0,  0,   0,   20,  0,   0,
+            0,   5,   69,  109, 105, 108, 121, 255, 255, 255, 253, 0,   0,  0,   4,   84,  111, 110,
+            121, 0,   0,   0,   40,  58,  48,  0,   0,   1,   0,   0,   0,  0,   0,   1,   0,   16,
+            0,   0,   0,   0,   0,   7,   0,   58,  48,  0,   0,   1,   0,  0,   0,   0,   0,   1,
+            0,   16,  0,   0,   0,   1,   0,   5,   0,   58,  48,  0,   0,  1,   0,   0,   0,   0,
+            0,   1,   0,   16,  0,   0,   0,   3,   0,   6,   0,   1,   0,  0,   0,   8,   0,   0,
+            0,   2,   0,   0,   0,   0,   20,  0,   0,   0,   0,   0,   0,  0,   10,  0,   0,   0,
+            22,  58,  48,  0,   0,   1,   0,   0,   0,   0,   0,   2,   0,  16,  0,   0,   0,   4,
+            0,   6,   0,   7,   0,   58,  48,  0,   0,   1,   0,   0,   0,  0,   0,   4,   0,   16,
+            0,   0,   0,   0,   0,   1,   0,   2,   0,   3,   0,   5,   0,  1,   0,   0,   0,   8,
+            0,   0,   0,   2,   1,   255, 255, 255, 248, 0,   0,   0,   0,  0,   0,   0,   0,   0,
+            0,   0,   1,   0,   0,   0,   22,  58,  48,  0,   0,   1,   0,  0,   0,   0,   0,   2,
+            0,   16,  0,   0,   0,   2,   0,   3,   0,   6,   0,   58,  48, 0,   0,   1,   0,   0,
+            0,   0,   0,   3,   0,   16,  0,   0,   0,   0,   0,   1,   0,  4,   0,   5,   0};
+        auto embedded_index = std::make_shared<Bytes>(embedded_bytes.size(), pool_.get());
+        memcpy(embedded_index->data(), reinterpret_cast<const void*>(embedded_bytes.data()),
+               embedded_bytes.size());
+        auto data_file_meta = std::make_shared<DataFileMeta>(
+            file_name, /*file_size=*/689,
+            /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
+            /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+            /*max_sequence_number=*/7, /*schema_id=*/0,
+            /*level=*/0,
+            /*extra_files=*/std::vector<std::optional<std::string>>(),
+            /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+            /*embedded_index=*/embedded_index, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
+        DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                       /*bucket_path=*/path + "bucket-0/", {data_file_meta});
+        ASSERT_OK_AND_ASSIGN(
+            split, builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    }
     CheckResultForBitmapWithSingleRowGroup(path, arrow_data_type, split);
 }
 
 TEST_P(ReadInteWithIndexTest, TestNoEmbeddingBitmapIndex) {
-    auto [file_format, enable_prefetch] = GetParam();
-    std::string path = GetDataDir() + "/" + file_format +
-                       "/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/";
-    std::string file_name;
-    if (file_format == "orc") {
-        file_name = "data-414509f5-e40c-4245-b992-bbf486778ac9-0.orc";
-    } else if (file_format == "parquet") {
-        file_name = "data-783929b2-49d4-4006-a898-194a62e3278d-0.parquet";
-    }
-
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),
@@ -1221,39 +1445,45 @@ TEST_P(ReadInteWithIndexTest, TestNoEmbeddingBitmapIndex) {
                                           DataField(3, arrow::field("f3", arrow::float64()))};
     std::shared_ptr<arrow::DataType> arrow_data_type =
         DataField::ConvertDataFieldsToArrowStructType(read_fields);
+    std::string path;
+    std::shared_ptr<DataSplitImpl> split;
+    if (data_source_ != IndexDataSource::Java) {
+        ASSERT_NO_FATAL_FAILURE(WriteCppTable(
+            read_fields, kBitmapData, {{"file-index.bitmap.columns", "f0,f1,f2"}}, &path, &split));
+    } else {
+        path = GetDataDir() + "/" + file_format_ +
+               "/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/";
+        std::string file_name;
+        if (file_format_ == "orc") {
+            file_name = "data-414509f5-e40c-4245-b992-bbf486778ac9-0.orc";
+        } else if (file_format_ == "parquet") {
+            file_name = "data-783929b2-49d4-4006-a898-194a62e3278d-0.parquet";
+        }
 
-    auto data_file_meta = std::make_shared<DataFileMeta>(
-        file_name, /*file_size=*/689,
-        /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
-        /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
-        /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
-        /*max_sequence_number=*/7, /*schema_id=*/0,
-        /*level=*/0,
-        /*extra_files=*/
-        std::vector<std::optional<std::string>>({file_name + ".index"}),
-        /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
-        /*embedded_index=*/nullptr, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
-        /*column_max_sequence_numbers=*/std::nullopt);
-    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
-                                   /*bucket_path=*/path + "bucket-0/", {data_file_meta});
-    ASSERT_OK_AND_ASSIGN(auto split,
-                         builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+        auto data_file_meta = std::make_shared<DataFileMeta>(
+            file_name, /*file_size=*/689,
+            /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
+            /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+            /*max_sequence_number=*/7, /*schema_id=*/0,
+            /*level=*/0,
+            /*extra_files=*/
+            std::vector<std::optional<std::string>>({file_name + ".index"}),
+            /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+            /*embedded_index=*/nullptr, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
+        DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                       /*bucket_path=*/path + "bucket-0/", {data_file_meta});
+        ASSERT_OK_AND_ASSIGN(
+            split, builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    }
     CheckResultForBitmapWithSingleRowGroup(path, arrow_data_type, split);
 }
 
 TEST_P(ReadInteWithIndexTest, TestBitmapIndexWithLateMaterializing) {
-    auto [file_format, enable_prefetch] = GetParam();
-    std::string path = GetDataDir() + "/" + file_format +
-                       "/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/";
-    std::string file_name;
-    if (file_format == "orc") {
-        file_name = "data-414509f5-e40c-4245-b992-bbf486778ac9-0.orc";
-    } else if (file_format == "parquet") {
-        file_name = "data-783929b2-49d4-4006-a898-194a62e3278d-0.parquet";
-    }
-
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),
@@ -1261,25 +1491,41 @@ TEST_P(ReadInteWithIndexTest, TestBitmapIndexWithLateMaterializing) {
                                           DataField(3, arrow::field("f3", arrow::float64()))};
     std::shared_ptr<arrow::DataType> arrow_data_type =
         DataField::ConvertDataFieldsToArrowStructType(read_fields);
+    std::string path;
+    std::shared_ptr<DataSplitImpl> split;
+    if (data_source_ != IndexDataSource::Java) {
+        ASSERT_NO_FATAL_FAILURE(WriteCppTable(
+            read_fields, kBitmapData, {{"file-index.bitmap.columns", "f0,f1,f2"}}, &path, &split));
+    } else {
+        path = GetDataDir() + "/" + file_format_ +
+               "/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/";
+        std::string file_name;
+        if (file_format_ == "orc") {
+            file_name = "data-414509f5-e40c-4245-b992-bbf486778ac9-0.orc";
+        } else if (file_format_ == "parquet") {
+            file_name = "data-783929b2-49d4-4006-a898-194a62e3278d-0.parquet";
+        }
 
-    auto data_file_meta = std::make_shared<DataFileMeta>(
-        file_name, /*file_size=*/689,
-        /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
-        /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
-        /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
-        /*max_sequence_number=*/7, /*schema_id=*/0,
-        /*level=*/0,
-        /*extra_files=*/
-        std::vector<std::optional<std::string>>({file_name + ".index"}),
-        /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
-        /*embedded_index=*/nullptr, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
-        /*column_max_sequence_numbers=*/std::nullopt);
-    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
-                                   /*bucket_path=*/path + "bucket-0/", {data_file_meta});
-    ASSERT_OK_AND_ASSIGN(auto split,
-                         builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+        auto data_file_meta = std::make_shared<DataFileMeta>(
+            file_name, /*file_size=*/689,
+            /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
+            /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+            /*max_sequence_number=*/7, /*schema_id=*/0,
+            /*level=*/0,
+            /*extra_files=*/
+            std::vector<std::optional<std::string>>({file_name + ".index"}),
+            /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+            /*embedded_index=*/nullptr, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
+        DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                       /*bucket_path=*/path + "bucket-0/", {data_file_meta});
+        ASSERT_OK_AND_ASSIGN(
+            split, builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    }
 
     std::string literal_str = "Bob";
     auto predicate = PredicateBuilder::Equal(
@@ -1292,7 +1538,7 @@ TEST_P(ReadInteWithIndexTest, TestBitmapIndexWithLateMaterializing) {
         .SetPredicate(predicate)
         .EnablePredicateFilter(true)
         .EnableLateMaterializing(true);
-    if (enable_prefetch) {
+    if (enable_prefetch_) {
         context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3);
     }
     ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
@@ -1314,22 +1560,6 @@ TEST_P(ReadInteWithIndexTest, TestBitmapIndexWithLateMaterializing) {
 }
 
 TEST_P(ReadInteWithIndexTest, TestNoEmbeddingBitmapIndexWithExternalPath) {
-    auto [file_format, enable_prefetch] = GetParam();
-    std::string path = GetDataDir() + "/" + file_format +
-                       "/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/";
-    std::string file_name, external_file_path;
-    if (file_format == "orc") {
-        file_name = "data-414509f5-e40c-4245-b992-bbf486778ac9-0.orc";
-        external_file_path = GetDataDir() + "/" + file_format +
-                             "/append_with_bitmap_no_embedding.db/external-path/bucket-0/" +
-                             file_name;
-    } else if (file_format == "parquet") {
-        file_name = "data-783929b2-49d4-4006-a898-194a62e3278d-0.parquet";
-        external_file_path = GetDataDir() + "/" + file_format +
-                             "/append_with_bitmap_no_embedding.db/external-path/bucket-0/" +
-                             file_name;
-    }
-
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),
@@ -1337,41 +1567,56 @@ TEST_P(ReadInteWithIndexTest, TestNoEmbeddingBitmapIndexWithExternalPath) {
                                           DataField(3, arrow::field("f3", arrow::float64()))};
     std::shared_ptr<arrow::DataType> arrow_data_type =
         DataField::ConvertDataFieldsToArrowStructType(read_fields);
+    std::string path;
+    std::shared_ptr<DataSplitImpl> split;
+    if (data_source_ != IndexDataSource::Java) {
+        ASSERT_NO_FATAL_FAILURE(WriteCppTable(
+            read_fields, kBitmapData,
+            {{"file-index.bitmap.columns", "f0,f1,f2"},
+             {Options::DATA_FILE_EXTERNAL_PATHS, "file://" + dir_->Str() + "/external"},
+             {Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY, "round-robin"}},
+            &path, &split));
+        ASSERT_EQ(1, split->DataFiles().size());
+        ASSERT_TRUE(split->DataFiles()[0]->external_path.has_value());
+    } else {
+        path = GetDataDir() + "/" + file_format_ +
+               "/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/";
+        std::string file_name, external_file_path;
+        if (file_format_ == "orc") {
+            file_name = "data-414509f5-e40c-4245-b992-bbf486778ac9-0.orc";
+            external_file_path = GetDataDir() + "/" + file_format_ +
+                                 "/append_with_bitmap_no_embedding.db/external-path/bucket-0/" +
+                                 file_name;
+        } else if (file_format_ == "parquet") {
+            file_name = "data-783929b2-49d4-4006-a898-194a62e3278d-0.parquet";
+            external_file_path = GetDataDir() + "/" + file_format_ +
+                                 "/append_with_bitmap_no_embedding.db/external-path/bucket-0/" +
+                                 file_name;
+        }
 
-    auto data_file_meta = std::make_shared<DataFileMeta>(
-        file_name, /*file_size=*/689,
-        /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
-        /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
-        /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
-        /*max_sequence_number=*/7, /*schema_id=*/0,
-        /*level=*/0,
-        /*extra_files=*/
-        std::vector<std::optional<std::string>>({file_name + ".index"}),
-        /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
-        /*embedded_index=*/nullptr, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/external_file_path, /*first_row_id=*/std::nullopt,
-        /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
-    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
-                                   /*bucket_path=*/path + "bucket-0/", {data_file_meta});
-    ASSERT_OK_AND_ASSIGN(auto split,
-                         builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+        auto data_file_meta = std::make_shared<DataFileMeta>(
+            file_name, /*file_size=*/689,
+            /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
+            /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+            /*max_sequence_number=*/7, /*schema_id=*/0,
+            /*level=*/0,
+            /*extra_files=*/
+            std::vector<std::optional<std::string>>({file_name + ".index"}),
+            /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+            /*embedded_index=*/nullptr, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/external_file_path, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
+        DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                       /*bucket_path=*/path + "bucket-0/", {data_file_meta});
+        ASSERT_OK_AND_ASSIGN(
+            split, builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    }
     CheckResultForBitmapWithSingleRowGroup(path, arrow_data_type, split);
 }
 
 TEST_P(ReadInteWithIndexTest, TestBitmapIndexWithDv) {
-    auto [file_format, enable_prefetch] = GetParam();
-    std::string path =
-        GetDataDir() + "/" + file_format + "/pk_with_bitmap_dv.db/pk_with_bitmap_dv/";
-    std::string file_name, deletion_file_path;
-    if (file_format == "orc") {
-        file_name = "data-0da0dc51-797c-4c82-9b08-269b2ce400dc-0.orc";
-        deletion_file_path = path + "/index/index-73ec72e9-489e-49d9-a96c-f6db9b56fa07-0";
-    } else if (file_format == "parquet") {
-        file_name = "data-bfa36b28-40e7-4026-893b-d23116973174-0.parquet";
-        deletion_file_path = path + "/index/index-daed25d2-8e68-45ba-aaad-ff342666ef27-0";
-    }
-
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("key", arrow::int32())),
                                           DataField(1, arrow::field("f0", arrow::utf8())),
@@ -1380,30 +1625,46 @@ TEST_P(ReadInteWithIndexTest, TestBitmapIndexWithDv) {
                                           DataField(4, arrow::field("f3", arrow::float64()))};
     std::shared_ptr<arrow::DataType> arrow_data_type =
         DataField::ConvertDataFieldsToArrowStructType(read_fields);
+    std::string path;
+    std::shared_ptr<DataSplitImpl> split;
+    if (data_source_ != IndexDataSource::Java) {
+        ASSERT_NO_FATAL_FAILURE(WriteCppTableWithDv(read_fields, &path, &split));
+    } else {
+        path = GetDataDir() + "/" + file_format_ + "/pk_with_bitmap_dv.db/pk_with_bitmap_dv/";
+        std::string file_name, deletion_file_path;
+        if (file_format_ == "orc") {
+            file_name = "data-0da0dc51-797c-4c82-9b08-269b2ce400dc-0.orc";
+            deletion_file_path = path + "/index/index-73ec72e9-489e-49d9-a96c-f6db9b56fa07-0";
+        } else if (file_format_ == "parquet") {
+            file_name = "data-bfa36b28-40e7-4026-893b-d23116973174-0.parquet";
+            deletion_file_path = path + "/index/index-daed25d2-8e68-45ba-aaad-ff342666ef27-0";
+        }
 
-    auto data_file_meta = std::make_shared<DataFileMeta>(
-        file_name, /*file_size=*/1001,
-        /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
-        /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
-        /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
-        /*max_sequence_number=*/7, /*schema_id=*/0,
-        /*level=*/5,
-        /*extra_files=*/
-        std::vector<std::optional<std::string>>({file_name + ".index"}),
-        /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
-        /*embedded_index=*/nullptr, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
-        /*column_max_sequence_numbers=*/std::nullopt);
-    DeletionFile deletion_file(deletion_file_path,
-                               /*offset=*/1, /*length=*/24, /*cardinality=*/2);
-    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
-                                   /*bucket_path=*/path + "bucket-0/", {data_file_meta});
-    ASSERT_OK_AND_ASSIGN(auto split, builder.WithSnapshot(4)
-                                         .WithDataDeletionFiles({deletion_file})
-                                         .IsStreaming(false)
-                                         .RawConvertible(true)
-                                         .Build());
+        auto data_file_meta = std::make_shared<DataFileMeta>(
+            file_name, /*file_size=*/1001,
+            /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
+            /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+            /*max_sequence_number=*/7, /*schema_id=*/0,
+            /*level=*/5,
+            /*extra_files=*/
+            std::vector<std::optional<std::string>>({file_name + ".index"}),
+            /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+            /*embedded_index=*/nullptr, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
+        DeletionFile deletion_file(deletion_file_path,
+                                   /*offset=*/1, /*length=*/24, /*cardinality=*/2);
+        DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                       /*bucket_path=*/path + "bucket-0/", {data_file_meta});
+        ASSERT_OK_AND_ASSIGN(split, builder.WithSnapshot(4)
+                                        .WithDataDeletionFiles({deletion_file})
+                                        .IsStreaming(false)
+                                        .RawConvertible(true)
+                                        .Build());
+    }
     {
         // test with non predicate
         std::shared_ptr<arrow::ChunkedArray> expected_array;
@@ -1462,18 +1723,6 @@ TEST_P(ReadInteWithIndexTest, TestBitmapIndexWithDv) {
 }
 
 TEST_P(ReadInteWithIndexTest, TestWithAlterTable) {
-    auto [file_format, enable_prefetch] = GetParam();
-    std::string path = GetDataDir() + "/" + file_format +
-                       "/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/";
-    std::string file_name1, file_name2;
-    if (file_format == "orc") {
-        file_name1 = "data-68014988-5451-478f-a18a-a1668214cf3d-0.orc";
-        file_name2 = "data-a29b7235-760d-4838-881c-39cbef585dd2-0.orc";
-    } else if (file_format == "parquet") {
-        file_name1 = "data-da667947-0604-4459-973b-9d364c3953a9-0.parquet";
-        file_name2 = "data-2efd8477-9fb4-44a6-9a39-901dd5a83b28-0.parquet";
-    }
-
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(1, arrow::field("f1", arrow::int64())),
                                           DataField(0, arrow::field("f4", arrow::utf8())),
@@ -1481,86 +1730,103 @@ TEST_P(ReadInteWithIndexTest, TestWithAlterTable) {
                                           DataField(4, arrow::field("f5", arrow::int32()))};
     std::shared_ptr<arrow::DataType> arrow_data_type =
         DataField::ConvertDataFieldsToArrowStructType(read_fields);
+    std::string path;
+    std::shared_ptr<DataSplitImpl> split;
+    if (data_source_ != IndexDataSource::Java) {
+        ASSERT_NO_FATAL_FAILURE(WriteCppTableWithAlter(read_fields, &path, &split));
+    } else {
+        path = GetDataDir() + "/" + file_format_ +
+               "/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/";
+        std::string file_name1, file_name2;
+        if (file_format_ == "orc") {
+            file_name1 = "data-68014988-5451-478f-a18a-a1668214cf3d-0.orc";
+            file_name2 = "data-a29b7235-760d-4838-881c-39cbef585dd2-0.orc";
+        } else if (file_format_ == "parquet") {
+            file_name1 = "data-da667947-0604-4459-973b-9d364c3953a9-0.parquet";
+            file_name2 = "data-2efd8477-9fb4-44a6-9a39-901dd5a83b28-0.parquet";
+        }
 
-    auto create_data_file_meta = [&](const std::string& file_name, int32_t schema_id,
-                                     const std::vector<uint8_t>& embedded_bytes) {
-        auto embedded_index = std::make_shared<Bytes>(embedded_bytes.size(), pool_.get());
-        memcpy(embedded_index->data(), reinterpret_cast<const void*>(embedded_bytes.data()),
-               embedded_bytes.size());
-        return std::make_shared<DataFileMeta>(
-            file_name, /*file_size=*/10,
-            /*row_count=*/10, /*min_key=*/BinaryRow::EmptyRow(),
-            /*max_key=*/BinaryRow::EmptyRow(),
-            /*key_stats=*/SimpleStats::EmptyStats(),
-            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
-            /*max_sequence_number=*/10, schema_id,
-            /*level=*/0,
-            /*extra_files=*/std::vector<std::optional<std::string>>(),
-            /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
-            /*embedded_index=*/embedded_index, FileSource::Append(),
-            /*value_stats_cols=*/std::nullopt,
-            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
-            /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
-    };
-
-    std::vector<uint8_t> embedded_bytes1 = {
-        0,  5,   78,  78,  208, 26,  53,  174, 0,   0,   0,   1,   0,   0,   0,   96,  0,   0,
-        0,  3,   0,   2,   102, 48,  0,   0,   0,   1,   0,   6,   98,  105, 116, 109, 97,  112,
-        0,  0,   0,   96,  0,   0,   0,   176, 0,   2,   102, 49,  0,   0,   0,   1,   0,   6,
-        98, 105, 116, 109, 97,  112, 0,   0,   1,   16,  0,   0,   0,   102, 0,   2,   102, 50,
-        0,  0,   0,   1,   0,   6,   98,  105, 116, 109, 97,  112, 0,   0,   1,   118, 0,   0,
-        0,  108, 0,   0,   0,   0,   2,   0,   0,   0,   8,   0,   0,   0,   5,   0,   0,   0,
-        0,  1,   0,   0,   0,   5,   65,  108, 105, 99,  101, 0,   0,   0,   0,   0,   0,   0,
-        85, 0,   0,   0,   5,   0,   0,   0,   5,   65,  108, 105, 99,  101, 0,   0,   0,   0,
-        0,  0,   0,   20,  0,   0,   0,   3,   66,  111, 98,  0,   0,   0,   20,  0,   0,   0,
-        20, 0,   0,   0,   5,   69,  109, 105, 108, 121, 255, 255, 255, 253, 255, 255, 255, 255,
-        0,  0,   0,   4,   76,  117, 99,  121, 255, 255, 255, 251, 255, 255, 255, 255, 0,   0,
-        0,  4,   84,  111, 110, 121, 0,   0,   0,   40,  0,   0,   0,   20,  58,  48,  0,   0,
-        1,  0,   0,   0,   0,   0,   1,   0,   16,  0,   0,   0,   0,   0,   7,   0,   58,  48,
-        0,  0,   1,   0,   0,   0,   0,   0,   1,   0,   16,  0,   0,   0,   1,   0,   5,   0,
-        58, 48,  0,   0,   1,   0,   0,   0,   0,   0,   1,   0,   16,  0,   0,   0,   3,   0,
-        6,  0,   2,   0,   0,   0,   8,   0,   0,   0,   2,   0,   0,   0,   0,   1,   0,   0,
-        0,  10,  0,   0,   0,   0,   0,   0,   0,   28,  0,   0,   0,   2,   0,   0,   0,   10,
-        0,  0,   0,   22,  0,   0,   0,   26,  0,   0,   0,   20,  0,   0,   0,   0,   0,   0,
-        0,  22,  58,  48,  0,   0,   1,   0,   0,   0,   0,   0,   2,   0,   16,  0,   0,   0,
-        4,  0,   6,   0,   7,   0,   58,  48,  0,   0,   1,   0,   0,   0,   0,   0,   4,   0,
-        16, 0,   0,   0,   0,   0,   1,   0,   2,   0,   3,   0,   5,   0,   2,   0,   0,   0,
-        8,  0,   0,   0,   2,   1,   255, 255, 255, 248, 0,   0,   0,   18,  0,   0,   0,   1,
-        0,  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   28,  0,   0,   0,   2,   0,   0,
-        0,  0,   0,   0,   0,   0,   0,   0,   0,   22,  0,   0,   0,   1,   0,   0,   0,   22,
-        0,  0,   0,   24,  58,  48,  0,   0,   1,   0,   0,   0,   0,   0,   2,   0,   16,  0,
-        0,  0,   2,   0,   3,   0,   6,   0,   58,  48,  0,   0,   1,   0,   0,   0,   0,   0,
-        3,  0,   16,  0,   0,   0,   0,   0,   1,   0,   4,   0,   5,   0};
-    std::vector<uint8_t> embedded_bytes2 = {
-        0,   5,   78,  78,  208, 26,  53,  174, 0,   0,   0,   1,   0,   0,   0,   96,  0,   0,
-        0,   3,   0,   2,   102, 49,  0,   0,   0,   1,   0,   6,   98,  105, 116, 109, 97,  112,
-        0,   0,   0,   96,  0,   0,   0,   102, 0,   2,   102, 52,  0,   0,   0,   1,   0,   6,
-        98,  105, 116, 109, 97,  112, 0,   0,   0,   198, 0,   0,   0,   101, 0,   2,   102, 53,
-        0,   0,   0,   1,   0,   6,   98,  105, 116, 109, 97,  112, 0,   0,   1,   43,  0,   0,
-        0,   82,  0,   0,   0,   0,   2,   0,   0,   0,   4,   0,   0,   0,   3,   0,   0,   0,
-        0,   1,   0,   0,   0,   0,   0,   0,   0,   10,  0,   0,   0,   0,   0,   0,   0,   52,
-        0,   0,   0,   3,   0,   0,   0,   0,   0,   0,   0,   10,  255, 255, 255, 253, 255, 255,
-        255, 255, 0,   0,   0,   0,   0,   0,   0,   20,  255, 255, 255, 254, 255, 255, 255, 255,
-        0,   0,   0,   0,   0,   0,   0,   30,  0,   0,   0,   0,   0,   0,   0,   20,  58,  48,
-        0,   0,   1,   0,   0,   0,   0,   0,   1,   0,   16,  0,   0,   0,   0,   0,   3,   0,
-        2,   0,   0,   0,   4,   0,   0,   0,   4,   0,   0,   0,   0,   1,   0,   0,   0,   5,
-        65,  108, 105, 99,  101, 0,   0,   0,   0,   0,   0,   0,   70,  0,   0,   0,   4,   0,
-        0,   0,   5,   65,  108, 105, 99,  101, 255, 255, 255, 255, 255, 255, 255, 255, 0,   0,
-        0,   3,   66,  111, 98,  255, 255, 255, 253, 255, 255, 255, 255, 0,   0,   0,   5,   68,
-        97,  118, 105, 100, 255, 255, 255, 252, 255, 255, 255, 255, 0,   0,   0,   5,   69,  109,
-        105, 108, 121, 255, 255, 255, 254, 255, 255, 255, 255, 2,   0,   0,   0,   4,   0,   0,
-        0,   2,   1,   255, 255, 255, 252, 0,   0,   0,   18,  0,   0,   0,   1,   0,   0,   0,
-        100, 0,   0,   0,   0,   0,   0,   0,   28,  0,   0,   0,   2,   0,   0,   0,   100, 0,
-        0,   0,   0,   0,   0,   0,   20,  0,   0,   0,   101, 255, 255, 255, 254, 255, 255, 255,
-        255, 58,  48,  0,   0,   1,   0,   0,   0,   0,   0,   1,   0,   16,  0,   0,   0,   0,
-        0,   2,   0};
-    auto data_file_meta1 = create_data_file_meta(file_name1, 0, embedded_bytes1);
-    auto data_file_meta2 = create_data_file_meta(file_name2, 1, embedded_bytes2);
-    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
-                                   /*bucket_path=*/path + "/bucket-0/",
-                                   {data_file_meta1, data_file_meta2});
-    ASSERT_OK_AND_ASSIGN(auto split,
-                         builder.WithSnapshot(2).IsStreaming(false).RawConvertible(true).Build());
+        auto create_data_file_meta = [&](const std::string& file_name, int32_t schema_id,
+                                         const std::vector<uint8_t>& embedded_bytes) {
+            auto embedded_index = std::make_shared<Bytes>(embedded_bytes.size(), pool_.get());
+            memcpy(embedded_index->data(), reinterpret_cast<const void*>(embedded_bytes.data()),
+                   embedded_bytes.size());
+            return std::make_shared<DataFileMeta>(
+                file_name, /*file_size=*/10,
+                /*row_count=*/10, /*min_key=*/BinaryRow::EmptyRow(),
+                /*max_key=*/BinaryRow::EmptyRow(),
+                /*key_stats=*/SimpleStats::EmptyStats(),
+                /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+                /*max_sequence_number=*/10, schema_id,
+                /*level=*/0,
+                /*extra_files=*/std::vector<std::optional<std::string>>(),
+                /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+                /*embedded_index=*/embedded_index, FileSource::Append(),
+                /*value_stats_cols=*/std::nullopt,
+                /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+                /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
+        };
+
+        std::vector<uint8_t> embedded_bytes1 = {
+            0,  5,   78,  78,  208, 26,  53,  174, 0,   0,   0,   1,   0,   0,   0,   96,  0,   0,
+            0,  3,   0,   2,   102, 48,  0,   0,   0,   1,   0,   6,   98,  105, 116, 109, 97,  112,
+            0,  0,   0,   96,  0,   0,   0,   176, 0,   2,   102, 49,  0,   0,   0,   1,   0,   6,
+            98, 105, 116, 109, 97,  112, 0,   0,   1,   16,  0,   0,   0,   102, 0,   2,   102, 50,
+            0,  0,   0,   1,   0,   6,   98,  105, 116, 109, 97,  112, 0,   0,   1,   118, 0,   0,
+            0,  108, 0,   0,   0,   0,   2,   0,   0,   0,   8,   0,   0,   0,   5,   0,   0,   0,
+            0,  1,   0,   0,   0,   5,   65,  108, 105, 99,  101, 0,   0,   0,   0,   0,   0,   0,
+            85, 0,   0,   0,   5,   0,   0,   0,   5,   65,  108, 105, 99,  101, 0,   0,   0,   0,
+            0,  0,   0,   20,  0,   0,   0,   3,   66,  111, 98,  0,   0,   0,   20,  0,   0,   0,
+            20, 0,   0,   0,   5,   69,  109, 105, 108, 121, 255, 255, 255, 253, 255, 255, 255, 255,
+            0,  0,   0,   4,   76,  117, 99,  121, 255, 255, 255, 251, 255, 255, 255, 255, 0,   0,
+            0,  4,   84,  111, 110, 121, 0,   0,   0,   40,  0,   0,   0,   20,  58,  48,  0,   0,
+            1,  0,   0,   0,   0,   0,   1,   0,   16,  0,   0,   0,   0,   0,   7,   0,   58,  48,
+            0,  0,   1,   0,   0,   0,   0,   0,   1,   0,   16,  0,   0,   0,   1,   0,   5,   0,
+            58, 48,  0,   0,   1,   0,   0,   0,   0,   0,   1,   0,   16,  0,   0,   0,   3,   0,
+            6,  0,   2,   0,   0,   0,   8,   0,   0,   0,   2,   0,   0,   0,   0,   1,   0,   0,
+            0,  10,  0,   0,   0,   0,   0,   0,   0,   28,  0,   0,   0,   2,   0,   0,   0,   10,
+            0,  0,   0,   22,  0,   0,   0,   26,  0,   0,   0,   20,  0,   0,   0,   0,   0,   0,
+            0,  22,  58,  48,  0,   0,   1,   0,   0,   0,   0,   0,   2,   0,   16,  0,   0,   0,
+            4,  0,   6,   0,   7,   0,   58,  48,  0,   0,   1,   0,   0,   0,   0,   0,   4,   0,
+            16, 0,   0,   0,   0,   0,   1,   0,   2,   0,   3,   0,   5,   0,   2,   0,   0,   0,
+            8,  0,   0,   0,   2,   1,   255, 255, 255, 248, 0,   0,   0,   18,  0,   0,   0,   1,
+            0,  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   28,  0,   0,   0,   2,   0,   0,
+            0,  0,   0,   0,   0,   0,   0,   0,   0,   22,  0,   0,   0,   1,   0,   0,   0,   22,
+            0,  0,   0,   24,  58,  48,  0,   0,   1,   0,   0,   0,   0,   0,   2,   0,   16,  0,
+            0,  0,   2,   0,   3,   0,   6,   0,   58,  48,  0,   0,   1,   0,   0,   0,   0,   0,
+            3,  0,   16,  0,   0,   0,   0,   0,   1,   0,   4,   0,   5,   0};
+        std::vector<uint8_t> embedded_bytes2 = {
+            0,   5,   78,  78,  208, 26,  53,  174, 0,   0,   0,   1,   0,   0,   0,   96,  0,
+            0,   0,   3,   0,   2,   102, 49,  0,   0,   0,   1,   0,   6,   98,  105, 116, 109,
+            97,  112, 0,   0,   0,   96,  0,   0,   0,   102, 0,   2,   102, 52,  0,   0,   0,
+            1,   0,   6,   98,  105, 116, 109, 97,  112, 0,   0,   0,   198, 0,   0,   0,   101,
+            0,   2,   102, 53,  0,   0,   0,   1,   0,   6,   98,  105, 116, 109, 97,  112, 0,
+            0,   1,   43,  0,   0,   0,   82,  0,   0,   0,   0,   2,   0,   0,   0,   4,   0,
+            0,   0,   3,   0,   0,   0,   0,   1,   0,   0,   0,   0,   0,   0,   0,   10,  0,
+            0,   0,   0,   0,   0,   0,   52,  0,   0,   0,   3,   0,   0,   0,   0,   0,   0,
+            0,   10,  255, 255, 255, 253, 255, 255, 255, 255, 0,   0,   0,   0,   0,   0,   0,
+            20,  255, 255, 255, 254, 255, 255, 255, 255, 0,   0,   0,   0,   0,   0,   0,   30,
+            0,   0,   0,   0,   0,   0,   0,   20,  58,  48,  0,   0,   1,   0,   0,   0,   0,
+            0,   1,   0,   16,  0,   0,   0,   0,   0,   3,   0,   2,   0,   0,   0,   4,   0,
+            0,   0,   4,   0,   0,   0,   0,   1,   0,   0,   0,   5,   65,  108, 105, 99,  101,
+            0,   0,   0,   0,   0,   0,   0,   70,  0,   0,   0,   4,   0,   0,   0,   5,   65,
+            108, 105, 99,  101, 255, 255, 255, 255, 255, 255, 255, 255, 0,   0,   0,   3,   66,
+            111, 98,  255, 255, 255, 253, 255, 255, 255, 255, 0,   0,   0,   5,   68,  97,  118,
+            105, 100, 255, 255, 255, 252, 255, 255, 255, 255, 0,   0,   0,   5,   69,  109, 105,
+            108, 121, 255, 255, 255, 254, 255, 255, 255, 255, 2,   0,   0,   0,   4,   0,   0,
+            0,   2,   1,   255, 255, 255, 252, 0,   0,   0,   18,  0,   0,   0,   1,   0,   0,
+            0,   100, 0,   0,   0,   0,   0,   0,   0,   28,  0,   0,   0,   2,   0,   0,   0,
+            100, 0,   0,   0,   0,   0,   0,   0,   20,  0,   0,   0,   101, 255, 255, 255, 254,
+            255, 255, 255, 255, 58,  48,  0,   0,   1,   0,   0,   0,   0,   0,   1,   0,   16,
+            0,   0,   0,   0,   0,   2,   0};
+        auto data_file_meta1 = create_data_file_meta(file_name1, 0, embedded_bytes1);
+        auto data_file_meta2 = create_data_file_meta(file_name2, 1, embedded_bytes2);
+        DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                       /*bucket_path=*/path + "/bucket-0/",
+                                       {data_file_meta1, data_file_meta2});
+        ASSERT_OK_AND_ASSIGN(
+            split, builder.WithSnapshot(2).IsStreaming(false).RawConvertible(true).Build());
+    }
     {
         // test with non predicate
         std::shared_ptr<arrow::ChunkedArray> expected_array;
@@ -1853,15 +2119,6 @@ TEST_P(ReadInteWithIndexTest, TestWithAlterTable) {
 }
 
 TEST_P(ReadInteWithIndexTest, TestWithBsiIndex) {
-    auto [file_format, enable_prefetch] = GetParam();
-    std::string path = GetDataDir() + "/" + file_format + "/append_with_bsi.db/append_with_bsi/";
-    std::string file_name;
-    if (file_format == "orc") {
-        file_name = "data-0befce14-587d-48e5-ad08-efeecceb0f09-0.orc";
-    } else if (file_format == "parquet") {
-        file_name = "data-af9e31b7-b44d-4a0e-85ae-5877d224ec9f-0.parquet";
-    }
-
     std::vector<DataField> read_fields = {
         SpecialFields::ValueKind(),
         DataField(0, arrow::field("f0", arrow::utf8())),
@@ -1871,24 +2128,40 @@ TEST_P(ReadInteWithIndexTest, TestWithBsiIndex) {
         DataField(4, arrow::field("f4", arrow::timestamp(arrow::TimeUnit::NANO)))};
     std::shared_ptr<arrow::DataType> arrow_data_type =
         DataField::ConvertDataFieldsToArrowStructType(read_fields);
-    auto data_file_meta = std::make_shared<DataFileMeta>(
-        file_name, /*file_size=*/932,
-        /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
-        /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
-        /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
-        /*max_sequence_number=*/7, /*schema_id=*/0,
-        /*level=*/0,
-        /*extra_files=*/
-        std::vector<std::optional<std::string>>({file_name + ".index"}),
-        /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
-        /*embedded_index=*/nullptr, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
-        /*column_max_sequence_numbers=*/std::nullopt);
-    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
-                                   /*bucket_path=*/path + "bucket-0/", {data_file_meta});
-    ASSERT_OK_AND_ASSIGN(auto split,
-                         builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    std::string path;
+    std::shared_ptr<DataSplitImpl> split;
+    if (data_source_ != IndexDataSource::Java) {
+        ASSERT_NO_FATAL_FAILURE(WriteCppTable(
+            read_fields, kNumericData, {{"file-index.bsi.columns", "f1,f2,f4"}}, &path, &split));
+    } else {
+        path = GetDataDir() + "/" + file_format_ + "/append_with_bsi.db/append_with_bsi/";
+        std::string file_name;
+        if (file_format_ == "orc") {
+            file_name = "data-0befce14-587d-48e5-ad08-efeecceb0f09-0.orc";
+        } else if (file_format_ == "parquet") {
+            file_name = "data-af9e31b7-b44d-4a0e-85ae-5877d224ec9f-0.parquet";
+        }
+
+        auto data_file_meta = std::make_shared<DataFileMeta>(
+            file_name, /*file_size=*/932,
+            /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
+            /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+            /*max_sequence_number=*/7, /*schema_id=*/0,
+            /*level=*/0,
+            /*extra_files=*/
+            std::vector<std::optional<std::string>>({file_name + ".index"}),
+            /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+            /*embedded_index=*/nullptr, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
+        DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                       /*bucket_path=*/path + "bucket-0/", {data_file_meta});
+        ASSERT_OK_AND_ASSIGN(
+            split, builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    }
     {
         // test equal predicate for f0, take no effective as bsi does not support string
         auto predicate =
@@ -1913,16 +2186,6 @@ TEST_P(ReadInteWithIndexTest, TestWithBsiIndex) {
 }
 
 TEST_P(ReadInteWithIndexTest, TestWithBloomFilterIndex) {
-    auto [file_format, enable_prefetch] = GetParam();
-    std::string path =
-        GetDataDir() + "/" + file_format + "/append_with_bloomfilter.db/append_with_bloomfilter/";
-    std::string file_name;
-    if (file_format == "orc") {
-        file_name = "data-34e8acb2-110b-4c32-9dc6-d6435178d0ad-0.orc";
-    } else if (file_format == "parquet") {
-        file_name = "data-79a27bc0-bcec-4062-9915-83ec8eb1622d-0.parquet";
-    }
-
     std::vector<DataField> read_fields = {
         SpecialFields::ValueKind(),
         DataField(0, arrow::field("f0", arrow::utf8())),
@@ -1932,24 +2195,42 @@ TEST_P(ReadInteWithIndexTest, TestWithBloomFilterIndex) {
         DataField(4, arrow::field("f4", arrow::timestamp(arrow::TimeUnit::NANO)))};
     std::shared_ptr<arrow::DataType> arrow_data_type =
         DataField::ConvertDataFieldsToArrowStructType(read_fields);
-    auto data_file_meta = std::make_shared<DataFileMeta>(
-        file_name, /*file_size=*/932,
-        /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
-        /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
-        /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
-        /*max_sequence_number=*/7, /*schema_id=*/0,
-        /*level=*/0,
-        /*extra_files=*/
-        std::vector<std::optional<std::string>>({file_name + ".index"}),
-        /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
-        /*embedded_index=*/nullptr, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
-        /*column_max_sequence_numbers=*/std::nullopt);
-    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
-                                   /*bucket_path=*/path + "bucket-0/", {data_file_meta});
-    ASSERT_OK_AND_ASSIGN(auto split,
-                         builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    std::string path;
+    std::shared_ptr<DataSplitImpl> split;
+    if (data_source_ != IndexDataSource::Java) {
+        ASSERT_NO_FATAL_FAILURE(
+            WriteCppTable(read_fields, kNumericData,
+                          {{"file-index.bloom-filter.columns", "f0,f1,f2,f3,f4"}}, &path, &split));
+    } else {
+        path = GetDataDir() + "/" + file_format_ +
+               "/append_with_bloomfilter.db/append_with_bloomfilter/";
+        std::string file_name;
+        if (file_format_ == "orc") {
+            file_name = "data-34e8acb2-110b-4c32-9dc6-d6435178d0ad-0.orc";
+        } else if (file_format_ == "parquet") {
+            file_name = "data-79a27bc0-bcec-4062-9915-83ec8eb1622d-0.parquet";
+        }
+
+        auto data_file_meta = std::make_shared<DataFileMeta>(
+            file_name, /*file_size=*/932,
+            /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
+            /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+            /*max_sequence_number=*/7, /*schema_id=*/0,
+            /*level=*/0,
+            /*extra_files=*/
+            std::vector<std::optional<std::string>>({file_name + ".index"}),
+            /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+            /*embedded_index=*/nullptr, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
+        DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                       /*bucket_path=*/path + "bucket-0/", {data_file_meta});
+        ASSERT_OK_AND_ASSIGN(
+            split, builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    }
 
     std::shared_ptr<arrow::ChunkedArray> all_array;
     auto array_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow_data_type, {R"([
@@ -2117,20 +2398,6 @@ TEST_P(ReadInteWithIndexTest, TestWithBloomFilterIndex) {
 }
 
 TEST_P(ReadInteWithIndexTest, TestBitmapPushDownWithMultiStripes) {
-    auto [file_format, enable_prefetch] = GetParam();
-    std::string path = paimon::test::GetDataDir() +
-                       "/append_with_bitmap_multi_stripes.db/append_with_bitmap_multi_stripes/";
-    std::string file_name, index_file_name;
-    if (file_format == "orc") {
-        // each record is a stripe
-        file_name = "data-55e21ed3-d118-4d75-a0fd-cd52039cb634-0.orc";
-        index_file_name = "data-55e21ed3-d118-4d75-a0fd-cd52039cb634-0.orc.index";
-    } else if (file_format == "parquet") {
-        // each record is a row group
-        file_name = "data-multi-row-groups.parquet";
-        index_file_name = "data-55e21ed3-d118-4d75-a0fd-cd52039cb634-0.orc.index";
-    }
-    // each record is a stripe
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),
@@ -2138,25 +2405,51 @@ TEST_P(ReadInteWithIndexTest, TestBitmapPushDownWithMultiStripes) {
                                           DataField(3, arrow::field("f3", arrow::float64()))};
     std::shared_ptr<arrow::DataType> arrow_data_type =
         DataField::ConvertDataFieldsToArrowStructType(read_fields);
+    std::string path;
+    std::shared_ptr<DataSplitImpl> split;
+    if (data_source_ != IndexDataSource::Java) {
+        ASSERT_NO_FATAL_FAILURE(WriteCppTable(read_fields, kBitmapData,
+                                              {{"file-index.bitmap.columns", "f0,f1,f2"},
+                                               {"write.batch-size", "1"},
+                                               {"orc.stripe.size", "1"},
+                                               {"orc.row.index.stride", "1"},
+                                               {"parquet.write.max-row-group-length", "1"}},
+                                              &path, &split, /*single_row_batches=*/true));
+    } else {
+        path = paimon::test::GetDataDir() +
+               "/append_with_bitmap_multi_stripes.db/append_with_bitmap_multi_stripes/";
+        std::string file_name, index_file_name;
+        if (file_format_ == "orc") {
+            // each record is a stripe
+            file_name = "data-55e21ed3-d118-4d75-a0fd-cd52039cb634-0.orc";
+            index_file_name = "data-55e21ed3-d118-4d75-a0fd-cd52039cb634-0.orc.index";
+        } else if (file_format_ == "parquet") {
+            // each record is a row group
+            file_name = "data-multi-row-groups.parquet";
+            index_file_name = "data-55e21ed3-d118-4d75-a0fd-cd52039cb634-0.orc.index";
+        }
+        // each record is a stripe
 
-    auto data_file_meta = std::make_shared<DataFileMeta>(
-        file_name, /*file_size=*/2637,
-        /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
-        /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
-        /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
-        /*max_sequence_number=*/7, /*schema_id=*/0,
-        /*level=*/0,
-        /*extra_files=*/
-        std::vector<std::optional<std::string>>({index_file_name}),
-        /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
-        /*embedded_index=*/nullptr, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
-        /*column_max_sequence_numbers=*/std::nullopt);
-    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
-                                   /*bucket_path=*/path + "bucket-0/", {data_file_meta});
-    ASSERT_OK_AND_ASSIGN(auto split,
-                         builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+        auto data_file_meta = std::make_shared<DataFileMeta>(
+            file_name, /*file_size=*/2637,
+            /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
+            /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+            /*max_sequence_number=*/7, /*schema_id=*/0,
+            /*level=*/0,
+            /*extra_files=*/
+            std::vector<std::optional<std::string>>({index_file_name}),
+            /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+            /*embedded_index=*/nullptr, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
+        DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                       /*bucket_path=*/path + "bucket-0/", {data_file_meta});
+        ASSERT_OK_AND_ASSIGN(
+            split, builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    }
 
     // test bitmap index takes effective
     CheckResultForBitmap(path, arrow_data_type, split);
@@ -2231,16 +2524,6 @@ TEST_P(ReadInteWithIndexTest, TestBitmapPushDownWithMultiStripes) {
 }
 
 TEST_P(ReadInteWithIndexTest, TestWithBitmapAndBsiAndBloomFilterIndex) {
-    auto [file_format, enable_prefetch] = GetParam();
-    std::string path = GetDataDir() + "/" + file_format +
-                       "/append_with_bsi_bitmap_bloomfilter.db/append_with_bsi_bitmap_bloomfilter/";
-    std::string file_name;
-    if (file_format == "orc") {
-        file_name = "data-d77394c5-8f82-4fca-93ca-d4da892e7f4f-0.orc";
-    } else if (file_format == "parquet") {
-        file_name = "data-15f89029-c10f-437f-b819-338c3e8ab2ee-0.parquet";
-    }
-
     std::vector<DataField> read_fields = {
         SpecialFields::ValueKind(),
         DataField(0, arrow::field("f0", arrow::utf8())),
@@ -2250,24 +2533,44 @@ TEST_P(ReadInteWithIndexTest, TestWithBitmapAndBsiAndBloomFilterIndex) {
         DataField(4, arrow::field("f4", arrow::timestamp(arrow::TimeUnit::NANO)))};
     std::shared_ptr<arrow::DataType> arrow_data_type =
         DataField::ConvertDataFieldsToArrowStructType(read_fields);
-    auto data_file_meta = std::make_shared<DataFileMeta>(
-        file_name, /*file_size=*/932,
-        /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
-        /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
-        /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
-        /*max_sequence_number=*/7, /*schema_id=*/0,
-        /*level=*/0,
-        /*extra_files=*/
-        std::vector<std::optional<std::string>>({file_name + ".index"}),
-        /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
-        /*embedded_index=*/nullptr, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
-        /*column_max_sequence_numbers=*/std::nullopt);
-    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
-                                   /*bucket_path=*/path + "bucket-0/", {data_file_meta});
-    ASSERT_OK_AND_ASSIGN(auto split,
-                         builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    std::string path;
+    std::shared_ptr<DataSplitImpl> split;
+    if (data_source_ != IndexDataSource::Java) {
+        ASSERT_NO_FATAL_FAILURE(
+            WriteCppTable(read_fields, kNumericData,
+                          {{"file-index.bsi.columns", "f1,f2,f4"},
+                           {"file-index.bloom-filter.columns", "f0,f1,f2,f3,f4"},
+                           {"file-index.bitmap.columns", "f0,f1,f2,f4"}},
+                          &path, &split));
+    } else {
+        path = GetDataDir() + "/" + file_format_ +
+               "/append_with_bsi_bitmap_bloomfilter.db/append_with_bsi_bitmap_bloomfilter/";
+        std::string file_name;
+        if (file_format_ == "orc") {
+            file_name = "data-d77394c5-8f82-4fca-93ca-d4da892e7f4f-0.orc";
+        } else if (file_format_ == "parquet") {
+            file_name = "data-15f89029-c10f-437f-b819-338c3e8ab2ee-0.parquet";
+        }
+        auto data_file_meta = std::make_shared<DataFileMeta>(
+            file_name, /*file_size=*/932,
+            /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
+            /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+            /*max_sequence_number=*/7, /*schema_id=*/0,
+            /*level=*/0,
+            /*extra_files=*/
+            std::vector<std::optional<std::string>>({file_name + ".index"}),
+            /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+            /*embedded_index=*/nullptr, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
+        DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                       /*bucket_path=*/path + "bucket-0/", {data_file_meta});
+        ASSERT_OK_AND_ASSIGN(
+            split, builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    }
 
     CheckResultForBsi(path, arrow_data_type, split);
     {
@@ -2312,17 +2615,6 @@ TEST_P(ReadInteWithIndexTest, TestWithBitmapAndBsiAndBloomFilterIndex) {
 }
 
 TEST_P(ReadInteWithIndexTest, TestWithIndexWithoutRegistered) {
-    auto [file_format, enable_prefetch] = GetParam();
-    std::string path = GetDataDir() + "/" + file_format +
-                       "/append_with_bsi_bitmap_bloomfilter.db/append_with_bsi_bitmap_bloomfilter/";
-    std::string file_name;
-    if (file_format == "orc") {
-        file_name = "data-d77394c5-8f82-4fca-93ca-d4da892e7f4f-0.orc";
-    } else if (file_format == "parquet") {
-        file_name = "data-15f89029-c10f-437f-b819-338c3e8ab2ee-0.parquet";
-    }
-
-    auto factory_creator = FactoryCreator::GetInstance();
     std::vector<DataField> read_fields = {
         SpecialFields::ValueKind(),
         DataField(0, arrow::field("f0", arrow::utf8())),
@@ -2332,25 +2624,47 @@ TEST_P(ReadInteWithIndexTest, TestWithIndexWithoutRegistered) {
         DataField(4, arrow::field("f4", arrow::timestamp(arrow::TimeUnit::NANO)))};
     std::shared_ptr<arrow::DataType> arrow_data_type =
         DataField::ConvertDataFieldsToArrowStructType(read_fields);
-    // index file contains bitmap & bsi & bloomfilter
-    auto data_file_meta = std::make_shared<DataFileMeta>(
-        file_name, /*file_size=*/932,
-        /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
-        /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
-        /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
-        /*max_sequence_number=*/7, /*schema_id=*/0,
-        /*level=*/0,
-        /*extra_files=*/
-        std::vector<std::optional<std::string>>({file_name + ".index"}),
-        /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
-        /*embedded_index=*/nullptr, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
-        /*column_max_sequence_numbers=*/std::nullopt);
-    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
-                                   /*bucket_path=*/path + "bucket-0/", {data_file_meta});
-    ASSERT_OK_AND_ASSIGN(auto split,
-                         builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    auto factory_creator = FactoryCreator::GetInstance();
+    std::string path;
+    std::shared_ptr<DataSplitImpl> split;
+    if (data_source_ != IndexDataSource::Java) {
+        ASSERT_NO_FATAL_FAILURE(
+            WriteCppTable(read_fields, kNumericData,
+                          {{"file-index.bsi.columns", "f1,f2,f4"},
+                           {"file-index.bloom-filter.columns", "f0,f1,f2,f3,f4"},
+                           {"file-index.bitmap.columns", "f0,f1,f2,f4"}},
+                          &path, &split));
+    } else {
+        path = GetDataDir() + "/" + file_format_ +
+               "/append_with_bsi_bitmap_bloomfilter.db/append_with_bsi_bitmap_bloomfilter/";
+        std::string file_name;
+        if (file_format_ == "orc") {
+            file_name = "data-d77394c5-8f82-4fca-93ca-d4da892e7f4f-0.orc";
+        } else if (file_format_ == "parquet") {
+            file_name = "data-15f89029-c10f-437f-b819-338c3e8ab2ee-0.parquet";
+        }
+
+        // index file contains bitmap & bsi & bloomfilter
+        auto data_file_meta = std::make_shared<DataFileMeta>(
+            file_name, /*file_size=*/932,
+            /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
+            /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+            /*max_sequence_number=*/7, /*schema_id=*/0,
+            /*level=*/0,
+            /*extra_files=*/
+            std::vector<std::optional<std::string>>({file_name + ".index"}),
+            /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+            /*embedded_index=*/nullptr, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
+        DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                       /*bucket_path=*/path + "bucket-0/", {data_file_meta});
+        ASSERT_OK_AND_ASSIGN(
+            split, builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    }
     {
         // only bitmap is registered
         factory_creator->TEST_Unregister("bsi");
@@ -2451,16 +2765,6 @@ TEST_P(ReadInteWithIndexTest, TestWithIndexWithoutRegistered) {
 }
 
 TEST_P(ReadInteWithIndexTest, TestRangeBitmapIndex) {
-    auto [file_format, enable_prefetch] = GetParam();
-    std::string path =
-        GetDataDir() + file_format + "/append_with_rangebitmap.db/append_with_rangebitmap/";
-    std::string file_name;
-    if (file_format == "orc") {
-        file_name = "data-5759403d-17fc-4031-b5bb-5e22b02fdb3b-0.orc";
-    } else if (file_format == "parquet") {
-        file_name = "data-2fb852e2-e4b5-4807-bf92-04401ed10560-0.parquet";
-    }
-
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::int32())),
                                           DataField(1, arrow::field("f1", arrow::int64())),
@@ -2470,43 +2774,48 @@ TEST_P(ReadInteWithIndexTest, TestRangeBitmapIndex) {
                                           DataField(5, arrow::field("f5", arrow::utf8()))};
     std::shared_ptr<arrow::DataType> arrow_data_type =
         DataField::ConvertDataFieldsToArrowStructType(read_fields);
+    std::string path;
+    std::shared_ptr<DataSplitImpl> split;
+    if (data_source_ != IndexDataSource::Java) {
+        ASSERT_NO_FATAL_FAILURE(
+            WriteCppTable(read_fields, kRangeBitmapData,
+                          {{"file-index.range-bitmap.columns", "f0,f1,f2,f3,f4"}}, &path, &split));
+    } else {
+        path = GetDataDir() + file_format_ + "/append_with_rangebitmap.db/append_with_rangebitmap/";
+        std::string file_name;
+        if (file_format_ == "orc") {
+            file_name = "data-5759403d-17fc-4031-b5bb-5e22b02fdb3b-0.orc";
+        } else if (file_format_ == "parquet") {
+            file_name = "data-2fb852e2-e4b5-4807-bf92-04401ed10560-0.parquet";
+        }
+
+        auto data_file_meta = std::make_shared<DataFileMeta>(
+            file_name, /*file_size=*/1288,
+            /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
+            /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+            /*max_sequence_number=*/7, /*schema_id=*/0,
+            /*level=*/0,
+            /*extra_files=*/
+            std::vector<std::optional<std::string>>({file_name + ".index"}),
+            /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+            /*embedded_index=*/nullptr, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
 
-    auto data_file_meta = std::make_shared<DataFileMeta>(
-        file_name, /*file_size=*/1288,
-        /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
-        /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
-        /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
-        /*max_sequence_number=*/7, /*schema_id=*/0,
-        /*level=*/0,
-        /*extra_files=*/
-        std::vector<std::optional<std::string>>({file_name + ".index"}),
-        /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
-        /*embedded_index=*/nullptr, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
-        /*column_max_sequence_numbers=*/std::nullopt);
-
-    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
-                                   /*bucket_path=*/path + "bucket-0/", {data_file_meta});
-    ASSERT_OK_AND_ASSIGN(auto split,
-                         builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+        DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                       /*bucket_path=*/path + "bucket-0/", {data_file_meta});
+        ASSERT_OK_AND_ASSIGN(
+            split, builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    }
 
     // Run comprehensive range bitmap index tests
     CheckResultForRangeBitmap(path, arrow_data_type, split);
 }
 
 TEST_P(ReadInteWithIndexTest, TestRangeBitmapIndexMultiChunk) {
-    auto [file_format, enable_prefetch] = GetParam();
-    std::string path =
-        GetDataDir() + file_format +
-        "/append_with_rangebitmap_multi_chunk.db/append_with_rangebitmap_multi_chunk/";
-    std::string file_name;
-    if (file_format == "orc") {
-        file_name = "data-b6b64e1e-e3d2-4f08-9a36-726a96cde1be-0.orc";
-    } else if (file_format == "parquet") {
-        file_name = "data-e4673af1-afcb-4b84-b69a-ae472ba517f2-0.parquet";
-    }
-
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::int32())),
                                           DataField(1, arrow::field("f1", arrow::int64())),
@@ -2516,42 +2825,55 @@ TEST_P(ReadInteWithIndexTest, TestRangeBitmapIndexMultiChunk) {
                                           DataField(5, arrow::field("f5", arrow::utf8()))};
     std::shared_ptr<arrow::DataType> arrow_data_type =
         DataField::ConvertDataFieldsToArrowStructType(read_fields);
+    std::string path;
+    std::shared_ptr<DataSplitImpl> split;
+    if (data_source_ != IndexDataSource::Java) {
+        ASSERT_NO_FATAL_FAILURE(
+            WriteCppTable(read_fields, kRangeBitmapData,
+                          {{"file-index.range-bitmap.columns", "f0,f1,f2,f3,f4"},
+                           {"file-index.range-bitmap.f0.chunk-size", "16B"},
+                           {"file-index.range-bitmap.f1.chunk-size", "16B"},
+                           {"file-index.range-bitmap.f2.chunk-size", "16B"},
+                           {"file-index.range-bitmap.f3.chunk-size", "16B"},
+                           {"file-index.range-bitmap.f4.chunk-size", "16B"}},
+                          &path, &split));
+    } else {
+        path = GetDataDir() + file_format_ +
+               "/append_with_rangebitmap_multi_chunk.db/append_with_rangebitmap_multi_chunk/";
+        std::string file_name;
+        if (file_format_ == "orc") {
+            file_name = "data-b6b64e1e-e3d2-4f08-9a36-726a96cde1be-0.orc";
+        } else if (file_format_ == "parquet") {
+            file_name = "data-e4673af1-afcb-4b84-b69a-ae472ba517f2-0.parquet";
+        }
+
+        auto data_file_meta = std::make_shared<DataFileMeta>(
+            file_name, /*file_size=*/1413,
+            /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
+            /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+            /*max_sequence_number=*/7, /*schema_id=*/0,
+            /*level=*/0,
+            /*extra_files=*/
+            std::vector<std::optional<std::string>>({file_name + ".index"}),
+            /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+            /*embedded_index=*/nullptr, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
 
-    auto data_file_meta = std::make_shared<DataFileMeta>(
-        file_name, /*file_size=*/1413,
-        /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
-        /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
-        /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
-        /*max_sequence_number=*/7, /*schema_id=*/0,
-        /*level=*/0,
-        /*extra_files=*/
-        std::vector<std::optional<std::string>>({file_name + ".index"}),
-        /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
-        /*embedded_index=*/nullptr, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
-        /*column_max_sequence_numbers=*/std::nullopt);
-
-    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
-                                   /*bucket_path=*/path + "bucket-0/", {data_file_meta});
-    ASSERT_OK_AND_ASSIGN(auto split,
-                         builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+        DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                       /*bucket_path=*/path + "bucket-0/", {data_file_meta});
+        ASSERT_OK_AND_ASSIGN(
+            split, builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    }
 
     // Run range bitmap index tests with multi-chunk test data
     CheckResultForRangeBitmap(path, arrow_data_type, split);
 }
 
 TEST_P(ReadInteWithIndexTest, TestWithIOException) {
-    auto [file_format, enable_prefetch] = GetParam();
-    std::string path = GetDataDir() + "/" + file_format +
-                       "/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/";
-    std::string file_name;
-    if (file_format == "orc") {
-        file_name = "data-414509f5-e40c-4245-b992-bbf486778ac9-0.orc";
-    } else if (file_format == "parquet") {
-        file_name = "data-783929b2-49d4-4006-a898-194a62e3278d-0.parquet";
-    }
-
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),
@@ -2559,25 +2881,41 @@ TEST_P(ReadInteWithIndexTest, TestWithIOException) {
                                           DataField(3, arrow::field("f3", arrow::float64()))};
     std::shared_ptr<arrow::DataType> arrow_data_type =
         DataField::ConvertDataFieldsToArrowStructType(read_fields);
+    std::string path;
+    std::shared_ptr<DataSplitImpl> split;
+    if (data_source_ != IndexDataSource::Java) {
+        ASSERT_NO_FATAL_FAILURE(WriteCppTable(
+            read_fields, kBitmapData, {{"file-index.bitmap.columns", "f0,f1,f2"}}, &path, &split));
+    } else {
+        path = GetDataDir() + "/" + file_format_ +
+               "/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/";
+        std::string file_name;
+        if (file_format_ == "orc") {
+            file_name = "data-414509f5-e40c-4245-b992-bbf486778ac9-0.orc";
+        } else if (file_format_ == "parquet") {
+            file_name = "data-783929b2-49d4-4006-a898-194a62e3278d-0.parquet";
+        }
 
-    auto data_file_meta = std::make_shared<DataFileMeta>(
-        file_name, /*file_size=*/689,
-        /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
-        /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
-        /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
-        /*max_sequence_number=*/7, /*schema_id=*/0,
-        /*level=*/0,
-        /*extra_files=*/
-        std::vector<std::optional<std::string>>({file_name + ".index"}),
-        /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
-        /*embedded_index=*/nullptr, FileSource::Append(),
-        /*value_stats_cols=*/std::nullopt,
-        /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt,
-        /*column_max_sequence_numbers=*/std::nullopt);
-    DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
-                                   /*bucket_path=*/path + "bucket-0/", {data_file_meta});
-    ASSERT_OK_AND_ASSIGN(auto split,
-                         builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+        auto data_file_meta = std::make_shared<DataFileMeta>(
+            file_name, /*file_size=*/689,
+            /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(),
+            /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+            /*max_sequence_number=*/7, /*schema_id=*/0,
+            /*level=*/0,
+            /*extra_files=*/
+            std::vector<std::optional<std::string>>({file_name + ".index"}),
+            /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0,
+            /*embedded_index=*/nullptr, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt,
+            /*column_max_sequence_numbers=*/std::nullopt);
+        DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                       /*bucket_path=*/path + "bucket-0/", {data_file_meta});
+        ASSERT_OK_AND_ASSIGN(
+            split, builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build());
+    }
 
     auto predicate =
         PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING,
@@ -2599,7 +2937,7 @@ TEST_P(ReadInteWithIndexTest, TestWithIOException) {
         context_builder.AddOption("read.batch-size", "2")
             .AddOption("test.enable-adaptive-prefetch-strategy", "false")
             .SetPredicate(predicate);
-        if (enable_prefetch) {
+        if (enable_prefetch_) {
             context_builder.EnablePrefetch(true).SetPrefetchBatchCount(3);
         }
         ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());

From 15ab4b8ad52527e7c64911c4b4ea6e260b15e265 Mon Sep 17 00:00:00 2001
From: "jinli.zjw" <jinli.zjw@alibaba-inc.com>
Date: Thu, 22 Oct 2026 17:18:29 +0800
Subject: [PATCH 06/10] refactor(file-index): centralize payload output stream
 offsets

Move OffsetOutputStream into common IO and wrap V2 payload streams at the container layer.

Keep LuminaFileWriter ownership and close behavior unchanged, adapt direct Lumina writes, and add focused OffsetOutputStream unit tests.
---
 include/paimon/file_index/file_index_format.h |  2 +
 include/paimon/file_index/file_index_writer.h |  1 +
 src/paimon/CMakeLists.txt                     |  2 +
 .../file_index/file_index_format_v2.cpp       |  5 +-
 src/paimon/common/io/offset_output_stream.cpp | 53 +++++++++++
 src/paimon/common/io/offset_output_stream.h   | 52 +++++++++++
 .../common/io/offset_output_stream_test.cpp   | 92 +++++++++++++++++++
 .../indexer/lumina/lumina_file_index.cpp      | 40 +-------
 .../indexer/lumina/lumina_file_index_test.cpp |  7 +-
 9 files changed, 214 insertions(+), 40 deletions(-)
 create mode 100644 src/paimon/common/io/offset_output_stream.cpp
 create mode 100644 src/paimon/common/io/offset_output_stream.h
 create mode 100644 src/paimon/common/io/offset_output_stream_test.cpp

diff --git a/include/paimon/file_index/file_index_format.h b/include/paimon/file_index/file_index_format.h
index ec4e3f00d..591757aa7 100644
--- a/include/paimon/file_index/file_index_format.h
+++ b/include/paimon/file_index/file_index_format.h
@@ -85,6 +85,8 @@ class PAIMON_EXPORT FileIndexFormat::Writer {
  public:
     /// Writes one payload without closing the stream. Returns false for an absent index
     /// (without writing bytes), or true after writing a non-empty payload.
+    /// The borrowed stream reports positions relative to the start of this payload, starting at 0.
+    /// The callback must not retain the stream.
     using PayloadWriter = std::function<Result<bool>(OutputStream*)>;
     virtual ~Writer() = default;
 
diff --git a/include/paimon/file_index/file_index_writer.h b/include/paimon/file_index/file_index_writer.h
index 43028bf40..116cec8e1 100644
--- a/include/paimon/file_index/file_index_writer.h
+++ b/include/paimon/file_index/file_index_writer.h
@@ -59,6 +59,7 @@ class PAIMON_EXPORT FileIndexWriter {
 
     /// Writes one index payload to the caller-owned stream without closing it.
     /// The stream is borrowed for this call and must not be retained by the writer.
+    /// The supplied stream must report positions relative to the start of this index payload.
     /// The default implementation writes SerializedBytes(); streaming writers can override this
     /// method to avoid materializing the complete payload. Write failures must be propagated.
     /// @return true if a non-empty index payload was produced, false if there is no index, or an
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index ef7ecec15..0c6bcaa30 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -97,6 +97,7 @@ set(PAIMON_COMMON_SRCS
     common/io/data_output_stream.cpp
     common/io/memory_segment_output_stream.cpp
     common/io/offset_input_stream.cpp
+    common/io/offset_output_stream.cpp
     common/io/cache/cache.cpp
     common/io/cache/cache_key.cpp
     common/io/cache/cache_manager.cpp
@@ -645,6 +646,7 @@ if(PAIMON_BUILD_TESTS)
                     common/io/memory_segment_output_stream_test.cpp
                     common/io/cache_input_stream_test.cpp
                     common/io/offset_input_stream_test.cpp
+                    common/io/offset_output_stream_test.cpp
                     common/lookup/sort/sort_lookup_store_test.cpp
                     common/logging/logging_test.cpp
                     common/metrics/metrics_impl_test.cpp
diff --git a/src/paimon/common/file_index/file_index_format_v2.cpp b/src/paimon/common/file_index/file_index_format_v2.cpp
index eb3a9fa29..d4bd08eab 100644
--- a/src/paimon/common/file_index/file_index_format_v2.cpp
+++ b/src/paimon/common/file_index/file_index_format_v2.cpp
@@ -21,6 +21,7 @@
 #include <utility>
 
 #include "paimon/common/io/data_output_stream.h"
+#include "paimon/common/io/offset_output_stream.h"
 #include "paimon/common/utils/math.h"
 #include "paimon/fs/file_system.h"
 #include "paimon/io/data_input_stream.h"
@@ -62,7 +63,9 @@ Status FileIndexFormatV2::Writer::WriteIndex(const std::string& column, const st
     int64_t length = 0;
     if (payload) {
         PAIMON_ASSIGN_OR_RAISE(start, output_->GetPos());
-        PAIMON_ASSIGN_OR_RAISE(bool written, payload(output_.get()));
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<OffsetOutputStream> payload_output,
+                               OffsetOutputStream::Create(output_.get()));
+        PAIMON_ASSIGN_OR_RAISE(bool written, payload(payload_output.get()));
         PAIMON_ASSIGN_OR_RAISE(int64_t end, output_->GetPos());
         length = end - start;
         if (!written) {
diff --git a/src/paimon/common/io/offset_output_stream.cpp b/src/paimon/common/io/offset_output_stream.cpp
new file mode 100644
index 000000000..e5cd922f0
--- /dev/null
+++ b/src/paimon/common/io/offset_output_stream.cpp
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/common/io/offset_output_stream.h"
+
+#include "paimon/macros.h"
+
+namespace paimon {
+Result<std::unique_ptr<OffsetOutputStream>> OffsetOutputStream::Create(OutputStream* output) {
+    PAIMON_ASSIGN_OR_RAISE(int64_t start, output->GetPos());
+    return std::unique_ptr<OffsetOutputStream>(new OffsetOutputStream(output, start));
+}
+
+OffsetOutputStream::OffsetOutputStream(OutputStream* output, int64_t start)
+    : output_(output), start_(start) {}
+
+Result<int64_t> OffsetOutputStream::Write(const char* buffer, int64_t size) {
+    return output_->Write(buffer, size);
+}
+
+Result<int64_t> OffsetOutputStream::GetPos() const {
+    PAIMON_ASSIGN_OR_RAISE(int64_t position, output_->GetPos());
+    return position - start_;
+}
+
+Result<std::string> OffsetOutputStream::GetUri() const {
+    return output_->GetUri();
+}
+
+Status OffsetOutputStream::Flush() {
+    return output_->Flush();
+}
+
+Status OffsetOutputStream::Close() {
+    return Status::OK();
+}
+}  // namespace paimon
diff --git a/src/paimon/common/io/offset_output_stream.h b/src/paimon/common/io/offset_output_stream.h
new file mode 100644
index 000000000..3a2410bb1
--- /dev/null
+++ b/src/paimon/common/io/offset_output_stream.h
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <string>
+
+#include "paimon/fs/file_system.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+#include "paimon/visibility.h"
+
+namespace paimon {
+/// A borrowed output view reporting positions relative to the wrapped stream's position at
+/// creation. Flush forwards to the wrapped stream; Close leaves it open. The wrapped stream must
+/// outlive this view.
+class PAIMON_EXPORT OffsetOutputStream : public OutputStream {
+ public:
+    static Result<std::unique_ptr<OffsetOutputStream>> Create(OutputStream* output);
+    ~OffsetOutputStream() override = default;
+
+    Result<int64_t> Write(const char* buffer, int64_t size) override;
+    Result<int64_t> GetPos() const override;
+    Result<std::string> GetUri() const override;
+    Status Flush() override;
+    Status Close() override;
+
+ private:
+    OffsetOutputStream(OutputStream* output, int64_t start);
+
+    OutputStream* output_;
+    const int64_t start_;
+};
+}  // namespace paimon
diff --git a/src/paimon/common/io/offset_output_stream_test.cpp b/src/paimon/common/io/offset_output_stream_test.cpp
new file mode 100644
index 000000000..d4ef8c14a
--- /dev/null
+++ b/src/paimon/common/io/offset_output_stream_test.cpp
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/common/io/offset_output_stream.h"
+
+#include <memory>
+#include <string>
+
+#include "gtest/gtest.h"
+#include "paimon/common/io/byte_array_output_stream.h"
+#include "paimon/common/io/memory_segment_output_stream.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+TEST(OffsetOutputStreamTest, TestRelativePositionsAndWrites) {
+    auto pool = GetDefaultPool();
+    auto output =
+        std::make_unique<ByteArrayOutputStream>(std::make_unique<MemorySegmentOutputStream>(
+            MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool));
+    ASSERT_OK_AND_ASSIGN(int64_t written, output->Write("prefix", 6));
+    ASSERT_EQ(6, written);
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<OffsetOutputStream> first,
+                         OffsetOutputStream::Create(output.get()));
+    ASSERT_OK_AND_ASSIGN(int64_t position, first->GetPos());
+    ASSERT_EQ(0, position);
+    ASSERT_OK_AND_ASSIGN(written, first->Write("first", 5));
+    ASSERT_EQ(5, written);
+    ASSERT_OK_AND_ASSIGN(position, first->GetPos());
+    ASSERT_EQ(5, position);
+    ASSERT_OK_AND_ASSIGN(position, output->GetPos());
+    ASSERT_EQ(11, position);
+
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<OffsetOutputStream> second,
+                         OffsetOutputStream::Create(output.get()));
+    ASSERT_OK_AND_ASSIGN(position, second->GetPos());
+    ASSERT_EQ(0, position);
+    ASSERT_OK_AND_ASSIGN(written, second->Write("second", 6));
+    ASSERT_EQ(6, written);
+    ASSERT_OK_AND_ASSIGN(position, second->GetPos());
+    ASSERT_EQ(6, position);
+    ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> bytes, output->Finish(pool.get()));
+    ASSERT_EQ("prefixfirstsecond", std::string(bytes->data(), bytes->size()));
+}
+
+TEST(OffsetOutputStreamTest, TestFlushAndClose) {
+    auto dir = UniqueTestDirectory::Create();
+    auto fs = dir->GetFileSystem();
+    std::string path = dir->Str() + "/offset-output";
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<OutputStream> output,
+                         fs->Create(path, /*overwrite=*/false));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<OffsetOutputStream> offset,
+                         OffsetOutputStream::Create(output.get()));
+    ASSERT_OK_AND_ASSIGN(std::string uri, offset->GetUri());
+    ASSERT_OK_AND_ASSIGN(std::string wrapped_uri, output->GetUri());
+    ASSERT_EQ(wrapped_uri, uri);
+
+    ASSERT_OK_AND_ASSIGN(int64_t written, offset->Write("payload", 7));
+    ASSERT_EQ(7, written);
+    ASSERT_OK(offset->Flush());
+    std::string content;
+    ASSERT_OK(fs->ReadFile(path, &content));
+    ASSERT_EQ("payload", content);
+
+    ASSERT_OK(offset->Close());
+    ASSERT_OK_AND_ASSIGN(written, output->Write("suffix", 6));
+    ASSERT_EQ(6, written);
+    ASSERT_OK(output->Close());
+    ASSERT_OK(fs->ReadFile(path, &content));
+    ASSERT_EQ("payloadsuffix", content);
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/indexer/lumina/lumina_file_index.cpp b/src/paimon/indexer/lumina/lumina_file_index.cpp
index 7ad70102c..6db365aa7 100644
--- a/src/paimon/indexer/lumina/lumina_file_index.cpp
+++ b/src/paimon/indexer/lumina/lumina_file_index.cpp
@@ -39,40 +39,6 @@
 namespace paimon::lumina {
 namespace {
 
-/// Reports positions relative to the wrapped stream's position at creation.
-/// Flush forwards to the wrapped stream; Close leaves it open so the container can continue
-/// writing.
-class OffsetOutputStream : public OutputStream {
- public:
-    static Result<std::unique_ptr<OffsetOutputStream>> Create(OutputStream* output) {
-        PAIMON_ASSIGN_OR_RAISE(int64_t start, output->GetPos());
-        return std::unique_ptr<OffsetOutputStream>(new OffsetOutputStream(output, start));
-    }
-
-    Result<int64_t> Write(const char* buffer, int64_t size) override {
-        return output_->Write(buffer, size);
-    }
-    Result<int64_t> GetPos() const override {
-        PAIMON_ASSIGN_OR_RAISE(int64_t position, output_->GetPos());
-        return position - start_;
-    }
-    Result<std::string> GetUri() const override {
-        return output_->GetUri();
-    }
-    Status Flush() override {
-        return output_->Flush();
-    }
-    Status Close() override {
-        return Status::OK();
-    }
-
- private:
-    OffsetOutputStream(OutputStream* output, int64_t start) : output_(output), start_(start) {}
-
-    OutputStream* output_;
-    int64_t start_;
-};
-
 Result<std::shared_ptr<arrow::Schema>> ImportVectorSchema(::ArrowSchema* c_schema,
                                                           const std::string& owner) {
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> schema,
@@ -158,15 +124,15 @@ Result<bool> LuminaFileIndexWriter::WriteTo(OutputStream* output) const {
     if (accumulator_.IndexedCount() == 0) {
         return false;
     }
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OffsetOutputStream> offset_output,
-                           OffsetOutputStream::Create(output));
     std::unique_ptr<::lumina::extensions::experimental::CkptManager> checkpoint_manager;
     PAIMON_ASSIGN_OR_RAISE(
         std::unique_ptr<LuminaIndexBuildContext> build_context,
         accumulator_.Build(builder_options_, index_info_.dimension, !tag_fields_.empty(),
                            std::move(checkpoint_manager), pool_.get()));
 
-    auto file_writer = std::make_unique<LuminaFileWriter>(offset_output);
+    // Dump completes synchronously; ownership of the supplied payload stream stays with the caller.
+    std::shared_ptr<OutputStream> borrowed_output(output, [](OutputStream*) {});
+    auto file_writer = std::make_unique<LuminaFileWriter>(borrowed_output);
     PAIMON_RETURN_NOT_OK_FROM_LUMINA(
         build_context->builder.Dump(std::move(file_writer), ::lumina::api::IOOptions()));
     return true;
diff --git a/src/paimon/indexer/lumina/lumina_file_index_test.cpp b/src/paimon/indexer/lumina/lumina_file_index_test.cpp
index fe4558db2..a3ddd8e66 100644
--- a/src/paimon/indexer/lumina/lumina_file_index_test.cpp
+++ b/src/paimon/indexer/lumina/lumina_file_index_test.cpp
@@ -29,6 +29,7 @@
 #include "arrow/ipc/json_simple.h"
 #include "gtest/gtest.h"
 #include "paimon/common/io/byte_array_output_stream.h"
+#include "paimon/common/io/offset_output_stream.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/io/byte_array_input_stream.h"
 #include "paimon/memory/memory_pool.h"
@@ -155,13 +156,15 @@ TEST(LuminaFileIndexTest, WriteToUsesRelativeOffsetsAndKeepsStreamOpen) {
     ASSERT_OK(writer->AddBatch(&c_batch));
 
     // Write after a prefix so the payload starts at a nonzero stream position.
-    // Lumina must record its internal offsets relative to this payload start.
+    // The caller supplies a view so Lumina sees positions relative to the payload start.
     auto output =
         std::make_shared<ByteArrayOutputStream>(std::make_unique<MemorySegmentOutputStream>(
             MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool));
     ASSERT_OK_AND_ASSIGN(int64_t prefix_length, output->Write("prefix", 6));
     ASSERT_EQ(6, prefix_length);
-    ASSERT_OK_AND_ASSIGN(bool written, writer->WriteTo(output.get()));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<OffsetOutputStream> payload_output,
+                         OffsetOutputStream::Create(output.get()));
+    ASSERT_OK_AND_ASSIGN(bool written, writer->WriteTo(payload_output.get()));
     ASSERT_TRUE(written);
     ASSERT_OK_AND_ASSIGN(int64_t end, output->GetPos());
     int64_t length = end - prefix_length;

From c28d7fe4e61c73236dfbc9c4f61446ebe84eee0c Mon Sep 17 00:00:00 2001
From: "jinli.zjw" <jinli.zjw@alibaba-inc.com>
Date: Thu, 22 Oct 2026 17:36:13 +0800
Subject: [PATCH 07/10] test(file-index): fix option validation expectations

---
 src/paimon/core/core_options_test.cpp          | 4 +++-
 src/paimon/core/io/file_index_options_test.cpp | 4 ++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp
index c533de71a..a74349c0e 100644
--- a/src/paimon/core/core_options_test.cpp
+++ b/src/paimon/core/core_options_test.cpp
@@ -529,10 +529,12 @@ TEST(CoreOptionsTest, TestInvalidCase) {
     ASSERT_NOK_WITH_MSG(
         CoreOptions::FromMap({{Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "2147483648B"}}),
         "file index inline threshold");
-    for (const std::string version : {"0", "3", "-1", "invalid"}) {
+    for (const std::string version : {"0", "3", "-1"}) {
         ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"file-index.format.version", version}}),
                             "file-index.format.version must be 1 or 2");
     }
+    ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"file-index.format.version", "invalid"}}),
+                        "Invalid Config [file-index.format.version: invalid]");
     ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{Options::TARGET_FILE_ROW_NUM, "0"}}),
                         "target-file-row-num should be at least 1");
     ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{Options::BUCKET, "3.5"}}),
diff --git a/src/paimon/core/io/file_index_options_test.cpp b/src/paimon/core/io/file_index_options_test.cpp
index 3fa188c05..2c3bdc999 100644
--- a/src/paimon/core/io/file_index_options_test.cpp
+++ b/src/paimon/core/io/file_index_options_test.cpp
@@ -88,7 +88,7 @@ TEST(FileIndexOptionsTest, TestRewriteFieldNamesPreservesOptions) {
         {"file-index.bitmap.columns", "title"},
         {Options::FILE_INDEX_READ_ENABLED, "true"},
         {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1KB"},
-        {Options::FILE_FORMAT, "missing-format"}};
+        {Options::FILE_FORMAT, "parquet"}};
     const auto original = options;
     ASSERT_OK_AND_ASSIGN(auto rewritten, FileIndexOptions::RewriteFieldNames(
                                              options, {{"vec", "embedding"}, {"title", "text"}}));
@@ -100,7 +100,7 @@ TEST(FileIndexOptionsTest, TestRewriteFieldNamesPreservesOptions) {
     ASSERT_EQ(0, rewritten.count("file-index.lumina.vec.index.dimension"));
     ASSERT_EQ(0, rewritten.count("file-index.lumina.added.index.dimension"));
     ASSERT_EQ("true", rewritten.at(Options::FILE_INDEX_READ_ENABLED));
-    ASSERT_EQ("missing-format", rewritten.at(Options::FILE_FORMAT));
+    ASSERT_EQ("parquet", rewritten.at(Options::FILE_FORMAT));
     ASSERT_OK_AND_ASSIGN(FileIndexOptions parsed, FileIndexOptions::FromMap(rewritten));
     ASSERT_EQ(1024, parsed.InManifestThreshold());
     ASSERT_EQ(2, parsed.Definitions().size());

From 34e0e24c93b27898fdc36e4fed212c4abb73a551 Mon Sep 17 00:00:00 2001
From: "jinli.zjw" <jinli.zjw@alibaba-inc.com>
Date: Thu, 22 Oct 2026 18:18:24 +0800
Subject: [PATCH 08/10] refactor(file-index): clarify payload writers and
 simplify format tests

---
 include/paimon/file_index/file_index_format.h |   2 +-
 .../file_index/file_index_format_test.cpp     | 266 ++++++++----------
 .../file_index/file_index_format_v1.cpp       |   6 +-
 .../common/file_index/file_index_format_v1.h  |   2 +-
 .../file_index/file_index_format_v2.cpp       |   6 +-
 .../common/file_index/file_index_format_v2.h  |   2 +-
 6 files changed, 120 insertions(+), 164 deletions(-)

diff --git a/include/paimon/file_index/file_index_format.h b/include/paimon/file_index/file_index_format.h
index 591757aa7..3777df0a6 100644
--- a/include/paimon/file_index/file_index_format.h
+++ b/include/paimon/file_index/file_index_format.h
@@ -98,7 +98,7 @@ class PAIMON_EXPORT FileIndexFormat::Writer {
     /// A true result indicates that a non-empty payload was written.
     /// Call Finish() after the last index. Payload failures are terminal.
     virtual Status WriteIndex(const std::string& column_name, const std::string& index_type,
-                              const PayloadWriter& payload) = 0;
+                              const PayloadWriter& payload_writer) = 0;
 
     /// Completes the container: writes V1's header and buffered bodies, or V2's footer and trailer.
     virtual Status Finish() = 0;
diff --git a/src/paimon/common/file_index/file_index_format_test.cpp b/src/paimon/common/file_index/file_index_format_test.cpp
index fbd82fbb3..e9ea6cf56 100644
--- a/src/paimon/common/file_index/file_index_format_test.cpp
+++ b/src/paimon/common/file_index/file_index_format_test.cpp
@@ -31,7 +31,6 @@
 #include "paimon/common/file_index/bloomfilter/bloom_filter_file_index.h"
 #include "paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h"
 #include "paimon/common/file_index/empty/empty_file_index_reader.h"
-#include "paimon/common/file_index/file_index_format_utils.h"
 #include "paimon/common/file_index/rangebitmap/range_bitmap_file_index.h"
 #include "paimon/common/io/byte_array_output_stream.h"
 #include "paimon/common/io/data_output_stream.h"
@@ -42,7 +41,6 @@
 #include "paimon/file_index/scored_file_index_result.h"
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/io/byte_array_input_stream.h"
-#include "paimon/io/data_input_stream.h"
 #include "paimon/memory/bytes.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/predicate/literal.h"
@@ -53,13 +51,104 @@
 
 namespace paimon::test {
 
+namespace {
+
+std::shared_ptr<ByteArrayOutputStream> NewIndexOutput(const std::shared_ptr<MemoryPool>& pool) {
+    return std::make_shared<ByteArrayOutputStream>(std::make_unique<MemorySegmentOutputStream>(
+        MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool));
+}
+
+// Generated with FileIndexFormatV2.Writer from Java Paimon.
+// Contains an empty index, a two-byte unknown index, and a present zero-byte index.
+const std::vector<uint8_t> kJavaV2Golden = {
+    0,   5,   78,  78,  208, 26, 53, 174, 0,  0,  0,   2,   42,  43,  0,   0,   0,   1,
+    0,   2,   99,  49,  0,   0,  0,  2,   0,  5,  101, 109, 112, 116, 121, 255, 255, 255,
+    255, 255, 255, 255, 255, 0,  0,  0,   0,  0,  0,   0,   0,   0,   7,   117, 110, 107,
+    110, 111, 119, 110, 0,   0,  0,  0,   0,  0,  0,   12,  0,   0,   0,   0,   0,   0,
+    0,   2,   0,   0,   0,   60, 80, 65,  70, 73, 68,  88,  48,  50,
+};
+
+struct SparseIndexState {
+    std::map<int64_t, std::string> segments;
+    int64_t length = 0;
+    bool discard = false;
+};
+
+class SparseIndexOutput : public MockOutputStream {
+ public:
+    explicit SparseIndexOutput(const std::shared_ptr<SparseIndexState>& state) : state_(state) {}
+    Result<int64_t> Write(const char* buffer, int64_t size) override {
+        if (!state_->discard) {
+            state_->segments[state_->length] = std::string(buffer, size);
+        }
+        state_->length += size;
+        return size;
+    }
+    Result<int64_t> GetPos() const override {
+        return state_->length;
+    }
+
+ private:
+    std::shared_ptr<SparseIndexState> state_;
+};
+
+class SparseIndexInput : public MockInputStream {
+ public:
+    explicit SparseIndexInput(const std::shared_ptr<SparseIndexState>& state) : state_(state) {}
+    Status Seek(int64_t offset, SeekOrigin origin) override {
+        if (origin == FS_SEEK_CUR) {
+            offset += position_;
+        }
+        if (origin == FS_SEEK_END) {
+            offset += state_->length;
+        }
+        if (offset < 0 || offset > state_->length) {
+            return Status::Invalid("Invalid seek");
+        }
+        position_ = offset;
+        return Status::OK();
+    }
+    Result<int64_t> GetPos() const override {
+        return position_;
+    }
+    Result<int64_t> Length() const override {
+        return state_->length;
+    }
+    Result<int64_t> Read(char* buffer, int64_t size) override {
+        PAIMON_ASSIGN_OR_RAISE(int64_t read, Read(buffer, size, position_));
+        position_ += read;
+        return read;
+    }
+    Result<int64_t> Read(char* buffer, int64_t size, int64_t offset) override {
+        if (offset < 0 || size < 0 || offset > state_->length) {
+            return Status::Invalid("Invalid read");
+        }
+        size = std::min(size, state_->length - offset);
+        std::memset(buffer, 0, size);
+        for (const auto& [start, bytes] : state_->segments) {
+            int64_t end = start + bytes.size();
+            int64_t from = std::max(start, offset);
+            int64_t to = std::min(end, offset + size);
+            if (from < to) {
+                std::memcpy(buffer + from - offset, bytes.data() + from - start, to - from);
+            }
+        }
+        return size;
+    }
+
+ private:
+    std::shared_ptr<SparseIndexState> state_;
+    int64_t position_ = 0;
+};
+}  // namespace
+
 class FileIndexFormatTest : public ::testing::Test {
  public:
     Status WriteIndexes(const FileIndexFormat::ColumnIndexes& indexes,
                         FileIndexFormat::Writer* writer) const {
         for (const auto& [column, entries] : indexes) {
             for (const auto& [type, bytes] : entries) {
-                FileIndexFormat::Writer::PayloadWriter payload =
+                FileIndexFormat::Writer::PayloadWriter payload_writer =
                     [bytes = bytes](OutputStream* output) -> Result<bool> {
                     if (!bytes || bytes->size() == 0) {
                         return false;
@@ -67,7 +156,7 @@ class FileIndexFormatTest : public ::testing::Test {
                     PAIMON_RETURN_NOT_OK(DataOutputStream(output).WriteBytes(bytes));
                     return true;
                 };
-                PAIMON_RETURN_NOT_OK(writer->WriteIndex(column, type, payload));
+                PAIMON_RETURN_NOT_OK(writer->WriteIndex(column, type, payload_writer));
             }
         }
         return Status::OK();
@@ -900,97 +989,6 @@ TEST_F(FileIndexFormatTest, TestBitmapIndexWithTimestamp) {
     check_nano("ts_tz_nano");
 }
 
-namespace {
-
-std::shared_ptr<ByteArrayOutputStream> NewIndexOutput(const std::shared_ptr<MemoryPool>& pool) {
-    return std::make_shared<ByteArrayOutputStream>(std::make_unique<MemorySegmentOutputStream>(
-        MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool));
-}
-
-// Generated with FileIndexFormatV2.Writer from Java Paimon.
-// Contains an empty index, a two-byte unknown index, and a present zero-byte index.
-const std::vector<uint8_t> kJavaV2Golden = {
-    0,   5,   78,  78,  208, 26, 53, 174, 0,  0,  0,   2,   42,  43,  0,   0,   0,   1,
-    0,   2,   99,  49,  0,   0,  0,  2,   0,  5,  101, 109, 112, 116, 121, 255, 255, 255,
-    255, 255, 255, 255, 255, 0,  0,  0,   0,  0,  0,   0,   0,   0,   7,   117, 110, 107,
-    110, 111, 119, 110, 0,   0,  0,  0,   0,  0,  0,   12,  0,   0,   0,   0,   0,   0,
-    0,   2,   0,   0,   0,   60, 80, 65,  70, 73, 68,  88,  48,  50,
-};
-
-struct SparseIndexState {
-    std::map<int64_t, std::string> segments;
-    int64_t length = 0;
-    bool discard = false;
-};
-
-class SparseIndexOutput : public MockOutputStream {
- public:
-    explicit SparseIndexOutput(const std::shared_ptr<SparseIndexState>& state) : state_(state) {}
-    Result<int64_t> Write(const char* buffer, int64_t size) override {
-        if (!state_->discard) {
-            state_->segments[state_->length] = std::string(buffer, size);
-        }
-        state_->length += size;
-        return size;
-    }
-    Result<int64_t> GetPos() const override {
-        return state_->length;
-    }
-
- private:
-    std::shared_ptr<SparseIndexState> state_;
-};
-
-class SparseIndexInput : public MockInputStream {
- public:
-    explicit SparseIndexInput(const std::shared_ptr<SparseIndexState>& state) : state_(state) {}
-    Status Seek(int64_t offset, SeekOrigin origin) override {
-        if (origin == FS_SEEK_CUR) {
-            offset += position_;
-        }
-        if (origin == FS_SEEK_END) {
-            offset += state_->length;
-        }
-        if (offset < 0 || offset > state_->length) {
-            return Status::Invalid("Invalid seek");
-        }
-        position_ = offset;
-        return Status::OK();
-    }
-    Result<int64_t> GetPos() const override {
-        return position_;
-    }
-    Result<int64_t> Length() const override {
-        return state_->length;
-    }
-    Result<int64_t> Read(char* buffer, int64_t size) override {
-        PAIMON_ASSIGN_OR_RAISE(int64_t read, Read(buffer, size, position_));
-        position_ += read;
-        return read;
-    }
-    Result<int64_t> Read(char* buffer, int64_t size, int64_t offset) override {
-        if (offset < 0 || size < 0 || offset > state_->length) {
-            return Status::Invalid("Invalid read");
-        }
-        size = std::min(size, state_->length - offset);
-        std::memset(buffer, 0, size);
-        for (const auto& [start, bytes] : state_->segments) {
-            int64_t end = start + bytes.size();
-            int64_t from = std::max(start, offset);
-            int64_t to = std::min(end, offset + size);
-            if (from < to) {
-                std::memcpy(buffer + from - offset, bytes.data() + from - start, to - from);
-            }
-        }
-        return size;
-    }
-
- private:
-    std::shared_ptr<SparseIndexState> state_;
-    int64_t position_ = 0;
-};
-}  // namespace
-
 TEST_F(FileIndexFormatTest, TestV2JavaGoldenBytes) {
     FileIndexFormat::ColumnIndexes indexes;
     indexes["c1"]["empty"] = nullptr;
@@ -1016,33 +1014,12 @@ TEST_F(FileIndexFormatTest, TestV2JavaGoldenBytes) {
     ASSERT_TRUE(dynamic_cast<EmptyFileIndexReader*>(readers[0].get()));
 }
 
-TEST_F(FileIndexFormatTest, TestV2InvalidContainers) {
-    auto check = [this](const std::vector<uint8_t>& bytes) {
-        auto input = std::make_shared<ByteArrayInputStream>(
-            bytes.empty() ? "" : reinterpret_cast<const char*>(bytes.data()), bytes.size());
-        ASSERT_NOK(FileIndexFormat::CreateReader(input, pool_, /*options=*/{}));
-    };
-    for (size_t size : {static_cast<size_t>(0), static_cast<size_t>(11), static_cast<size_t>(24),
-                        kJavaV2Golden.size() - 1}) {
-        check(std::vector<uint8_t>(kJavaV2Golden.begin(), kJavaV2Golden.begin() + size));
-    }
-    for (const auto& [offset, value] : std::vector<std::pair<size_t, uint8_t>>{
-             {11, 3},
-             {kJavaV2Golden.size() - 1, 0},
-             {kJavaV2Golden.size() - 12, 0x7f},     // Impossible footer length.
-             {kJavaV2Golden.size() - 12, 0xff}}) {  // Negative footer length.
-        auto bytes = kJavaV2Golden;
-        bytes[offset] = value;
-        check(bytes);
-    }
-}
-
 TEST_F(FileIndexFormatTest, TestV2StreamingLargeContainer) {
     auto state = std::make_shared<SparseIndexState>();
     auto output = std::make_shared<SparseIndexOutput>(state);
     ASSERT_OK_AND_ASSIGN(auto writer,
                          FileIndexFormat::CreateWriter(output, FileIndexFormat::V_2, pool_));
-    // Stream a virtual >2 GiB payload in bounded chunks; the mock discards only this gap.
+    // Write a virtual 2 GiB payload in bounded chunks; the mock discards only this gap.
     const int64_t gap_length = int64_t{1} << 31;
     ASSERT_OK(writer->WriteIndex("gap", "unknown", [state](OutputStream* stream) -> Result<bool> {
         state->discard = true;
@@ -1070,6 +1047,7 @@ TEST_F(FileIndexFormatTest, TestV2StreamingLargeContainer) {
                                                   R"([[1], [2], [1]])")
             .ValueOrDie();
     auto batch = checked_pointer_cast<arrow::StructArray>(array);
+    // Build and stream the real indexes for [1, 2, 1] after the gap.
     for (const auto& [type, indexer] : indexers) {
         ASSERT_OK_AND_ASSIGN(auto index_writer,
                              indexer->CreateWriter(CreateArrowSchema(schema).get(), pool_));
@@ -1083,6 +1061,7 @@ TEST_F(FileIndexFormatTest, TestV2StreamingLargeContainer) {
     ASSERT_OK(writer->Finish());
     ASSERT_OK(writer->Close());
     ASSERT_GT(state->length, gap_length);
+    // Read the container and query each real index at an offset beyond INT32_MAX.
     auto input = std::make_shared<SparseIndexInput>(state);
     ASSERT_OK_AND_ASSIGN(auto reader, FileIndexFormat::CreateReader(input, pool_, /*options=*/{}));
     for (const auto& [type, indexer] : indexers) {
@@ -1102,33 +1081,17 @@ TEST_F(FileIndexFormatTest, TestV2StreamingLargeContainer) {
     }
 }
 
-TEST_F(FileIndexFormatTest, TestPluginRangeLimits) {
-    auto schema = arrow::schema({arrow::field("f0", arrow::int32())});
-    auto state = std::make_shared<SparseIndexState>();
-    state->length = int64_t{1} << 32;
-    auto input = std::make_shared<SparseIndexInput>(state);
-    BitmapFileIndex bitmap({});
-    RangeBitmapFileIndex range({});
-    const int64_t oversized = int64_t{1} << 31;
-    for (FileIndexer* indexer : std::vector<FileIndexer*>{&bitmap, &range}) {
-        auto c_schema = CreateArrowSchema(schema);
-        ASSERT_NOK_WITH_MSG(indexer->CreateReader(c_schema.get(), 0, oversized, input, pool_),
-                            "INT32_MAX");
-        if (c_schema->release) {
-            c_schema->release(c_schema.get());
-        }
-    }
-}
-
 TEST_F(FileIndexFormatTest, TestV2EmptyContainer) {
     auto output = NewIndexOutput(pool_);
     ASSERT_OK_AND_ASSIGN(auto writer,
                          FileIndexFormat::CreateWriter(output, FileIndexFormat::V_2, pool_));
     ASSERT_OK_AND_ASSIGN(int64_t prefix_length, output->GetPos());
+    // Prefix: 8-byte magic + 4-byte version.
     ASSERT_EQ(12, prefix_length);
     ASSERT_OK(writer->Finish());
     ASSERT_OK(writer->Close());
     ASSERT_OK_AND_ASSIGN(auto bytes, output->Finish(pool_.get()));
+    // Prefix (12) + empty footer's column count (4) + footer length (4) + tail magic (8).
     ASSERT_EQ(28, bytes->size());
     auto input = std::make_shared<ByteArrayInputStream>(bytes->data(), bytes->size());
     ASSERT_OK_AND_ASSIGN(auto reader, FileIndexFormat::CreateReader(input, pool_, /*options=*/{}));
@@ -1142,19 +1105,12 @@ TEST_F(FileIndexFormatTest, TestV1BuffersPayloadsUntilFinish) {
                          FileIndexFormat::CreateWriter(output, FileIndexFormat::V_1, pool_));
     ASSERT_OK(WriteIndexes(indexes, writer.get()));
     ASSERT_OK_AND_ASSIGN(int64_t before_finish, output->GetPos());
+    // V1 buffers payloads until Finish(), so the destination is still empty.
     ASSERT_EQ(0, before_finish);
     ASSERT_OK(writer->Finish());
+    ASSERT_OK_AND_ASSIGN(int64_t after_finish, output->GetPos());
+    ASSERT_GT(after_finish, 0);
     ASSERT_OK(writer->Close());
-    ASSERT_OK_AND_ASSIGN(auto actual, output->Finish(pool_.get()));
-    auto input = std::make_shared<ByteArrayInputStream>(actual->data(), actual->size());
-    DataInputStream in(input);
-    ASSERT_OK(in.Seek(12));
-    ASSERT_OK_AND_ASSIGN(int32_t head_length, in.ReadValue<int32_t>());
-    ASSERT_OK_AND_ASSIGN(FileIndexFormatUtils::FileIndexEntries entries,
-                         FileIndexFormatUtils::ReadFileIndexEntries(in, FileIndexFormat::V_1));
-    ASSERT_EQ(std::make_pair(int64_t{head_length}, int64_t{7}), entries.at("f0").at("bitmap"));
-    ASSERT_EQ(std::make_pair(int64_t{-1}, int64_t{0}), entries.at("f0").at("empty"));
-    ASSERT_EQ("payload", std::string(actual->data() + head_length, 7));
 }
 
 TEST_F(FileIndexFormatTest, TestAbsentPayloadMustNotWriteBytes) {
@@ -1162,17 +1118,17 @@ TEST_F(FileIndexFormatTest, TestAbsentPayloadMustNotWriteBytes) {
         SCOPED_TRACE(version);
         ASSERT_OK_AND_ASSIGN(auto writer,
                              FileIndexFormat::CreateWriter(NewIndexOutput(pool_), version, pool_));
-        ASSERT_NOK_WITH_MSG(writer->WriteIndex("f0", "bad",
-                                               [](OutputStream* output) -> Result<bool> {
-                                                   PAIMON_ASSIGN_OR_RAISE(int64_t written,
-                                                                          output->Write("x", 1));
-                                                   if (written != 1) {
-                                                       return Status::IOError("short write");
-                                                   }
-                                                   return false;
-                                               }),
+        // This callback violates the contract: it writes a byte, then reports no index payload.
+        auto payload_writer = [](OutputStream* output) -> Result<bool> {
+            PAIMON_ASSIGN_OR_RAISE(int64_t written, output->Write("x", 1));
+            if (written != 1) {
+                return Status::IOError("short write");
+            }
+            return false;
+        };
+        ASSERT_NOK_WITH_MSG(writer->WriteIndex("f0", "bad", payload_writer),
                             "Absent file index payload must not write bytes");
-        ASSERT_NOK(writer->Finish());
+        ASSERT_NOK_WITH_MSG(writer->Finish(), "File index writer is not writable");
         ASSERT_OK(writer->Close());
     }
 }
diff --git a/src/paimon/common/file_index/file_index_format_v1.cpp b/src/paimon/common/file_index/file_index_format_v1.cpp
index 25ce9be5d..ccd26987e 100644
--- a/src/paimon/common/file_index/file_index_format_v1.cpp
+++ b/src/paimon/common/file_index/file_index_format_v1.cpp
@@ -46,7 +46,7 @@ FileIndexFormatV1::Writer::Writer(const std::shared_ptr<OutputStream>& output,
 }
 
 Status FileIndexFormatV1::Writer::WriteIndex(const std::string& column, const std::string& type,
-                                             const PayloadWriter& payload) {
+                                             const PayloadWriter& payload_writer) {
     if (finished_ || closed_ || failed_) {
         return Status::Invalid("File index writer is not writable");
     }
@@ -54,12 +54,12 @@ Status FileIndexFormatV1::Writer::WriteIndex(const std::string& column, const st
         return Status::Invalid("Duplicate file index entry");
     }
     std::shared_ptr<Bytes> bytes;
-    if (payload) {
+    if (payload_writer) {
         auto output =
             std::make_shared<ByteArrayOutputStream>(std::make_unique<MemorySegmentOutputStream>(
                 MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_));
         failed_ = true;
-        PAIMON_ASSIGN_OR_RAISE(bool written, payload(output.get()));
+        PAIMON_ASSIGN_OR_RAISE(bool written, payload_writer(output.get()));
         PAIMON_ASSIGN_OR_RAISE(int64_t length, output->GetPos());
         if (!written && length != 0) {
             return Status::Invalid("Absent file index payload must not write bytes");
diff --git a/src/paimon/common/file_index/file_index_format_v1.h b/src/paimon/common/file_index/file_index_format_v1.h
index 96af5f68c..951f6570c 100644
--- a/src/paimon/common/file_index/file_index_format_v1.h
+++ b/src/paimon/common/file_index/file_index_format_v1.h
@@ -98,7 +98,7 @@ class FileIndexFormatV1::Writer : public FileIndexFormat::Writer {
  public:
     Writer(const std::shared_ptr<OutputStream>& output, const std::shared_ptr<MemoryPool>& pool);
     Status WriteIndex(const std::string& column, const std::string& type,
-                      const PayloadWriter& payload) override;
+                      const PayloadWriter& payload_writer) override;
     Status Finish() override;
     Status Close() override;
 
diff --git a/src/paimon/common/file_index/file_index_format_v2.cpp b/src/paimon/common/file_index/file_index_format_v2.cpp
index d4bd08eab..69a35251c 100644
--- a/src/paimon/common/file_index/file_index_format_v2.cpp
+++ b/src/paimon/common/file_index/file_index_format_v2.cpp
@@ -48,7 +48,7 @@ Result<std::unique_ptr<FileIndexFormatV2::Writer>> FileIndexFormatV2::Writer::Cr
 }
 
 Status FileIndexFormatV2::Writer::WriteIndex(const std::string& column, const std::string& type,
-                                             const PayloadWriter& payload) {
+                                             const PayloadWriter& payload_writer) {
     if (failed_ || finished_ || closed_) {
         return Status::Invalid("File index writer is not writable");
     }
@@ -61,11 +61,11 @@ Status FileIndexFormatV2::Writer::WriteIndex(const std::string& column, const st
     failed_ = true;
     int64_t start = FileIndexFormat::EMPTY_INDEX_FLAG;
     int64_t length = 0;
-    if (payload) {
+    if (payload_writer) {
         PAIMON_ASSIGN_OR_RAISE(start, output_->GetPos());
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<OffsetOutputStream> payload_output,
                                OffsetOutputStream::Create(output_.get()));
-        PAIMON_ASSIGN_OR_RAISE(bool written, payload(payload_output.get()));
+        PAIMON_ASSIGN_OR_RAISE(bool written, payload_writer(payload_output.get()));
         PAIMON_ASSIGN_OR_RAISE(int64_t end, output_->GetPos());
         length = end - start;
         if (!written) {
diff --git a/src/paimon/common/file_index/file_index_format_v2.h b/src/paimon/common/file_index/file_index_format_v2.h
index 11e621839..81a4567d5 100644
--- a/src/paimon/common/file_index/file_index_format_v2.h
+++ b/src/paimon/common/file_index/file_index_format_v2.h
@@ -94,7 +94,7 @@ class FileIndexFormatV2::Writer : public FileIndexFormat::Writer {
     static Result<std::unique_ptr<Writer>> Create(const std::shared_ptr<OutputStream>& output);
     Status Close() override;
     Status WriteIndex(const std::string& column, const std::string& type,
-                      const PayloadWriter& payload) override;
+                      const PayloadWriter& payload_writer) override;
     Status Finish() override;
 
  private:

From 1f99223781c3b82df95a6c8a8dcad5bab113a5c4 Mon Sep 17 00:00:00 2001
From: "jinli.zjw" <jinli.zjw@alibaba-inc.com>
Date: Thu, 22 Oct 2026 18:36:51 +0800
Subject: [PATCH 09/10] test(file-index): rename write and read integration
 tests

---
 test/inte/CMakeLists.txt                      |  2 +-
 ...> write_and_read_inte_with_index_test.cpp} | 44 +++++++++----------
 2 files changed, 23 insertions(+), 23 deletions(-)
 rename test/inte/{read_inte_with_index_test.cpp => write_and_read_inte_with_index_test.cpp} (99%)

diff --git a/test/inte/CMakeLists.txt b/test/inte/CMakeLists.txt
index 271a52c62..0f7a3f0d3 100644
--- a/test/inte/CMakeLists.txt
+++ b/test/inte/CMakeLists.txt
@@ -106,7 +106,7 @@ if(PAIMON_BUILD_TESTS)
                     test_utils_static
                     ${GTEST_LINK_TOOLCHAIN})
 
-    add_paimon_test(read_inte_with_index_test
+    add_paimon_test(write_and_read_inte_with_index_test
                     STATIC_LINK_LIBS
                     paimon_shared
                     ${TEST_STATIC_LINK_LIBS}
diff --git a/test/inte/read_inte_with_index_test.cpp b/test/inte/write_and_read_inte_with_index_test.cpp
similarity index 99%
rename from test/inte/read_inte_with_index_test.cpp
rename to test/inte/write_and_read_inte_with_index_test.cpp
index 8a7a2b3fd..955e068a0 100644
--- a/test/inte/read_inte_with_index_test.cpp
+++ b/test/inte/write_and_read_inte_with_index_test.cpp
@@ -114,7 +114,7 @@ constexpr char kRangeBitmapData[] = R"([
     [10, 600, 6.6, 66.66, 19732, "row7"]
 ])";
 
-class ReadInteWithIndexTest
+class WriteAndReadInteWithIndexTest
     : public testing::Test,
       public ::testing::WithParamInterface<std::tuple<std::string, bool, IndexDataSource>> {
  public:
@@ -1097,7 +1097,7 @@ class ReadInteWithIndexTest
     std::unique_ptr<UniqueTestDirectory> dir_;
 };
 
-std::vector<std::string> GetFileFormatsForReadInteWithIndexTest() {
+std::vector<std::string> GetFileFormatsForWriteAndReadInteWithIndexTest() {
     std::vector<std::string> values = {"parquet"};
 #ifdef PAIMON_ENABLE_ORC
     values.emplace_back("orc");
@@ -1106,12 +1106,12 @@ std::vector<std::string> GetFileFormatsForReadInteWithIndexTest() {
 }
 
 INSTANTIATE_TEST_SUITE_P(
-    FileFormatAndPrefetchAndDataSource, ReadInteWithIndexTest,
-    ::testing::Combine(::testing::ValuesIn(GetFileFormatsForReadInteWithIndexTest()),
+    FileFormatAndPrefetchAndDataSource, WriteAndReadInteWithIndexTest,
+    ::testing::Combine(::testing::ValuesIn(GetFileFormatsForWriteAndReadInteWithIndexTest()),
                        ::testing::Bool(),
                        ::testing::Values(IndexDataSource::CppV1, IndexDataSource::CppV2,
                                          IndexDataSource::Java)),
-    [](const ::testing::TestParamInfo<ReadInteWithIndexTest::ParamType>& info) {
+    [](const ::testing::TestParamInfo<WriteAndReadInteWithIndexTest::ParamType>& info) {
         IndexDataSource source = std::get<2>(info.param);
         std::string source_name = source == IndexDataSource::Java
                                       ? "Java"
@@ -1120,7 +1120,7 @@ INSTANTIATE_TEST_SUITE_P(
                source_name;
     });
 
-TEST_P(ReadInteWithIndexTest, TestSimple) {
+TEST_P(WriteAndReadInteWithIndexTest, TestSimple) {
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),
@@ -1201,7 +1201,7 @@ TEST_P(ReadInteWithIndexTest, TestSimple) {
     }
 }
 
-TEST_P(ReadInteWithIndexTest, TestReadWithLimits) {
+TEST_P(WriteAndReadInteWithIndexTest, TestReadWithLimits) {
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),
@@ -1288,7 +1288,7 @@ TEST_P(ReadInteWithIndexTest, TestReadWithLimits) {
     }
 }
 
-TEST_P(ReadInteWithIndexTest, TestEmbeddingBitmapIndex) {
+TEST_P(WriteAndReadInteWithIndexTest, TestEmbeddingBitmapIndex) {
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),
@@ -1363,7 +1363,7 @@ TEST_P(ReadInteWithIndexTest, TestEmbeddingBitmapIndex) {
     CheckResultForBitmapWithSingleRowGroup(path, arrow_data_type, split);
 }
 
-TEST_P(ReadInteWithIndexTest, TestBitmapWithV1) {
+TEST_P(WriteAndReadInteWithIndexTest, TestBitmapWithV1) {
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),
@@ -1437,7 +1437,7 @@ TEST_P(ReadInteWithIndexTest, TestBitmapWithV1) {
     CheckResultForBitmapWithSingleRowGroup(path, arrow_data_type, split);
 }
 
-TEST_P(ReadInteWithIndexTest, TestNoEmbeddingBitmapIndex) {
+TEST_P(WriteAndReadInteWithIndexTest, TestNoEmbeddingBitmapIndex) {
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),
@@ -1483,7 +1483,7 @@ TEST_P(ReadInteWithIndexTest, TestNoEmbeddingBitmapIndex) {
     CheckResultForBitmapWithSingleRowGroup(path, arrow_data_type, split);
 }
 
-TEST_P(ReadInteWithIndexTest, TestBitmapIndexWithLateMaterializing) {
+TEST_P(WriteAndReadInteWithIndexTest, TestBitmapIndexWithLateMaterializing) {
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),
@@ -1559,7 +1559,7 @@ TEST_P(ReadInteWithIndexTest, TestBitmapIndexWithLateMaterializing) {
     ASSERT_TRUE(result_array->Equals(*expected_array)) << result_array->ToString();
 }
 
-TEST_P(ReadInteWithIndexTest, TestNoEmbeddingBitmapIndexWithExternalPath) {
+TEST_P(WriteAndReadInteWithIndexTest, TestNoEmbeddingBitmapIndexWithExternalPath) {
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),
@@ -1616,7 +1616,7 @@ TEST_P(ReadInteWithIndexTest, TestNoEmbeddingBitmapIndexWithExternalPath) {
     CheckResultForBitmapWithSingleRowGroup(path, arrow_data_type, split);
 }
 
-TEST_P(ReadInteWithIndexTest, TestBitmapIndexWithDv) {
+TEST_P(WriteAndReadInteWithIndexTest, TestBitmapIndexWithDv) {
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("key", arrow::int32())),
                                           DataField(1, arrow::field("f0", arrow::utf8())),
@@ -1722,7 +1722,7 @@ TEST_P(ReadInteWithIndexTest, TestBitmapIndexWithDv) {
     }
 }
 
-TEST_P(ReadInteWithIndexTest, TestWithAlterTable) {
+TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) {
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(1, arrow::field("f1", arrow::int64())),
                                           DataField(0, arrow::field("f4", arrow::utf8())),
@@ -2118,7 +2118,7 @@ TEST_P(ReadInteWithIndexTest, TestWithAlterTable) {
     }
 }
 
-TEST_P(ReadInteWithIndexTest, TestWithBsiIndex) {
+TEST_P(WriteAndReadInteWithIndexTest, TestWithBsiIndex) {
     std::vector<DataField> read_fields = {
         SpecialFields::ValueKind(),
         DataField(0, arrow::field("f0", arrow::utf8())),
@@ -2185,7 +2185,7 @@ TEST_P(ReadInteWithIndexTest, TestWithBsiIndex) {
     CheckResultForBsi(path, arrow_data_type, {split});
 }
 
-TEST_P(ReadInteWithIndexTest, TestWithBloomFilterIndex) {
+TEST_P(WriteAndReadInteWithIndexTest, TestWithBloomFilterIndex) {
     std::vector<DataField> read_fields = {
         SpecialFields::ValueKind(),
         DataField(0, arrow::field("f0", arrow::utf8())),
@@ -2397,7 +2397,7 @@ TEST_P(ReadInteWithIndexTest, TestWithBloomFilterIndex) {
     }
 }
 
-TEST_P(ReadInteWithIndexTest, TestBitmapPushDownWithMultiStripes) {
+TEST_P(WriteAndReadInteWithIndexTest, TestBitmapPushDownWithMultiStripes) {
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),
@@ -2523,7 +2523,7 @@ TEST_P(ReadInteWithIndexTest, TestBitmapPushDownWithMultiStripes) {
     }
 }
 
-TEST_P(ReadInteWithIndexTest, TestWithBitmapAndBsiAndBloomFilterIndex) {
+TEST_P(WriteAndReadInteWithIndexTest, TestWithBitmapAndBsiAndBloomFilterIndex) {
     std::vector<DataField> read_fields = {
         SpecialFields::ValueKind(),
         DataField(0, arrow::field("f0", arrow::utf8())),
@@ -2614,7 +2614,7 @@ TEST_P(ReadInteWithIndexTest, TestWithBitmapAndBsiAndBloomFilterIndex) {
     }
 }
 
-TEST_P(ReadInteWithIndexTest, TestWithIndexWithoutRegistered) {
+TEST_P(WriteAndReadInteWithIndexTest, TestWithIndexWithoutRegistered) {
     std::vector<DataField> read_fields = {
         SpecialFields::ValueKind(),
         DataField(0, arrow::field("f0", arrow::utf8())),
@@ -2764,7 +2764,7 @@ TEST_P(ReadInteWithIndexTest, TestWithIndexWithoutRegistered) {
     }
 }
 
-TEST_P(ReadInteWithIndexTest, TestRangeBitmapIndex) {
+TEST_P(WriteAndReadInteWithIndexTest, TestRangeBitmapIndex) {
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::int32())),
                                           DataField(1, arrow::field("f1", arrow::int64())),
@@ -2815,7 +2815,7 @@ TEST_P(ReadInteWithIndexTest, TestRangeBitmapIndex) {
     CheckResultForRangeBitmap(path, arrow_data_type, split);
 }
 
-TEST_P(ReadInteWithIndexTest, TestRangeBitmapIndexMultiChunk) {
+TEST_P(WriteAndReadInteWithIndexTest, TestRangeBitmapIndexMultiChunk) {
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::int32())),
                                           DataField(1, arrow::field("f1", arrow::int64())),
@@ -2873,7 +2873,7 @@ TEST_P(ReadInteWithIndexTest, TestRangeBitmapIndexMultiChunk) {
     CheckResultForRangeBitmap(path, arrow_data_type, split);
 }
 
-TEST_P(ReadInteWithIndexTest, TestWithIOException) {
+TEST_P(WriteAndReadInteWithIndexTest, TestWithIOException) {
     std::vector<DataField> read_fields = {SpecialFields::ValueKind(),
                                           DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::int32())),

From 21b76f65d0fd293a0c60d4e9005fe464bbd900ff Mon Sep 17 00:00:00 2001
From: "jinli.zjw" <jinli.zjw@alibaba-inc.com>
Date: Fri, 23 Oct 2026 10:01:48 +0800
Subject: [PATCH 10/10] test(file-index): reserve capacity for integration test
 formats

---
 test/inte/write_and_read_inte_with_index_test.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/inte/write_and_read_inte_with_index_test.cpp b/test/inte/write_and_read_inte_with_index_test.cpp
index 955e068a0..efda3de03 100644
--- a/test/inte/write_and_read_inte_with_index_test.cpp
+++ b/test/inte/write_and_read_inte_with_index_test.cpp
@@ -1099,6 +1099,7 @@ class WriteAndReadInteWithIndexTest
 
 std::vector<std::string> GetFileFormatsForWriteAndReadInteWithIndexTest() {
     std::vector<std::string> values = {"parquet"};
+    values.reserve(10);
 #ifdef PAIMON_ENABLE_ORC
     values.emplace_back("orc");
 #endif