From 7c6fec5922004d76d58b1e829f0bbfc15aa9bd42 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 20 Oct 2026 14:58:24 +0800 Subject: [PATCH 01/19] refactor table --- docs/source/user_guide/format_table.rst | 28 +- include/paimon/catalog/catalog.h | 33 +- include/paimon/catalog/table.h | 47 ++- include/paimon/commit_context.h | 2 +- include/paimon/read_context.h | 2 +- include/paimon/scan_context.h | 2 +- include/paimon/table/format/format_table.h | 74 ++-- include/paimon/write_context.h | 2 +- src/paimon/CMakeLists.txt | 1 - src/paimon/core/catalog/catalog.cpp | 17 - src/paimon/core/catalog/catalog_utils.cpp | 20 - src/paimon/core/catalog/catalog_utils.h | 25 +- .../core/catalog/file_system_catalog.cpp | 30 +- src/paimon/core/catalog/file_system_catalog.h | 7 - .../core/catalog/file_system_catalog_test.cpp | 21 +- src/paimon/core/table/file_store_table.h | 65 +++ src/paimon/core/table/format/format_table.cpp | 195 ++++++++- .../core/table/format/format_table_loader.cpp | 87 ---- .../core/table/format/format_table_loader.h | 71 ---- .../core/table/format/format_table_test.cpp | 155 +++++++ src/paimon/core/table/table.cpp | 389 +++++++----------- src/paimon/core/table/table_test.cpp | 18 +- src/paimon/rest/rest_catalog.cpp | 37 +- src/paimon/rest/rest_catalog.h | 6 - src/paimon/rest/rest_catalog_test.cpp | 45 +- src/paimon/testing/mock/mock_catalog.h | 3 +- 26 files changed, 755 insertions(+), 627 deletions(-) create mode 100644 src/paimon/core/table/file_store_table.h delete mode 100644 src/paimon/core/table/format/format_table_loader.cpp delete mode 100644 src/paimon/core/table/format/format_table_loader.h diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 492b5f0d0..4be21a1dd 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -40,8 +40,9 @@ format table is an insert. Reading and writing ------------------- -A format table is not served through :cpp:func:`Catalog::GetTable`, which describes a managed -table; use :cpp:func:`Catalog::GetFormatTable` instead. +A format table is served through the same :cpp:func:`Catalog::GetTable` as every other table: +its ``type`` option says what it is, and the call hands back the ``FormatTable`` it describes, +through the ``Table`` interface the two kinds of table share. Reading and writing go through the entry points every other table uses: ``TableScan::Create``, ``TableRead::Create``, ``FileStoreWrite::Create`` and ``FileStoreCommit::Create``, each from its @@ -58,8 +59,11 @@ a constructor taking one - which is what Java does through ``FormatTable.newRead .. code-block:: cpp - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - catalog->GetFormatTable(paimon::Identifier("db", "tbl"))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded, + catalog->GetTable(paimon::Identifier("db", "tbl"))); + // The table's `type` option is `format-table`, so this is a `FormatTable`. + std::shared_ptr table = + std::dynamic_pointer_cast(loaded); paimon::ScanContextBuilder scan_builder(table); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); @@ -71,6 +75,12 @@ from one refuses ``SetTableSchema()``, ``WithFileSystem()``, ``WithFileSystemSchemeToIdentifierMap()``, ``WithCatalog()`` and a branch. Options given at the call still win over the ones the schema stored, as they do everywhere else. +``FormatTable`` is a derived class of ``Table``, so the table serves the same calls itself: a +caller holding one through a ``std::shared_ptr`` calls ``NewScan()``, +``NewRead()``, ``NewWrite()`` or ``NewCommit()`` on it, and the operators come back for the table +it is, whether or not the context carries one. This is the C++ side of what Java serves through +one ``Table.newReadBuilder()`` for both kinds of table. + A batch comes back in the table's column order, or the projection's when the read names one, behind the ``_VALUE_KIND`` field every ``BatchReader`` carries. Which columns a *file* is asked for and in what order is its own business: they are matched to the table's by name, as Java matches @@ -89,12 +99,10 @@ outside the projection is never read, which is how Java scopes the same checks. named here only to describe what happens. A caller never needs them: a plan comes back as ``Plan``, a split as ``Split``, and a commit message as ``CommitMessage``. -A catalog that can load a format table itself overrides ``Catalog::LoadFormatTable()``, the -protected hook :cpp:func:`Catalog::GetFormatTable` calls. The file system catalog uses it to say -its metadata lives under the table location, and the REST catalog to take the location and the -schema from one response instead of two that could disagree. A catalog that does not override it -still serves format tables, by reading the location and the schema through the virtuals every -catalog has. +Each catalog serves a format table from its own ``GetTable``. The file system catalog reads the +schema under the table location, so it says the location carries the table's own metadata and a +scan leaves the ``schema`` and ``branch`` directories below it alone; the REST catalog takes the +location and the schema from one response, so the two cannot disagree. ``TableScan::ListPartitions()`` lists the partitions a scan can see. A format table answers it by listing directories; every other table type returns ``NotImplemented`` for now. diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 69eb6187c..d501bc7e3 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -40,7 +40,6 @@ namespace paimon { using Instant = std::variant; class Database; -class FormatTable; class Table; class View; class Schema; @@ -148,6 +147,11 @@ class PAIMON_EXPORT Catalog { /// Gets a table. /// + /// The table's `type` option decides what is handed back: a `format-table` comes back as + /// the `FormatTable` it is - a directory of data files, with no snapshot and no manifest - + /// and every other implemented type as the managed table it is. Both go through this one + /// `Table` interface: `NewScan()`, `NewRead()`, `NewWrite()` and `NewCommit()` serve either. + /// /// @param identifier Identifier of the table to get. /// @return A result containing the table, or an error status. virtual Result> GetTable(const Identifier& identifier) const = 0; @@ -246,33 +250,6 @@ class PAIMON_EXPORT Catalog { /// @note Implementations returning true must also implement the internal /// `VersionManagedCatalog`. virtual bool SupportsVersionManagement() const; - - /// Gets a format table: a directory of data files laid out like a standard Hive table. - /// - /// A format table carries no snapshots and no manifests, so it is loaded through its own - /// method rather than `GetTable()`. Reading and writing it go through the same `TableScan`, - /// `TableRead`, `FileStoreWrite` and `FileStoreCommit` entry points every other table uses: - /// hand the table back to `ScanContextBuilder`, `ReadContextBuilder`, `WriteContextBuilder` or - /// `CommitContextBuilder`, each of which takes one. - /// - /// @param identifier Identifier of the table to get. - /// @return A result containing the format table, or an error status if the table does not - /// exist or its `type` option is not `format-table`. - Result> GetFormatTable(const Identifier& identifier) const; - - protected: - /// Loads `identifier` as a format table, which is what `GetFormatTable()` hands back. - /// - /// The default reads the location and the schema through the virtuals above - two requests - /// that can disagree - and treats every directory below the location as table content. An - /// override answers both questions directly: whether location and schema come back in one - /// round trip, and whether this catalog keeps the table's metadata under the table path. - /// - /// @param identifier Identifier of the table to load. - /// @return A result containing the format table, or an error status if the table does not - /// exist or its `type` option is not `format-table`. - virtual Result> LoadFormatTable( - const Identifier& identifier) const; }; } // namespace paimon diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index e542c9aee..d18bd07b8 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -33,20 +33,25 @@ namespace paimon { -class TableScan; +class CommitContext; +class FileStoreCommit; +class FileStoreWrite; +class ReadContext; +class ScanContext; class TableRead; +class TableScan; +class WriteContext; /// A table provides basic abstraction for table type. +/// +/// The state a table of every kind carries - file system, root path, schema and identifier - lives +/// here, and each kind of table fills the operations in: the operators below create this table's +/// scan, read, write or commit, from a context the caller finishes through its builder. What the +/// context asks for that this table cannot honour is refused by name by the implementation rather +/// than silently dropped. class PAIMON_EXPORT Table { public: - /// @param file_system File system the table's data is accessed through. - /// @param root_path Root path of the table in the file system. - /// @param uuid Metastore UUID, or empty if unavailable. - Table(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = ""); - - ~Table() = default; + virtual ~Table() = default; /// Returns the file system the table was opened with, which serves the table's /// files. A catalog that hands out per-table file systems opens a table with the @@ -99,33 +104,45 @@ class PAIMON_EXPORT Table { /// that `TableScan::Create()` also builds through. The scan is derived from the context. /// @param context The finished scan context. /// @return The scan, or the error building it reports. - Result> NewScan(std::unique_ptr context) const; + virtual Result> NewScan( + std::unique_ptr context) const = 0; /// Creates a read of this table from a finished read context, the table-first entry point /// that `TableRead::Create()` also builds through. The read is derived from the context. /// @param context The finished read context. /// @return The read, or the error building it reports. - Result> NewRead(std::unique_ptr context) const; + virtual Result> NewRead( + std::unique_ptr context) const = 0; /// Creates a write of this table from a finished write context, the table-first entry point /// that `FileStoreWrite::Create()` also builds through. The write is derived from the context. /// @param context The finished write context. /// @return The write, or the error building it reports. - Result> NewWrite(std::unique_ptr context) const; + virtual Result> NewWrite( + std::unique_ptr context) const = 0; /// Creates a commit of this table from a finished commit context, the table-first entry point /// that `FileStoreCommit::Create()` also builds through. The commit is derived from the /// context. /// @param context The finished commit context. /// @return The commit, or the error building it reports. - Result> NewCommit( - std::unique_ptr context) const; + virtual Result> NewCommit( + std::unique_ptr context) const = 0; + + protected: + /// @param file_system File system the table's data is accessed through. + /// @param root_path Root path of the table in the file system. + /// @param uuid Metastore UUID, or empty if unavailable. + Table(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid = ""); - private: std::shared_ptr file_system_; std::string root_path_; std::shared_ptr schema_; Identifier identifier_; + + private: std::string uuid_; }; diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index ffd40ac07..1ec784f76 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -151,7 +151,7 @@ class PAIMON_EXPORT CommitContextBuilder { /// /// There is no commit user: a format table keeps no snapshot to record one in. /// - /// @param table The format table to commit to, as `Catalog::GetFormatTable()` hands it back. + /// @param table The format table to commit to, as `Catalog::GetTable()` hands it back. explicit CommitContextBuilder(const std::shared_ptr& table); ~CommitContextBuilder(); diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index e62386c65..b9c534065 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -246,7 +246,7 @@ class PAIMON_EXPORT ReadContextBuilder { /// `SetTableSchema()`, `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()` and a /// branch are refused here rather than ignored. /// - /// @param table The format table to read, as `Catalog::GetFormatTable()` hands it back. + /// @param table The format table to read, as `Catalog::GetTable()` hands it back. explicit ReadContextBuilder(const std::shared_ptr& table); ~ReadContextBuilder(); diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 6fbb006f0..5be7c9c4e 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -183,7 +183,7 @@ class PAIMON_EXPORT ScanContextBuilder { /// table a REST catalog serves. The table carries what such a location does not say, so /// `SetTableSchema()` and `WithFileSystem()` are refused here rather than ignored. /// - /// @param table The format table to scan, as `Catalog::GetFormatTable()` hands it back. + /// @param table The format table to scan, as `Catalog::GetTable()` hands it back. explicit ScanContextBuilder(const std::shared_ptr& table); ~ScanContextBuilder(); diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index a4c310254..d2db6aa4a 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -24,6 +24,7 @@ #include #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/result.h" #include "paimon/schema/schema.h" #include "paimon/status.h" @@ -47,13 +48,18 @@ class FileSystem; /// Writes only insert - there is nowhere to record an update or a delete - so a read fills /// `_VALUE_KIND` with inserts throughout. /// -/// Reading and writing go through the entry points every other table uses: `TableScan`, -/// `TableRead`, `FileStoreWrite` and `FileStoreCommit`. Each recognises a format table from the -/// schema under the table path, or takes one already loaded through the `FormatTable` constructor -/// every context builder has, which is how a table whose schema lives in a metastore is reached. +/// A format table is a `Table`, so the operators it inherits create its scan, read, write and +/// commit from this table, with the context's options applied on top of the ones it carries. The +/// entry points every other table uses remain open too - `TableScan::Create()` and friends - each +/// of which recognises a format table from the schema under the table path, or takes one already +/// loaded through the `FormatTable` constructor every context builder has, which is how a table +/// whose schema lives in a metastore is reached. +/// +/// Every instance lives in a `shared_ptr`, which is what its operators hand to +/// `FormatTable::Copy()`. /// /// See `docs/source/user_guide/format_table.rst` for what is not supported yet. -class PAIMON_EXPORT FormatTable { +class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_this { public: /// Formats a format table's files can be in. enum class Format { @@ -72,7 +78,7 @@ class PAIMON_EXPORT FormatTable { /// /// This needs a schema file under the table directory, which a table created through /// `SchemaManager` or a file system catalog has. A table whose schema lives in a metastore has - /// none, and is loaded through `Catalog::GetFormatTable()` instead. + /// none, and is loaded through `Catalog::GetTable()` instead. /// /// @param file_system File system holding the table directory. /// @param table_path Root path of the table, which is also its data location. @@ -119,7 +125,7 @@ class PAIMON_EXPORT FormatTable { /// Directory the data files live in. const std::string& Location() const { - return location_; + return GetRootPath(); } /// Format of every data file in the directory. @@ -155,27 +161,16 @@ class PAIMON_EXPORT FormatTable { return options_; } - /// A name to identify this table. - std::string Name() const { - return identifier_.GetObjectName(); - } - - /// Full name of the table, database.tableName. - std::string FullName() const; - - /// Schema of the table, including its partition fields. + /// Schema of the table, including its partition fields. It hides `Table::LatestSchema()`, + /// which returns the same schema as the base every table sees; this one hands it back as the + /// `DataSchema` a format table is always created with. std::shared_ptr LatestSchema() const { - return schema_; + return std::static_pointer_cast(schema_); } /// Schema of the table as an arrow schema, including its partition fields. Result> GetArrowSchema() const; - /// File system holding the table directory. - std::shared_ptr GetFileSystem() const { - return file_system_; - } - /// Whether this table's own metadata lives under its location, as told by whoever loaded it. /// /// Only then are the `schema` and `branch` directories below the location table metadata @@ -185,6 +180,28 @@ class PAIMON_EXPORT FormatTable { return location_carries_paimon_metadata_; } + /// Creates a scan of this table from a finished scan context, refusing by name what a format + /// table cannot honour. The context's options are applied on top of the ones this table + /// carries, the precedence every context builder promises. + Result> NewScan(std::unique_ptr context) const override; + + /// Creates a read of this table from a finished read context, refusing by name what a format + /// table cannot honour. The context's options are applied on top of the ones this table + /// carries, the precedence every context builder promises. + Result> NewRead(std::unique_ptr context) const override; + + /// Creates a write of this table from a finished write context, refusing by name what a format + /// table cannot honour. The context's options are applied on top of the ones this table + /// carries, the precedence every context builder promises. + Result> NewWrite( + std::unique_ptr context) const override; + + /// Creates a commit of this table from a finished commit context, refusing by name what a + /// format table cannot honour. The context's options are applied on top of the ones this + /// table carries, the precedence every context builder promises. + Result> NewCommit( + std::unique_ptr context) const override; + private: FormatTable(const std::shared_ptr& file_system, const std::string& location, const Identifier& identifier, const std::shared_ptr& schema, @@ -192,10 +209,15 @@ class PAIMON_EXPORT FormatTable { const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata); - std::shared_ptr file_system_; - std::string location_; - Identifier identifier_; - std::shared_ptr schema_; + /// This table as a `shared_ptr`, which the operators above hand to `FormatTable::Copy()`. + /// + /// `shared_from_this()` on a const method hands back a `shared_ptr`, while + /// `Copy()` takes a mutable one; the cast is safe because `Copy()` never writes to the table + /// it is given. + std::shared_ptr SharedSelf() const { + return std::const_pointer_cast(shared_from_this()); + } + std::map options_; Format format_; std::string file_compression_; diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index e0a940eb0..58c0b3f97 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -178,7 +178,7 @@ class PAIMON_EXPORT WriteContextBuilder { /// /// There is no commit user: a format table keeps no snapshot to record one in. /// - /// @param table The format table to write to, as `Catalog::GetFormatTable()` hands it back. + /// @param table The format table to write to, as `Catalog::GetTable()` hands it back. explicit WriteContextBuilder(const std::shared_ptr& table); ~WriteContextBuilder(); diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index f92450d39..ad3d65f90 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -452,7 +452,6 @@ set(PAIMON_CORE_SRCS core/table/format/format_table_commit.cpp core/table/format/format_table_file_store_commit.cpp core/table/format/format_table_file_store_write.cpp - core/table/format/format_table_loader.cpp core/table/format/format_table_read.cpp core/table/format/format_table_scan.cpp core/table/format/format_table_write.cpp diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index 978c20e9d..318125ec6 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -26,11 +26,9 @@ #include "fmt/format.h" #include "paimon/catalog_options.h" #include "paimon/common/utils/string_utils.h" -#include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/core_options.h" #include "paimon/schema/schema.h" -#include "paimon/table/format/format_table.h" #ifdef PAIMON_ENABLE_REST #include "paimon/rest/rest_catalog.h" #endif @@ -74,19 +72,4 @@ bool Catalog::SupportsVersionManagement() const { return false; } -Result> Catalog::GetFormatTable(const Identifier& identifier) const { - // Refused here rather than in each override, so every catalog rejects a system table the same - // way and in the same order: the database first, so that an identifier of "sys" is rejected - // without being parsed as a table name. - PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "getFormatTable")); - return LoadFormatTable(identifier); -} - -Result> Catalog::LoadFormatTable(const Identifier& identifier) const { - // The location and the schema are read separately, and every directory below the location is - // table content: a catalog that knows better overrides this and says so. - return CatalogUtils::LoadFormatTableInTwoRequests(*this, identifier, - /*location_carries_paimon_metadata=*/false); -} - } // namespace paimon diff --git a/src/paimon/core/catalog/catalog_utils.cpp b/src/paimon/core/catalog/catalog_utils.cpp index 1f2a3800a..721942a88 100644 --- a/src/paimon/core/catalog/catalog_utils.cpp +++ b/src/paimon/core/catalog/catalog_utils.cpp @@ -31,7 +31,6 @@ #include "paimon/defs.h" #include "paimon/result.h" #include "paimon/schema/schema.h" -#include "paimon/table/format/format_table.h" namespace paimon { @@ -117,12 +116,6 @@ Status CatalogUtils::CheckManagedTableType(const Identifier& identifier, } PAIMON_ASSIGN_OR_RAISE(TableType table_type, TableTypeDefine::FromOptions(data_schema->Options())); - if (table_type == TableType::FORMAT_TABLE) { - return Status::Invalid( - fmt::format("Cannot open format table '{}' as a Table in '{}', please use " - "'Catalog::GetFormatTable' or 'FormatTable::Create'.", - identifier.ToString(), action)); - } // A materialized table is a managed table that also carries the SQL it materializes. if (table_type != TableType::TABLE && table_type != TableType::MATERIALIZED_TABLE) { const std::map& options = data_schema->Options(); @@ -136,17 +129,4 @@ Status CatalogUtils::CheckManagedTableType(const Identifier& identifier, return Status::OK(); } -Result> CatalogUtils::LoadFormatTableInTwoRequests( - const Catalog& catalog, const Identifier& identifier, bool location_carries_paimon_metadata) { - PAIMON_ASSIGN_OR_RAISE(std::string location, catalog.GetTableLocation(identifier)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, catalog.LoadTableSchema(identifier)); - std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); - if (data_schema == nullptr) { - return Status::Invalid(fmt::format("{} is not a data table, so it cannot be a format table", - identifier.GetFullName())); - } - return FormatTable::Create(catalog.GetFileSystem(), location, identifier, data_schema, - location_carries_paimon_metadata, /*dynamic_options=*/{}); -} - } // namespace paimon diff --git a/src/paimon/core/catalog/catalog_utils.h b/src/paimon/core/catalog/catalog_utils.h index 565a6aef5..23cb77203 100644 --- a/src/paimon/core/catalog/catalog_utils.h +++ b/src/paimon/core/catalog/catalog_utils.h @@ -66,32 +66,19 @@ class CatalogUtils { /// branch name, system table name) cannot be used as a single path component. static Status CheckValidTableName(const Identifier& identifier); - /// Fails when `schema` is not a table a `Table` can describe. + /// Fails when `schema` is not a table a managed `Table` can describe. /// - /// Only a managed table is one: a format table is loaded through `GetFormatTable()`, and the - /// remaining table types are stored differently and are not implemented here, so handing one - /// back as a `Table` would promise snapshots it never had. A schema that is not a data - /// table's, such as a system table's, passes. + /// Only a managed table is one: the remaining table types are stored differently and are + /// not implemented here, so handing one back as a `Table` would promise snapshots it never + /// had. A schema that is not a data table's, such as a system table's, passes. A format + /// table is served by the caller - `GetTable()` hands one back as the `FormatTable` it is - + /// so its schema only reaches this check by mistake and falls to the refusal below. /// /// @param action The call being refused, qualified as a caller would write it, e.g. /// `Catalog::GetTable`. It names the entry point in the message. static Status CheckManagedTableType(const Identifier& identifier, const std::shared_ptr& schema, const std::string& action); - - /// Loads `identifier` as a format table by reading its location and its schema separately. - /// - /// What every catalog can do, so it is what `Catalog::GetFormatTable()` falls back to. A - /// catalog that can get both from one response should not use it, since two requests can - /// disagree. - /// - /// @param location_carries_paimon_metadata Whether this catalog put the table's own metadata - /// under the table path, which is what tells a `schema` or `branch` directory below the - /// location from a partition directory of the same name. See - /// `FormatTable::LocationCarriesPaimonMetadata()`. - static Result> LoadFormatTableInTwoRequests( - const Catalog& catalog, const Identifier& identifier, - bool location_carries_paimon_metadata); }; } // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 87c89de8c..14293021e 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -34,7 +34,9 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" @@ -44,6 +46,7 @@ #include "paimon/fs/file_system.h" #include "paimon/logging.h" #include "paimon/result.h" +#include "paimon/table/format/format_table.h" namespace arrow { class Schema; @@ -343,7 +346,7 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); Identifier data_identifier(identifier.GetDatabaseName(), data_table_name); PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); - return std::make_shared(fs_, table_path, schema, identifier); + return std::make_shared(fs_, table_path, schema, identifier); } // The schema is read on the table's own branch: `root/branch/branch-` below the // table path, which `main` resolves to itself. `GetBranchNameOrDefault` folds any @@ -353,14 +356,27 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide if (!latest_schema) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); + // The schema's `type` says what the table is. A format table is the same directory read as + // plain data, so the schema read above is all it takes to serve one: no manifest is looked + // for, and the checks a format table adds run in `FormatTable::Create`. + std::shared_ptr data_schema = checked_pointer_cast(*latest_schema); + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(data_schema->Options())); + if (is_format_table) { + // This catalog keeps a table's own metadata under the table path, so a `schema` or + // `branch` directory there is metadata rather than table content. + return FormatTable::Create(fs_, table_path, identifier, data_schema, + /*location_carries_paimon_metadata=*/true, + /*dynamic_options=*/{}); + } // Only a managed table is a `Table`; describing another type as one would hand back // snapshots and manifests it never had, so the failure belongs to the call that opens the // table rather than at the first read. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); - return std::make_shared
(fs_, table_path, checked_pointer_cast(*latest_schema), - identifier); + return std::make_shared( + fs_, table_path, checked_pointer_cast(*latest_schema), identifier); } Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, @@ -579,10 +595,4 @@ Result> FileSystemCatalog::GetLatestSnapshot( return std::optional(snapshot->ToSnapshotInfo()); } -Result> FileSystemCatalog::LoadFormatTable( - const Identifier& identifier) const { - return CatalogUtils::LoadFormatTableInTwoRequests(*this, identifier, - /*location_carries_paimon_metadata=*/true); -} - } // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index ccc45f666..fd4360755 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -71,13 +71,6 @@ class FileSystemCatalog : public Catalog { Result> GetLatestSnapshot(const Identifier& identifier, const std::string& branch) const override; - protected: - /// This catalog keeps a table's schema under the table's own path, so the `schema` and - /// `branch` directories there are metadata rather than table content. Nothing else differs, - /// so the location and the schema are still read separately. - Result> LoadFormatTable( - const Identifier& identifier) const override; - private: /// Fails when `db_name` cannot be used as a single path component, so that the returned /// path always stays under `warehouse`. diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 51e7c642d..cfe361bae 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -629,7 +629,7 @@ TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { ArrowSchemaRelease(&schema); } -TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { +TEST(FileSystemCatalogTest, TestGetTableServesAFormatTable) { std::map options; options[Options::FILE_SYSTEM] = "local"; options[Options::FILE_FORMAT] = "orc"; @@ -650,11 +650,11 @@ TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - // A format table has no snapshots and no manifests, so handing it back as a Table would - // describe it as something it is not. - ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db1", "fmt")), "Cannot open format table"); - ASSERT_OK_AND_ASSIGN(std::shared_ptr format_table, - catalog.GetFormatTable(Identifier("db1", "fmt"))); + // The table's `type` says what it is, so the one call that opens a table hands a format + // table back as the `FormatTable` it is, through the `Table` interface the two share. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db1", "fmt"))); + std::shared_ptr format_table = std::dynamic_pointer_cast(table); + ASSERT_NE(nullptr, format_table); ASSERT_EQ(format_table->GetFormat(), FormatTable::Format::PARQUET); ASSERT_EQ(format_table->FullName(), "db1.fmt"); @@ -662,15 +662,6 @@ TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { // `branch` directories there are its metadata and a scan leaves them alone. A catalog that // keeps schemas elsewhere says no, and then every directory below the location is data. ASSERT_TRUE(format_table->LocationCarriesPaimonMetadata()); - - // A system table is refused by `Catalog::GetFormatTable()` itself, so every catalog answers - // alike. The database is checked before the table name is parsed, so a malformed name under - // "sys" is refused as a system table rather than surfacing a parse error. - ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("db1", "fmt$snapshots")), - "for system table"); - ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("sys", "all_table_options")), - "for system table"); - ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("sys", "a$b$c$d")), "for system table"); } TEST(FileSystemCatalogTest, TestInvalidCreateTable) { diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h new file mode 100644 index 000000000..409fefca9 --- /dev/null +++ b/src/paimon/core/table/file_store_table.h @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class Schema; + +/// The managed table a catalog hands out: one with snapshots and manifests, which the schema under +/// its root path describes. A format table - a directory of plain data files - is a `FormatTable`, +/// the other kind of `Table`. +/// +/// The operators create through the context-only entry points every table serves, so a context +/// built without a table reaches the same operators: `TableScan::Create()`, `TableRead::Create()`, +/// `FileStoreWrite::Create()` and `FileStoreCommit::Create()`, which read the table's schema to +/// dispatch on its type. +class FileStoreTable : public Table { + public: + /// The constructor a catalog builds a managed table through, so a `shared_ptr` can hold one; + /// the base constructor is protected, which an inherited constructor would stay. + /// @param file_system File system the table's data is accessed through. + /// @param root_path Root path of the table in the file system. + /// @param uuid Metastore UUID, or empty if unavailable. + FileStoreTable(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid = "") + : Table(file_system, root_path, schema, identifier, uuid) {} + + Result> NewScan(std::unique_ptr context) const override; + + Result> NewRead(std::unique_ptr context) const override; + + Result> NewWrite( + std::unique_ptr context) const override; + + Result> NewCommit( + std::unique_ptr context) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index a039baa4b..53d53b967 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -18,9 +18,15 @@ #include "paimon/table/format/format_table.h" +#include +#include #include +#include +#include +#include #include "fmt/format.h" +#include "paimon/commit_context.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/string_utils.h" @@ -29,8 +35,20 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/format/format_table_file_store_commit.h" +#include "paimon/core/table/format/format_table_file_store_write.h" +#include "paimon/core/table/format/format_table_read.h" +#include "paimon/core/table/format/format_table_scan.h" #include "paimon/defs.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" +#include "paimon/status.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/write_context.h" namespace paimon { @@ -45,10 +63,7 @@ FormatTable::FormatTable(const std::shared_ptr& file_system, const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata) - : file_system_(file_system), - location_(location), - identifier_(identifier), - schema_(schema), + : Table(file_system, location, schema, identifier), options_(options), format_(format), file_compression_(file_compression), @@ -192,20 +207,180 @@ Result> FormatTable::Copy( for (const auto& [key, value] : dynamic_options) { options[key] = value; } - return Create(table->file_system_, table->location_, table->identifier_, table->schema_, + return Create(table->file_system_, table->root_path_, table->identifier_, + std::static_pointer_cast(table->schema_), table->location_carries_paimon_metadata_, options); } const std::vector& FormatTable::PartitionKeys() const { - return schema_->PartitionKeys(); -} - -std::string FormatTable::FullName() const { - return identifier_.GetFullName(); + return LatestSchema()->PartitionKeys(); } Result> FormatTable::GetArrowSchema() const { return schema_->GetArrowSchema(); } +Result> FormatTable::NewScan( + std::unique_ptr context) const { + std::shared_ptr shared_context = std::move(context); + if (shared_context == nullptr) { + return Status::Invalid("scan context is null pointer"); + } + if (shared_context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (shared_context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + FormatTable::Copy(SharedSelf(), shared_context->GetOptions())); + // Anything a `ScanContext` carries that a format table cannot honour is refused rather than + // silently dropped. + if (shared_context->IsStreamingMode()) { + return Status::NotImplemented( + "a format table has no snapshots, so there is nothing for a streaming scan to follow"); + } + if (shared_context->GetRealtimeContext() != nullptr) { + return Status::NotImplemented( + "a format table has no real-time store to union with what is on disk"); + } + if (shared_context->GetGlobalIndexResult() != nullptr) { + return Status::NotImplemented("a format table carries no global index"); + } + std::map partition_filter; + std::shared_ptr filters = shared_context->GetScanFilters(); + if (filters != nullptr) { + if (filters->GetPredicate() != nullptr) { + return Status::NotImplemented( + "a format table scan does not take a predicate: its files carry no statistics to " + "skip by, and pruning by the partition columns a predicate names is not " + "implemented yet; give the partition values as a partition filter instead"); + } + if (filters->GetBucketFilter()) { + return Status::NotImplemented("a format table has no buckets to filter by"); + } + const std::vector>& partition_filters = + filters->GetPartitionFilters(); + if (partition_filters.size() > 1) { + return Status::NotImplemented("a format table scan takes at most one partition filter"); + } + if (partition_filters.size() == 1) { + partition_filter = partition_filters.front(); + } + } + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + FormatTableScan::Create(table, partition_filter, shared_context->GetLimit())); + return std::unique_ptr(std::move(scan)); +} + +Result> FormatTable::NewRead( + std::unique_ptr context) const { + std::shared_ptr read_context = std::move(context); + if (read_context == nullptr) { + return Status::Invalid("read context is null pointer"); + } + if (read_context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (read_context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + if (read_context->HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch is not supported for format tables"); + } + if (PredicateUtils::ContainsFullTextMatch(read_context->GetPredicate())) { + return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + FormatTable::Copy(SharedSelf(), read_context->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + FormatTableRead::Create(table, read_context)); + return std::unique_ptr(std::move(read)); +} + +Result> FormatTable::NewWrite( + std::unique_ptr context) const { + if (context == nullptr) { + return Status::Invalid("write context is null pointer"); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + FormatTable::Copy(SharedSelf(), context->GetOptions())); + if (context->GetCatalog() != nullptr) { + return Status::Invalid( + "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " + "format table"); + } + if (context->IsStreamingMode()) { + return Status::NotImplemented( + "a format table has no snapshots, so there is nothing a streaming write could " + "commit against"); + } + if (context->GetRealtimeContext() != nullptr) { + return Status::NotImplemented("a format table has no real-time store to write into"); + } + if (!context->GetWriteSchema().empty()) { + return Status::NotImplemented( + "a format table write takes the table's own columns; a write schema naming a " + "subset of them is not supported yet"); + } + // A write id prefixes a postpone-bucket writer's files so that one compaction reader can + // put them back in order. A format table has no buckets, so it would identify nothing. + if (context->GetWriteId().has_value()) { + return Status::NotImplemented( + "a format table has no buckets, so a write id would name nothing"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, + FormatTableFileStoreWrite::Create(table, context->GetMemoryPool())); + return std::unique_ptr(std::move(format_write)); +} + +Result> FormatTable::NewCommit( + std::unique_ptr context) const { + if (context == nullptr) { + return Status::Invalid("commit context is null pointer"); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, + FormatTable::Copy(SharedSelf(), context->GetOptions())); + if (!context->IgnoreEmptyCommit()) { + return Status::NotImplemented( + "a format table cannot record an empty commit: keeping one means writing a " + "snapshot that adds no files, and there are no snapshots here"); + } + if (context->UseRESTCatalogCommit()) { + return Status::NotImplemented( + "a format table commits by renaming files into place, not by sending a snapshot " + "to a rest catalog"); + } + if (context->GetTableId()) { + return Status::NotImplemented( + "a format table sends no commit table request, so there is nowhere to put a table id"); + } + if (context->GetCatalog() != nullptr) { + return Status::NotImplemented( + "a format table keeps no snapshot for a catalog to take, so it commits by renaming " + "files into place rather than through the catalog it was loaded from"); + } + if (context->AppendCommitCheckConflict()) { + return Status::NotImplemented( + "a format table has no manifests to check a concurrent commit against"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, + FormatTableFileStoreCommit::Create(given_table)); + return std::unique_ptr(std::move(format_commit)); +} + } // namespace paimon diff --git a/src/paimon/core/table/format/format_table_loader.cpp b/src/paimon/core/table/format/format_table_loader.cpp deleted file mode 100644 index 797bc3351..000000000 --- a/src/paimon/core/table/format/format_table_loader.cpp +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "paimon/core/table/format/format_table_loader.h" - -#include -#include - -#include "fmt/format.h" -#include "paimon/catalog/identifier.h" -#include "paimon/common/utils/checked_cast.h" -#include "paimon/common/utils/path_util.h" -#include "paimon/core/options/table_type.h" -#include "paimon/core/schema/schema_manager.h" -#include "paimon/core/schema/table_schema.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/schema/schema.h" -#include "paimon/table/format/format_table.h" - -namespace paimon { - -Result> FormatTableLoader::TryLoad( - const std::shared_ptr& file_system, const std::string& table_path, - const std::string& branch, const std::map& options, - const std::optional& specific_table_schema, const SchemaManager* schema_manager, - std::shared_ptr* table_schema_out) { - assert(table_schema_out != nullptr); - table_schema_out->reset(); - - std::shared_ptr table_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - // Through the caller's manager when it has one, so that the read warms the cache it goes - // on to use rather than a cache that dies with this call. - SchemaManager own_schema_manager(file_system, table_path, branch); - const SchemaManager& reader = - schema_manager != nullptr ? *schema_manager : own_schema_manager; - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - reader.Latest()); - if (!latest_schema) { - return std::shared_ptr(); - } - table_schema = *latest_schema; - } - *table_schema_out = table_schema; - - // From the schema alone: see `FormatTable::Create()`. - PAIMON_ASSIGN_OR_RAISE(bool is_format_table, - TableTypeDefine::IsFormatTable(table_schema->Options())); - if (!is_format_table) { - return std::shared_ptr(); - } - // A format table has no branches, so it is refused rather than served from the table path - // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. - if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { - return Status::Invalid( - fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", - table_path, branch)); - } - // A context names a path, never a catalog identifier, so the table is known by the directory - // it sits in. - Identifier identifier(Identifier::kUnknownDatabase, PathUtil::GetName(table_path)); - return FormatTable::Create(file_system, table_path, identifier, - checked_pointer_cast(table_schema), - /*location_carries_paimon_metadata=*/true, options); -} - -} // namespace paimon diff --git a/src/paimon/core/table/format/format_table_loader.h b/src/paimon/core/table/format/format_table_loader.h deleted file mode 100644 index 290ee94a7..000000000 --- a/src/paimon/core/table/format/format_table_loader.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include - -#include "paimon/result.h" - -namespace paimon { - -class FileSystem; -class FormatTable; -class SchemaManager; -class TableSchema; - -/// Recognises a format table behind a table path. -/// -/// The scan, read, write and commit entry points all take a path and have to answer the same -/// question before they can dispatch, so one place answers it and they cannot disagree. -class FormatTableLoader { - public: - FormatTableLoader() = delete; - ~FormatTableLoader() = delete; - - /// Loads `table_path` as a format table, or returns null when it is not one. - /// - /// Null is also the answer when there is no schema under the path at all: what to say about a - /// table that is not there is the managed path's to decide, and it says it in more detail. - /// - /// The table type is read from the schema, never from `options`: `type` is structural, and one - /// call's options must not decide what kind of table this is. - /// - /// @param branch Branch the schema is read from. Take it from the same place the managed path - /// in the same entry point takes it, or the two dispatch on different schemas. Anything - /// but the main branch is refused once the schema turns out to be a format table's, - /// which has no branches to read. - /// @param options Options given at the call, which win over the ones the schema stored. - /// @param specific_table_schema A schema the caller already holds, as json, which saves - /// reading one from under the path. Used only on the main branch, as on the managed - /// read path. - /// @param schema_manager The manager to read the schema through, or null to use one of this - /// call's own. A caller that has one must pass it: a manager caches what it read. - /// @param table_schema_out Set to the schema this read, or null when there is none, so that a - /// caller falling back to the managed path does not read it twice. - static Result> TryLoad( - const std::shared_ptr& file_system, const std::string& table_path, - const std::string& branch, const std::map& options, - const std::optional& specific_table_schema, - const SchemaManager* schema_manager, std::shared_ptr* table_schema_out); -}; - -} // namespace paimon diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index e8391f68c..2430ffbda 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -34,6 +34,7 @@ #include "arrow/c/bridge.h" #include "gtest/gtest.h" #include "paimon/cache/cache.h" +#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" #include "paimon/common/table/special_fields.h" @@ -58,6 +59,7 @@ #include "paimon/memory/memory_pool.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" +#include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" #include "paimon/record_batch.h" #include "paimon/scan_context.h" @@ -2843,6 +2845,159 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { ASSERT_EQ(rows, 3); } +TEST(FormatTableTest, TestATableServesAFormatTableThroughItsOwnEntryPoints) { + // `NewScan()` and its kin take the table they were called on as the one to serve, the way Java + // Paimon's `Table` serves both kinds through one `newReadBuilder()`, so a caller holding a + // format table as the `Table` a catalog hands out reaches it through them as well. The + // contexts name a path only and carry no format table, so it is the table itself that + // answers. + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::shared_ptr
table; + ASSERT_OK_AND_ASSIGN(table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + + WriteContextBuilder write_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + MakeBatch({1, 2}, {"alice", "bob"}, "20240101", {{"dt", "20240101"}})); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + + CommitContextBuilder commit_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + ScanContextBuilder scan_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1u); + + ReadContextBuilder read_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); + std::vector rows; + while (true) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch read_batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(read_batch)) { + break; + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr record_batch, + ImportBatch(read_batch)); + // `_VALUE_KIND` first, then the table's own columns, exactly as the generic entry points + // give them. + auto ids = checked_pointer_cast(record_batch->column(1)); + auto names = checked_pointer_cast(record_batch->column(2)); + auto dts = checked_pointer_cast(record_batch->column(3)); + for (int64_t i = 0; i < record_batch->num_rows(); i++) { + rows.push_back(std::to_string(ids->Value(i)) + "|" + names->GetString(i) + "|" + + dts->GetString(i)); + } + } + reader->Close(); + ASSERT_EQ(rows, (std::vector{"1|alice|20240101", "2|bob|20240101"})); +} + +TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) { + // A refusal is a promise about what was not done, so the table's own entry points must make + // the same promises the generic ones do. Each context is handed to both, and the answer has + // to come out the same whichever way in the caller takes. + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr format_table, + CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + std::shared_ptr
table = format_table; + + // A context that never was is refused first of all, and by name, exactly as the generic + // entry points refuse one. + ASSERT_NOK_WITH_MSG(TableScan::Create(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(TableRead::Create(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewRead(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(nullptr), "write context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewWrite(nullptr), "write context is null pointer"); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(nullptr), "commit context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewCommit(nullptr), "commit context is null pointer"); + + // A streaming scan follows snapshots, which a format table keeps none of. + auto streaming_scan_context = [&dir]() -> Result> { + ScanContextBuilder builder(dir->Str()); + builder.WithStreamingMode(true); + return builder.Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context, streaming_scan_context()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(streaming_context)), + "nothing for a streaming scan to follow"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context_again, + streaming_scan_context()); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(streaming_context_again)), + "nothing for a streaming scan to follow"); + + // Vector search needs an index a format table never builds. + auto vector_read_context = [&dir]() -> Result> { + ReadContextBuilder builder(dir->Str()); + builder.SetVectorSearch(std::make_shared( + /*field_name=*/"name", /*limit=*/1, std::vector{0.0f}, /*pre_filter=*/nullptr, + /*predicate=*/nullptr, /*distance_type=*/std::nullopt, + /*options=*/std::map{})); + return builder.Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context, vector_read_context()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(vector_context)), + "not supported for format tables"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context_again, vector_read_context()); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(vector_context_again)), + "not supported for format tables"); + + // A catalog takes the snapshot a native table commits, and a format table has none to give. + auto catalog = std::make_shared(); + catalog->SetTableSchema(format_table->LatestSchema()); + + auto catalog_write_context = [&dir, &catalog]() -> Result> { + WriteContextBuilder builder(dir->Str(), "test-user"); + return builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_write, catalog_write_context()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(catalog_write)), + "use WriteContextBuilder(FormatTable)"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_write_again, + catalog_write_context()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(catalog_write_again)), + "use WriteContextBuilder(FormatTable)"); + + auto catalog_commit_context = [&dir, &catalog]() -> Result> { + CommitContextBuilder builder(dir->Str(), "test-user"); + return builder.WithCatalog(catalog, Identifier("db", "fmt")).Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_commit, catalog_commit_context()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(catalog_commit)), + "keeps no snapshot for a catalog to take"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_commit_again, + catalog_commit_context()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(catalog_commit_again)), + "keeps no snapshot for a catalog to take"); + + // A write id prefixes a postpone-bucket writer's files; a format table has no buckets. + auto write_id_context = [&dir]() -> Result> { + WriteContextBuilder builder(dir->Str(), "test-user"); + builder.WithWriteId(3); + return builder.Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id, write_id_context()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_id)), + "a write id would name nothing"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id_again, write_id_context()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_id_again)), + "a write id would name nothing"); +} + TEST(FormatTableTest, TestAContextBuiltFromAFormatTableRefusesASecondAnswer) { // The table already carries its schema and the file system it was loaded through, from a // source the context cannot see behind, so a second answer is refused rather than dropped. diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index 05751f92e..3b50d2e85 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -30,14 +30,17 @@ #include "arrow/c/bridge.h" #include "fmt/format.h" #include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" #include "paimon/commit_context.h" #include "paimon/common/global_index/btree/btree_defs.h" #include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/binary_row_partition_computer.h" +#include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/fields_comparator.h" #include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_snapshot_commit.h" #include "paimon/core/catalog/catalog_utils.h" @@ -65,6 +68,7 @@ #include "paimon/core/operation/key_value_file_store_scan.h" #include "paimon/core/operation/key_value_file_store_write.h" #include "paimon/core/options/merge_engine.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/postpone/postpone_bucket_file_store_write.h" #include "paimon/core/realtime/realtime_context_impl.h" #include "paimon/core/realtime/realtime_schema_layout.h" @@ -72,11 +76,7 @@ #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/bucket_mode.h" -#include "paimon/core/table/format/format_table_file_store_commit.h" -#include "paimon/core/table/format/format_table_file_store_write.h" -#include "paimon/core/table/format/format_table_loader.h" -#include "paimon/core/table/format/format_table_read.h" -#include "paimon/core/table/format/format_table_scan.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/abstract_table_scan.h" #include "paimon/core/table/source/append_only_split_generator.h" #include "paimon/core/table/source/append_only_table_read.h" @@ -130,6 +130,45 @@ class MergeFunctionWrapper; namespace { +/// Recognises a format table behind `table_path`, or returns null when it is not one. +/// +/// The scan, read, write and commit entry points below all take a path and have to answer the +/// same question before they can dispatch, so one place answers it and they cannot disagree. +/// Null is also the answer when no schema was found under the path at all: what to say about a +/// table that is not there is the managed path's to decide, and it says it in more detail. +/// +/// The table type is read from the schema, never from `options`: `type` is structural, and one +/// call's options must not decide what kind of table this is. `branch` names the branch the +/// schema was read from: anything but the main branch is refused once the schema turns out to be +/// a format table's, which has no branches to read. +Result> TryLoadFormatTable( + const std::shared_ptr& file_system, const std::string& table_path, + const std::string& branch, const std::map& options, + const std::shared_ptr& table_schema) { + if (table_schema == nullptr) { + return std::shared_ptr(); + } + // From the schema alone: see `FormatTable::Create()`. + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(table_schema->Options())); + if (!is_format_table) { + return std::shared_ptr(); + } + // A format table has no branches, so it is refused rather than served from the table path + // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. + if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { + return Status::Invalid( + fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", + table_path, branch)); + } + // A context names a path, never a catalog identifier, so the table is known by the directory + // it sits in. + Identifier identifier(Identifier::kUnknownDatabase, PathUtil::GetName(table_path)); + return FormatTable::Create(file_system, table_path, identifier, + checked_pointer_cast(table_schema), + /*location_carries_paimon_metadata=*/true, options); +} + Status RestoreRealtimeCommittedProgress(const std::shared_ptr& realtime_context, const std::shared_ptr& snapshot_manager, const CoreOptions& options) { @@ -148,39 +187,6 @@ Status RestoreRealtimeCommittedProgress(const std::shared_ptr& return Status::OK(); } -/// Maps a `WriteContext` onto `FormatTableFileStoreWrite`, refusing by name what a format table -/// cannot honour rather than silently dropping it. -Result> NewFormatTableWrite( - const std::shared_ptr& table, const WriteContext& ctx) { - if (ctx.GetCatalog() != nullptr) { - return Status::Invalid( - "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " - "format table"); - } - if (ctx.IsStreamingMode()) { - return Status::NotImplemented( - "a format table has no snapshots, so there is nothing a streaming write could " - "commit against"); - } - if (ctx.GetRealtimeContext() != nullptr) { - return Status::NotImplemented("a format table has no real-time store to write into"); - } - if (!ctx.GetWriteSchema().empty()) { - return Status::NotImplemented( - "a format table write takes the table's own columns; a write schema naming a " - "subset of them is not supported yet"); - } - // A write id prefixes a postpone-bucket writer's files so that one compaction reader can - // put them back in order. A format table has no buckets, so it would identify nothing. - if (ctx.GetWriteId().has_value()) { - return Status::NotImplemented( - "a format table has no buckets, so a write id would name nothing"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, - FormatTableFileStoreWrite::Create(table, ctx.GetMemoryPool())); - return std::unique_ptr(std::move(format_write)); -} - Result HasHistoricalPrimaryKeyBTreeDefinition( const std::shared_ptr& schema_manager, int64_t current_schema_id) { if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { @@ -214,20 +220,22 @@ Table::Table(const std::shared_ptr& file_system, const std::string& identifier_(identifier), uuid_(uuid) {} -Result> Table::NewScan(std::unique_ptr context) const { +Result> FileStoreTable::NewScan( + std::unique_ptr context) const { return TableScan::Create(std::move(context)); } -Result> Table::NewRead(std::unique_ptr context) const { +Result> FileStoreTable::NewRead( + std::unique_ptr context) const { return TableRead::Create(std::move(context)); } -Result> Table::NewWrite( +Result> FileStoreTable::NewWrite( std::unique_ptr context) const { return FileStoreWrite::Create(std::move(context)); } -Result> Table::NewCommit( +Result> FileStoreTable::NewCommit( std::unique_ptr context) const { return FileStoreCommit::Create(std::move(context)); } @@ -245,9 +253,7 @@ Result> FileStoreWrite::Create(std::unique_ptrGetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return NewFormatTableWrite(given_table, *ctx); + return ctx->GetFormatTable()->NewWrite(std::move(ctx)); } PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); @@ -292,16 +298,28 @@ Result> FileStoreWrite::Create(std::unique_ptr(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch, - ctx->GetOptions(), catalog_table_schema, schema_manager.get(), - &latest_schema)); + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(catalog_table_schema.value())); + } else { + // Through the manager built above, so the read warms the cache the managed path below + // goes on to use. + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager->Latest()); + if (latest) { + latest_schema = *latest; + } + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + TryLoadFormatTable(tmp_options.GetFileSystem(), ctx->GetRootPath(), + branch, ctx->GetOptions(), latest_schema)); if (format_table != nullptr) { - return NewFormatTableWrite(format_table, *ctx); + return format_table->NewWrite(std::move(ctx)); } - // The schema the dispatch above already read through `schema_manager`, rather than a second - // read of the same file. + // The schema the branch read above through `schema_manager`, rather than a second read of + // the same file. if (latest_schema == nullptr) { return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); } @@ -678,53 +696,6 @@ class TableScanImpl { Result> NewDataTableScan( const std::shared_ptr& context, const std::shared_ptr& loaded_schema); -/// Maps a `ScanContext` onto `FormatTableScan`, which takes one partition filter and a limit and -/// nothing else. -/// -/// A format table has no buckets, so a bucket filter has nothing to act on. A predicate is -/// refused too, which is narrower than Java: `FormatReadBuilder.newScan()` hands the partition -/// half of one to the scan to prune directories with, which this does not do yet. -Result> NewFormatTableScan(const std::shared_ptr& table, - const std::shared_ptr& context) { - // Anything a `ScanContext` carries that a format table cannot honour is refused rather than - // silently dropped. - if (context->IsStreamingMode()) { - return Status::NotImplemented( - "a format table has no snapshots, so there is nothing for a streaming scan to follow"); - } - if (context->GetRealtimeContext() != nullptr) { - return Status::NotImplemented( - "a format table has no real-time store to union with what is on disk"); - } - if (context->GetGlobalIndexResult() != nullptr) { - return Status::NotImplemented("a format table carries no global index"); - } - std::map partition_filter; - std::shared_ptr filters = context->GetScanFilters(); - if (filters != nullptr) { - if (filters->GetPredicate() != nullptr) { - return Status::NotImplemented( - "a format table scan does not take a predicate: its files carry no statistics to " - "skip by, and pruning by the partition columns a predicate names is not " - "implemented yet; give the partition values as a partition filter instead"); - } - if (filters->GetBucketFilter()) { - return Status::NotImplemented("a format table has no buckets to filter by"); - } - const std::vector>& partition_filters = - filters->GetPartitionFilters(); - if (partition_filters.size() > 1) { - return Status::NotImplemented("a format table scan takes at most one partition filter"); - } - if (partition_filters.size() == 1) { - partition_filter = partition_filters.front(); - } - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - FormatTableScan::Create(table, partition_filter, context->GetLimit())); - return std::unique_ptr(std::move(scan)); -} - } // namespace Result> TableScan::Create(std::unique_ptr context) { @@ -738,43 +709,49 @@ Result> TableScan::Create(std::unique_ptr shared_context = std::move(context); // A table the caller already loaded says what it is, so nothing is read to find out. - if (shared_context->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTable::Copy(shared_context->GetFormatTable(), shared_context->GetOptions())); - return NewFormatTableScan(format_table, shared_context); + if (context->GetFormatTable() != nullptr) { + return context->GetFormatTable()->NewScan(std::move(context)); } // load schema - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(shared_context->GetOptions(), - shared_context->GetSpecificFileSystem(), {})); + PAIMON_ASSIGN_OR_RAISE( + CoreOptions tmp_options, + CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(shared_context->GetPath())); + SystemTableLoader::TryParsePath(context->GetPath())); if (system_table_path) { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), shared_context->GetPath(), - shared_context->GetOptions())); - return system_table->NewScan(shared_context); + SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), context->GetPath(), + context->GetOptions())); + return system_table->NewScan(std::move(context)); } // A format table is planned by listing directories, so it never reaches the manifest path // below. One `TableScan` interface serves both, as Java Paimon serves both through one // `ReadBuilder`. + const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), shared_context->GetPath(), - BranchManager::NormalizeBranch(tmp_options.GetBranch()), - shared_context->GetOptions(), - shared_context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && context->GetSpecificTableSchema()) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE( + latest_schema, TableSchema::CreateFromJson(context->GetSpecificTableSchema().value())); + } else { + SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; + } + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + TryLoadFormatTable(tmp_options.GetFileSystem(), context->GetPath(), + branch, context->GetOptions(), latest_schema)); if (format_table != nullptr) { - return NewFormatTableScan(format_table, shared_context); + return format_table->NewScan(std::move(context)); } - // With the schema the dispatch already read, so the managed path does not read it again. - return NewDataTableScan(shared_context, latest_schema); + // With the schema already read above, so the managed path does not read it again. + return NewDataTableScan(std::move(context), latest_schema); } namespace { @@ -821,29 +798,16 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& return Status::OK(); } +/// @param loaded_schema The schema `TableScan::Create()` already read while dispatching on the +/// table type, or null when neither the caller's json nor the branch's own directory had +/// one. Result> NewDataTableScan( const std::shared_ptr& context, const std::shared_ptr& loaded_schema) { - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); - std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - std::shared_ptr table_schema; - const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (loaded_schema != nullptr) { - table_schema = loaded_schema; - } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_table_schema, - schema_manager.Latest()); - if (latest_table_schema == std::nullopt) { - return Status::Invalid("not found latest schema"); - } - table_schema = latest_table_schema.value(); + if (loaded_schema == nullptr) { + return Status::Invalid("not found latest schema"); } + const std::shared_ptr& table_schema = loaded_schema; PAIMON_ASSIGN_OR_RAISE(bool read_optimized, OptionsUtils::GetValueFromMap(context->GetOptions(), kReadOptimizedScanOption, false)); @@ -1053,84 +1017,69 @@ Result> NewDataTableRead( std::move(fallback_table_read), arrow_pool); } -/// Maps a `ReadContext` onto `FormatTableRead`, which reads everything it needs out of the context -/// itself: the columns, the predicate, the pool and executor, and what a file is opened with. It -/// refuses by name what a format table cannot honour. -Result> NewFormatTableRead(const std::shared_ptr& table, - const std::shared_ptr& context) { - if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { - return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - FormatTableRead::Create(table, context)); - return std::unique_ptr(std::move(read)); -} - } // namespace Result> TableRead::Create(std::unique_ptr ctx) { - std::shared_ptr context = std::move(ctx); - if (context == nullptr) { + if (ctx == nullptr) { return Status::Invalid("read context is null pointer"); } - if (context->GetMemoryPool() == nullptr) { + if (ctx->GetMemoryPool() == nullptr) { return Status::Invalid("memory pool is null pointer"); } - if (context->GetExecutor() == nullptr) { + if (ctx->GetExecutor() == nullptr) { return Status::Invalid("executor is null pointer"); } // A table the caller already loaded says what it is, so nothing is read to find out. - if (context->GetFormatTable() != nullptr) { - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for format tables"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); - return NewFormatTableRead(format_table, context); + if (ctx->GetFormatTable() != nullptr) { + return ctx->GetFormatTable()->NewRead(std::move(ctx)); } - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_core_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, + CoreOptions::FromMap(ctx->GetOptions(), ctx->GetSpecificFileSystem(), + ctx->GetFileSystemSchemeToIdentifierMap())); PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(context->GetPath())); + SystemTableLoader::TryParsePath(ctx->GetPath())); if (system_table_path) { - if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { + if (PredicateUtils::ContainsFullTextMatch(ctx->GetPredicate())) { return Status::NotImplemented( "FullTextMatch predicate execution is not implemented yet"); } - if (context->HasVectorOrFullTextSearch()) { + if (ctx->HasVectorOrFullTextSearch()) { return Status::NotImplemented( "VectorSearch or FullTextSearch is not supported for system tables"); } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetOptions())); - return system_table->NewRead(context); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), + ctx->GetPath(), ctx->GetOptions())); + return system_table->NewRead(std::move(ctx)); } // A format table has no manifest and a file another engine wrote carries no field ids, so // which files there are and how a row is put back together is its own. How a file is opened // is not: both paths go through `DataFileReaderFactory`. + const std::string branch = ctx->GetBranch(); std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetBranch(), context->GetOptions(), - context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); - if (format_table != nullptr) { - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for format tables"); + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx->GetSpecificTableSchema()) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(ctx->GetSpecificTableSchema().value())); + } else { + SchemaManager schema_manager(tmp_core_options.GetFileSystem(), ctx->GetPath(), branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; } - return NewFormatTableRead(format_table, context); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + TryLoadFormatTable(tmp_core_options.GetFileSystem(), ctx->GetPath(), + branch, ctx->GetOptions(), latest_schema)); + if (format_table != nullptr) { + return format_table->NewRead(std::move(ctx)); } - // With the schema the dispatch already read, so the managed path does not read it again. - return NewDataTableRead(context, latest_schema); + // With the schema already read above, so the managed path does not read it again. + return NewDataTableRead(std::move(ctx), latest_schema); } namespace { @@ -1206,37 +1155,6 @@ Result> NewSnapshotCommit( std::make_shared(fs, snapshot_manager)); } -/// Creates a format-table commit, rejecting non-default options that require snapshots. -Result> NewFormatTableCommit( - const std::shared_ptr& table, const CommitContext& ctx) { - if (!ctx.IgnoreEmptyCommit()) { - return Status::NotImplemented( - "a format table cannot record an empty commit: keeping one means writing a " - "snapshot that adds no files, and there are no snapshots here"); - } - if (ctx.UseRESTCatalogCommit()) { - return Status::NotImplemented( - "a format table commits by renaming files into place, not by sending a snapshot " - "to a rest catalog"); - } - if (ctx.GetTableId()) { - return Status::NotImplemented( - "a format table sends no commit table request, so there is nowhere to put a table id"); - } - if (ctx.GetCatalog() != nullptr) { - return Status::NotImplemented( - "a format table keeps no snapshot for a catalog to take, so it commits by renaming " - "files into place rather than through the catalog it was loaded from"); - } - if (ctx.AppendCommitCheckConflict()) { - return Status::NotImplemented( - "a format table has no manifests to check a concurrent commit against"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, - FormatTableFileStoreCommit::Create(table)); - return std::unique_ptr(std::move(format_commit)); -} - } // namespace Result> FileStoreCommit::Create( @@ -1253,9 +1171,7 @@ Result> FileStoreCommit::Create( // A table the caller already loaded says what it is, so nothing is read to find out. if (ctx->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return NewFormatTableCommit(given_table, *ctx); + return ctx->GetFormatTable()->NewCommit(std::move(ctx)); } PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); @@ -1320,12 +1236,25 @@ Result> FileStoreCommit::Create( auto schema_manager = std::make_shared(tmp_options.GetFileSystem(), root_path, branch); std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(catalog_table_schema.value())); + } else { + // Through the manager built above, so the read warms the cache the managed path below + // goes on to use. + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager->Latest()); + if (latest) { + latest_schema = *latest; + } + } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTableLoader::TryLoad( - tmp_options.GetFileSystem(), root_path, branch, ctx->GetOptions(), - catalog_table_schema, schema_manager.get(), &latest_schema)); + TryLoadFormatTable(tmp_options.GetFileSystem(), root_path, branch, + ctx->GetOptions(), latest_schema)); if (format_table != nullptr) { - return NewFormatTableCommit(format_table, *ctx); + return format_table->NewCommit(std::move(ctx)); } if (latest_schema == nullptr) { return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 4792a8461..78d01e5a5 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -31,11 +31,13 @@ #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" #include "paimon/schema/schema.h" +#include "paimon/table/format/format_table.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_catalog.h" @@ -57,7 +59,7 @@ Result> CreateManagedTable(const std::shared_ptr created, schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{})); - return std::make_shared
( + return std::make_shared( file_system, path, checked_pointer_cast(std::shared_ptr(std::move(created))), identifier); } @@ -164,7 +166,7 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { SCOPED_TRACE(main_identifier.ToString()); // A catalog that loaded the schema itself calls the constructor directly, which a // rest catalog is shaped like. The alias resolves to the main branch as well. - Table table(fs, dir->Str(), schema, main_identifier); + FileStoreTable table(fs, dir->Str(), schema, main_identifier); ASSERT_OK_AND_ASSIGN(std::string branch, table.CurrentBranch()); EXPECT_EQ("main", branch); } @@ -211,10 +213,9 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { EXPECT_EQ(opened->GetRootPath(), ""); } -TEST(TableTest, TestGetTableRejectsATableTypeThatIsNotAManagedTable) { - // A `Table` describes a managed paimon table. Handing back another type as one would promise - // snapshots and manifests it never had, so the failure belongs at the call that opens the - // table rather than at the first read. +TEST(TableTest, TestGetTableServesEachKindOfTableAsWhatItIs) { + // The table's `type` option says what it is, so the one call that opens a table hands each + // kind back as what it is, through the `Table` interface the two kinds share. auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); @@ -230,7 +231,10 @@ TEST(TableTest, TestGetTableRejectsATableTypeThatIsNotAManagedTable) { format_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {{"type", "format-table"}, {"file.format", "parquet"}})); - ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db", "format")), "Cannot open format table"); + // A format table comes back as the `FormatTable` it is. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, + catalog.GetTable(Identifier("db", "format"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(opened)); // A managed table with no `type` option at all is what a `Table` is for, and still opens. const std::string managed_dir = dir->Str() + "/db.db/managed"; diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 66898d8af..fbd9521c4 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -32,8 +32,10 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" @@ -430,36 +432,35 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i return checked_pointer_cast(schema); } -Result> RestCatalog::LoadFormatTable( - const Identifier& identifier) const { - // A system table is already refused by `Catalog::GetFormatTable()`, which every caller goes - // through, so this only has to load. - PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); - branch = NormalizeBranch(std::move(branch)); - PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); - std::string location; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr schema, - LoadDataTableSchema(load_identifier, branch, &location, /*table_id=*/nullptr)); - // A rest catalog holds the schema itself, so everything below the location is data. - return FormatTable::Create(fs_, location, identifier, checked_pointer_cast(schema), - /*location_carries_paimon_metadata=*/false, - /*dynamic_options=*/{}); -} - Result> RestCatalog::GetTable(const Identifier& identifier) const { // Read UUID, path and schema together to avoid mixing table generations after recreation. std::string uuid; std::string table_path; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier, &uuid, &table_path)); + // The schema the same response carried says what the table is: a format table is handed + // back as the `FormatTable` it is, built from the location of the same response, rather + // than refused as the managed table it is not. + std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); + if (data_schema != nullptr) { + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(data_schema->Options())); + if (is_format_table) { + // A rest catalog holds the schema itself, so everything below the location is data. + // The catalog-wide file system, as a format table reads no metadata of its own. + return FormatTable::Create(fs_, table_path, identifier, data_schema, + /*location_carries_paimon_metadata=*/false, + /*dynamic_options=*/{}); + } + } PAIMON_RETURN_NOT_OK( CatalogUtils::CheckManagedTableType(identifier, schema, "Catalog::GetTable")); // The table holds the file system its own data is served by, which a server issuing data // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); - return std::make_shared
(table_file_system, table_path, schema, identifier, uuid); + return std::make_shared(table_file_system, table_path, schema, identifier, + uuid); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 051b90d38..6b7f22cda 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -110,12 +110,6 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { /// Options merged with the server side config. const std::map& GetOptions() const override; - protected: - /// Loads the location and the schema from one `GetTable` response, so the table cannot be - /// built from a location and a schema that two requests disagreed on. - Result> LoadFormatTable( - const Identifier& identifier) const override; - private: RestCatalog(std::shared_ptr api, const std::shared_ptr& fs, const std::string& warehouse, bool data_token_enabled, bool fs_explicitly_supplied, diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 4b878250d..48d5bf19c 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -1146,8 +1146,10 @@ TEST_F(RestCatalogTest, CreateRefusesAFormatTableThisClientCouldNotOpen) { // And a schema this library can open is created and opens. ASSERT_OK(create("fine", good, {"dt"}, format_table)); - ASSERT_OK_AND_ASSIGN(std::shared_ptr table, - catalog->GetFormatTable(Identifier("db1", "fine"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, + catalog->GetTable(Identifier("db1", "fine"))); + std::shared_ptr table = std::dynamic_pointer_cast(loaded); + ASSERT_NE(nullptr, table); ASSERT_EQ(FormatTable::Format::PARQUET, table->GetFormat()); } @@ -1169,7 +1171,7 @@ TEST_F(RestCatalogTest, TableResponseWithoutAPathIsRejected) { // would be checked against an empty one has no boundary at all, so the response is refused // rather than turned into a table. What the response said is not repeated back: a body may // carry credentials, so a failure to read one names the request and nothing else. - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "no_path")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "no_path")), "failed to deserialize the response"); ASSERT_NOK_WITH_MSG(catalog->GetTableLocation(Identifier("db1", "no_path")), "failed to deserialize the response"); @@ -1192,8 +1194,10 @@ TEST_F(RestCatalogTest, FormatTableIsLoadedInOneRequest) { state_->get_table_requests = 0; } - ASSERT_OK_AND_ASSIGN(std::shared_ptr table, - catalog->GetFormatTable(Identifier("db1", "fmt"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, + catalog->GetTable(Identifier("db1", "fmt"))); + std::shared_ptr table = std::dynamic_pointer_cast(loaded); + ASSERT_NE(nullptr, table); ASSERT_EQ("wh1/db1.db/fmt", table->Location()); ASSERT_EQ(FormatTable::Format::PARQUET, table->GetFormat()); ASSERT_EQ((std::vector{"dt"}), table->PartitionKeys()); @@ -1208,18 +1212,11 @@ TEST_F(RestCatalogTest, FormatTableIsLoadedInOneRequest) { // directory named "schema" there is a partition value, not metadata to skip. ASSERT_FALSE(table->LocationCarriesPaimonMetadata()); - ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "fmt")), "Cannot open format table"); - - // A managed table stays out of this path, and so does a system table. + // A managed table comes back through the same call, as the managed table it is. ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "t1")), "is not a format table"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "t1$snapshots")), - "for system table"); - // The database is checked before the table name is parsed, so a malformed name under "sys" - // is refused as a system table rather than surfacing a parse error. - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("sys", "all_table_options")), - "for system table"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("sys", "a$b$c$d")), "for system table"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
managed, + catalog->GetTable(Identifier("db1", "t1"))); + ASSERT_EQ(nullptr, std::dynamic_pointer_cast(managed)); } TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { @@ -1245,24 +1242,23 @@ TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { // A format table format with no reader here. seed("csv_table", R"({"type": "format-table", "file.format": "csv"})", R"(["dt"])", default_fields); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "csv_table")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "csv_table")), "not supported by paimon-cpp yet"); // A row count no file could ever reach. seed("bad_rows", R"({"type": "format-table", "file.format": "parquet", "target-file-row-num": "0"})", R"(["dt"])", default_fields); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "bad_rows")), - "should be at least 1"); + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "bad_rows")), "should be at least 1"); // The failure names the table, which is all a caller holding several has to go on. - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "bad_rows")), "db1.bad_rows"); + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "bad_rows")), "db1.bad_rows"); // The structural rules run here too, not only the format-table ones: a partition key that is // not a field of the schema is a table nothing could ever read. seed("bad_partition_key", R"({"type": "format-table", "file.format": "parquet"})", R"(["nosuchfield"])", default_fields); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "bad_partition_key")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "bad_partition_key")), "should include all partition fields"); // Two schemas this library cannot open, refused when the table is opened rather than when a @@ -1272,12 +1268,12 @@ TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { R"([{"id": 0, "name": "f0", "type": "INT NOT NULL"},)" // Not a raw string: `TIMESTAMP(6)"` holds the sequence that would end one. " {\"id\": 1, \"name\": \"dt\", \"type\": \"TIMESTAMP(6)\"}]"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "timestamp_partition")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "timestamp_partition")), "cannot be TIMESTAMP/DECIMAL/BLOB"); seed("only_partitions", R"({"type": "format-table", "file.format": "parquet"})", R"(["dt"])", R"([{"id": 0, "name": "dt", "type": "STRING"}])"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "only_partitions")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "only_partitions")), "every one of its columns"); // An option this library does not honour keeps its own status code, so a caller can still @@ -1286,8 +1282,7 @@ TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { R"({"type": "format-table", "file.format": "parquet",)" R"( "metastore.partitioned-table": "true"})", R"(["dt"])", default_fields); - Status not_implemented = - catalog->GetFormatTable(Identifier("db1", "catalog_partitions")).status(); + Status not_implemented = catalog->GetTable(Identifier("db1", "catalog_partitions")).status(); ASSERT_TRUE(not_implemented.IsNotImplemented()) << not_implemented.ToString(); } diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index 67449abf2..c4a200239 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -34,6 +34,7 @@ #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/result.h" #include "paimon/schema/schema.h" #include "paimon/status.h" @@ -98,7 +99,7 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { // Serves the per-table file system when one was set, matching `GetTableFileSystem` // without recording the request, so a test can still tell the two apart. No location: // this mock's `GetTableLocation` is the unsupported stub. - return std::make_shared
( + return std::make_shared( table_file_system_ != nullptr ? table_file_system_ : file_system_, /*root_path=*/"", table_schema_, identifier, table_uuid_); } From 3fbf1bbe1c3cc40523414e30e8b87465eb4f0953 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 20 Oct 2026 16:18:14 +0800 Subject: [PATCH 02/19] fix --- include/paimon/catalog/catalog.h | 6 +- include/paimon/catalog/table.h | 14 ++-- include/paimon/table/format/format_table.h | 8 ++ .../core/catalog/file_system_catalog.cpp | 67 ++++++++------- src/paimon/core/catalog/file_system_catalog.h | 5 ++ .../core/catalog/file_system_catalog_test.cpp | 83 ++++++++++++++++++- src/paimon/core/table/file_store_table.h | 8 ++ .../table/system/audit_log_system_table.cpp | 9 +- .../table/system/audit_log_system_table.h | 6 +- .../core/table/system/binlog_system_table.cpp | 3 +- .../core/table/system/binlog_system_table.h | 3 +- .../table/system/global_system_tables.cpp | 16 ++-- .../table/system/in_memory_system_table.cpp | 11 ++- .../table/system/in_memory_system_table.h | 9 +- .../table/system/metadata_system_tables.cpp | 19 +++-- .../table/system/metadata_system_tables.h | 3 +- .../system/read_optimized_system_table.cpp | 12 ++- .../system/read_optimized_system_table.h | 10 +-- src/paimon/core/table/system/system_table.cpp | 41 +++++++-- src/paimon/core/table/system/system_table.h | 46 +++++++--- .../core/table/system/system_table_test.cpp | 2 +- src/paimon/rest/rest_catalog.cpp | 65 ++++++++------- src/paimon/rest/rest_catalog.h | 7 ++ src/paimon/rest/rest_catalog_test.cpp | 9 +- test/inte/read_inte_test.cpp | 8 +- 25 files changed, 330 insertions(+), 140 deletions(-) diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index d501bc7e3..0bb578383 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -149,8 +149,10 @@ class PAIMON_EXPORT Catalog { /// /// The table's `type` option decides what is handed back: a `format-table` comes back as /// the `FormatTable` it is - a directory of data files, with no snapshot and no manifest - - /// and every other implemented type as the managed table it is. Both go through this one - /// `Table` interface: `NewScan()`, `NewRead()`, `NewWrite()` and `NewCommit()` serve either. + /// and every other implemented type as the managed table it is. A `T$system` identifier + /// names a read-only view instead, which comes back as the `SystemTable` it is. All go + /// through this one `Table` interface: `NewScan()` and `NewRead()` serve any of them, + /// `NewWrite()` and `NewCommit()` the writable ones. /// /// @param identifier Identifier of the table to get. /// @return A result containing the table, or an error status. diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index d18bd07b8..17b3b3d21 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -66,15 +66,13 @@ class PAIMON_EXPORT Table { return root_path_; } - /// A name to identify this table. - std::string Name() const { - return identifier_.GetObjectName(); - } + /// A name to identify this table, which each kind of table answers for itself: a managed or + /// format table is named by its identifier, a system table by its own kind. + virtual std::string Name() const = 0; - /// Full name of the table, default is database.tableName. - std::string FullName() const { - return identifier_.GetFullName(); - } + /// Full name of the table, which each kind of table answers for itself. A managed or format + /// table answers database.tableName, a system table the same name as `Name()`. + virtual std::string FullName() const = 0; /// Returns the branch the table was opened for, which its identifier carries; `main` /// when it names none or a spelling of it, which its managers are bound to. A diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index d2db6aa4a..d15a8f6ce 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -123,6 +123,14 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t ~FormatTable(); + std::string Name() const override { + return identifier_.GetObjectName(); + } + + std::string FullName() const override { + return identifier_.GetFullName(); + } + /// Directory the data files live in. const std::string& Location() const { return GetRootPath(); diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 14293021e..46131f729 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -294,40 +294,16 @@ Result> FileSystemCatalog::LoadTableSchema( context.catalog_options = catalog_options_; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + return system_table->LatestSchema(); } // The system table suffix is dropped when the data table identifier is rebuilt below, so the // identifier is validated as a whole here. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckValidTableName(identifier)); PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, - identifier.GetSystemTableName()); - if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { - return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); - } - PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); - PAIMON_ASSIGN_OR_RAISE( - Identifier data_identifier, - CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - TableSchemaExists(data_identifier)); - if (!latest_schema) { - return Status::NotExist(fmt::format("{} not exist", data_identifier.ToString())); - } - std::map dynamic_options; - if (branch) { - dynamic_options[Options::BRANCH] = branch.value(); - } - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::Load(system_table_name.value(), fs_, table_path, - latest_schema.value(), dynamic_options)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + LoadSystemTable(identifier)); + return system_table->LatestSchema(); } PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, TableSchemaExists(identifier)); @@ -340,13 +316,9 @@ Result> FileSystemCatalog::LoadTableSchema( Result> FileSystemCatalog::GetTable(const Identifier& identifier) const { PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier)); - // A system table reads the files of the data table it belongs to, so its root path - // is that table's. - PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); - Identifier data_identifier(identifier.GetDatabaseName(), data_table_name); - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); - return std::make_shared(fs_, table_path, schema, identifier); + // A system table reads the files of the data table it belongs to, through the file + // system of that table's catalog, and the schema it exposes is its own read-only one. + return LoadSystemTable(identifier); } // The schema is read on the table's own branch: `root/branch/branch-` below the // table path, which `main` resolves to itself. `GetBranchNameOrDefault` folds any @@ -379,6 +351,33 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide fs_, table_path, checked_pointer_cast(*latest_schema), identifier); } +Result> FileSystemCatalog::LoadSystemTable( + const Identifier& identifier) const { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + PAIMON_ASSIGN_OR_RAISE( + Identifier data_identifier, + CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, + TableSchemaExists(data_identifier)); + if (!latest_schema) { + return Status::NotExist(fmt::format("{} not exist", data_identifier.ToString())); + } + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + // A system table reads the files of the data table it belongs to, so its root path is + // that table's. + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); + return SystemTableLoader::Load(system_table_name.value(), fs_, table_path, + latest_schema.value(), dynamic_options); +} + Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, bool cascade) { PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemDatabase(name, "dropDatabase")); diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index fd4360755..203e1b92e 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -37,6 +37,7 @@ class TableSchema; class FileSystem; class Identifier; class Logger; +class SystemTable; class FileSystemCatalog : public Catalog { public: @@ -85,6 +86,10 @@ class FileSystemCatalog : public Catalog { Result>> TableSchemaExists( const Identifier& identifier) const; + /// Loads the system table a `T$system` identifier names, on the branch the identifier + /// carries. The base table it belongs to must exist and carry a schema. + Result> LoadSystemTable(const Identifier& identifier) const; + Status CreateDatabaseImpl(const std::string& db_name, const std::map& options); Result TableExistsInFileSystem(const std::string& table_path) const; diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index cfe361bae..f31c0391d 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -34,14 +34,21 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" #include "paimon/fs/file_system_factory.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" #include "paimon/snapshot/snapshot_info.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/write_context.h" namespace paimon::test { @@ -223,10 +230,15 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_FALSE(loaded_schema->field(1)->nullable()); ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(options_identifier)); - ASSERT_EQ(system_table->Name(), "tbl1$options"); - // A system table reads the files of the data table it belongs to, so the table opened - // for it carries that table's root path. + // A system table comes back as the `SystemTable` it is: it answers for itself with its own + // read-only schema and the root path of the data table it reads. + ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); + ASSERT_TRUE(std::dynamic_pointer_cast(system_table) == nullptr); + ASSERT_EQ("options", system_table->Name()); + ASSERT_EQ("options", system_table->FullName()); ASSERT_EQ(system_table->GetRootPath(), table_path); + ASSERT_EQ((std::vector{"key", "value"}), + system_table->LatestSchema()->FieldNames()); ASSERT_NOK_WITH_MSG(catalog.LoadTableSchema(Identifier("db1", "tbl1$unknown")), "not exist"); ASSERT_NOK_WITH_MSG(catalog.LoadTableSchema(Identifier("db1", "missing$options")), "not exist"); @@ -242,6 +254,71 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { "Cannot 'renameTable' for system table"); } +// A `T$system` identifier opens the read-only view it names, which serves scans and reads of +// the data table it belongs to through the same `Table` interface, and refuses the operations +// that would modify the data table. +TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { + std::map options; + options[Options::FILE_SYSTEM] = "local"; + options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); + + auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32(), /*nullable=*/false), + arrow::field("dt", arrow::utf8())}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog.CreateTable(Identifier("db1", "tbl1"), &schema, /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + // A managed table with no `type` option still opens, as the managed table it is. + ASSERT_OK_AND_ASSIGN(auto managed, catalog.GetTable(Identifier("db1", "tbl1"))); + ASSERT_TRUE(std::dynamic_pointer_cast(managed) == nullptr); + + ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(Identifier("db1", "tbl1$options"))); + ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); + ASSERT_EQ("options", system_table->Name()); + + // The scan and the read work from a context of the caller's, like every other table. + ScanContextBuilder scan_builder(system_table->GetRootPath()); + scan_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan, system_table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); + + ReadContextBuilder read_builder(system_table->GetRootPath()); + read_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto read, system_table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto table_result, reader->NextBatch()); + + // The options the table was created with come back, one row per key. + auto& [c_array, c_schema] = table_result; + std::shared_ptr batch; + auto maybe_batch = arrow::ImportRecordBatch(c_array.get(), c_schema.get()); + ASSERT_TRUE(maybe_batch.ok()) << maybe_batch.status().ToString(); + batch = std::move(maybe_batch).ValueUnsafe(); + ASSERT_GE(batch->num_rows(), 2); + + // The operations that would modify the data table underneath are refused by name. + WriteContextBuilder write_builder(system_table->GetRootPath(), "commit_user"); + write_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(system_table->NewWrite(std::move(write_context)), + "options system table is read-only: writing is not supported"); + + CommitContextBuilder commit_builder(system_table->GetRootPath(), "commit_user"); + commit_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto commit_context, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(system_table->NewCommit(std::move(commit_context)), + "options system table is read-only: committing for it is not supported"); +} + TEST(FileSystemCatalogTest, TestBranchTableLoadsBranchSchema) { std::map options; options[Options::FILE_SYSTEM] = "local"; diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 409fefca9..f02b4e528 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -51,6 +51,14 @@ class FileStoreTable : public Table { const std::string& uuid = "") : Table(file_system, root_path, schema, identifier, uuid) {} + std::string Name() const override { + return identifier_.GetObjectName(); + } + + std::string FullName() const override { + return identifier_.GetFullName(); + } + Result> NewScan(std::unique_ptr context) const override; Result> NewRead(std::unique_ptr context) const override; diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 8de9742e1..3d3f7b3e5 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -358,7 +358,8 @@ std::unique_ptr CreateChangelogBatchReader( AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, std::map options) - : fs_(std::move(fs)), + : SystemTable(fs, table_path), + fs_(std::move(fs)), table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} @@ -383,7 +384,8 @@ Result> AuditLogSystemTable::ArrowSchema() const } Result> AuditLogSystemTable::NewScan( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { return Status::NotImplemented("audit_log system table predicate pushdown is not supported"); } @@ -411,7 +413,8 @@ Result> AuditLogSystemTable::NewScan( } Result> AuditLogSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); return NewChangelogRead(context, std::make_shared()); } diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h index 780f5fa72..7395c9797 100644 --- a/src/paimon/core/table/system/audit_log_system_table.h +++ b/src/paimon/core/table/system/audit_log_system_table.h @@ -60,10 +60,8 @@ class AuditLogSystemTable : public SystemTable { std::string Name() const override; Result> ArrowSchema() const override; - Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewScan(std::unique_ptr context) const override; + Result> NewRead(std::unique_ptr context) const override; protected: Result> NewChangelogRead( diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index 88541194d..8a821d45f 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -98,7 +98,8 @@ Result> BinlogSystemTable::ArrowSchema() const { } Result> BinlogSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); return NewChangelogRead(context, CreateBinlogBatchConverter()); } diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h index 3b4f45bd0..3a939fcef 100644 --- a/src/paimon/core/table/system/binlog_system_table.h +++ b/src/paimon/core/table/system/binlog_system_table.h @@ -42,8 +42,7 @@ class BinlogSystemTable : public AuditLogSystemTable { std::string Name() const override; Result> ArrowSchema() const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewRead(std::unique_ptr context) const override; }; } // namespace paimon diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 647da76c3..a66362b39 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -300,7 +300,13 @@ Result> GlobalSystemTableLoader::Load( return Status::NotImplemented("global system table requires catalog context: ", table_name); } - return entry.factory(context); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + entry.factory(context)); + // The table is handed out through the `Table` interface, which serves the schema + // the base holds, so its own read-only schema is adopted before it leaves the + // loader. + PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); + return system_table; } } return Status::NotImplemented("unsupported global system table: ", table_name); @@ -324,7 +330,7 @@ Result> GlobalSystemTableLoader::GetSupportedTableNames // ============================================================================= CatalogOptionsSystemTable::CatalogOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/catalog_options"), context_(std::move(context)) {} + : InMemorySystemTable(context.fs, "sys/catalog_options"), context_(std::move(context)) {} std::string CatalogOptionsSystemTable::Name() const { return kName; @@ -360,7 +366,7 @@ Result> CatalogOptionsSystemTable::BuildRows() const { // ============================================================================= AllTableOptionsSystemTable::AllTableOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/all_table_options"), context_(std::move(context)) {} + : InMemorySystemTable(context.fs, "sys/all_table_options"), context_(std::move(context)) {} std::string AllTableOptionsSystemTable::Name() const { return kName; @@ -399,7 +405,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { // ============================================================================= TablesSystemTable::TablesSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/tables"), context_(std::move(context)) {} + : InMemorySystemTable(context.fs, "sys/tables"), context_(std::move(context)) {} std::string TablesSystemTable::Name() const { return kName; @@ -471,7 +477,7 @@ Result> TablesSystemTable::BuildRows() const { // ============================================================================= PartitionsSystemTable::PartitionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/partitions"), context_(std::move(context)) {} + : InMemorySystemTable(context.fs, "sys/partitions"), context_(std::move(context)) {} std::string PartitionsSystemTable::Name() const { return kName; diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index 8d6111d79..773f621b8 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -31,6 +31,7 @@ #include "paimon/core/table/system/system_table_scan.h" #include "paimon/memory/memory_pool.h" #include "paimon/read_context.h" +#include "paimon/scan_context.h" #include "paimon/status.h" #include "paimon/table/source/table_read.h" @@ -104,16 +105,18 @@ class InMemorySystemTableRead : public TableRead { } // namespace -InMemorySystemTable::InMemorySystemTable(std::string table_path) - : table_path_(std::move(table_path)) {} +InMemorySystemTable::InMemorySystemTable(const std::shared_ptr& fs, + std::string table_path) + : SystemTable(fs, table_path), table_path_(std::move(table_path)) {} Result> InMemorySystemTable::NewScan( - const std::shared_ptr& /*context*/) const { + std::unique_ptr /*context*/) const { return std::make_unique(table_path_); } Result> InMemorySystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); return std::make_unique( checked_pointer_cast(shared_from_this()), context->GetMemoryPool()); diff --git a/src/paimon/core/table/system/in_memory_system_table.h b/src/paimon/core/table/system/in_memory_system_table.h index f9fcebdb3..90cd46d84 100644 --- a/src/paimon/core/table/system/in_memory_system_table.h +++ b/src/paimon/core/table/system/in_memory_system_table.h @@ -28,18 +28,19 @@ namespace paimon { +class FileSystem; + /// Base class for system tables whose result can be materialized as a single in-memory /// RecordBatch. /// /// It provides the common singleton split scan and one-shot batch reader. class InMemorySystemTable : public SystemTable { public: - explicit InMemorySystemTable(std::string table_path); + InMemorySystemTable(const std::shared_ptr& fs, std::string table_path); Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + std::unique_ptr /*context*/) const override; + Result> NewRead(std::unique_ptr context) const override; virtual Result> BuildRows() const = 0; protected: diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index 5279e6871..e9e00ea42 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -504,9 +504,10 @@ Result> WriteColsValue( } // namespace -OptionsSystemTable::OptionsSystemTable(std::string table_path, +OptionsSystemTable::OptionsSystemTable(const std::shared_ptr& fs, + std::string table_path, std::shared_ptr table_schema) - : InMemorySystemTable(std::move(table_path)), table_schema_(std::move(table_schema)) {} + : InMemorySystemTable(fs, table_path), table_schema_(std::move(table_schema)) {} std::string OptionsSystemTable::Name() const { return kName; @@ -532,7 +533,7 @@ Result> OptionsSystemTable::BuildRows() const { SnapshotsSystemTable::SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} std::string SnapshotsSystemTable::Name() const { @@ -594,7 +595,7 @@ Result> SnapshotsSystemTable::BuildRows() const { SchemasSystemTable::SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} std::string SchemasSystemTable::Name() const { @@ -650,7 +651,7 @@ Result> SchemasSystemTable::BuildRows() const { TagsSystemTable::TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} std::string TagsSystemTable::Name() const { @@ -700,7 +701,7 @@ Result> TagsSystemTable::BuildRows() const { BranchesSystemTable::BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} std::string BranchesSystemTable::Name() const { @@ -739,7 +740,7 @@ Result> BranchesSystemTable::BuildRows() const { ConsumersSystemTable::ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} std::string ConsumersSystemTable::Name() const { @@ -774,7 +775,7 @@ ManifestsSystemTable::ManifestsSystemTable(std::shared_ptr fs, std:: std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} @@ -843,7 +844,7 @@ Result> ManifestsSystemTable::BuildRows() const { FilesSystemTable::FilesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} diff --git a/src/paimon/core/table/system/metadata_system_tables.h b/src/paimon/core/table/system/metadata_system_tables.h index e3961e476..ae2e5155b 100644 --- a/src/paimon/core/table/system/metadata_system_tables.h +++ b/src/paimon/core/table/system/metadata_system_tables.h @@ -35,7 +35,8 @@ class OptionsSystemTable : public InMemorySystemTable { public: static constexpr const char* kName = "options"; - OptionsSystemTable(std::string table_path, std::shared_ptr table_schema); + OptionsSystemTable(const std::shared_ptr& fs, std::string table_path, + std::shared_ptr table_schema); std::string Name() const override; Result> ArrowSchema() const override; diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 8d5696447..172fc964c 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -35,10 +35,12 @@ namespace paimon { -ReadOptimizedSystemTable::ReadOptimizedSystemTable(std::string table_path, +ReadOptimizedSystemTable::ReadOptimizedSystemTable(const std::shared_ptr& fs, + std::string table_path, std::shared_ptr table_schema, std::map options) - : table_path_(std::move(table_path)), + : SystemTable(fs, table_path), + table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} @@ -57,7 +59,8 @@ std::map ReadOptimizedSystemTable::ReadOptimizedOption } Result> ReadOptimizedSystemTable::NewScan( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); if (context->GetRealtimeContext() && !table_schema_->PrimaryKeys().empty()) { return Status::NotImplemented( "PK real-time union read does not support read-optimized scans"); @@ -133,7 +136,8 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon } Result> ReadOptimizedSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, CreateDataReadContext(context)); return TableRead::Create(std::move(base_context)); diff --git a/src/paimon/core/table/system/read_optimized_system_table.h b/src/paimon/core/table/system/read_optimized_system_table.h index 3123fe03b..7f92e04e2 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.h +++ b/src/paimon/core/table/system/read_optimized_system_table.h @@ -26,6 +26,7 @@ #include "paimon/core/table/system/system_table.h" namespace paimon { +class FileSystem; class TableSchema; /// System table for `T$ro`, exposing read-optimized data. @@ -33,15 +34,14 @@ class ReadOptimizedSystemTable : public SystemTable { public: static constexpr const char* kName = "ro"; - ReadOptimizedSystemTable(std::string table_path, std::shared_ptr table_schema, + ReadOptimizedSystemTable(const std::shared_ptr& fs, std::string table_path, + std::shared_ptr table_schema, std::map options); std::string Name() const override; Result> ArrowSchema() const override; - Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewScan(std::unique_ptr context) const override; + Result> NewRead(std::unique_ptr context) const override; private: /// Build the context for the data table underneath this read-optimized view. diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 5a1a1de00..dd00b7bcd 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -28,6 +28,7 @@ #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/schema/schema_manager.h" @@ -37,8 +38,12 @@ #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/metadata_system_tables.h" #include "paimon/core/table/system/read_optimized_system_table.h" +#include "paimon/core/table/system/system_table_schema.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/status.h" +#include "paimon/write_context.h" namespace paimon { namespace { @@ -70,11 +75,11 @@ std::string LoadBranch(const std::map& options) { const std::vector& SystemTableRegistry() { static const std::vector registry = { {OptionsSystemTable::kName, - [](const std::shared_ptr& /*fs*/, const std::string& table_path, + [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, const std::map& /*dynamic_options*/) -> Result> { - return std::make_shared(table_path, table_schema); + return std::make_shared(fs, table_path, table_schema); }}, {AuditLogSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, @@ -93,12 +98,12 @@ const std::vector& SystemTableRegistry() { fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); }}, {ReadOptimizedSystemTable::kName, - [](const std::shared_ptr& /*fs*/, const std::string& table_path, + [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, const std::map& dynamic_options) -> Result> { return std::make_shared( - table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); }}, {SnapshotsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, @@ -176,6 +181,27 @@ std::optional FindSystemTableFactory(const std::string& syst } // namespace +SystemTable::SystemTable(const std::shared_ptr& fs, const std::string& table_path) + : Table(fs, table_path, /*schema=*/nullptr, + Identifier(Identifier::kUnknownDatabase, "system_table"), /*uuid=*/"") {} + +Status SystemTable::AdoptOwnSchema() { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, ArrowSchema()); + schema_ = std::make_shared(std::move(arrow_schema)); + return Status::OK(); +} + +Result> SystemTable::NewWrite( + std::unique_ptr /*context*/) const { + return Status::NotImplemented(Name(), " system table is read-only: writing is not supported"); +} + +Result> SystemTable::NewCommit( + std::unique_ptr /*context*/) const { + return Status::NotImplemented(Name(), + " system table is read-only: committing for it is not supported"); +} + bool SystemTableLoader::IsSupported(const std::string& system_table_name) { return FindSystemTableFactory(system_table_name).has_value(); } @@ -186,7 +212,12 @@ Result> SystemTableLoader::Load( const std::map& dynamic_options) { std::optional factory = FindSystemTableFactory(system_table_name); if (factory) { - return factory.value()(fs, table_path, table_schema, dynamic_options); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + factory.value()(fs, table_path, table_schema, dynamic_options)); + // The table is handed out through the `Table` interface, which serves the schema the + // base holds, so its own read-only schema is adopted before it leaves the loader. + PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); + return system_table; } return Status::NotImplemented("unsupported system table: ", system_table_name); } diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 39e1da31b..2e4dff55d 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -25,6 +25,8 @@ #include #include "arrow/api.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/result.h" #include "paimon/status.h" #include "paimon/type_fwd.h" @@ -33,9 +35,8 @@ namespace paimon { class FileSystem; class ReadContext; class ScanContext; -class TableScan; -class TableRead; class TableSchema; +class SystemTableSchema; /// Parsed information for a table-scoped system table path. struct SystemTablePath { @@ -49,19 +50,42 @@ struct SystemTablePath { bool is_global = false; }; -/// Base interface for table-scoped system tables such as `T$options` and `T$snapshots`. +/// Base class for table-scoped system tables such as `T$options` and `T$snapshots`. /// -/// Implementations expose a read-only schema and create their own scan/read objects. -class SystemTable : public std::enable_shared_from_this { +/// A system table is the read-only view a `T$system` identifier names: it exposes a schema of +/// its own and serves scans and reads from the data table it belongs to, through the same +/// `Table` interface as every other kind of table. `Name()` - which each kind of system table +/// answers for itself - is its full name as well: the identifier it is opened by belongs to the +/// data table underneath and would name that one. +class SystemTable : public Table, public std::enable_shared_from_this { public: - virtual ~SystemTable() = default; + std::string FullName() const override { + return Name(); + } - virtual std::string Name() const = 0; + /// The read-only schema this system table exposes, which is not the data table's schema. virtual Result> ArrowSchema() const = 0; - virtual Result> NewScan( - const std::shared_ptr& context) const = 0; - virtual Result> NewRead( - const std::shared_ptr& context) const = 0; + + /// A system table is read-only: writing it or committing for it is refused rather than + /// routed to the data table underneath. + Result> NewWrite( + std::unique_ptr context) const override; + + Result> NewCommit( + std::unique_ptr context) const override; + + protected: + /// @param fs File system the data table underneath is read through. + /// @param table_path Root path of the data table this system table belongs to. + SystemTable(const std::shared_ptr& fs, const std::string& table_path); + + /// Loaders call this once the object exists, because `ArrowSchema()` is virtual and cannot + /// run from a constructor: it fills the schema the base hands out with this table's own + /// read-only `SystemTableSchema`. + Status AdoptOwnSchema(); + + friend class SystemTableLoader; + friend class GlobalSystemTableLoader; }; /// Loads system table implementations from parsed table identifiers or table paths. diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 7591e53bf..49cf6838f 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -249,7 +249,7 @@ TEST(SystemTableTest, TestNewReadPropagatesWarmupLevel) { AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); - ReadOptimizedSystemTable read_optimized("/tmp/table", table_schema, options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); for (WarmupLevel level : {WarmupLevel::NONE, WarmupLevel::RAW, WarmupLevel::DECODED}) { ReadContextBuilder builder("/tmp/table"); diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index fbd9521c4..3ff2ed756 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -390,49 +390,58 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i context.catalog_options = api_->GetMergedOptions(); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + return system_table->LatestSchema(); } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); branch = NormalizeBranch(std::move(branch)); PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); if (is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, - identifier.GetSystemTableName()); - if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { - return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); - } - std::string data_table_path; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr latest_schema, - LoadDataTableSchema(load_identifier, branch, &data_table_path, - /*table_id=*/nullptr)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + LoadSystemTable(identifier, load_identifier, branch)); if (table_path != nullptr) { - *table_path = data_table_path; - } - std::map dynamic_options; - if (branch) { - dynamic_options[Options::BRANCH] = branch.value(); + *table_path = system_table->GetRootPath(); } - // The file system is only used to build the system table, whose schema does not - // read any file, so the catalog wide one is enough here. The credentials of the - // table are applied when its rows are read, through the file system of the read or - // scan context. - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::Load(system_table_name.value(), fs_, data_table_path, latest_schema, - dynamic_options)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + return system_table->LatestSchema(); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadDataTableSchema(load_identifier, branch, table_path, table_id)); return checked_pointer_cast(schema); } +Result> RestCatalog::LoadSystemTable( + const Identifier& identifier, const Identifier& load_identifier, + const std::optional& branch) const { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::string data_table_path; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr latest_schema, + LoadDataTableSchema(load_identifier, branch, &data_table_path, + /*table_id=*/nullptr)); + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + // The file system is only used to build the system table, whose schema does not + // read any file, so the catalog wide one is enough here. The credentials of the + // table are applied when its rows are read, through the file system of the read or + // scan context. + return SystemTableLoader::Load(system_table_name.value(), fs_, data_table_path, latest_schema, + dynamic_options); +} + Result> RestCatalog::GetTable(const Identifier& identifier) const { + // A `T$system` identifier opens the read-only view it names, which reads the data table it + // belongs to and answers for itself through the same `Table` interface. + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + if (is_system_table) { + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + return LoadSystemTable(identifier, load_identifier, NormalizeBranch(std::move(branch))); + } // Read UUID, path and schema together to avoid mixing table generations after recreation. std::string uuid; std::string table_path; diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 6b7f22cda..9a4e6eb06 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -36,6 +36,7 @@ struct ArrowSchema; namespace paimon { class FileSystem; +class SystemTable; class TableSchema; /// A catalog backed by a REST catalog server. Metadata operations are delegated to @@ -127,6 +128,12 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { const Identifier& data_identifier, const std::optional& branch, std::string* table_path, std::string* table_id) const; + /// Loads the system table a `T$system` identifier names, on the branch the identifier + /// carries, from the same response that carries the data table's schema and path. + Result> LoadSystemTable( + const Identifier& identifier, const Identifier& load_identifier, + const std::optional& branch) const; + /// Converts a `GetTableResponse` into a `TableSchema`, applying the table path, audit /// and branch options. static Result> ToTableSchema( diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 48d5bf19c..c76aa4b33 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -44,6 +44,7 @@ #include "paimon/core/catalog/commit_table_request.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" @@ -994,9 +995,13 @@ TEST_F(RestCatalogTest, SystemTableSchema) { requests_before = state_->get_table_requests; } ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(system_identifier)); - ASSERT_EQ("t1$options", table->Name()); + // A system table comes back as the read-only `SystemTable` it is, answering for itself + // with the kind it serves rather than the identifier it was opened by. + std::shared_ptr system_table = std::dynamic_pointer_cast(table); + ASSERT_NE(nullptr, system_table); + ASSERT_EQ("options", system_table->Name()); ASSERT_EQ((std::vector{"key", "value"}), table->LatestSchema()->FieldNames()); - ASSERT_EQ("db1.t1$options", table->Uuid()); + ASSERT_EQ("options", table->Uuid()); { std::lock_guard lock(state_->mutex); ASSERT_EQ(requests_before + 1, state_->get_table_requests); diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 284cd362f..2c24607a2 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -4190,18 +4190,18 @@ Result> ReadGlobalSystemTable( ScanContextBuilder scan_context_builder(sys_path); scan_context_builder.SetOptions(options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr scan_context, + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - system_table->NewScan(scan_context)); + system_table->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); ReadContextBuilder read_context_builder(sys_path); read_context_builder.SetOptions(options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - system_table->NewRead(read_context)); + system_table->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, From a2ccafd8cdef9cb656250de9e22b1b52f900a794 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 20 Oct 2026 17:49:07 +0800 Subject: [PATCH 03/19] tmp --- include/paimon/catalog/table.h | 10 +- include/paimon/file_store_commit.h | 3 +- include/paimon/file_store_write.h | 3 +- include/paimon/table/source/table_read.h | 3 +- include/paimon/table/source/table_scan.h | 3 +- src/paimon/core/table/file_store_table.h | 10 +- src/paimon/core/table/table.cpp | 594 +++++++++++++---------- src/paimon/core/table/table_test.cpp | 23 + 8 files changed, 391 insertions(+), 258 deletions(-) diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index 17b3b3d21..69abbe6d5 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -99,29 +99,29 @@ class PAIMON_EXPORT Table { } /// Creates a scan of this table from a finished scan context, the table-first entry point - /// that `TableScan::Create()` also builds through. The scan is derived from the context. + /// that `TableScan::Create()` loads the table the context names and hands the context to. /// @param context The finished scan context. /// @return The scan, or the error building it reports. virtual Result> NewScan( std::unique_ptr context) const = 0; /// Creates a read of this table from a finished read context, the table-first entry point - /// that `TableRead::Create()` also builds through. The read is derived from the context. + /// that `TableRead::Create()` loads the table the context names and hands the context to. /// @param context The finished read context. /// @return The read, or the error building it reports. virtual Result> NewRead( std::unique_ptr context) const = 0; /// Creates a write of this table from a finished write context, the table-first entry point - /// that `FileStoreWrite::Create()` also builds through. The write is derived from the context. + /// that `FileStoreWrite::Create()` loads the table the context names and hands the context to. /// @param context The finished write context. /// @return The write, or the error building it reports. virtual Result> NewWrite( std::unique_ptr context) const = 0; /// Creates a commit of this table from a finished commit context, the table-first entry point - /// that `FileStoreCommit::Create()` also builds through. The commit is derived from the - /// context. + /// that `FileStoreCommit::Create()` loads the table the context names and hands the context + /// to. /// @param context The finished commit context. /// @return The commit, or the error building it reports. virtual Result> NewCommit( diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index fcb094e05..581cadf2d 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -50,7 +50,8 @@ class CommitMessage; /// manages versions. `UseRESTCatalogCommit()` only prepares a request for the caller to send. class PAIMON_EXPORT FileStoreCommit { public: - /// Create an instance of `FileStoreCommit`. + /// Create an instance of `FileStoreCommit`: the table the context names is loaded first, then + /// the context is handed to that table's `NewCommit`. /// /// @param context A unique pointer to the `CommitContext` used for commit operations. /// diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index 8859e4f3f..5f74d6536 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -42,7 +42,8 @@ class WriteContext; /// Interface for write operations in a file store. class PAIMON_EXPORT FileStoreWrite { public: - /// Create an instance of `FileStoreWrite`. + /// Create an instance of `FileStoreWrite`: the table the context names is loaded first, then + /// the context is handed to that table's `NewWrite`. /// /// @param context A unique pointer to the `WriteContext` used for write operations. /// diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index c77b323da..2bfd84fff 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -42,7 +42,8 @@ class PAIMON_EXPORT TableRead { public: virtual ~TableRead() = default; - /// Create an instance of `TableRead`. + /// Create an instance of `TableRead`: the table the context names is loaded first, then the + /// context is handed to that table's `NewRead`. /// /// @param context A unique pointer to the `ReadContext` used for read operations. /// @return A Result containing a unique pointer to the `TableRead` instance. diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h index 8bae74c41..39f0453c8 100644 --- a/include/paimon/table/source/table_scan.h +++ b/include/paimon/table/source/table_scan.h @@ -36,7 +36,8 @@ class ScanContext; /// A scanner interface for reading table's meta and create a plan. class PAIMON_EXPORT TableScan { public: - /// Create an instance of `TableScan`. + /// Create an instance of `TableScan`: the table the context names is loaded first, then the + /// context is handed to that table's `NewScan`. /// /// @param context A unique pointer to the `ScanContext` used for scan operations. /// @return A Result containing a unique pointer to the `TableScan` instance. diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index f02b4e528..428513e16 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -35,10 +35,12 @@ class Schema; /// its root path describes. A format table - a directory of plain data files - is a `FormatTable`, /// the other kind of `Table`. /// -/// The operators create through the context-only entry points every table serves, so a context -/// built without a table reaches the same operators: `TableScan::Create()`, `TableRead::Create()`, -/// `FileStoreWrite::Create()` and `FileStoreCommit::Create()`, which read the table's schema to -/// dispatch on its type. +/// Its `NewScan`/`NewRead`/`NewWrite`/`NewCommit` build the operators here, from the context plus +/// the schema the table was constructed with, so they serve the table as it was loaded. The +/// context-only entry points - `TableScan::Create()`, `TableRead::Create()`, +/// `FileStoreWrite::Create()` and `FileStoreCommit::Create()` - load the table the context names +/// and hand the context to that table's `NewXxx`, which is how a context built without a table +/// reaches the same operators. class FileStoreTable : public Table { public: /// The constructor a catalog builds a managed table through, so a `shared_ptr` can hold one; diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index 3b50d2e85..f9cf78545 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -209,94 +209,166 @@ Result HasHistoricalPrimaryKeyBTreeDefinition( return false; } -} // namespace - -Table::Table(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid) - : file_system_(file_system), - root_path_(root_path), - schema_(schema), - identifier_(identifier), - uuid_(uuid) {} - -Result> FileStoreTable::NewScan( - std::unique_ptr context) const { - return TableScan::Create(std::move(context)); -} - -Result> FileStoreTable::NewRead( - std::unique_ptr context) const { - return TableRead::Create(std::move(context)); -} - -Result> FileStoreTable::NewWrite( - std::unique_ptr context) const { - return FileStoreWrite::Create(std::move(context)); -} - -Result> FileStoreTable::NewCommit( - std::unique_ptr context) const { - return FileStoreCommit::Create(std::move(context)); +/// The tail every `TableOf` ends with: the schema one of them read says what the path names. A +/// format table is handed back as the `FormatTable` it is, everything else as the +/// `FileStoreTable` a managed table is, carrying the schema for its `NewXxx` to build from. A +/// null schema is handed over as it is: what to say about a table that is not there is the +/// managed path's to decide, and it says it in more detail. +Result> TableOfLoadedSchema( + const std::shared_ptr& file_system, const std::string& path, + const std::string& branch, const std::map& options, + const std::shared_ptr& latest_schema, const std::optional& identifier, + const std::string& uuid = "") { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + TryLoadFormatTable(file_system, path, branch, options, latest_schema)); + if (format_table != nullptr) { + return std::shared_ptr
(std::move(format_table)); + } + // A context names a path, never a catalog identifier, so the table is known by the directory + // it sits in - the same convention `TryLoadFormatTable` follows above. + Identifier table_identifier = + identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); + return std::make_shared(file_system, path, latest_schema, table_identifier, + uuid); } -Result> FileStoreWrite::Create(std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("write context is null pointer"); +/// The table a scan context names, read from what the context carries: a format table the +/// builder was handed, a system table behind a `T$system` path, or the table - format or +/// managed - the schema under the context's path says is there. +Result> TableOf(const ScanContext& ctx) { + // A table the caller already loaded says what it is, so nothing is read to find out. + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), {})); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(ctx.GetPath())); + if (system_table_path) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), + ctx.GetPath(), ctx.GetOptions())); + return std::shared_ptr
(std::move(system_table)); } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); + // A format table is planned by listing directories, so it never reaches the manifest path + // below. One `TableScan` interface serves both, as Java Paimon serves both through one + // `ReadBuilder`. + const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx.GetSpecificTableSchema()) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(ctx.GetSpecificTableSchema().value())); + } else { + SchemaManager schema_manager(tmp_options.GetFileSystem(), ctx.GetPath(), branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; + } } + return TableOfLoadedSchema(tmp_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), + latest_schema, ctx.GetIdentifier()); +} +/// The table a read context names, read from what the context carries: a format table the +/// builder was handed, a system table behind a `T$system` path, or the table - format or +/// managed - the schema under the context's path says is there. +Result> TableOf(const ReadContext& ctx) { // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - return ctx->GetFormatTable()->NewWrite(std::move(ctx)); + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, + CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), + ctx.GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(ctx.GetPath())); + if (system_table_path) { + if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { + return Status::NotImplemented( + "FullTextMatch predicate execution is not implemented yet"); + } + if (ctx.HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch is not supported for system tables"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), + ctx.GetPath(), ctx.GetOptions())); + return std::shared_ptr
(std::move(system_table)); } - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && - ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { + // A format table has no manifest and a file another engine wrote carries no field ids, so + // which files there are and how a row is put back together is its own. How a file is opened + // is not: both paths go through `DataFileReaderFactory`. + const std::string branch = ctx.GetBranch(); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx.GetSpecificTableSchema()) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(ctx.GetSpecificTableSchema().value())); + } else { + SchemaManager schema_manager(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; + } + } + return TableOfLoadedSchema(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch, + ctx.GetOptions(), latest_schema, ctx.GetIdentifier()); +} + +/// The table a write context names, read from what the context carries: a format table the +/// builder was handed, or the table - format or managed - the schema under the context's root +/// path says is there. +Result> TableOf(const WriteContext& ctx) { + // A table the caller already loaded says what it is, so nothing is read to find out. + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); + std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx.GetFileSystemSchemeToIdentifierMap().empty() && + ctx.GetCatalog() != nullptr) { + if (!ctx.GetIdentifier()) { return Status::Invalid("a catalog write requires a table identifier"); } // A catalog issuing per-table temporary credentials only hands them out through // GetTableFileSystem, so the write uses the credentials of this table. PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); + ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); } PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs, - ctx->GetFileSystemSchemeToIdentifierMap())); - std::string branch = ctx->GetBranch(); + CoreOptions::FromMap(ctx.GetOptions(), specific_fs, + ctx.GetFileSystemSchemeToIdentifierMap())); + std::string branch = ctx.GetBranch(); std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); + if (ctx.GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + CatalogUtils::BranchIdentifier(ctx.GetIdentifier().value(), branch)); catalog_identifier.emplace(std::move(branch_identifier)); } // Only the main branch may have a current schema the catalog alone holds. A branch publishes // its own under `branch/branch-`, which is where the load below and a read of the branch // both read it, so a catalog answering for no branch schema does not fail a write to one. std::optional catalog_table_schema; - if (ctx->GetCatalog() != nullptr) { + if (ctx.GetCatalog() != nullptr) { if (!catalog_identifier) { return Status::Invalid("a catalog write requires a table identifier"); } if (BranchManager::IsMainBranch(branch)) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); + ctx.GetCatalog()->LoadTableSchema(catalog_identifier.value())); PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, schema->GetJsonSchema()); } } // A format table writes plain data files into a directory, so it never reaches the manifest // path below. - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); + SchemaManager schema_manager(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch); std::shared_ptr latest_schema; if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { // Handing the schema over only saves reading it. A context always names a table path and @@ -304,22 +376,136 @@ Result> FileStoreWrite::Create(std::unique_ptr> latest, - schema_manager->Latest()); + schema_manager.Latest()); if (latest) { latest_schema = *latest; } } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - TryLoadFormatTable(tmp_options.GetFileSystem(), ctx->GetRootPath(), - branch, ctx->GetOptions(), latest_schema)); - if (format_table != nullptr) { - return format_table->NewWrite(std::move(ctx)); + return TableOfLoadedSchema(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch, + ctx.GetOptions(), latest_schema, ctx.GetIdentifier()); +} + +/// The table a commit context names, read from what the context carries: a format table the +/// builder was handed, or the table - format or managed - the schema under the context's root +/// path says is there. +Result> TableOf(const CommitContext& ctx) { + // A table the caller already loaded says what it is, so nothing is read to find out. + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); + // Use catalog credentials unless the caller supplied a file system. + std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx.GetCatalog() != nullptr) { + if (!ctx.GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + // Use the credentials of this table, which a catalog issuing per-table temporary + // ones only hands out through GetTableFileSystem. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), specific_fs)); + const std::string& root_path = ctx.GetRootPath(); + // The branch the context was built for, already validated by `CommitContextBuilder`. + PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( + ctx.GetIdentifier(), ctx.GetOptions(), + /*explicit_branch=*/std::nullopt, "commit")); + std::optional catalog_identifier; + if (ctx.GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + CatalogUtils::BranchIdentifier(ctx.GetIdentifier().value(), branch)); + catalog_identifier.emplace(std::move(branch_identifier)); + } + // Catalog-managed schemas may be absent from the table directory, but only on the main branch: + // a branch publishes its own under `branch/branch-`, which is where the load below and a + // read of the branch both read it, so a catalog answering for no branch schema does not fail + // a commit to one. + std::optional catalog_table_schema; + if (ctx.GetCatalog() != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, + ctx.GetCatalog()->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); + catalog_table_schema = std::move(schema_json); + } + } + // A format table commits by renaming files into place, so it never reaches the snapshot path + // below. The managed path here reads the commit's branch, so this reads the same one: the two + // must not dispatch on different schemas. + SchemaManager schema_manager(tmp_options.GetFileSystem(), root_path, branch); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(catalog_table_schema.value())); + } else { + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; + } } - // The schema the branch read above through `schema_manager`, rather than a second read of - // the same file. + return TableOfLoadedSchema(tmp_options.GetFileSystem(), root_path, branch, ctx.GetOptions(), + latest_schema, ctx.GetIdentifier(), ctx.GetTableId().value_or("")); +} + +} // namespace + +Table::Table(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid) + : file_system_(file_system), + root_path_(root_path), + schema_(schema), + identifier_(identifier), + uuid_(uuid) {} + +Result> FileStoreTable::NewWrite( + std::unique_ptr ctx) const { + if (ctx == nullptr) { + return Status::Invalid("write context is null pointer"); + } + if (ctx->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (ctx->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && + ctx->GetCatalog() != nullptr) { + if (!ctx->GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); + } + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the write uses the credentials of this table. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx->GetOptions(), specific_fs, + ctx->GetFileSystemSchemeToIdentifierMap())); + std::string branch = ctx->GetBranch(); + std::optional catalog_identifier; + if (ctx->GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE( + Identifier branch_identifier, + CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); + catalog_identifier.emplace(std::move(branch_identifier)); + } + auto schema_manager = + std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); + // The schema the table was constructed with, so a write serves the table as it was loaded + // rather than reading the schema again. + std::shared_ptr latest_schema = std::static_pointer_cast(schema_); if (latest_schema == nullptr) { return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); } @@ -586,6 +772,21 @@ Result> FileStoreWrite::Create(std::unique_ptr> FileStoreWrite::Create(std::unique_ptr ctx) { + if (ctx == nullptr) { + return Status::Invalid("write context is null pointer"); + } + if (ctx->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (ctx->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + // First the table the context names, then the context handed to that table's `NewWrite`. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + return table->NewWrite(std::move(ctx)); +} + namespace { class TableScanImpl { @@ -691,71 +892,6 @@ class TableScanImpl { } }; -/// @param loaded_schema The table's schema, when the caller already read it, or null when it has -/// to be read here. `TableScan::Create()` reads one to dispatch on its table type. -Result> NewDataTableScan( - const std::shared_ptr& context, const std::shared_ptr& loaded_schema); - -} // namespace - -Result> TableScan::Create(std::unique_ptr context) { - if (context == nullptr) { - return Status::Invalid("scan context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (context->GetFormatTable() != nullptr) { - return context->GetFormatTable()->NewScan(std::move(context)); - } - // load schema - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(context->GetPath())); - if (system_table_path) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), context->GetPath(), - context->GetOptions())); - return system_table->NewScan(std::move(context)); - } - // A format table is planned by listing directories, so it never reaches the manifest path - // below. One `TableScan` interface serves both, as Java Paimon serves both through one - // `ReadBuilder`. - const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && context->GetSpecificTableSchema()) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE( - latest_schema, TableSchema::CreateFromJson(context->GetSpecificTableSchema().value())); - } else { - SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; - } - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - TryLoadFormatTable(tmp_options.GetFileSystem(), context->GetPath(), - branch, context->GetOptions(), latest_schema)); - if (format_table != nullptr) { - return format_table->NewScan(std::move(context)); - } - // With the schema already read above, so the managed path does not read it again. - return NewDataTableScan(std::move(context), latest_schema); -} - -namespace { - Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options, const ScanContext& context, bool read_optimized) { if (!context.GetRealtimeContext()) { @@ -798,12 +934,22 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& return Status::OK(); } -/// @param loaded_schema The schema `TableScan::Create()` already read while dispatching on the -/// table type, or null when neither the caller's json nor the branch's own directory had -/// one. -Result> NewDataTableScan( - const std::shared_ptr& context, - const std::shared_ptr& loaded_schema) { +} // namespace + +Result> FileStoreTable::NewScan( + std::unique_ptr context) const { + if (context == nullptr) { + return Status::Invalid("scan context is null pointer"); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + // The schema the table was constructed with, so a scan serves the table as it was loaded + // rather than reading the schema again. + std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); if (loaded_schema == nullptr) { return Status::Invalid("not found latest schema"); } @@ -906,12 +1052,25 @@ Result> NewDataTableScan( return batch_scan; } -} // namespace +Result> TableScan::Create(std::unique_ptr context) { + if (context == nullptr) { + return Status::Invalid("scan context is null pointer"); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + // First the table the context names, then the context handed to that table's `NewScan`. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*context)); + return table->NewScan(std::move(context)); +} namespace { /// @param loaded_schema The schema of `branch`, when the caller already read it, or null when it -/// has to be read here: `TableRead::Create()` reads one to dispatch on the table type, and +/// has to be read here: `FileStoreTable::NewRead()` hands the table's own schema over, and /// reading it again would cost a second listing and read. Result> CreateInternalReadContext( const std::shared_ptr& context, const std::string& branch, @@ -987,15 +1146,30 @@ Result> CreateTableRead( return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); } -Result> NewDataTableRead( - const std::shared_ptr& context, - const std::shared_ptr& loaded_schema) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, - CreateInternalReadContext(context, context->GetBranch(), loaded_schema)); +} // namespace +Result> FileStoreTable::NewRead( + std::unique_ptr context) const { + std::shared_ptr read_context = std::move(context); + if (read_context == nullptr) { + return Status::Invalid("read context is null pointer"); + } + if (read_context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (read_context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + // The schema the table was constructed with, so a read serves the table as it was loaded + // rather than reading the schema again. + std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr table_read, - CreateTableRead(internal_context, context->GetMemoryPool(), context->GetExecutor())); + std::shared_ptr internal_context, + CreateInternalReadContext(read_context, read_context->GetBranch(), loaded_schema)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + CreateTableRead(internal_context, read_context->GetMemoryPool(), + read_context->GetExecutor())); std::optional scan_fallback_branch = internal_context->GetCoreOptions().GetScanFallbackBranch(); @@ -1006,19 +1180,17 @@ Result> NewDataTableRead( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr fallback_context, - CreateInternalReadContext(context, /*branch=*/scan_fallback_branch.value(), + CreateInternalReadContext(read_context, /*branch=*/scan_fallback_branch.value(), /*loaded_schema=*/nullptr)); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr fallback_table_read, - CreateTableRead(fallback_context, context->GetMemoryPool(), context->GetExecutor())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, + CreateTableRead(fallback_context, read_context->GetMemoryPool(), + read_context->GetExecutor())); const std::shared_ptr& arrow_pool = internal_context->GetArrowMemoryPool(); return std::make_unique(std::move(table_read), std::move(fallback_table_read), arrow_pool); } -} // namespace - Result> TableRead::Create(std::unique_ptr ctx) { if (ctx == nullptr) { return Status::Invalid("read context is null pointer"); @@ -1029,57 +1201,9 @@ Result> TableRead::Create(std::unique_ptrGetExecutor() == nullptr) { return Status::Invalid("executor is null pointer"); } - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - return ctx->GetFormatTable()->NewRead(std::move(ctx)); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, - CoreOptions::FromMap(ctx->GetOptions(), ctx->GetSpecificFileSystem(), - ctx->GetFileSystemSchemeToIdentifierMap())); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(ctx->GetPath())); - if (system_table_path) { - if (PredicateUtils::ContainsFullTextMatch(ctx->GetPredicate())) { - return Status::NotImplemented( - "FullTextMatch predicate execution is not implemented yet"); - } - if (ctx->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for system tables"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), - ctx->GetPath(), ctx->GetOptions())); - return system_table->NewRead(std::move(ctx)); - } - - // A format table has no manifest and a file another engine wrote carries no field ids, so - // which files there are and how a row is put back together is its own. How a file is opened - // is not: both paths go through `DataFileReaderFactory`. - const std::string branch = ctx->GetBranch(); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx->GetSpecificTableSchema()) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(ctx->GetSpecificTableSchema().value())); - } else { - SchemaManager schema_manager(tmp_core_options.GetFileSystem(), ctx->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; - } - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - TryLoadFormatTable(tmp_core_options.GetFileSystem(), ctx->GetPath(), - branch, ctx->GetOptions(), latest_schema)); - if (format_table != nullptr) { - return format_table->NewRead(std::move(ctx)); - } - - // With the schema already read above, so the managed path does not read it again. - return NewDataTableRead(std::move(ctx), latest_schema); + // First the table the context names, then the context handed to that table's `NewRead`. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + return table->NewRead(std::move(ctx)); } namespace { @@ -1157,8 +1281,8 @@ Result> NewSnapshotCommit( } // namespace -Result> FileStoreCommit::Create( - std::unique_ptr ctx) { +Result> FileStoreTable::NewCommit( + std::unique_ptr ctx) const { if (ctx == nullptr) { return Status::Invalid("commit context is null pointer"); } @@ -1168,12 +1292,6 @@ Result> FileStoreCommit::Create( if (ctx->GetExecutor() == nullptr) { return Status::Invalid("executor is null pointer"); } - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - return ctx->GetFormatTable()->NewCommit(std::move(ctx)); - } - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); // Use catalog credentials unless the caller supplied a file system. std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); @@ -1200,22 +1318,13 @@ Result> FileStoreCommit::Create( CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); catalog_identifier.emplace(std::move(branch_identifier)); } - // Catalog-managed schemas may be absent from the table directory, but only on the main branch: - // a branch publishes its own under `branch/branch-`, which is where the load below and a - // read of the branch both read it, so a catalog answering for no branch schema does not fail - // a commit to one. - std::optional catalog_table_schema; + // The id a snapshot records the catalog's current schema by, read through the loader below + // rather than trusted from a schema this commit will not re-read. FileStoreCommitImpl::SchemaIdLoader schema_id_loader; if (ctx->GetCatalog() != nullptr) { if (!catalog_identifier) { return Status::Invalid("a catalog commit requires a table identifier"); } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); - catalog_table_schema = std::move(schema_json); - } std::shared_ptr catalog = ctx->GetCatalog(); Identifier identifier = catalog_identifier.value(); schema_id_loader = [catalog, identifier]() -> Result { @@ -1230,32 +1339,11 @@ Result> FileStoreCommit::Create( return data_schema->Id(); }; } - // A format table commits by renaming files into place, so it never reaches the snapshot path - // below. The managed path here reads the commit's branch, so this reads the same one: the two - // must not dispatch on different schemas. auto schema_manager = std::make_shared(tmp_options.GetFileSystem(), root_path, branch); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(catalog_table_schema.value())); - } else { - // Through the manager built above, so the read warms the cache the managed path below - // goes on to use. - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager->Latest()); - if (latest) { - latest_schema = *latest; - } - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - TryLoadFormatTable(tmp_options.GetFileSystem(), root_path, branch, - ctx->GetOptions(), latest_schema)); - if (format_table != nullptr) { - return format_table->NewCommit(std::move(ctx)); - } + // The schema the table was constructed with, so a commit serves the table as it was loaded + // rather than reading the schema again. + std::shared_ptr latest_schema = std::static_pointer_cast(schema_); if (latest_schema == nullptr) { return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); } @@ -1358,4 +1446,20 @@ Result> FileStoreCommit::Create( expire_snapshots, schema_manager, std::move(scan_supplier)); } +Result> FileStoreCommit::Create( + std::unique_ptr ctx) { + if (ctx == nullptr) { + return Status::Invalid("commit context is null pointer"); + } + if (ctx->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (ctx->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + // First the table the context names, then the context handed to that table's `NewCommit`. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + return table->NewCommit(std::move(ctx)); +} + } // namespace paimon diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 78d01e5a5..1ad8d9cd3 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -291,6 +291,29 @@ TEST(TableTest, TestNewScanCreatesATableScan) { table->NewScan(std::move(ctx))); } +TEST(TableTest, TestNewScanServesTheSchemaTheTableWasLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier)); + + // With the schema file gone the table still serves a scan: it carries the schema it was + // loaded with, which is what a scan is built from. + SchemaManager schema_manager(fs, dir->Str()); + ASSERT_OK(fs->Delete(schema_manager.ToSchemaPath(0), /*recursive=*/false)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr held_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + table->NewScan(std::move(held_ctx))); + + // A context without a table still reads the schema from the directory, which no longer has + // one. + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ScanContextBuilder(dir->Str()).Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); +} + TEST(TableTest, TestNewReadCreatesATableRead) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); From 5cac87bd9fa9020b95fa3bf6e872ed5c135d5486 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Thu, 22 Oct 2026 15:05:34 +0800 Subject: [PATCH 04/19] tmp --- include/paimon/catalog/table.h | 22 +- include/paimon/table/format/format_table.h | 35 +- src/paimon/CMakeLists.txt | 1 + .../core/catalog/file_system_catalog.cpp | 8 +- ...eted_primary_key_index_maintainer_test.cpp | 26 +- .../append_only_file_store_write_test.cpp | 22 +- .../core/operation/internal_read_context.cpp | 28 +- .../core/operation/internal_read_context.h | 14 +- src/paimon/core/schema/table_schema.cpp | 42 +- src/paimon/core/schema/table_schema.h | 13 + src/paimon/core/table/file_store.cpp | 100 +++ src/paimon/core/table/file_store.h | 125 +++ src/paimon/core/table/file_store_table.h | 59 +- src/paimon/core/table/format/format_table.cpp | 33 +- .../core/table/format/format_table_test.cpp | 26 +- .../core/table/source/table_read_test.cpp | 54 +- src/paimon/core/table/system/system_table.cpp | 5 + src/paimon/core/table/system/system_table.h | 6 + .../core/table/system/system_table_test.cpp | 14 + src/paimon/core/table/table.cpp | 717 +++++++++--------- src/paimon/core/table/table_test.cpp | 434 ++++++++++- src/paimon/rest/rest_catalog.cpp | 8 +- src/paimon/rest/rest_catalog_test.cpp | 6 +- test/inte/realtime_write_inte_test.cpp | 57 +- test/inte/scan_and_read_inte_test.cpp | 64 +- test/inte/scan_inte_test.cpp | 27 +- .../write_and_read_inte_with_index_test.cpp | 3 +- 27 files changed, 1449 insertions(+), 500 deletions(-) create mode 100644 src/paimon/core/table/file_store.cpp create mode 100644 src/paimon/core/table/file_store.h diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index 69abbe6d5..a31d3100d 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -55,13 +55,15 @@ class PAIMON_EXPORT Table { /// Returns the file system the table was opened with, which serves the table's /// files. A catalog that hands out per-table file systems opens a table with the - /// one for that table; it is null when the opener had none. + /// one for that table; it is null when the opener had none. The `NewXxx` + /// operations read the table through it. std::shared_ptr GetFileSystem() const { return file_system_; } /// Returns the root path of the table in its file system. It is empty when the /// opener had no location for the table, which a mock in a test is one case of. + /// The `NewXxx` operations serve the table from it. const std::string& GetRootPath() const { return root_path_; } @@ -74,13 +76,6 @@ class PAIMON_EXPORT Table { /// table answers database.tableName, a system table the same name as `Name()`. virtual std::string FullName() const = 0; - /// Returns the branch the table was opened for, which its identifier carries; `main` - /// when it names none or a spelling of it, which its managers are bound to. A - /// malformed object name that hides the branch is reported. - Result CurrentBranch() const { - return identifier_.GetBranchNameOrDefault(); - } - /// Returns the metastore UUID, falling back to `FullName()`. /// @note Use `CatalogUuid()` for commit requests to avoid sending a table name as an ID. std::string Uuid() const { @@ -98,6 +93,17 @@ class PAIMON_EXPORT Table { return schema_; } + /// Copies this table with `dynamic_options` on top of the options it carries, which is + /// the precedence every context builder promises for a table it was handed rather than loaded + /// itself: an option given at the call wins over the one the schema stored. + /// + /// `type` is not overridable: it is structural and is read from the schema alone. + /// + /// @param dynamic_options Options given at the call. + /// @return A result containing the copied table, or the error copying it reports. + virtual Result> Copy( + const std::map& dynamic_options) const = 0; + /// Creates a scan of this table from a finished scan context, the table-first entry point /// that `TableScan::Create()` loads the table the context names and hands the context to. /// @param context The finished scan context. diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index d15a8f6ce..88444d360 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -55,8 +55,7 @@ class FileSystem; /// loaded through the `FormatTable` constructor every context builder has, which is how a table /// whose schema lives in a metastore is reached. /// -/// Every instance lives in a `shared_ptr`, which is what its operators hand to -/// `FormatTable::Copy()`. +/// Every instance lives in a `shared_ptr`: a copy that adds nothing hands the same one back. /// /// See `docs/source/user_guide/format_table.rst` for what is not supported yet. class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_this { @@ -108,20 +107,7 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t bool location_carries_paimon_metadata, const std::map& dynamic_options); - /// Copies `table` with `dynamic_options` on top of the options it already carries, which is - /// the precedence every context builder promises for a table it was handed rather than loaded - /// itself. `table` comes back as it is when there is nothing to add. - /// - /// `type` is not overridable: it is structural and is read from the schema alone. - /// - /// @param table The table to copy. - /// @param dynamic_options Options given at the call. - /// @return A result containing the copied table, or an error status. - static Result> Copy( - const std::shared_ptr& table, - const std::map& dynamic_options); - - ~FormatTable(); + ~FormatTable() override; std::string Name() const override { return identifier_.GetObjectName(); @@ -131,6 +117,11 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t return identifier_.GetFullName(); } + /// Copies the table with `dynamic_options` on top of the options it carries, the precedence + /// every context builder promises. The same table comes back when there is nothing to add. + Result> Copy( + const std::map& dynamic_options) const override; + /// Directory the data files live in. const std::string& Location() const { return GetRootPath(); @@ -217,15 +208,21 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata); - /// This table as a `shared_ptr`, which the operators above hand to `FormatTable::Copy()`. + /// This table as a `shared_ptr`, which a copy that adds nothing hands back. /// /// `shared_from_this()` on a const method hands back a `shared_ptr`, while - /// `Copy()` takes a mutable one; the cast is safe because `Copy()` never writes to the table - /// it is given. + /// the copy below needs a mutable one; the cast is safe because the table is never written to. std::shared_ptr SharedSelf() const { return std::const_pointer_cast(shared_from_this()); } + /// Copies this table with `dynamic_options` on top of the options it carries, as a format + /// table rather than the `Table` `Copy()` hands back, which is what its operators need. The + /// same table comes back when there is nothing to add; `type` is not overridable, and + /// `Create()` keeps the schema's own. + Result> CopySelf( + const std::map& dynamic_options) const; + std::map options_; Format format_; std::string file_compression_; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index ad3d65f90..261987aea 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -445,6 +445,7 @@ set(PAIMON_CORE_SRCS core/stats/simple_stats_evolution.cpp core/table/table.cpp core/table/bucket_mode.cpp + core/table/file_store.cpp core/table/format/format_file_listing.cpp core/table/format/format_file_naming.cpp core/table/format/format_path_validation.cpp diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 46131f729..36721a148 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -347,8 +347,12 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide // table rather than at the first read. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); - return std::make_shared( - fs_, table_path, checked_pointer_cast(*latest_schema), identifier); + // The branch the schema above was read on, which the table carries so its managers read + // the same branch: `GetBranchNameOrDefault` folds any spelling of `main`. + PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); + return std::make_shared(fs_, table_path, + checked_pointer_cast(*latest_schema), + identifier, /*uuid=*/"", branch); } Result> FileSystemCatalog::LoadSystemTable( diff --git a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp index 81803ed90..769bbf326 100644 --- a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp +++ b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp @@ -33,6 +33,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/global_index/indexed_split_impl.h" @@ -473,15 +474,21 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsRestoresAndReplacesCompacted ASSERT_EQ(0, data_split_count); ReadContextBuilder read_builder(table_path_); - // The source build above keeps lazy decoding enabled. Disable it only for the final data-row - // assertion, whose predicate reader is outside the maintenance path covered by this test. read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) - .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + // The source build above keeps lazy decoding enabled. Disable it only for the final data-row + // assertion, whose predicate reader is outside the maintenance path covered by this test. + // The table is copied with it off because a read through the loaded table refuses an option + // that contradicts the one it carries. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(directory_->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded_table, + catalog->GetTable(Identifier("db", "table"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied_table, + loaded_table->Copy({{"orc.read.enable-lazy-decoding", "false"}})); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + copied_table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -707,11 +714,18 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, ContinuesCompactionWhenIndexBuildF ReadContextBuilder read_builder(table_path_); read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) - .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + // Like the test above, the assertion reads with lazy decoding disabled, so it reads through + // a copy of the table carrying it off: a read through the loaded table refuses an option that + // contradicts the one it carries. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(directory_->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded_table, + catalog->GetTable(Identifier("db", "table"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied_table, + loaded_table->Copy({{"orc.read.enable-lazy-decoding", "false"}})); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + copied_table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp index bfeaefced..3ce48ee2c 100644 --- a/src/paimon/core/operation/append_only_file_store_write_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp @@ -36,6 +36,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" @@ -513,10 +514,25 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingNewWriterIgnoresExisting /*commit_identifier=*/0); Commit(table_path, avro_options, avro_commit_msgs); - auto second_commit_msgs = WriteAndPrepare(table_path, logical_schema, shredding_options, R"([ + // The second batch is written through a copy of the table carrying the shredding options: + // an option on the context would contradict the table the catalog loaded, which a write + // through that table refuses. + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir->Str(), avro_options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
avro_table, + catalog->GetTable(Identifier("foo", "bar"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
shredded_table, + avro_table->Copy(shredding_options)); + WriteContextBuilder shredded_builder(table_path, commit_user_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr shredded_context, shredded_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto shredded_write, + shredded_table->NewWrite(std::move(shredded_context))); + ASSERT_OK(shredded_write->Write(MakeBatch(logical_schema, R"([ [2, [["c", 3], ["d", 4], ["e", 5]]] - ])", - /*commit_identifier=*/1); + ])"))); + ASSERT_OK_AND_ASSIGN(auto second_commit_msgs, + shredded_write->PrepareCommit(/*wait_compaction=*/false, + /*commit_identifier=*/1)); + ASSERT_OK(shredded_write->Close()); auto second_file_schema = ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), shredding_options); auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/1); diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index f36a15d71..6c0c43341 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -275,10 +275,15 @@ std::optional InternalReadContext::TryResolveSpecialFieldByName( Result> InternalReadContext::Create( const std::shared_ptr& context, const std::shared_ptr& table_schema, - const std::map& options) { - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); + const std::map& options, + const std::shared_ptr& file_system, const std::string* path) { + // The file system a table reads through, or null for the read context's own, which callers + // without a table hand over. + PAIMON_ASSIGN_OR_RAISE( + CoreOptions core_options, + CoreOptions::FromMap( + options, file_system != nullptr ? file_system : context->GetSpecificFileSystem(), + context->GetFileSystemSchemeToIdentifierMap())); core_options.WithCache(context->GetCache()); if (context->HasVectorOrFullTextSearch()) { if (!core_options.FileIndexReadEnabled()) { @@ -390,10 +395,11 @@ Result> InternalReadContext::Create( return Status::Invalid("memory pool is null"); } std::shared_ptr arrow_pool = GetArrowPool(context->GetMemoryPool()); - auto internal_context = std::unique_ptr( - new InternalReadContext(context, table_schema, read_schema, core_options, arrow_pool)); + auto internal_context = std::unique_ptr(new InternalReadContext( + context, table_schema, read_schema, core_options, arrow_pool, + path != nullptr ? std::optional(*path) : std::nullopt)); PAIMON_RETURN_NOT_OK(internal_context->BindIndexScores()); - return internal_context; + return internal_context } Status InternalReadContext::BindIndexScores() { @@ -471,12 +477,14 @@ InternalReadContext::InternalReadContext(const std::shared_ptr& rea const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, const CoreOptions& options, - const std::shared_ptr& arrow_pool) + const std::shared_ptr& arrow_pool, + std::optional path) : read_context_(read_context), table_schema_(table_schema), read_schema_(read_schema), arrow_pool_(arrow_pool), - options_(options) {} + options_(options), + path_(std::move(path)) {} Result> InternalReadContext::CreateWithSchema( const std::shared_ptr& original, @@ -490,7 +498,7 @@ Result> InternalReadContext::CreateWithSche // The new read_schema is the minimal column set for COUNT(*). return std::shared_ptr( new InternalReadContext(original->read_context_, original->table_schema_, new_read_schema, - original->options_, original->arrow_pool_)); + original->options_, original->arrow_pool_, original->path_)); } } // namespace paimon diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index 9de415d5f..b1619c5bc 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,7 @@ class Schema; namespace paimon { struct CatalogContext; class Executor; +class FileSystem; class MemoryPool; class Predicate; class RealtimeContext; @@ -59,7 +61,9 @@ class InternalReadContext { static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, - const std::map& options); + const std::map& options, + const std::shared_ptr& file_system = nullptr, + const std::string* path = nullptr); /// Maps score output positions in the read schema to their logical search sources. const std::map& GetIndexScoreBindings() const { @@ -79,7 +83,7 @@ class InternalReadContext { return table_schema_; } const std::string& GetPath() const { - return read_context_->GetPath(); + return path_ ? *path_ : read_context_->GetPath(); } std::shared_ptr CreateDependencyCatalogContext() const; @@ -164,7 +168,8 @@ class InternalReadContext { const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, const CoreOptions& options, - const std::shared_ptr& arrow_pool); + const std::shared_ptr& arrow_pool, + std::optional path = std::nullopt); static std::optional TryResolveSpecialFieldById(int32_t field_id, const CoreOptions& core_options, @@ -184,6 +189,9 @@ class InternalReadContext { std::shared_ptr arrow_pool_; CoreOptions options_; std::map index_score_bindings_; + /// The root path a table reads through, or null for the read context's own, which callers + /// without a table hand over. + std::optional path_; }; } // namespace paimon diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index 1d69c033a..e6151d4c7 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -194,6 +194,35 @@ TableSchema::TableSchema(int32_t version, int64_t id, const std::vector> TableSchema::MergeOptions( + const std::map& dynamic_options) const { + std::map options = options_; + for (const auto& [key, value] : dynamic_options) { + options[key] = value; + } + auto merged = std::unique_ptr( + new TableSchema(version_, id_, fields_, highest_field_id_, partition_keys_, primary_keys_, + options, comment_, time_millis_)); + // The derived members follow the merged options: a copy the options restyle must serve the + // bucket keys and the bucket count it carries, not the ones the base schema carried. + PAIMON_RETURN_NOT_OK(merged->DeriveBucketInfo()); + return merged; +} + +Status TableSchema::DeriveBucketInfo() { + PAIMON_ASSIGN_OR_RAISE(std::vector keys, TrimmedPrimaryKeys()); + + // Try to validate bucket keys + PAIMON_ASSIGN_OR_RAISE(std::vector bucket_keys, OriginalBucketKeys()); + if (bucket_keys.empty()) { + bucket_keys = keys; + } + bucket_keys_ = bucket_keys; + PAIMON_ASSIGN_OR_RAISE(num_bucket_, + OptionsUtils::GetValueFromMap(Options(), Options::BUCKET, -1)); + return Status::OK(); +} + bool TableSchema::operator==(const TableSchema& other) const { return version_ == other.version_ && fields_ == other.fields_ && partition_keys_ == other.partition_keys_ && primary_keys_ == other.primary_keys_ && @@ -332,18 +361,7 @@ Result> TableSchema::InitSchema( auto table_schema = std::unique_ptr( new TableSchema(TableSchema::CURRENT_VERSION, schema_id, fields, highest_field_id, partition_keys, primary_keys, options, comment, time_millis)); - PAIMON_ASSIGN_OR_RAISE(std::vector keys, table_schema->TrimmedPrimaryKeys()); - - // Try to validate bucket keys - PAIMON_ASSIGN_OR_RAISE(std::vector bucket_keys, - table_schema->OriginalBucketKeys()); - if (bucket_keys.empty()) { - bucket_keys = keys; - } - table_schema->bucket_keys_ = bucket_keys; - PAIMON_ASSIGN_OR_RAISE( - table_schema->num_bucket_, - OptionsUtils::GetValueFromMap(table_schema->Options(), Options::BUCKET, -1)); + PAIMON_RETURN_NOT_OK(table_schema->DeriveBucketInfo()); return table_schema; } diff --git a/src/paimon/core/schema/table_schema.h b/src/paimon/core/schema/table_schema.h index eab109cf1..aff4ee920 100644 --- a/src/paimon/core/schema/table_schema.h +++ b/src/paimon/core/schema/table_schema.h @@ -101,6 +101,15 @@ class TableSchema : public DataSchema, public Jsonizable { const std::map& Options() const override { return options_; } + + /// A copy of this schema with `dynamic_options` on top of the options it carries, which is + /// the precedence every copy of a table promises. Everything else - id, fields, keys, + /// comment - is copied as it is. + /// + /// @param dynamic_options Options given at the call, which win over the schema's own. + /// @return A result containing the merged schema, or an error status. + Result> MergeOptions( + const std::map& dynamic_options) const; const std::vector& Fields() const { return fields_; } @@ -142,6 +151,10 @@ class TableSchema : public DataSchema, public Jsonizable { Result> OriginalBucketKeys() const; + /// Derives `bucket_keys_` and `num_bucket_` from the options the schema carries: every + /// constructor path must call this, as both members serve scans and copies. + Status DeriveBucketInfo(); + static Result> AssignFieldIdsRecursively( const std::shared_ptr& field, bool set_field_id, int32_t* field_id); diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp new file mode 100644 index 000000000..e70e85c1d --- /dev/null +++ b/src/paimon/core/table/file_store.cpp @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/file_store.h" + +#include +#include +#include +#include + +#include "paimon/catalog/catalog.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +FileStore::FileStore(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) + : file_system_(file_system), + root_path_(root_path), + schema_(schema), + branch_(BranchManager::NormalizeBranch(branch)), + schema_manager_(schema_manager), + snapshot_manager_(snapshot_manager) { + // A manager handed over points at `root_path` on `branch` already; a missing one is built + // here, so the operators always read through the managers the store carries. + if (schema_manager_ == nullptr) { + schema_manager_ = std::make_shared(file_system_, root_path_, branch_); + } + if (snapshot_manager_ == nullptr) { + snapshot_manager_ = std::make_shared(file_system_, root_path_, branch_); + } +} + +Result> FileStore::NewPathFactory( + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::string& file_format_identifier, + const std::shared_ptr& memory_pool) const { + return MakePathFactory(root_path_, arrow_schema, schema_->PartitionKeys(), options, + file_format_identifier, memory_pool); +} + +Result> FileStore::NewSnapshotManager( + const CoreOptions& options, const std::shared_ptr& catalog, + const std::optional& catalog_identifier) const { + auto snapshot_manager = + std::make_shared(options.GetFileSystem(), root_path_, branch_); + if (VersionManagedCatalog* versioned = AsVersionManaged(catalog)) { + if (!catalog_identifier) { + return Status::Invalid("a version-managed catalog requires a table identifier"); + } + Identifier identifier = catalog_identifier.value(); + snapshot_manager->SetSnapshotLoader( + [catalog, versioned, identifier]() -> Result> { + return versioned->LoadSnapshot(identifier); + }); + } + return snapshot_manager; +} + +Result> FileStore::MakePathFactory( + const std::string& root_path, const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, + const std::string& file_format_identifier, const std::shared_ptr& memory_pool) { + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + options.CreateGlobalIndexExternalPath()); + return FileStorePathFactory::Create( + root_path, arrow_schema, partition_keys, options.GetPartitionDefaultName(), + file_format_identifier, options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_path, options.IndexFileInDataFileDir(), memory_pool); +} + +} // namespace paimon diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h new file mode 100644 index 000000000..ec3bc155c --- /dev/null +++ b/src/paimon/core/table/file_store.h @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class Catalog; +class CoreOptions; +class TableSchema; + +/// The storage a managed table is served through: the file system, root path, schema and branch +/// it was loaded with, and the schema and snapshot managers that read its root path on that +/// branch - the state Java Paimon's `AbstractFileStore` carries. `FileStoreTable` hands the +/// state it was loaded with to one of these and asks it back for the pieces every operator +/// opens with. +/// +/// The factory methods build a fresh instance on every call, never a cached one: a path factory +/// names each file it builds, and a snapshot manager a write or a commit arms with a catalog +/// loader is not one a scan should be served through afterwards. +class FileStore { + public: + /// @param file_system File system the table's data is accessed through. + /// @param root_path Root path of the table in the file system. + /// @param schema The schema the table was loaded with, whose partition keys name the + /// partitions the paths below are built for. + /// @param branch The branch the store serves, normalized here. + /// @param schema_manager A schema manager to carry instead of building one - the one a + /// context dispatch read the schema through, whose cache the operators then serve. + /// @param snapshot_manager A snapshot manager to carry instead of building one. + /// A null manager is built over `root_path` on `branch`, so the operators always read + /// through the managers the store carries. + FileStore(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const std::string& branch, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); + + /// The branch this store serves, as resolved when it was built. + const std::string& branch() const { + return branch_; + } + + /// The schema manager the operators read through. It carries the schema cache of `root_path` + /// on `branch()`. + const std::shared_ptr& schema_manager() const { + return schema_manager_; + } + + /// The snapshot manager a scan reads through. A write or a commit builds its own: see + /// `NewSnapshotManager`. + const std::shared_ptr& snapshot_manager() const { + return snapshot_manager_; + } + + /// The path factory an operator builds its metadata paths through: over this store's root + /// path and the partition keys of its schema. The file format identifier is the caller's to + /// name: a write names the format its level-0 files are written in, which the options may + /// set per level, while a scan, a read and a commit name the table's format. + Result> NewPathFactory( + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::string& file_format_identifier, + const std::shared_ptr& memory_pool) const; + + /// The snapshot manager a write or a commit reads snapshots through, over this store's root + /// path and branch. The file system is the caller's to name, from the options its operator + /// serves: a table opened without one has its options build the resolving file system. A + /// catalog that version-manages the table hands its snapshots out itself, so its loader is + /// attached, the catalog captured to keep the versioned interface - and the loader with it + /// - alive. Such a catalog names the table by an identifier, which the context must have + /// supplied. + Result> NewSnapshotManager( + const CoreOptions& options, const std::shared_ptr& catalog, + const std::optional& catalog_identifier) const; + + /// The assembly the factory methods above share: a path factory over a root path and + /// partition keys named by the caller rather than the store's own, which the read path + /// needs - the schema it serves may be a fallback branch's, not the table's. + static Result> MakePathFactory( + const std::string& root_path, const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, + const std::string& file_format_identifier, const std::shared_ptr& memory_pool); + + private: + std::shared_ptr file_system_; + std::string root_path_; + std::shared_ptr schema_; + std::string branch_; + std::shared_ptr schema_manager_; + std::shared_ptr snapshot_manager_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 428513e16..51c7c0b95 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -19,11 +19,16 @@ #pragma once +#include #include #include #include "paimon/catalog/identifier.h" #include "paimon/catalog/table.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/snapshot_manager.h" #include "paimon/result.h" #include "paimon/type_fwd.h" @@ -36,7 +41,8 @@ class Schema; /// the other kind of `Table`. /// /// Its `NewScan`/`NewRead`/`NewWrite`/`NewCommit` build the operators here, from the context plus -/// the schema the table was constructed with, so they serve the table as it was loaded. The +/// the schema, file system, root path, branch and managers the table was constructed with, so +/// they serve the table as it was loaded. The /// context-only entry points - `TableScan::Create()`, `TableRead::Create()`, /// `FileStoreWrite::Create()` and `FileStoreCommit::Create()` - load the table the context names /// and hand the context to that table's `NewXxx`, which is how a context built without a table @@ -48,10 +54,39 @@ class FileStoreTable : public Table { /// @param file_system File system the table's data is accessed through. /// @param root_path Root path of the table in the file system. /// @param uuid Metastore UUID, or empty if unavailable. + /// @param branch The branch the table serves, which the managers below read and write on. + /// @param schema_manager A schema manager to carry instead of building one - the one a + /// context dispatch read the schema through, whose cache the operators then serve. + /// @param snapshot_manager A snapshot manager to carry instead of building one, the way + /// `Copy` hands its table's over so both share one. + /// A null manager is built over `root_path` on `branch`, so the operators always read + /// through the managers the table carries. FileStoreTable(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = "") - : Table(file_system, root_path, schema, identifier, uuid) {} + const std::string& uuid = "", + const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); + + /// The branch this table serves, as resolved when the table was built: the schema the table + /// was loaded with came from it, and its managers point at it. + const std::string& CurrentBranch() const { + return store_->branch(); + } + + /// The schema manager the operators read through. It carries the schema cache of `root_path` + /// on `CurrentBranch()`, which every operator serves rather than each reading the schema + /// files again. + const std::shared_ptr& GetSchemaManager() const { + return store_->schema_manager(); + } + + /// The snapshot manager a scan reads through. A write or a commit builds its own: it arms + /// the manager with a catalog snapshot loader, which a shared one would keep serving to + /// scans afterwards. + const std::shared_ptr& GetSnapshotManager() const { + return store_->snapshot_manager(); + } std::string Name() const override { return identifier_.GetObjectName(); @@ -61,6 +96,19 @@ class FileStoreTable : public Table { return identifier_.GetFullName(); } + /// Copies the table with `dynamic_options` on top of the ones its schema stored, the + /// precedence `Copy` promises. The file system, root path and identifier stay as they are: + /// they name this table, which an option does not rename. + Result> Copy( + const std::map& dynamic_options) const override; + + /// Returns the table of `branch`, as Java Paimon's `switchToBranch` does. The branch switched + /// to is served as it is: its schema is read from its own directory, so the dynamic options + /// of this table are not carried over. Switching to the branch already served hands back a + /// copy of this table, schema and managers shared, and a branch with no schema does not + /// exist. + Result> SwitchToBranch(const std::string& branch) const; + Result> NewScan(std::unique_ptr context) const override; Result> NewRead(std::unique_ptr context) const override; @@ -70,6 +118,11 @@ class FileStoreTable : public Table { Result> NewCommit( std::unique_ptr context) const override; + + private: + /// The storage this table serves: the file system, root path, schema, branch and managers it + /// was loaded with, and the factories that build the pieces every operator opens with. + std::shared_ptr store_; }; } // namespace paimon diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index 53d53b967..6f2ac16a4 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -192,24 +192,26 @@ Result> FormatTable::Create( core_options.FormatTablePartitionOnlyValueInPath(), location_carries_paimon_metadata)); } -Result> FormatTable::Copy( - const std::shared_ptr& table, - const std::map& dynamic_options) { - if (table == nullptr) { - return Status::Invalid("format table requires a table to copy"); - } +Result> FormatTable::CopySelf( + const std::map& dynamic_options) const { if (dynamic_options.empty()) { - return table; + return SharedSelf(); } // The options this table already carries go back in as dynamic ones, so an option an earlier // call set is not lost to the schema's own value underneath it. - std::map options = table->options_; + std::map options = options_; for (const auto& [key, value] : dynamic_options) { options[key] = value; } - return Create(table->file_system_, table->root_path_, table->identifier_, - std::static_pointer_cast(table->schema_), - table->location_carries_paimon_metadata_, options); + return Create(file_system_, root_path_, identifier_, + std::static_pointer_cast(schema_), location_carries_paimon_metadata_, + options); +} + +Result> FormatTable::Copy( + const std::map& dynamic_options) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr copied, CopySelf(dynamic_options)); + return std::shared_ptr
(std::move(copied)); } const std::vector& FormatTable::PartitionKeys() const { @@ -233,7 +235,7 @@ Result> FormatTable::NewScan( return Status::Invalid("executor is null pointer"); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - FormatTable::Copy(SharedSelf(), shared_context->GetOptions())); + CopySelf(shared_context->GetOptions())); // Anything a `ScanContext` carries that a format table cannot honour is refused rather than // silently dropped. if (shared_context->IsStreamingMode()) { @@ -294,7 +296,7 @@ Result> FormatTable::NewRead( return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - FormatTable::Copy(SharedSelf(), read_context->GetOptions())); + CopySelf(read_context->GetOptions())); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, FormatTableRead::Create(table, read_context)); return std::unique_ptr(std::move(read)); @@ -311,8 +313,7 @@ Result> FormatTable::NewWrite( if (context->GetExecutor() == nullptr) { return Status::Invalid("executor is null pointer"); } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - FormatTable::Copy(SharedSelf(), context->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, CopySelf(context->GetOptions())); if (context->GetCatalog() != nullptr) { return Status::Invalid( "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " @@ -354,7 +355,7 @@ Result> FormatTable::NewCommit( return Status::Invalid("executor is null pointer"); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(SharedSelf(), context->GetOptions())); + CopySelf(context->GetOptions())); if (!context->IgnoreEmptyCommit()) { return Status::NotImplemented( "a format table cannot record an empty commit: keeping one means writing a " diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index 2430ffbda..2dac6c102 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -3071,7 +3071,7 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) ASSERT_OK(WriteAndCommit(table, std::move(batch))); const std::map branch = {{Options::BRANCH, "dev"}}; - ASSERT_NOK_WITH_MSG(FormatTable::Copy(table, branch), "has no branches"); + ASSERT_NOK_WITH_MSG(table->Copy(branch), "has no branches"); ASSERT_NOK_WITH_MSG( FormatTable::Create(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"), branch), "has no branches"); @@ -3108,10 +3108,32 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) // The branch that names no branch at all is the main one, and is left alone: refusing it // would refuse the default every context carries. for (const char* main_branch : {"main", ""}) { - ASSERT_OK(FormatTable::Copy(table, {{Options::BRANCH, main_branch}})) << main_branch; + ASSERT_OK(table->Copy({{Options::BRANCH, main_branch}})) << main_branch; } } +TEST(FormatTableTest, TestATableCopiesItselfWithTheOptionsGivenAtTheCall) { + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + + // The copy overlays what the call gives on the options the schema stored, the precedence + // `Copy()` promises: a probe option lands, `type` - structural - does not. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, + table->Copy({{Options::TYPE, "table"}, {"probe", "on-top"}})); + std::shared_ptr copied_table = std::static_pointer_cast(copied); + EXPECT_EQ(copied_table->Options().at("probe"), "on-top"); + EXPECT_EQ(copied_table->Options().at(Options::TYPE), "format-table"); + EXPECT_EQ(copied_table->FullName(), table->FullName()); + EXPECT_EQ(copied_table->Location(), table->Location()); + EXPECT_EQ(copied_table->GetFileSystem(), table->GetFileSystem()); + + // The refusal `Copy(table, ...)` reports is the one the table itself reports: the copy goes + // through the same entry point, not a shortcut around its checks. + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BRANCH, "dev"}}), "has no branches"); +} + TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFromTheTableRoot) { // A context that names a path takes its branch from `WithBranch()`, and the schema of that // branch is what the entry point dispatches on. A format table's data is the files under the diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index ead31b17a..3d580d64d 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -250,25 +250,41 @@ TEST(TableReadTest, TestCreateAppendOnlyTableRead) { TEST(TableReadTest, TestMergeOptions) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); - context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - context_builder.AddOption("read.batch-size", "2"); - context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); - context_builder.AddOption("bucket", "10"); - context_builder.AddOption("bucket-key", "f3"); - ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - auto append_only_table_read = dynamic_cast(table_read.get()); - ASSERT_TRUE(append_only_table_read); - auto* typed_split_read = - dynamic_cast(append_only_table_read->split_reads_[0].get()); - ASSERT_TRUE(typed_split_read); - const auto& core_options = typed_split_read->options_; - std::map expected_options = { - {"read.batch-size", "2"}, {"orc.read.enable-lazy-decoding", "true"}, - {"bucket", "10"}, {"bucket-key", "f3"}, - {"manifest.format", "orc"}, {"file.format", "orc"}}; - ASSERT_EQ(expected_options, core_options.ToMap()); + { + // The options the table was loaded with stand as they are, and a runtime setting the + // context adds on top - the table was loaded with neither of these keys - is served as + // given. + ReadContextBuilder context_builder(path); + context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); + context_builder.AddOption("read.batch-size", "2"); + context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + auto append_only_table_read = dynamic_cast(table_read.get()); + ASSERT_TRUE(append_only_table_read); + auto* typed_split_read = + dynamic_cast(append_only_table_read->split_reads_[0].get()); + ASSERT_TRUE(typed_split_read); + const auto& core_options = typed_split_read->options_; + std::map expected_options = { + {"read.batch-size", "2"}, + {"orc.read.enable-lazy-decoding", "true"}, + {"bucket", "2"}, + {"bucket-key", "f2"}, + {"manifest.format", "orc"}, + {"file.format", "orc"}}; + ASSERT_EQ(expected_options, core_options.ToMap()); + } + { + // An option that contradicts the table is refused: the read serves the options the table + // was loaded with, and a caller that means to change them copies the table. + ReadContextBuilder context_builder(path); + context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); + context_builder.AddOption("bucket", "10"); + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + "option 'bucket' is '10' in the context but '2' on the table"); + } } TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index dd00b7bcd..cf4864071 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -191,6 +191,11 @@ Status SystemTable::AdoptOwnSchema() { return Status::OK(); } +Result> SystemTable::Copy( + const std::map& /*dynamic_options*/) const { + return Status::NotImplemented(Name(), " system table is read-only: copying is not supported"); +} + Result> SystemTable::NewWrite( std::unique_ptr /*context*/) const { return Status::NotImplemented(Name(), " system table is read-only: writing is not supported"); diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 2e4dff55d..96eead153 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -66,6 +66,12 @@ class SystemTable : public Table, public std::enable_shared_from_this> ArrowSchema() const = 0; + /// A system table has no options of its own to override: its schema is the read-only view it + /// exposes, which is not reconfigured by a copy. The data table underneath is copied through + /// its own `Table`. + Result> Copy( + const std::map& dynamic_options) const override; + /// A system table is read-only: writing it or committing for it is refused rather than /// routed to the data table underneath. Result> NewWrite( diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 49cf6838f..0a9f85a2e 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -227,6 +227,20 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { ASSERT_EQ(arrow_schema->field(1)->type()->id(), arrow::Type::INT32); } +TEST(SystemTableTest, TestCopyIsRefused) { + // A system table is a read-only view over the data table underneath it, and the schema it + // exposes is not one a copy reconfigures. The data table is copied through its own `Table`. + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr system_table, + SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, + "/tmp/table", table_schema, + /*dynamic_options=*/{})); + ASSERT_NOK_WITH_MSG(system_table->Copy({{"probe", "on-top"}}), "copying is not supported"); +} + TEST(SystemTableTest, TestReadOptimizedSystemTablePathParsing) { ASSERT_OK_AND_ASSIGN(std::optional parsed, SystemTableLoader::TryParsePath("/tmp/db.db/t$branch_audit$ro")); diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index f9cf78545..cb86e4281 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -76,6 +76,7 @@ #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" #include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/abstract_table_scan.h" #include "paimon/core/table/source/append_only_split_generator.h" @@ -217,6 +218,7 @@ Result HasHistoricalPrimaryKeyBTreeDefinition( Result> TableOfLoadedSchema( const std::shared_ptr& file_system, const std::string& path, const std::string& branch, const std::map& options, + const std::shared_ptr& schema_manager, const std::shared_ptr& latest_schema, const std::optional& identifier, const std::string& uuid = "") { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, @@ -229,7 +231,72 @@ Result> TableOfLoadedSchema( Identifier table_identifier = identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); return std::make_shared(file_system, path, latest_schema, table_identifier, - uuid); + uuid, branch, schema_manager); +} + +/// The managed table a context names behind a path: the schema is read through a manager of the +/// context's branch - handed over instead when the caller already read it - and what the schema +/// says is there decides the table `TableOfLoadedSchema` hands back. +/// +/// @param schema_json The schema of the table, when the caller already read it, or null when it +/// has to be read here. It is honoured on the main branch only: a branch publishes its own +/// under `branch/branch-`, which is where the read below goes. +Result> ManagedTableOf(const std::shared_ptr& file_system, + const std::string& path, const std::string& branch, + const std::map& options, + const std::optional& schema_json, + const std::optional& identifier, + const std::string& uuid = "") { + auto schema_manager = std::make_shared(file_system, path, branch); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, TableSchema::CreateFromJson(schema_json.value())); + } else { + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager->Latest()); + if (latest) { + latest_schema = *latest; + } + } + return TableOfLoadedSchema(file_system, path, branch, options, schema_manager, latest_schema, + identifier, uuid); +} + +/// The identifier a catalog knows the table by, on `branch`: a non-main branch is folded into +/// the identifier, so the catalog is asked for the table on that branch rather than the main one. +Result> MakeBranchCatalogIdentifier( + const std::optional& identifier, const std::string& branch) { + if (!identifier) { + return std::optional(); + } + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + CatalogUtils::BranchIdentifier(identifier.value(), branch)); + return std::optional(std::move(branch_identifier)); +} + +/// The schema a catalog may hold for the table, read from it on the main branch only and null +/// otherwise. A branch publishes its own schema under `branch/branch-`, which is where +/// `ManagedTableOf` reads instead, so a catalog answering for no branch schema does not fail the +/// `kind` operation to one. `kind` also names the operator the missing-identifier message speaks +/// of. +Result> CatalogMainBranchSchemaJson( + const std::shared_ptr& catalog, const std::optional& catalog_identifier, + const std::string& branch, const char* kind) { + if (catalog == nullptr) { + return std::optional(); + } + if (!catalog_identifier) { + return Status::Invalid(fmt::format("a catalog {} requires a table identifier", kind)); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + catalog->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(std::string schema_json, schema->GetJsonSchema()); + return std::optional(std::move(schema_json)); + } + return std::optional(); } /// The table a scan context names, read from what the context carries: a format table the @@ -254,22 +321,8 @@ Result> TableOf(const ScanContext& ctx) { // below. One `TableScan` interface serves both, as Java Paimon serves both through one // `ReadBuilder`. const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx.GetSpecificTableSchema()) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(ctx.GetSpecificTableSchema().value())); - } else { - SchemaManager schema_manager(tmp_options.GetFileSystem(), ctx.GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; - } - } - return TableOfLoadedSchema(tmp_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), - latest_schema, ctx.GetIdentifier()); + return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), + ctx.GetSpecificTableSchema(), ctx.GetIdentifier()); } /// The table a read context names, read from what the context carries: a format table the @@ -304,22 +357,8 @@ Result> TableOf(const ReadContext& ctx) { // which files there are and how a row is put back together is its own. How a file is opened // is not: both paths go through `DataFileReaderFactory`. const std::string branch = ctx.GetBranch(); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx.GetSpecificTableSchema()) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(ctx.GetSpecificTableSchema().value())); - } else { - SchemaManager schema_manager(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; - } - } - return TableOfLoadedSchema(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch, - ctx.GetOptions(), latest_schema, ctx.GetIdentifier()); + return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), + ctx.GetSpecificTableSchema(), ctx.GetIdentifier()); } /// The table a write context names, read from what the context carries: a format table the @@ -346,44 +385,18 @@ Result> TableOf(const WriteContext& ctx) { CoreOptions::FromMap(ctx.GetOptions(), specific_fs, ctx.GetFileSystemSchemeToIdentifierMap())); std::string branch = ctx.GetBranch(); - std::optional catalog_identifier; - if (ctx.GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx.GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + MakeBranchCatalogIdentifier(ctx.GetIdentifier(), branch)); // Only the main branch may have a current schema the catalog alone holds. A branch publishes // its own under `branch/branch-`, which is where the load below and a read of the branch // both read it, so a catalog answering for no branch schema does not fail a write to one. - std::optional catalog_table_schema; - if (ctx.GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog write requires a table identifier"); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - ctx.GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, schema->GetJsonSchema()); - } - } + PAIMON_ASSIGN_OR_RAISE( + std::optional catalog_table_schema, + CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "write")); // A format table writes plain data files into a directory, so it never reaches the manifest // path below. - SchemaManager schema_manager(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(catalog_table_schema.value())); - } else { - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; - } - } - return TableOfLoadedSchema(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch, - ctx.GetOptions(), latest_schema, ctx.GetIdentifier()); + return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch, ctx.GetOptions(), + catalog_table_schema, ctx.GetIdentifier()); } /// The table a commit context names, read from what the context carries: a format table the @@ -413,47 +426,83 @@ Result> TableOf(const CommitContext& ctx) { PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( ctx.GetIdentifier(), ctx.GetOptions(), /*explicit_branch=*/std::nullopt, "commit")); - std::optional catalog_identifier; - if (ctx.GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx.GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + MakeBranchCatalogIdentifier(ctx.GetIdentifier(), branch)); // Catalog-managed schemas may be absent from the table directory, but only on the main branch: // a branch publishes its own under `branch/branch-`, which is where the load below and a // read of the branch both read it, so a catalog answering for no branch schema does not fail // a commit to one. - std::optional catalog_table_schema; - if (ctx.GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, - ctx.GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); - catalog_table_schema = std::move(schema_json); - } - } + PAIMON_ASSIGN_OR_RAISE( + std::optional catalog_table_schema, + CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "commit")); // A format table commits by renaming files into place, so it never reaches the snapshot path // below. The managed path here reads the commit's branch, so this reads the same one: the two // must not dispatch on different schemas. - SchemaManager schema_manager(tmp_options.GetFileSystem(), root_path, branch); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(catalog_table_schema.value())); - } else { - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; + return ManagedTableOf(tmp_options.GetFileSystem(), root_path, branch, ctx.GetOptions(), + catalog_table_schema, ctx.GetIdentifier(), ctx.GetTableId().value_or("")); +} + +/// The options a context carries against the ones the table was loaded with. A table serves the +/// options it was loaded with, so an option the context contradicts is refused rather than +/// honoured - a caller that means to change them copies the table. A key the table was not loaded +/// with is no contradiction: it names a runtime setting - a scan mode, a batch size - which is +/// none of the table's business, and is served as given. +Status CheckOptionsAgreeWithTable(const TableSchema& table_schema, + const std::map& context_options) { + const std::map& table_options = table_schema.Options(); + for (const auto& [key, value] : context_options) { + auto table_option = table_options.find(key); + if (table_option != table_options.end() && table_option->second != value) { + return Status::Invalid( + fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, value, + table_option->second)); } } - return TableOfLoadedSchema(tmp_options.GetFileSystem(), root_path, branch, ctx.GetOptions(), - latest_schema, ctx.GetIdentifier(), ctx.GetTableId().value_or("")); + return Status::OK(); +} + +/// The checks every operator entry point opens with: a context, a memory pool and an executor +/// must all be there for the operator to be built at all. `kind` names the operator the first +/// message speaks of, which is all that sets the entry points' messages apart. +template +Status CheckOperatorContext(const ContextPointer& context, const char* kind) { + if (context == nullptr) { + return Status::Invalid(fmt::format("{} context is null pointer", kind)); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + return Status::OK(); +} + +/// The schema the table was constructed with, which a write or a commit serves as the table was +/// loaded rather than reading the schema again. +Result> LatestTableSchemaOfBranch( + const std::shared_ptr& schema, const std::string& branch) { + std::shared_ptr latest_schema = std::static_pointer_cast(schema); + if (latest_schema == nullptr) { + return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); + } + return latest_schema; +} + +/// The options a write or a commit serves: the ones the table was loaded with - an option the +/// context contradicts is refused by `CheckOptionsAgreeWithTable` above - the ones the context +/// adds on top, and the table's branch, which the metadata paths below are built for: what reads +/// through these options, whose cache keys are scoped by it, has to read the same one. +Result> MergedOperatorOptions( + const TableSchema& table_schema, const std::map& context_options, + const std::string& branch) { + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(table_schema, context_options)); + auto opts = table_schema.Options(); + for (const auto& [key, value] : context_options) { + opts[key] = value; + } + opts[Options::BRANCH] = branch; + return opts; } } // namespace @@ -467,60 +516,97 @@ Table::Table(const std::shared_ptr& file_system, const std::string& identifier_(identifier), uuid_(uuid) {} +FileStoreTable::FileStoreTable(const std::shared_ptr& file_system, + const std::string& root_path, const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid, + const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) + : Table(file_system, root_path, schema, identifier, uuid), + store_(std::make_shared(file_system, root_path, + std::static_pointer_cast(schema), branch, + schema_manager, snapshot_manager)) {} + +Result> FileStoreTable::Copy( + const std::map& dynamic_options) const { + std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); + if (loaded_schema == nullptr) { + return Status::Invalid(fmt::format("not found latest schema of {}", FullName())); + } + // `type` is structural and read from the schema alone, the promise `Copy` makes: an option + // given at the call must not restyle the table. + std::map overridable = dynamic_options; + overridable.erase(Options::TYPE); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, + loaded_schema->MergeOptions(overridable)); + // The managers are handed over, not rebuilt: the copy serves the same root path and branch, + // so its operators read the schema cache this table already warmed. + return std::shared_ptr
(std::make_shared( + file_system_, root_path_, std::shared_ptr(std::move(merged_schema)), + identifier_, CatalogUuid().value_or(""), CurrentBranch(), GetSchemaManager(), + GetSnapshotManager())); +} + +Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { + const std::string target_branch = BranchManager::NormalizeBranch(branch); + if (target_branch == CurrentBranch()) { + // The table already serves this branch: a copy of it - schema and managers shared - is + // the same table to the caller. + return std::shared_ptr
(std::make_shared( + file_system_, root_path_, schema_, identifier_, CatalogUuid().value_or(""), + CurrentBranch(), GetSchemaManager(), GetSnapshotManager())); + } + // The schema of the branch switched to, read from its own directory, which a branch without + // one does not have. + auto schema_manager = std::make_shared(file_system_, root_path_, target_branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager->Latest()); + if (!latest) { + return Status::Invalid(fmt::format("branch {} does not exist", target_branch)); + } + // A branch other than main is named in the options the schema carries - the way a read + // context names it - so what reads through them serves the branch the managers point at. + std::map overridable; + if (target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { + overridable[Options::BRANCH] = target_branch; + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr branch_schema, + (*latest)->MergeOptions(overridable)); + return std::shared_ptr
(std::make_shared( + file_system_, root_path_, std::shared_ptr(std::move(branch_schema)), + identifier_, CatalogUuid().value_or(""), target_branch, schema_manager)); +} + Result> FileStoreTable::NewWrite( std::unique_ptr ctx) const { - if (ctx == nullptr) { - return Status::Invalid("write context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "write")); PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + // A catalog write names its table; the file system itself comes from the table, which its + // catalog's credentials already opened. if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && - ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog write requires a table identifier"); - } - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the write uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); + ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs, - ctx->GetFileSystemSchemeToIdentifierMap())); - std::string branch = ctx->GetBranch(); - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); + // The write goes to the branch the table serves: a table is the branch it was loaded for, + // so the context's own branch is not consulted. + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + MakeBranchCatalogIdentifier(ctx->GetIdentifier(), CurrentBranch())); // The schema the table was constructed with, so a write serves the table as it was loaded // rather than reading the schema again. - std::shared_ptr latest_schema = std::static_pointer_cast(schema_); - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - const std::shared_ptr& schema = latest_schema; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + LatestTableSchemaOfBranch(schema_, CurrentBranch())); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; - } - // The metadata paths above are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; + // The options the table was loaded with, with a runtime setting the context adds on top: an + // option the context would have changed is refused rather than honoured, and the branch is + // the table's - the metadata paths below are built for it, so what reads through these + // options, whose cache keys are scoped by it, has to read the same one. + std::map opts; + PAIMON_ASSIGN_OR_RAISE(opts, + MergedOperatorOptions(*schema, ctx->GetOptions(), CurrentBranch())); PAIMON_ASSIGN_OR_RAISE( CoreOptions options, - CoreOptions::FromMap(opts, specific_fs, ctx->GetFileSystemSchemeToIdentifierMap())); + CoreOptions::FromMap(opts, file_system_, ctx->GetFileSystemSchemeToIdentifierMap())); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); if (options.GetVectorFileFormat() && !ctx->GetWriteSchema().empty()) { @@ -530,29 +616,14 @@ Result> FileStoreTable::NewWrite( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); - PAIMON_ASSIGN_OR_RAISE( std::shared_ptr file_store_path_factory, - FileStorePathFactory::Create(ctx->GetRootPath(), arrow_schema, schema->PartitionKeys(), - options.GetPartitionDefaultName(), - options.GetWriteFileFormat(/*level=*/0)->Identifier(), - options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), ctx->GetMemoryPool())); - auto snapshot_manager = - std::make_shared(options.GetFileSystem(), ctx->GetRootPath(), branch); - if (VersionManagedCatalog* versioned = AsVersionManaged(ctx->GetCatalog())) { - std::shared_ptr catalog = ctx->GetCatalog(); - Identifier identifier = catalog_identifier.value(); - // Keep the catalog alive while the writer uses its snapshot loader. - snapshot_manager->SetSnapshotLoader( - [catalog, versioned, identifier]() -> Result> { - return versioned->LoadSnapshot(identifier); - }); - } + store_->NewPathFactory(arrow_schema, options, + options.GetWriteFileFormat(/*level=*/0)->Identifier(), + ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr snapshot_manager, + store_->NewSnapshotManager(options, ctx->GetCatalog(), catalog_identifier)); std::shared_ptr io_manager; const auto& io_temp_dir = ctx->GetTempDirectory(); if (!io_temp_dir.empty()) { @@ -632,8 +703,8 @@ Result> FileStoreTable::NewWrite( } auto file_store_write = std::make_unique( - file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), - ctx->GetRootPath(), schema, arrow_schema, write_schema, realtime_schema_layout, + file_store_path_factory, snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), + root_path_, schema, arrow_schema, write_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, io_manager, options, ignore_previous_files, ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->GetRealtimeContext(), ctx->GetExecutor(), ctx->GetMemoryPool()); @@ -662,11 +733,11 @@ Result> FileStoreTable::NewWrite( } if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { return PostponeBucketFileStoreWrite::Create( - snapshot_manager, schema_manager, ctx->GetCommitUser(), ctx->GetRootPath(), schema, + snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), root_path_, schema, arrow_schema, partition_schema, io_manager, options, ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->GetWriteId(), ctx->GetFileSystemSchemeToIdentifierMap(), ctx->GetExecutor(), ctx->GetMemoryPool(), - specific_fs); + file_system_); } if (options.GetBucket() <= 0) { return Status::Invalid( @@ -710,7 +781,7 @@ Result> FileStoreTable::NewWrite( if (!has_btree_index) { PAIMON_ASSIGN_OR_RAISE( bool has_historical_btree_index, - HasHistoricalPrimaryKeyBTreeDefinition(schema_manager, schema->Id())); + HasHistoricalPrimaryKeyBTreeDefinition(GetSchemaManager(), schema->Id())); if (has_historical_btree_index) { PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); has_existing_index_manifest = @@ -749,9 +820,10 @@ Result> FileStoreTable::NewWrite( if (has_btree_index || has_restored_btree_index) { primary_key_index_maintainer_factory = std::make_shared( - ctx->GetRootPath(), branch, schema, primary_key_index_definitions.Definitions(), - file_store_path_factory, index_file_handler, options, io_manager, - ctx->EnableMultiThreadSpill(), ctx->GetExecutor(), ctx->GetMemoryPool()); + root_path_, CurrentBranch(), schema, + primary_key_index_definitions.Definitions(), file_store_path_factory, + index_file_handler, options, io_manager, ctx->EnableMultiThreadSpill(), + ctx->GetExecutor(), ctx->GetMemoryPool()); } std::shared_ptr realtime_schema_layout; @@ -763,8 +835,8 @@ Result> FileStoreTable::NewWrite( } return std::make_unique( - file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), - ctx->GetRootPath(), schema, arrow_schema, realtime_schema_layout, partition_schema, + file_store_path_factory, snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), + root_path_, schema, arrow_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, primary_key_index_maintainer_factory, io_manager, key_comparator, sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->EnableMultiThreadSpill(), @@ -773,15 +845,7 @@ Result> FileStoreTable::NewWrite( } Result> FileStoreWrite::Create(std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("write context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "write")); // First the table the context names, then the context handed to that table's `NewWrite`. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); return table->NewWrite(std::move(ctx)); @@ -791,18 +855,19 @@ namespace { class TableScanImpl { public: + /// @param snapshot_manager The snapshot manager the scan reads through - the table's own. + /// @param schema_manager The schema manager the scan reads through, likewise. static Result> CreateFileStoreScan( const std::shared_ptr& path_factory, const std::shared_ptr& arrow_schema, const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& schema_manager, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, - const ScanContext* context) { + const ScanContext* context, const std::string& root_path) { auto fs = core_options.GetFileSystem(); auto manifest_file_format = core_options.GetManifestFormat(); - std::string branch = BranchManager::NormalizeBranch(core_options.GetBranch()); - auto snapshot_manager = std::make_shared(fs, context->GetPath(), branch); // TODO(liancheng.lsz): support fallback branch in scan - auto schema_manager = std::make_shared(fs, context->GetPath(), branch); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, ManifestList::Create(fs, manifest_file_format, core_options.GetManifestCompression(), @@ -837,12 +902,12 @@ class TableScanImpl { context->GetScanFilters(), core_options, executor, memory_pool)); } - return WithTablePath(std::move(scan), context); + return WithTablePath(std::move(scan), root_path); } static std::unique_ptr WithTablePath(std::unique_ptr&& scan, - const ScanContext* context) { - scan->WithTablePath(context->GetPath()); + const std::string& root_path) { + scan->WithTablePath(root_path); return std::move(scan); } @@ -938,15 +1003,7 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& Result> FileStoreTable::NewScan( std::unique_ptr context) const { - if (context == nullptr) { - return Status::Invalid("scan context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(context, "scan")); // The schema the table was constructed with, so a scan serves the table as it was loaded // rather than reading the schema again. std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); @@ -954,16 +1011,29 @@ Result> FileStoreTable::NewScan( return Status::Invalid("not found latest schema"); } const std::shared_ptr& table_schema = loaded_schema; + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*table_schema, context->GetOptions())); PAIMON_ASSIGN_OR_RAISE(bool read_optimized, OptionsUtils::GetValueFromMap(context->GetOptions(), kReadOptimizedScanOption, false)); - // merge options + // The options the table was loaded with, with a runtime setting the context adds on top: an + // option the context would have changed is refused above, so the table's own stand as they + // are. The branch is the table's, not the context's: a table is the branch it was loaded + // for, so a branch the context carries is not honoured. auto options = table_schema->Options(); for (const auto& [key, value] : context->GetOptions()) { - options[key] = value; + if (key != Options::BRANCH) { + options[key] = value; + } + } + if (BranchManager::IsMainBranch(CurrentBranch())) { + options.erase(Options::BRANCH); + } else { + options[Options::BRANCH] = CurrentBranch(); } + // The file system and root path the table was opened with, so a scan serves the table as it + // was loaded rather than resolving them from the context. PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, context->GetSpecificFileSystem(), {})); + CoreOptions::FromMap(options, file_system_, {})); core_options.WithCache(context->GetCache()); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); @@ -989,24 +1059,17 @@ Result> FileStoreTable::NewScan( *arrow_schema, context->GetScanFilters()->GetPredicate(), /*validate_field_idx=*/false)); } - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, - core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - FileStorePathFactory::Create( - context->GetPath(), arrow_schema, table_schema->PartitionKeys(), - core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), - core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - context->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory, + store_->NewPathFactory(arrow_schema, core_options, + core_options.GetFileFormat()->Identifier(), + context->GetMemoryPool())); + // The managers the table carries, whose schema cache the dispatch already warmed. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_store_scan, TableScanImpl::CreateFileStoreScan( path_factory, arrow_schema, table_schema, core_options, - context->GetExecutor(), context->GetMemoryPool(), context.get())); + GetSnapshotManager(), GetSchemaManager(), context->GetExecutor(), + context->GetMemoryPool(), context.get(), root_path_)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr split_generator, TableScanImpl::CreateSplitGenerator(table_schema, core_options, context.get())); @@ -1036,7 +1099,7 @@ Result> FileStoreTable::NewScan( } if (core_options.DataEvolutionEnabled()) { return std::make_unique( - context->GetPath(), snapshot_reader, std::move(batch_scan), table_schema, + root_path_, snapshot_reader, std::move(batch_scan), table_schema, context->GetGlobalIndexResult(), core_options, context->GetMemoryPool(), context->GetExecutor()); } @@ -1053,15 +1116,7 @@ Result> FileStoreTable::NewScan( } Result> TableScan::Create(std::unique_ptr context) { - if (context == nullptr) { - return Status::Invalid("scan context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(context, "scan")); // First the table the context names, then the context handed to that table's `NewScan`. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*context)); return table->NewScan(std::move(context)); @@ -1069,13 +1124,20 @@ Result> TableScan::Create(std::unique_ptr> CreateInternalReadContext( const std::shared_ptr& context, const std::string& branch, - const std::shared_ptr& loaded_schema) { - std::map tmp_options = context->GetOptions(); + const std::shared_ptr& schema_manager, + const std::shared_ptr& loaded_schema, + const std::shared_ptr& file_system, const std::string& root_path) { std::shared_ptr table_schema; const auto& specific_table_schema = context->GetSpecificTableSchema(); if (loaded_schema != nullptr) { @@ -1084,29 +1146,29 @@ Result> CreateInternalReadContext( PAIMON_ASSIGN_OR_RAISE(table_schema, TableSchema::CreateFromJson(specific_table_schema.value())); } else { - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, - CoreOptions::FromMap(tmp_options, context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - SchemaManager schema_manager(tmp_core_options.GetFileSystem(), context->GetPath(), branch); PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - schema_manager.Latest()); + schema_manager->Latest()); if (!latest_schema) { - return Status::Invalid(fmt::format("schema file not found in path {}, branch {}", - context->GetPath(), branch)); + return Status::Invalid( + fmt::format("schema file not found in path {}, branch {}", root_path, branch)); } table_schema = latest_schema.value(); } assert(table_schema); - // merge options + // The options the schema was loaded with, with a runtime setting the context adds on top - + // and the fallback below serves the ones its own schema was loaded with. The branch is the + // one named below, not the context's: a branch the context carries is not honoured here. auto options = table_schema->Options(); - for (const auto& [key, value] : tmp_options) { - options[key] = value; + for (const auto& [key, value] : context->GetOptions()) { + if (key != Options::BRANCH) { + options[key] = value; + } } if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { options[Options::BRANCH] = branch; } - return InternalReadContext::Create(context, table_schema, options); + return InternalReadContext::Create(context, table_schema, options, file_system, &root_path); } Result> CreateTableRead( @@ -1125,19 +1187,11 @@ Result> CreateTableRead( } PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, - core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); - PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, - FileStorePathFactory::Create( - internal_context->GetPath(), arrow_schema, table_schema->PartitionKeys(), - core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), - core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - memory_pool)); + FileStore::MakePathFactory(internal_context->GetPath(), arrow_schema, + table_schema->PartitionKeys(), core_options, + core_options.GetFileFormat()->Identifier(), memory_pool)); if (internal_context->GetPrimaryKeys().empty()) { return std::make_unique(path_factory, internal_context, memory_pool, @@ -1151,21 +1205,21 @@ Result> CreateTableRead( Result> FileStoreTable::NewRead( std::unique_ptr context) const { std::shared_ptr read_context = std::move(context); - if (read_context == nullptr) { - return Status::Invalid("read context is null pointer"); - } - if (read_context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (read_context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(read_context, "read")); // The schema the table was constructed with, so a read serves the table as it was loaded - // rather than reading the schema again. + // rather than reading the schema again. Its options are the ones the read serves: an option + // the context would have changed is refused rather than honoured. std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); + if (loaded_schema == nullptr) { + return Status::Invalid("not found latest schema"); + } + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); + // The branch the table serves, which the read reads on: a table is the branch it was loaded + // for, so the context's own branch is not consulted. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr internal_context, - CreateInternalReadContext(read_context, read_context->GetBranch(), loaded_schema)); + CreateInternalReadContext(read_context, CurrentBranch(), GetSchemaManager(), loaded_schema, + file_system_, root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, CreateTableRead(internal_context, read_context->GetMemoryPool(), @@ -1178,10 +1232,14 @@ Result> FileStoreTable::NewRead( return std::move(table_read); } + // The fallback branch is not the table's, so its schema is read through a manager of its + // own rather than the table's. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr fallback_context, - CreateInternalReadContext(read_context, /*branch=*/scan_fallback_branch.value(), - /*loaded_schema=*/nullptr)); + CreateInternalReadContext( + read_context, /*branch=*/scan_fallback_branch.value(), + std::make_shared(file_system_, root_path_, scan_fallback_branch.value()), + /*loaded_schema=*/nullptr, file_system_, root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, CreateTableRead(fallback_context, read_context->GetMemoryPool(), @@ -1192,15 +1250,7 @@ Result> FileStoreTable::NewRead( } Result> TableRead::Create(std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("read context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "read")); // First the table the context names, then the context handed to that table's `NewRead`. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); return table->NewRead(std::move(ctx)); @@ -1283,41 +1333,19 @@ Result> NewSnapshotCommit( Result> FileStoreTable::NewCommit( std::unique_ptr ctx) const { - if (ctx == nullptr) { - return Status::Invalid("commit context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "commit")); PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - // Use catalog credentials unless the caller supplied a file system. std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - // Use the credentials of this table, which a catalog issuing per-table temporary - // ones only hands out through GetTableFileSystem. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs)); - const std::string& root_path = ctx->GetRootPath(); - // The branch the context was built for, already validated by `CommitContextBuilder`. - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - ctx->GetIdentifier(), ctx->GetOptions(), - /*explicit_branch=*/std::nullopt, "commit")); - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } + // A catalog commit names its table; the file system itself comes from the table, which its + // catalog's credentials already opened. + if (specific_fs == nullptr && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + // A commit commits to the root path the table was opened with and the branch the table + // serves, in place of the ones named by the context: a table is the branch it was loaded + // for, so the context's own branch is not consulted. + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + MakeBranchCatalogIdentifier(ctx->GetIdentifier(), CurrentBranch())); // The id a snapshot records the catalog's current schema by, read through the loader below // rather than trusted from a schema this commit will not re-read. FileStoreCommitImpl::SchemaIdLoader schema_id_loader; @@ -1339,33 +1367,30 @@ Result> FileStoreTable::NewCommit( return data_schema->Id(); }; } - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), root_path, branch); // The schema the table was constructed with, so a commit serves the table as it was loaded // rather than reading the schema again. - std::shared_ptr latest_schema = std::static_pointer_cast(schema_); - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - const std::shared_ptr& schema = latest_schema; - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; - } - // The metadata paths below are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + LatestTableSchemaOfBranch(schema_, CurrentBranch())); std::shared_ptr arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, specific_fs)); + // The options the table was loaded with, with a runtime setting the context adds on top: an + // option the context would have changed is refused rather than honoured, and the branch is + // the table's - the metadata paths below are built for it, so what reads through these + // options, whose cache keys are scoped by it, has to read the same one. + std::map opts; + PAIMON_ASSIGN_OR_RAISE(opts, + MergedOperatorOptions(*schema, ctx->GetOptions(), CurrentBranch())); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, file_system_)); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); assert(options.GetFileSystem()); assert(options.GetFileFormat()); PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); - PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path)); - // Catalog commits do not require atomic rename on the object store. + PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path_)); + // Catalog commits do not require atomic rename on the object store. The inte-test marker is + // not a table option, so it is read from the context the commit was handed. if (is_object_store && !ctx->UseRESTCatalogCommit() && !IsCatalogCommit(*ctx) && - opts.find("enable-object-store-commit-in-inte-test") == opts.end()) { + ctx->GetOptions().find("enable-object-store-commit-in-inte-test") == + ctx->GetOptions().end()) { return Status::NotImplemented( "commit operation does not support object store file system for now"); } @@ -1374,30 +1399,14 @@ Result> FileStoreTable::NewCommit( options.GetPartitionDefaultName(), options.LegacyPartitionNameEnabled(), ctx->GetMemoryPool())); - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); - PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, - FileStorePathFactory::Create( - root_path, arrow_schema, schema->PartitionKeys(), options.GetPartitionDefaultName(), - options.GetFileFormat()->Identifier(), options.DataFilePrefix(), - options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), ctx->GetMemoryPool())); - - auto snapshot_manager = - std::make_shared(options.GetFileSystem(), root_path, branch); - if (IsCatalogCommit(*ctx)) { - std::shared_ptr catalog = ctx->GetCatalog(); - VersionManagedCatalog* versioned = AsVersionManaged(catalog); - Identifier identifier = catalog_identifier.value(); - // Capture the catalog to keep the versioned interface alive. - snapshot_manager->SetSnapshotLoader( - [catalog, versioned, identifier]() -> Result> { - return versioned->LoadSnapshot(identifier); - }); - } + store_->NewPathFactory(arrow_schema, options, options.GetFileFormat()->Identifier(), + ctx->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr snapshot_manager, + store_->NewSnapshotManager(options, ctx->GetCatalog(), catalog_identifier)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), @@ -1425,11 +1434,11 @@ Result> FileStoreTable::NewCommit( CommitScanner::ScanSupplier scan_supplier; if (schema->PrimaryKeys().empty()) { - scan_supplier = CreateAppendScanSupplier(snapshot_manager, schema_manager, manifest_list, - manifest_file, schema, arrow_schema, options, - ctx->GetExecutor(), ctx->GetMemoryPool()); + scan_supplier = CreateAppendScanSupplier(snapshot_manager, GetSchemaManager(), + manifest_list, manifest_file, schema, arrow_schema, + options, ctx->GetExecutor(), ctx->GetMemoryPool()); } else { - scan_supplier = CreatePkScanSupplier(snapshot_manager, schema_manager, manifest_list, + scan_supplier = CreatePkScanSupplier(snapshot_manager, GetSchemaManager(), manifest_list, manifest_file, schema, arrow_schema, options, ctx->GetExecutor(), ctx->GetMemoryPool()); } @@ -1439,24 +1448,16 @@ Result> FileStoreTable::NewCommit( NewSnapshotCommit(*ctx, catalog_identifier, options.GetFileSystem(), snapshot_manager)); return std::make_unique( - ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path, ctx->GetCommitUser(), + ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path_, ctx->GetCommitUser(), options, path_factory, std::move(partition_computer), snapshot_manager, snapshot_commit, ctx->IgnoreEmptyCommit(), ctx->AppendCommitCheckConflict(), schema, std::move(schema_id_loader), manifest_file, manifest_list, index_manifest_file, - expire_snapshots, schema_manager, std::move(scan_supplier)); + expire_snapshots, GetSchemaManager(), std::move(scan_supplier)); } Result> FileStoreCommit::Create( std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("commit context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "commit")); // First the table the context names, then the context handed to that table's `NewCommit`. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); return table->NewCommit(std::move(ctx)); diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 1ad8d9cd3..01a7ce27e 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -25,22 +25,29 @@ #include #include "arrow/api.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" #include "gtest/gtest.h" #include "paimon/commit_context.h" +#include "paimon/commit_message.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/file_store_table.h" +#include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/read_context.h" +#include "paimon/record_batch.h" #include "paimon/scan_context.h" #include "paimon/schema/schema.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/plan.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_catalog.h" +#include "paimon/testing/mock/mock_file_system.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" @@ -49,23 +56,198 @@ namespace paimon::test { namespace { // Creates a managed table directory and a `Table` over it, holding the schema written to -// `branch`'s directory under `identifier`. -Result> CreateManagedTable(const std::shared_ptr& file_system, - const std::string& path, - const Identifier& identifier, - const std::string& branch = "main") { +// `branch`'s directory under `identifier` with `options`. +Result> CreateManagedTable( + const std::shared_ptr& file_system, const std::string& path, + const Identifier& identifier, const std::string& branch = "main", + const std::map& options = {}) { SchemaManager schema_manager(file_system, path, branch); auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr created, schema_manager.CreateTable(schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, /*options=*/{})); + /*primary_keys=*/{}, options)); return std::make_shared( file_system, path, - checked_pointer_cast(std::shared_ptr(std::move(created))), identifier); + checked_pointer_cast(std::shared_ptr(std::move(created))), identifier, + /*uuid=*/"", branch); } } // namespace +TEST(TableTest, TestTableCarriesManagersOfItsBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(fs, path, Identifier(Identifier::kUnknownDatabase, "tbl"), + /*branch=*/"dev")); + auto file_store_table = std::dynamic_pointer_cast(table); + ASSERT_TRUE(file_store_table != nullptr); + // The branch the table was built for is the one its managers point at. + EXPECT_EQ(file_store_table->CurrentBranch(), "dev"); + ASSERT_TRUE(file_store_table->GetSchemaManager() != nullptr); + ASSERT_TRUE(file_store_table->GetSnapshotManager() != nullptr); + EXPECT_EQ(file_store_table->GetSnapshotManager()->Branch(), "dev"); + EXPECT_EQ(file_store_table->GetSnapshotManager()->RootPath(), path); + EXPECT_EQ(file_store_table->GetSnapshotManager()->Fs(), fs); + // A copy serves the same branch through the same managers, whose cache it shares. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({})); + auto file_store_copy = std::dynamic_pointer_cast(copy); + ASSERT_TRUE(file_store_copy != nullptr); + EXPECT_EQ(file_store_copy->CurrentBranch(), "dev"); + EXPECT_EQ(file_store_copy->GetSchemaManager(), file_store_table->GetSchemaManager()); + EXPECT_EQ(file_store_copy->GetSnapshotManager(), file_store_table->GetSnapshotManager()); +} + +TEST(TableTest, TestSwitchToBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier(Identifier::kUnknownDatabase, "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, CreateManagedTable(fs, path, identifier)); + // A branch of its own, with a schema of its own. + SchemaManager dev_schema_manager(fs, path, "dev"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr dev_schema, + dev_schema_manager.CreateTable(arrow::schema({arrow::field("c0", arrow::int32())}), + /*partition_keys=*/{}, /*primary_keys=*/{}, {})); + + // Switching to a branch serves that branch's schema and managers. + auto main_file_store_table = std::dynamic_pointer_cast(table); + ASSERT_TRUE(main_file_store_table != nullptr); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_table, + main_file_store_table->SwitchToBranch("dev")); + auto dev_file_store_table = std::dynamic_pointer_cast(dev_table); + ASSERT_TRUE(dev_file_store_table != nullptr); + EXPECT_EQ("dev", dev_file_store_table->CurrentBranch()); + ASSERT_EQ((std::vector{"c0"}), dev_table->LatestSchema()->FieldNames()); + auto dev_table_schema = std::dynamic_pointer_cast(dev_table->LatestSchema()); + ASSERT_TRUE(dev_table_schema != nullptr); + ASSERT_EQ("dev", dev_table_schema->Options().at(Options::BRANCH)); + ASSERT_TRUE(dev_file_store_table->GetSchemaManager() != nullptr); + ASSERT_TRUE(dev_file_store_table->GetSnapshotManager() != nullptr); + EXPECT_EQ("dev", dev_file_store_table->GetSnapshotManager()->Branch()); + + // Switching to the branch already served hands back the same table, managers shared. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
same_table, + dev_file_store_table->SwitchToBranch("dev")); + auto same_file_store_table = std::dynamic_pointer_cast(same_table); + ASSERT_TRUE(same_file_store_table != nullptr); + EXPECT_EQ(same_table->LatestSchema(), dev_table->LatestSchema()); + EXPECT_EQ(same_file_store_table->GetSchemaManager(), dev_file_store_table->GetSchemaManager()); + EXPECT_EQ(same_file_store_table->GetSnapshotManager(), + dev_file_store_table->GetSnapshotManager()); + + // Switching back serves the main branch's schema again. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_again, + dev_file_store_table->SwitchToBranch("main")); + auto main_again_file_store_table = std::dynamic_pointer_cast(main_again); + ASSERT_TRUE(main_again_file_store_table != nullptr); + EXPECT_EQ("main", main_again_file_store_table->CurrentBranch()); + ASSERT_EQ((std::vector{"id"}), main_again->LatestSchema()->FieldNames()); + auto main_again_schema = std::dynamic_pointer_cast(main_again->LatestSchema()); + ASSERT_TRUE(main_again_schema != nullptr); + EXPECT_EQ(0, main_again_schema->Options().count(Options::BRANCH)); + + // A branch with no schema does not exist. + ASSERT_NOK_WITH_MSG(main_file_store_table->SwitchToBranch("missing"), "branch"); +} + +TEST(TableTest, TestNewWriteServesTheBranchTheTableWasLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"dev", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // A context that names no branch is the one a caller hands straight to the table's write, + // which serves the branch the table was loaded with rather than the context's own main. + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::vector> messages, + write->PrepareCommit()); + ASSERT_OK(write->Close()); + + // The files land under the branch's own directory, which a write of the main branch - the + // branch the context alone would name - would never touch. + std::vector branch_entries; + ASSERT_OK(fs->ListDir(BranchManager::BranchPath(dir->Str(), "dev"), &branch_entries)); + EXPECT_FALSE(branch_entries.empty()); +} + +TEST(TableTest, TestOperatorsServeTheBranchTheTableWasLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"dev", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // The write and the commit below carry no branch, and both serve the branch the table was + // loaded with, so the data they publish lands on it. + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + ASSERT_OK(write->Close()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + + // The scan finds what they wrote, reading through the table's own managers on its branch. + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + EXPECT_FALSE(plan->Splits().empty()); + + // A branch the scan context carries does not win either: the table's own does, so the data + // it finds is still the branch's. + ScanContextBuilder main_scan_builder(dir->Str()); + main_scan_builder.AddOption(Options::BRANCH, "main"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan_ctx, main_scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan, + table->NewScan(std::move(main_scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr main_plan, main_scan->CreatePlan()); + EXPECT_FALSE(main_plan->Splits().empty()); +} + TEST(TableTest, TestGetTableWithUnknownDatabase) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -166,9 +348,9 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { SCOPED_TRACE(main_identifier.ToString()); // A catalog that loaded the schema itself calls the constructor directly, which a // rest catalog is shaped like. The alias resolves to the main branch as well. - FileStoreTable table(fs, dir->Str(), schema, main_identifier); - ASSERT_OK_AND_ASSIGN(std::string branch, table.CurrentBranch()); - EXPECT_EQ("main", branch); + ASSERT_OK_AND_ASSIGN(std::string branch, main_identifier.GetBranchNameOrDefault()); + FileStoreTable table(fs, dir->Str(), schema, main_identifier, /*uuid=*/"", branch); + EXPECT_EQ("main", table.CurrentBranch()); } } @@ -180,15 +362,17 @@ TEST(TableTest, TestCurrentBranch) { ASSERT_OK_AND_ASSIGN( std::shared_ptr
main_table, CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); - ASSERT_OK_AND_ASSIGN(std::string branch, main_table->CurrentBranch()); - EXPECT_EQ(branch, "main"); + auto main_file_store_table = std::dynamic_pointer_cast(main_table); + ASSERT_TRUE(main_file_store_table != nullptr); + EXPECT_EQ(main_file_store_table->CurrentBranch(), "main"); - // The identifier a table was opened with names the branch it is for. + // The branch a table was created on is the one it carries. ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/branch_tbl", Identifier("db", "tbl", "dev"), /*branch=*/"dev")); - ASSERT_OK_AND_ASSIGN(branch, branch_table->CurrentBranch()); - EXPECT_EQ(branch, "dev"); + auto branch_file_store_table = std::dynamic_pointer_cast(branch_table); + ASSERT_TRUE(branch_file_store_table != nullptr); + EXPECT_EQ(branch_file_store_table->CurrentBranch(), "dev"); } TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { @@ -314,6 +498,110 @@ TEST(TableTest, TestNewScanServesTheSchemaTheTableWasLoadedWith) { ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); } +TEST(TableTest, TestNewScanServesTheFileSystemTheTableWasOpenedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // A file system is only observable once a scan has data to read, so two rows are written and + // committed through the table's own write and commit. + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + // The table was opened with the local file system, so a scan serves its data through that + // one even when the context hands over a mock that can read nothing. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr held_ctx, + ScanContextBuilder(dir->Str()).WithFileSystem(std::make_shared()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(held_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + + // A context without a table resolves the file system itself, which is the mock and reads no + // schema at all: the file system counterpart of the schema case above. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr ctx, + ScanContextBuilder(dir->Str()).WithFileSystem(std::make_shared()).Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); +} + +TEST(TableTest, TestNewScanServesTheRootPathTheTableWasOpenedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // Two rows are written and committed through the table's own write and commit, so the scan + // below has data to find. + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + // The table was opened with this directory as its root path, so a scan serves its data from + // there even when the context names a different one. + ASSERT_OK_AND_ASSIGN(std::unique_ptr held_ctx, + ScanContextBuilder(dir->Str() + "/not-the-table").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(held_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + + // A context without a table reads the schema from the directory it names, which is not a + // table at all: the root path counterpart of the schema case above. + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + ScanContextBuilder(dir->Str() + "/not-the-table").Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); +} + TEST(TableTest, TestNewReadCreatesATableRead) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -383,6 +671,122 @@ TEST(TableTest, TestNewScanReadWriteCommitRejectANullContext) { ASSERT_NOK_WITH_MSG(table->NewCommit(nullptr), "commit context is null pointer"); } +TEST(TableTest, TestOperatorsRefuseAnOptionThatContradictsTheTable) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // The table was loaded with `bucket` = 1, so every operator refuses a context that names + // another value for it: the table serves the options it was loaded with, and a caller that + // means to change them copies the table. + { + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(ctx)), + "option 'bucket' is '2' in the context but '1' on the table"); + } + { + ReadContextBuilder read_builder(dir->Str()); + read_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, read_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(ctx)), + "option 'bucket' is '2' in the context but '1' on the table"); + } + { + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + write_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(ctx)), + "option 'bucket' is '2' in the context but '1' on the table"); + } + { + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), + "option 'bucket' is '2' in the context but '1' on the table"); + } + + // The copy made with the option in question agrees with the context, so it serves it. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "2"}})); + ScanContextBuilder agreeing_scan_builder(dir->Str()); + agreeing_scan_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, agreeing_scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + copied->NewScan(std::move(ctx))); +} + +TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // Two rows are written and committed through the table's own write and commit, so the copy + // below has data to find. + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + // The copy overlays what the call gives on the options the schema stored; the schema itself + // and the identity the table was opened with are copied as they are. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, + table->Copy({{Options::BUCKET, "2"}, {Options::TYPE, "format-table"}})); + std::shared_ptr copied_schema = + std::dynamic_pointer_cast(copied->LatestSchema()); + ASSERT_TRUE(copied_schema != nullptr); + EXPECT_EQ(copied_schema->Options().at(Options::BUCKET), "2"); + // `type` is structural and read from the schema alone, so the copy carries none of it: the + // option given at the call does not restyle a managed table. + EXPECT_EQ(copied_schema->Options().count(Options::TYPE), 0); + EXPECT_EQ(copied_schema->Options().at(Options::BUCKET_KEY), "id"); + EXPECT_EQ(copied_schema->Id(), + std::dynamic_pointer_cast(table->LatestSchema())->Id()); + EXPECT_EQ(copied->FullName(), table->FullName()); + EXPECT_EQ(copied->GetRootPath(), table->GetRootPath()); + EXPECT_EQ(copied->GetFileSystem(), table->GetFileSystem()); + // The original keeps the options it was loaded with. + EXPECT_EQ( + std::dynamic_pointer_cast(table->LatestSchema())->Options().at(Options::BUCKET), + "1"); + + // The copy is a table like the one it came from, and serves its data as one. + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, copied->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); +} + TEST(TableTest, TestCreateBuildsTheOperator) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 3ff2ed756..32ebefc1e 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -468,8 +468,12 @@ Result> RestCatalog::GetTable(const Identifier& identifie // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); - return std::make_shared(table_file_system, table_path, schema, identifier, - uuid); + // The branch the identifier carries, which the schema and path above were read on, so the + // table and its managers serve it. + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + return std::make_shared( + table_file_system, table_path, schema, identifier, uuid, + NormalizeBranch(std::move(branch)).value_or(Identifier::kDefaultMainBranch)); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index c76aa4b33..68081328e 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -44,6 +44,7 @@ #include "paimon/core/catalog/commit_table_request.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" @@ -1072,8 +1073,9 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_EQ(0, main_case_schema->Options().count(Options::BRANCH)); // the branch the alias spells resolves to the main branch, not to a branch // directory named after the alias - ASSERT_OK_AND_ASSIGN(std::string main_case_branch, main_case_table->CurrentBranch()); - ASSERT_EQ("main", main_case_branch); + auto main_case_file_store_table = std::dynamic_pointer_cast(main_case_table); + ASSERT_NE(nullptr, main_case_file_store_table); + ASSERT_EQ("main", main_case_file_store_table->CurrentBranch()); // a system table on a branch resolves against the branch's data table: the system // suffix is stripped while the branch stays in the identifier sent to the server diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 5de6c9fa7..fd9b49650 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -43,6 +43,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/table/special_fields.h" @@ -559,6 +560,41 @@ class RealtimeWriteInteTest : public ::testing::Test { return FileStoreWrite::Create(std::move(context)); } + // A writer that means to change an option the table was created with writes through a copy + // carrying it, which the context then agrees with. + Result> CopiedTable( + const std::map& overrides) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, Catalog::Create(dir_->Str(), {})); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + catalog->GetTable(Identifier("foo", "bar"))); + return table->Copy(overrides); + } + + // Evolves the table schema to the current options with its fields unchanged, the way an + // ALTER TABLE changing an option would: the table itself then serves the changed option. + // The fields are carried over from the latest schema so their attributes survive: rebuilt + // raw fields would lose the non-nullable primary keys the table schema enforces. + Status EvolveSchemaWithOptions() const { + SchemaManager schema_manager(dir_->GetFileSystem(), table_path_); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (!latest.has_value()) { + return Status::Invalid("table schema does not exist"); + } + return TestHelper::WriteNextSchema(dir_->GetFileSystem(), table_path_, + latest.value()->Fields(), + latest.value()->HighestFieldId(), options_); + } + + Result> CreateRealtimeWriter( + const std::shared_ptr& realtime_context, + const std::shared_ptr
& table) const { + WriteContextBuilder builder(table_path_, commit_user_); + builder.SetOptions(options_).WithStreamingMode(true).WithRealtimeContext(realtime_context); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + return table->NewWrite(std::move(context)); + } + Result> CreateRealtimeWriter() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context, RealtimeContext::Create()); @@ -1249,32 +1285,36 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeOperationsRequireEnabledOption) { disabled_options[Options::REALTIME_ENABLED] = "false"; ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); + // The table was created enabled, so each operation below goes through a copy carrying it + // disabled, whose contexts then agree: each reports that real-time needs the option itself. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
disabled_table, + CopiedTable({{Options::REALTIME_ENABLED, "false"}})); WriteContextBuilder write_builder(table_path_, commit_user_); write_builder.SetOptions(disabled_options) .WithStreamingMode(true) .WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + ASSERT_NOK_WITH_MSG(disabled_table->NewWrite(std::move(write_context)), "real-time write requires realtime.enabled=true"); ScanContextBuilder scan_builder(table_path_); scan_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(disabled_table->NewScan(std::move(scan_context)), "real-time scan requires realtime.enabled=true"); ReadContextBuilder read_builder(table_path_); read_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(disabled_table->NewRead(std::move(read_context)), "real-time read requires realtime.enabled=true"); CommitContextBuilder commit_builder(table_path_, commit_user_); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.SetOptions(disabled_options).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + disabled_table->NewCommit(std::move(commit_context))); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(/*realtime_commits=*/{}, /*commit_identifier=*/0, /*watermark=*/std::nullopt), @@ -2084,6 +2124,9 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateAcrossMultipleFileIndexStor ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; + // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: + // the schema is evolved to it, so every later write, commit, and read serves it. + ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_batch, @@ -3766,6 +3809,9 @@ TEST_F(RealtimeWriteInteTest, TestAppendRealtimeReadUsesEmbeddedAndExternalBitma ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; + // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: + // the schema is evolved to it, so every later write, commit, and read serves it. + ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN( @@ -4083,6 +4129,9 @@ TEST_F(RealtimeWriteInteTest, TestPkDvRealtimeReadUsesEmbeddedAndExternalBitmapI ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; + // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: + // the schema is evolved to it, so every later write, commit, and read serves it. + ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_batch, diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index b7cc73b96..41d35de8d 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -36,6 +36,9 @@ #include "fmt/ranges.h" #include "gtest/gtest.h" #include "paimon/bucket/bucket_id_calculator.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/table/special_fields.h" @@ -796,19 +799,27 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { } ASSERT_GE(other_index, 0); // These keys share a current two-bucket selection but need different historical buckets. + // The data was written with four buckets, so reading a rescaled layout reads through a + // copy of the table carrying the rescaled bucket count: an option on the context would + // contradict the table the catalog loaded. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + Catalog::Create(test_dir->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("foo", "bar"))); for (int32_t lookup_index : {32, other_index}) { - // The data was written with four buckets. Read it with rescaled table options too. for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); const std::string lookup_key = fmt::format("key{:03}", lookup_index); auto lookup_predicate = PredicateBuilder::Equal( 0, "rowkey", field_type, Literal(field_type, lookup_key.data(), lookup_key.size())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, + table->Copy({{Options::BUCKET, current_bucket_count}})); ScanContextBuilder scan_builder(table_path); scan_builder.SetPredicate(lookup_predicate) .AddOption(Options::BUCKET, current_bucket_count); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_builder)); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, rescaled_table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); if (EnableSnapshotLiveManifestCache()) { ASSERT_OK_AND_ASSIGN( @@ -820,7 +831,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { .AddOption(Options::BUCKET, current_bucket_count); ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); ASSERT_OK_AND_ASSIGN(auto cached_scan, - TableScan::Create(std::move(cached_context))); + rescaled_table->NewScan(std::move(cached_context))); ASSERT_OK_AND_ASSIGN(plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); @@ -3086,12 +3097,21 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice0", 6)); + // The table was loaded with its index enabled, so the index-disabled pass below goes through + // a copy carrying it disabled: an option on the context would contradict the table. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + Catalog::Create(paimon::test::GetDataDir() + file_format, {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("append_with_bitmap", "append_with_bitmap"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
disabled_table, + table->Copy({{"file-index.read.enabled", "false"}})); + // scan ScanContextBuilder scan_context_builder(table_path); ASSERT_OK_AND_ASSIGN( auto scan_context, scan_context_builder.AddOption("file-index.read.enabled", "false").Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, disabled_table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -3102,7 +3122,7 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { read_context_builder.AddOption("file-index.read.enabled", "false"); AddReadOptionsForPrefetch(&read_context_builder); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, disabled_table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3158,6 +3178,13 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, std::nullopt)); ASSERT_FALSE(commit_messages.empty()); + // The statistics-only baseline below disables the file index the table carries, so it scans + // through a copy with it disabled: an option on the context would contradict the table. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(dir->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("foo", "bar"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
stats_only_table, + table->Copy({{Options::FILE_INDEX_READ_ENABLED, "false"}})); + // 15 falls inside the file's [10, 30] min/max range, so statistics alone must retain the // file. Disabling file-index reads verifies that baseline explicitly. const std::shared_ptr absent_predicate = PredicateBuilder::Equal( @@ -3169,7 +3196,7 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - TableScan::Create(std::move(stats_only_scan_context))); + stats_only_table->NewScan(std::move(stats_only_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); @@ -3262,6 +3289,13 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc const std::shared_ptr predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"renamed_value", FieldType::INT, Literal(15)); + // The statistics-only baseline below disables the file index the table carries, so it scans + // through a copy with it disabled: an option on the context would contradict the table. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(dir->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("foo", "bar"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
stats_only_table, + table->Copy({{Options::FILE_INDEX_READ_ENABLED, "false"}})); + // The evolved min/max range is still [10, 30], so statistics alone retain the old file. ScanContextBuilder stats_only_scan_builder(table_path); stats_only_scan_builder.SetOptions(options) @@ -3270,7 +3304,7 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - TableScan::Create(std::move(stats_only_scan_context))); + stats_only_table->NewScan(std::move(stats_only_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); @@ -4021,16 +4055,25 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT, Literal(static_cast(0))); - // Historical files use two buckets even when the current option has changed. + // Historical files use two buckets even when the current option has changed, so the scan + // goes through a copy of the table carrying the bucket count it selects by: an option on + // the context would contradict the table the catalog loaded. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + Catalog::Create(paimon::test::GetDataDir() + file_format, {})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + catalog->GetTable(Identifier("pk_table_scan_and_read_dv", "pk_table_scan_and_read_dv"))); for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, + table->Copy({{Options::BUCKET, current_bucket_count}})); ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); scan_context_builder.AddOption(Options::BUCKET, current_bucket_count); scan_context_builder.SetPartitionFilter({{{"f1", "10"}}}); scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, rescaled_table->NewScan(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); AddReadOptionsForPrefetch(&read_context_builder); @@ -4049,7 +4092,8 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { .SetPartitionFilter({{{"f1", "10"}}}) .SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); - ASSERT_OK_AND_ASSIGN(auto cached_scan, TableScan::Create(std::move(cached_context))); + ASSERT_OK_AND_ASSIGN(auto cached_scan, + rescaled_table->NewScan(std::move(cached_context))); ASSERT_OK_AND_ASSIGN(result_plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index 3f949484d..1502c39a7 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -32,6 +32,9 @@ #include "arrow/api.h" #include "gtest/gtest.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/data_define.h" #include "paimon/common/utils/decimal_utils.h" @@ -1161,10 +1164,18 @@ TEST_P(ScanInteTest, TestScanAppendWithInvalidOptions) { "scan.snapshot-id or scan.tag-name must be set when startup mode is FROM_SNAPSHOT"); } { + // `bucket` = -2 is refused by the scan itself, which a copy of the table carrying it + // reaches: an option on the context would contradict the table the catalog loaded, + // which a scan through that table refuses. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + Catalog::Create(paimon::test::GetDataDir() + "orc", {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("append_09", "append_09"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "-2"}})); ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::BUCKET, "-2").WithStreamingMode(true); + context_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(copied->NewScan(std::move(scan_context)), "do not support bucket=-2 in scan process"); } } @@ -1726,10 +1737,18 @@ TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { TEST_P(ScanInteTest, TestScanPKWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; + // Like the append case above, the invalid bucket count rides on a copy of the table: the + // scan refuses it where the bucket count is validated. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + Catalog::Create(paimon::test::GetDataDir() + "orc", {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("pk_table_with_total_buckets", + "pk_table_with_total_buckets"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "-1"}})); ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::BUCKET, "-1").WithStreamingMode(true); + context_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(copied->NewScan(std::move(scan_context)), "do not support pk table bucket=-1 in scan process"); } diff --git a/test/inte/write_and_read_inte_with_index_test.cpp b/test/inte/write_and_read_inte_with_index_test.cpp index efda3de03..9123a3385 100644 --- a/test/inte/write_and_read_inte_with_index_test.cpp +++ b/test/inte/write_and_read_inte_with_index_test.cpp @@ -1250,8 +1250,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestReadWithLimits) { PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice", 5)); ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") + context_builder.AddOption("test.enable-adaptive-prefetch-strategy", "false") .AddOption(Options::READ_BATCH_SIZE, "1") .AddOption("orc.read.enable-metrics", "true") .SetPredicate(predicate); From 54367bb7bcc18fc1fa9f4b66b0afae9435b82dd7 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Thu, 22 Oct 2026 17:16:15 +0800 Subject: [PATCH 05/19] tmp --- src/paimon/CMakeLists.txt | 4 +- .../core/catalog/file_system_catalog.cpp | 5 +- .../table/append_only_file_store_table.cpp | 209 ++++++ .../core/table/append_only_file_store_table.h | 91 +++ src/paimon/core/table/file_store.cpp | 75 ++- src/paimon/core/table/file_store.h | 56 +- .../table/{table.cpp => file_store_table.cpp} | 596 ++++-------------- src/paimon/core/table/file_store_table.h | 124 +++- .../table/primary_key_file_store_table.cpp | 291 +++++++++ .../core/table/primary_key_file_store_table.h | 91 +++ src/paimon/core/table/table_test.cpp | 10 +- src/paimon/rest/rest_catalog.cpp | 2 +- src/paimon/testing/mock/mock_catalog.h | 2 +- 13 files changed, 1018 insertions(+), 538 deletions(-) create mode 100644 src/paimon/core/table/append_only_file_store_table.cpp create mode 100644 src/paimon/core/table/append_only_file_store_table.h rename src/paimon/core/table/{table.cpp => file_store_table.cpp} (66%) create mode 100644 src/paimon/core/table/primary_key_file_store_table.cpp create mode 100644 src/paimon/core/table/primary_key_file_store_table.h diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 261987aea..8d1234c4d 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -443,9 +443,10 @@ set(PAIMON_CORE_SRCS core/stats/simple_stats_converter.cpp core/stats/simple_stats.cpp core/stats/simple_stats_evolution.cpp - core/table/table.cpp + core/table/append_only_file_store_table.cpp core/table/bucket_mode.cpp core/table/file_store.cpp + core/table/file_store_table.cpp core/table/format/format_file_listing.cpp core/table/format/format_file_naming.cpp core/table/format/format_path_validation.cpp @@ -457,6 +458,7 @@ set(PAIMON_CORE_SRCS core/table/format/format_table_scan.cpp core/table/format/format_table_write.cpp core/table/format/lazy_concat_batch_reader.cpp + core/table/primary_key_file_store_table.cpp core/table/sink/commit_message.cpp core/table/sink/commit_message_impl.cpp core/table/sink/commit_message_serializer.cpp diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 36721a148..d3c6bd92e 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -350,9 +350,8 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide // The branch the schema above was read on, which the table carries so its managers read // the same branch: `GetBranchNameOrDefault` folds any spelling of `main`. PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); - return std::make_shared(fs_, table_path, - checked_pointer_cast(*latest_schema), - identifier, /*uuid=*/"", branch); + return FileStoreTable::Create(fs_, table_path, checked_pointer_cast(*latest_schema), + identifier, /*uuid=*/"", branch); } Result> FileSystemCatalog::LoadSystemTable( diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp new file mode 100644 index 000000000..0846aef90 --- /dev/null +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/append_only_file_store_table.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/operation/append_only_file_store_scan.h" +#include "paimon/core/operation/append_only_file_store_write.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/data_evolution_file_store_scan.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/realtime/realtime_schema_layout.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/source/append_only_split_generator.h" +#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/table/source/data_evolution_split_generator.h" +#include "paimon/core/table/source/split_generator.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/write_context.h" + +namespace paimon { + +AppendOnlyFileStoreTable::AppendOnlyFileStoreTable( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, + const std::string& branch, const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) + : FileStoreTable(file_system, root_path, schema, identifier, uuid, branch, schema_manager, + snapshot_manager) {} + +Result> AppendOnlyFileStoreTable::CreateWrite( + const WriteContext& ctx, const std::shared_ptr& schema, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const { + // append table + const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); + bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); + if (options.GetBucket() == -1) { + need_dv_maintainer_factory = false; + ignore_previous_files = true; + } else if (options.GetBucket() <= 0) { + return Status::Invalid( + fmt::format("not support bucket {} in append table", options.GetBucket())); + } + if (ctx.GetRealtimeContext()) { + if (!ctx.GetWriteSchema().empty()) { + return Status::Invalid("real-time append write does not support a custom schema"); + } + if (!realtime_deduplicate) { + PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx.GetRealtimeContext(), + snapshot_manager, options)); + } + } else if (realtime_deduplicate) { + return Status::Invalid("real-time deduplicate write requires a real-time context"); + } + std::shared_ptr write_schema = arrow_schema; + const auto& write_field_names = ctx.GetWriteSchema(); + if (!write_field_names.empty()) { + arrow::FieldVector write_fields; + write_fields.reserve(write_field_names.size()); + for (const auto& field_name : write_field_names) { + auto field = arrow_schema->GetFieldByName(field_name); + if (!field) { + // TODO(xinyu.lxy): support _ROW_ID and _SEQUENCE_NUMBER + return Status::Invalid( + fmt::format("write field {} does not exist in table schema", field_name)); + } + write_fields.push_back(field); + } + write_schema = arrow::schema(write_fields); + } + + std::shared_ptr realtime_schema_layout; + if (ctx.GetRealtimeContext()) { + const RealtimeStoreMode mode = + realtime_deduplicate ? RealtimeStoreMode::DEDUPLICATE : RealtimeStoreMode::APPEND_ONLY; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr layout, + RealtimeSchemaLayout::Create(mode, write_schema)); + realtime_schema_layout = std::move(layout); + } + + std::shared_ptr dv_maintainer_factory; + if (need_dv_maintainer_factory) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_file_handler, + store_->NewIndexFileHandler(options, path_factory, ctx.GetMemoryPool())); + dv_maintainer_factory = + std::make_shared(std::move(index_file_handler)); + } + + auto file_store_write = std::make_unique( + path_factory, snapshot_manager, GetSchemaManager(), ctx.GetCommitUser(), root_path_, schema, + arrow_schema, write_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, + io_manager, options, ignore_previous_files, ctx.IsStreamingMode(), + ctx.IgnoreNumBucketCheck(), ctx.GetRealtimeContext(), ctx.GetExecutor(), + ctx.GetMemoryPool()); + if (ctx.GetRealtimeContext() && realtime_deduplicate) { + PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, + snapshot_manager->LatestSnapshot()); + if (latest_snapshot) { + PAIMON_RETURN_NOT_OK(file_store_write->RefreshCommittedSnapshot(latest_snapshot->Id())); + } + } + return std::unique_ptr(std::move(file_store_write)); +} + +Result> AppendOnlyFileStoreTable::NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const { + // TODO(liancheng.lsz): support fallback branch in scan + std::unique_ptr scan; + if (core_options.DataEvolutionEnabled()) { + PAIMON_ASSIGN_OR_RAISE( + scan, DataEvolutionFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), + manifest_list, manifest_file, table_schema, + arrow_schema, context->GetScanFilters(), + core_options, executor, memory_pool)); + } else { + PAIMON_ASSIGN_OR_RAISE( + scan, AppendOnlyFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), + manifest_list, manifest_file, table_schema, + arrow_schema, context->GetScanFilters(), + core_options, executor, memory_pool)); + } + return WithTablePath(std::move(scan), root_path_); +} + +Result> AppendOnlyFileStoreTable::NewSplitGenerator( + const std::shared_ptr& table_schema, const CoreOptions& core_options) const { + auto source_split_target_size = core_options.GetSourceSplitTargetSize(); + auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); + if (core_options.DataEvolutionEnabled()) { + return std::make_unique(source_split_target_size, + source_split_open_file_cost, + core_options.BlobSplitByFileSize()); + } + BucketMode bucket_mode = + (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE : BucketMode::HASH_FIXED); + return std::make_unique(source_split_target_size, + source_split_open_file_cost, bucket_mode); +} + +Result> AppendOnlyFileStoreTable::NewTableRead( + const std::shared_ptr& path_factory, + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const { + return std::make_unique(path_factory, internal_context, memory_pool, + executor); +} + +CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) const { + // The schema manager the table carries, which the scans this supplier hands out read the + // schema through. + const std::shared_ptr& schema_manager = GetSchemaManager(); + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +} // namespace paimon diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h new file mode 100644 index 000000000..ea5374ba3 --- /dev/null +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class CoreOptions; +class FileStoreScan; +class IOManager; +class SplitGenerator; +class TableSchema; + +/// The managed table a schema without primary keys describes, Java Paimon's +/// `AppendOnlyFileStoreTable`. Its write appends the rows it is given, and its scan, read and +/// commit walk the files as they were written. The operator flows are the ones `FileStoreTable` +/// carries; the pieces those flows hand down are built here. +class AppendOnlyFileStoreTable : public FileStoreTable { + public: + /// The constructor `FileStoreTable::Create` builds an append-only table through; the + /// parameters are the ones it takes, and the base class carries them. + AppendOnlyFileStoreTable(const std::shared_ptr& file_system, + const std::string& root_path, const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid = "", + const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); + + private: + Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& schema, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const override; + + Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const override; + + Result> NewSplitGenerator( + const std::shared_ptr& table_schema, + const CoreOptions& core_options) const override; + + Result> NewTableRead( + const std::shared_ptr& path_factory, + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const override; + + CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index e70e85c1d..80b6afb05 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -26,12 +26,18 @@ #include "paimon/catalog/catalog.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/binary_row_partition_computer.h" #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/core_options.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/manifest/manifest_file.h" +#include "paimon/core/manifest/manifest_list.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/index_file_path_factories.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/result.h" #include "paimon/status.h" @@ -62,8 +68,13 @@ Result> FileStore::NewPathFactory( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::string& file_format_identifier, const std::shared_ptr& memory_pool) const { - return MakePathFactory(root_path_, arrow_schema, schema_->PartitionKeys(), options, - file_format_identifier, memory_pool); + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + options.CreateGlobalIndexExternalPath()); + return FileStorePathFactory::Create( + root_path_, arrow_schema, schema_->PartitionKeys(), options.GetPartitionDefaultName(), + file_format_identifier, options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_path, options.IndexFileInDataFileDir(), memory_pool); } Result> FileStore::NewSnapshotManager( @@ -84,17 +95,55 @@ Result> FileStore::NewSnapshotManager( return snapshot_manager; } -Result> FileStore::MakePathFactory( - const std::string& root_path, const std::shared_ptr& arrow_schema, - const std::vector& partition_keys, const CoreOptions& options, - const std::string& file_format_identifier, const std::shared_ptr& memory_pool) { - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); - return FileStorePathFactory::Create( - root_path, arrow_schema, partition_keys, options.GetPartitionDefaultName(), - file_format_identifier, options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, options.IndexFileInDataFileDir(), memory_pool); +Result> FileStore::NewManifestList( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_list, + ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, options.GetCache(), + memory_pool)); + return manifest_list; +} + +Result> FileStore::NewManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& partition_schema, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_file, + ManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, + options.GetManifestTargetFileSize(), memory_pool, options, + partition_schema)); + return manifest_file; +} + +Result> FileStore::NewIndexManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const { + return IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, + options.GetBucket(), memory_pool, options); +} + +Result> FileStore::NewIndexFileHandler( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_manifest_file, + NewIndexManifestFile(options, path_factory, memory_pool)); + return std::make_unique( + options.GetFileSystem(), std::move(index_manifest_file), + std::make_shared(path_factory), options.DeletionVectorsBitmap64(), + memory_pool); +} + +Result> FileStore::NewPartitionComputer( + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& memory_pool) const { + return BinaryRowPartitionComputer::Create(schema_->PartitionKeys(), arrow_schema, + options.GetPartitionDefaultName(), + options.LegacyPartitionNameEnabled(), memory_pool); } } // namespace paimon diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index ec3bc155c..83c34b1e7 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -22,11 +22,9 @@ #include #include #include -#include #include "paimon/catalog/identifier.h" #include "paimon/core/schema/schema_manager.h" -#include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/result.h" @@ -38,8 +36,13 @@ class Schema; namespace paimon { +class BinaryRowPartitionComputer; class Catalog; class CoreOptions; +class IndexFileHandler; +class IndexManifestFile; +class ManifestFile; +class ManifestList; class TableSchema; /// The storage a managed table is served through: the file system, root path, schema and branch @@ -86,9 +89,11 @@ class FileStore { } /// The path factory an operator builds its metadata paths through: over this store's root - /// path and the partition keys of its schema. The file format identifier is the caller's to - /// name: a write names the format its level-0 files are written in, which the options may - /// set per level, while a scan, a read and a commit name the table's format. + /// path and the partition keys of its schema - which no schema change can alter, so a read + /// serving a fallback branch's schema still names the same partitions. The file format + /// identifier is the caller's to name: a write names the format its level-0 files are + /// written in, which the options may set per level, while a scan, a read and a commit name + /// the table's format. Result> NewPathFactory( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::string& file_format_identifier, @@ -105,13 +110,40 @@ class FileStore { const CoreOptions& options, const std::shared_ptr& catalog, const std::optional& catalog_identifier) const; - /// The assembly the factory methods above share: a path factory over a root path and - /// partition keys named by the caller rather than the store's own, which the read path - /// needs - the schema it serves may be a fallback branch's, not the table's. - static Result> MakePathFactory( - const std::string& root_path, const std::shared_ptr& arrow_schema, - const std::vector& partition_keys, const CoreOptions& options, - const std::string& file_format_identifier, const std::shared_ptr& memory_pool); + /// The manifest list a scan or a commit reads the table's manifest entries through, over + /// the caller's path factory: an operator's manifests sit in the paths its path factory + /// names. + Result> NewManifestList( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const; + + /// The manifest file a scan or a commit reads a single manifest's entries through, over + /// the caller's path factory and the partition schema its entries are pruned by. + Result> NewManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& partition_schema, + const std::shared_ptr& memory_pool) const; + + /// The index manifest file a write, a scan or a commit reads the table's index manifest + /// through, over the caller's path factory. + Result> NewIndexManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const; + + /// The index file handler a write's deletion vectors and indexes are maintained through, or + /// a scan reads them through: the index manifest file above and the path factories over the + /// caller's path factory, in one handler. + Result> NewIndexFileHandler( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const; + + /// The partition computer a commit names the partitions it writes with, over the partition + /// keys of this store's schema - which no schema change can alter, so a commit serving a + /// schema read after the table was loaded still names the same partitions - and the + /// caller's arrow schema and options. + Result> NewPartitionComputer( + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& memory_pool) const; private: std::shared_ptr file_system_; diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/file_store_table.cpp similarity index 66% rename from src/paimon/core/table/table.cpp rename to src/paimon/core/table/file_store_table.cpp index cb86e4281..502556990 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -9,17 +9,16 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -#include "paimon/catalog/table.h" +#include "paimon/core/table/file_store_table.h" -#include -#include #include #include #include @@ -27,18 +26,14 @@ #include #include -#include "arrow/c/bridge.h" #include "fmt/format.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" #include "paimon/commit_context.h" -#include "paimon/common/global_index/btree/btree_defs.h" #include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/types/data_field.h" -#include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/binary_row_partition_computer.h" #include "paimon/common/utils/checked_cast.h" -#include "paimon/common/utils/fields_comparator.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" @@ -50,57 +45,36 @@ #include "paimon/core/core_options.h" #include "paimon/core/disk/io_manager.h" #include "paimon/core/index/index_file_handler.h" -#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h" #include "paimon/core/index/pk/primary_key_index_definitions.h" -#include "paimon/core/manifest/index_manifest_entry.h" #include "paimon/core/manifest/index_manifest_file.h" -#include "paimon/core/mergetree/compact/lookup_merge_function.h" -#include "paimon/core/mergetree/compact/merge_function.h" -#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" -#include "paimon/core/operation/append_only_file_store_scan.h" -#include "paimon/core/operation/append_only_file_store_write.h" #include "paimon/core/operation/commit/realtime_commit_properties.h" -#include "paimon/core/operation/data_evolution_file_store_scan.h" #include "paimon/core/operation/expire_snapshots.h" #include "paimon/core/operation/file_store_commit_impl.h" #include "paimon/core/operation/file_store_scan.h" #include "paimon/core/operation/internal_read_context.h" -#include "paimon/core/operation/key_value_file_store_scan.h" -#include "paimon/core/operation/key_value_file_store_write.h" -#include "paimon/core/options/merge_engine.h" #include "paimon/core/options/table_type.h" -#include "paimon/core/postpone/postpone_bucket_file_store_write.h" #include "paimon/core/realtime/realtime_context_impl.h" -#include "paimon/core/realtime/realtime_schema_layout.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" -#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/append_only_file_store_table.h" #include "paimon/core/table/file_store.h" -#include "paimon/core/table/file_store_table.h" -#include "paimon/core/table/source/abstract_table_scan.h" -#include "paimon/core/table/source/append_only_split_generator.h" -#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/table/primary_key_file_store_table.h" #include "paimon/core/table/source/data_evolution_batch_scan.h" -#include "paimon/core/table/source/data_evolution_split_generator.h" #include "paimon/core/table/source/data_table_batch_scan.h" #include "paimon/core/table/source/data_table_stream_scan.h" #include "paimon/core/table/source/fallback_table_read.h" -#include "paimon/core/table/source/key_value_table_read.h" -#include "paimon/core/table/source/merge_tree_split_generator.h" #include "paimon/core/table/source/primary_key_index_batch_scan.h" #include "paimon/core/table/source/read_optimized_scan_options.h" #include "paimon/core/table/source/realtime_table_scan.h" #include "paimon/core/table/source/snapshot/snapshot_reader.h" -#include "paimon/core/table/source/split_generator.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/file_store_path_factory.h" -#include "paimon/core/utils/index_file_path_factories.h" #include "paimon/core/utils/primary_key_table_utils.h" #include "paimon/core/utils/snapshot_manager.h" -#include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/format/file_format.h" @@ -122,12 +96,9 @@ class Schema; } // namespace arrow namespace paimon { -class DataSplit; + class Executor; -struct KeyValue; class MemoryPool; -template -class MergeFunctionWrapper; namespace { @@ -170,46 +141,6 @@ Result> TryLoadFormatTable( /*location_carries_paimon_metadata=*/true, options); } -Status RestoreRealtimeCommittedProgress(const std::shared_ptr& realtime_context, - const std::shared_ptr& snapshot_manager, - const CoreOptions& options) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, - RealtimeContextImpl::Cast(realtime_context)); - PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager->LatestSnapshot()); - if (latest_snapshot) { - PAIMON_ASSIGN_OR_RAISE( - RealtimeOffsetMap realtime_committed_offsets, - RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem())); - PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress( - latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{})); - } - return Status::OK(); -} - -Result HasHistoricalPrimaryKeyBTreeDefinition( - const std::shared_ptr& schema_manager, int64_t current_schema_id) { - if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { - return false; - } - PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager->ListAllIds()); - for (int64_t schema_id : schema_ids) { - if (schema_id >= current_schema_id) { - continue; - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical_schema, - schema_manager->ReadSchema(schema_id)); - const auto& historical_options = historical_schema->Options(); - auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS); - if (option != historical_options.end() && - !StringUtils::IsNullOrWhitespaceOnly(option->second)) { - return true; - } - } - return false; -} - /// The tail every `TableOf` ends with: the schema one of them read says what the path names. A /// format table is handed back as the `FormatTable` it is, everything else as the /// `FileStoreTable` a managed table is, carrying the schema for its `NewXxx` to build from. A @@ -230,8 +161,8 @@ Result> TableOfLoadedSchema( // it sits in - the same convention `TryLoadFormatTable` follows above. Identifier table_identifier = identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); - return std::make_shared(file_system, path, latest_schema, table_identifier, - uuid, branch, schema_manager); + return FileStoreTable::Create(file_system, path, latest_schema, table_identifier, uuid, branch, + schema_manager); } /// The managed table a context names behind a path: the schema is read through a manager of the @@ -430,8 +361,8 @@ Result> TableOf(const CommitContext& ctx) { MakeBranchCatalogIdentifier(ctx.GetIdentifier(), branch)); // Catalog-managed schemas may be absent from the table directory, but only on the main branch: // a branch publishes its own under `branch/branch-`, which is where the load below and a - // read of the branch both read it, so a catalog answering for no branch schema does not fail - // a commit to one. + // read of the branch both read it, so a catalog answering for no branch schema does not fail a + // commit to one. PAIMON_ASSIGN_OR_RAISE( std::optional catalog_table_schema, CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "commit")); @@ -527,6 +458,25 @@ FileStoreTable::FileStoreTable(const std::shared_ptr& file_system, std::static_pointer_cast(schema), branch, schema_manager, snapshot_manager)) {} +Result> FileStoreTable::Create( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, + const std::string& branch, const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) { + // The kind the schema's primary keys name, which no schema change can add or drop, so the + // kind the table is built as is the kind every schema its root path ever held describes. A + // caller without a schema gets the append-only kind, whose operators then report that no + // schema was found. + std::shared_ptr table_schema = std::static_pointer_cast(schema); + if (table_schema != nullptr && !table_schema->PrimaryKeys().empty()) { + return std::make_shared(file_system, root_path, schema, + identifier, uuid, branch, schema_manager, + snapshot_manager); + } + return std::make_shared( + file_system, root_path, schema, identifier, uuid, branch, schema_manager, snapshot_manager); +} + Result> FileStoreTable::Copy( const std::map& dynamic_options) const { std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); @@ -541,10 +491,10 @@ Result> FileStoreTable::Copy( loaded_schema->MergeOptions(overridable)); // The managers are handed over, not rebuilt: the copy serves the same root path and branch, // so its operators read the schema cache this table already warmed. - return std::shared_ptr
(std::make_shared( - file_system_, root_path_, std::shared_ptr(std::move(merged_schema)), - identifier_, CatalogUuid().value_or(""), CurrentBranch(), GetSchemaManager(), - GetSnapshotManager())); + return FileStoreTable::Create(file_system_, root_path_, + std::shared_ptr(std::move(merged_schema)), + identifier_, CatalogUuid().value_or(""), CurrentBranch(), + GetSchemaManager(), GetSnapshotManager()); } Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { @@ -552,9 +502,9 @@ Result> FileStoreTable::SwitchToBranch(const std::string& if (target_branch == CurrentBranch()) { // The table already serves this branch: a copy of it - schema and managers shared - is // the same table to the caller. - return std::shared_ptr
(std::make_shared( - file_system_, root_path_, schema_, identifier_, CatalogUuid().value_or(""), - CurrentBranch(), GetSchemaManager(), GetSnapshotManager())); + return FileStoreTable::Create(file_system_, root_path_, schema_, identifier_, + CatalogUuid().value_or(""), CurrentBranch(), + GetSchemaManager(), GetSnapshotManager()); } // The schema of the branch switched to, read from its own directory, which a branch without // one does not have. @@ -572,9 +522,33 @@ Result> FileStoreTable::SwitchToBranch(const std::string& } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr branch_schema, (*latest)->MergeOptions(overridable)); - return std::shared_ptr
(std::make_shared( + return FileStoreTable::Create( file_system_, root_path_, std::shared_ptr(std::move(branch_schema)), - identifier_, CatalogUuid().value_or(""), target_branch, schema_manager)); + identifier_, CatalogUuid().value_or(""), target_branch, schema_manager); +} + +Status FileStoreTable::RestoreRealtimeCommittedProgress( + const std::shared_ptr& realtime_context, + const std::shared_ptr& snapshot_manager, const CoreOptions& options) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, + RealtimeContextImpl::Cast(realtime_context)); + PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); + PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, + snapshot_manager->LatestSnapshot()); + if (latest_snapshot) { + PAIMON_ASSIGN_OR_RAISE( + RealtimeOffsetMap realtime_committed_offsets, + RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem())); + PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress( + latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{})); + } + return Status::OK(); +} + +std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr&& scan, + const std::string& root_path) { + scan->WithTablePath(root_path); + return std::move(scan); } Result> FileStoreTable::NewWrite( @@ -639,209 +613,9 @@ Result> FileStoreTable::NewWrite( RealtimeContextImpl::Cast(ctx->GetRealtimeContext())); PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); } - if (schema->PrimaryKeys().empty()) { - // append table - const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); - bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); - if (options.GetBucket() == -1) { - need_dv_maintainer_factory = false; - ignore_previous_files = true; - } else if (options.GetBucket() <= 0) { - return Status::Invalid( - fmt::format("not support bucket {} in append table", options.GetBucket())); - } - if (ctx->GetRealtimeContext()) { - if (!ctx->GetWriteSchema().empty()) { - return Status::Invalid("real-time append write does not support a custom schema"); - } - if (!realtime_deduplicate) { - PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), - snapshot_manager, options)); - } - } else if (realtime_deduplicate) { - return Status::Invalid("real-time deduplicate write requires a real-time context"); - } - std::shared_ptr write_schema = arrow_schema; - const auto& write_field_names = ctx->GetWriteSchema(); - if (!write_field_names.empty()) { - arrow::FieldVector write_fields; - write_fields.reserve(write_field_names.size()); - for (const auto& field_name : write_field_names) { - auto field = arrow_schema->GetFieldByName(field_name); - if (!field) { - // TODO(xinyu.lxy): support _ROW_ID and _SEQUENCE_NUMBER - return Status::Invalid( - fmt::format("write field {} does not exist in table schema", field_name)); - } - write_fields.push_back(field); - } - write_schema = arrow::schema(write_fields); - } - - std::shared_ptr realtime_schema_layout; - if (ctx->GetRealtimeContext()) { - const RealtimeStoreMode mode = realtime_deduplicate ? RealtimeStoreMode::DEDUPLICATE - : RealtimeStoreMode::APPEND_ONLY; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr layout, - RealtimeSchemaLayout::Create(mode, write_schema)); - realtime_schema_layout = std::move(layout); - } - - std::shared_ptr dv_maintainer_factory; - if (need_dv_maintainer_factory) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), file_store_path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); - auto index_file_handler = std::make_shared( - options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(file_store_path_factory), - options.DeletionVectorsBitmap64(), ctx->GetMemoryPool()); - dv_maintainer_factory = - std::make_shared(index_file_handler); - } - - auto file_store_write = std::make_unique( - file_store_path_factory, snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), - root_path_, schema, arrow_schema, write_schema, realtime_schema_layout, - partition_schema, dv_maintainer_factory, io_manager, options, ignore_previous_files, - ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->GetRealtimeContext(), - ctx->GetExecutor(), ctx->GetMemoryPool()); - if (ctx->GetRealtimeContext() && realtime_deduplicate) { - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager->LatestSnapshot()); - if (latest_snapshot) { - PAIMON_RETURN_NOT_OK( - file_store_write->RefreshCommittedSnapshot(latest_snapshot->Id())); - } - } - return std::unique_ptr(std::move(file_store_write)); - } else { - // pk table - if (ctx->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); - if (ignore_previous_files) { - return Status::NotImplemented( - "PK realtime requires restore from the latest snapshot"); - } - if (!ctx->GetWriteSchema().empty()) { - return Status::NotImplemented("PK realtime does not support a custom write schema"); - } - PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), - snapshot_manager, options)); - } - if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { - return PostponeBucketFileStoreWrite::Create( - snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), root_path_, schema, - arrow_schema, partition_schema, io_manager, options, ctx->IsStreamingMode(), - ctx->IgnoreNumBucketCheck(), ctx->GetWriteId(), - ctx->GetFileSystemSchemeToIdentifierMap(), ctx->GetExecutor(), ctx->GetMemoryPool(), - file_system_); - } - if (options.GetBucket() <= 0) { - return Status::Invalid( - fmt::format("not support bucket {} in key value table", options.GetBucket())); - } - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - schema->TrimmedPrimaryKeys()); - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, - schema->GetFields(trimmed_primary_keys)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr key_comparator, - FieldsComparator::Create(trimmed_primary_key_fields, - options.SequenceFieldSortOrderIsAscending())); - auto primary_keys = schema->PrimaryKeys(); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, - PrimaryKeyTableUtils::CreateMergeFunction( - arrow_schema, primary_keys, options, ctx->GetMemoryPool())); - if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { - // don't wrap first row, it is already OK - merge_function = std::make_unique(std::move(merge_function)); - } - std::shared_ptr> merge_function_wrapper = - std::make_shared(std::move(merge_function)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr sequence_fields_comparator, - PrimaryKeyTableUtils::CreateSequenceFieldsComparator(schema->Fields(), options)); - - std::shared_ptr dv_maintainer_factory; - PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions primary_key_index_definitions, - PrimaryKeyIndexDefinitions::Create(*schema)); - bool has_btree_index = false; - for (const PrimaryKeyIndexDefinition& definition : - primary_key_index_definitions.Definitions()) { - if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE) { - has_btree_index = true; - break; - } - } - std::optional latest_snapshot; - bool has_existing_index_manifest = false; - if (!has_btree_index) { - PAIMON_ASSIGN_OR_RAISE( - bool has_historical_btree_index, - HasHistoricalPrimaryKeyBTreeDefinition(GetSchemaManager(), schema->Id())); - if (has_historical_btree_index) { - PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); - has_existing_index_manifest = - latest_snapshot.has_value() && latest_snapshot->IndexManifest().has_value(); - } - } - std::shared_ptr index_file_handler; - if (options.DeletionVectorsEnabled() || has_btree_index || has_existing_index_manifest) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), file_store_path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); - index_file_handler = std::make_shared( - options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(file_store_path_factory), - options.DeletionVectorsBitmap64(), ctx->GetMemoryPool()); - } - bool has_restored_btree_index = false; - if (!has_btree_index && has_existing_index_manifest) { - PAIMON_ASSIGN_OR_RAISE( - std::vector restored_btree_entries, - index_file_handler->Scan( - latest_snapshot.value(), [](const IndexManifestEntry& entry) -> bool { - return entry.index_file->IndexType() == BtreeDefs::kIdentifier && - IndexFileHandler::IsPrimaryKeySourceIndex(*entry.index_file); - })); - has_restored_btree_index = !restored_btree_entries.empty(); - } - if (options.DeletionVectorsEnabled()) { - dv_maintainer_factory = - std::make_shared(index_file_handler); - } - std::shared_ptr - primary_key_index_maintainer_factory; - if (has_btree_index || has_restored_btree_index) { - primary_key_index_maintainer_factory = - std::make_shared( - root_path_, CurrentBranch(), schema, - primary_key_index_definitions.Definitions(), file_store_path_factory, - index_file_handler, options, io_manager, ctx->EnableMultiThreadSpill(), - ctx->GetExecutor(), ctx->GetMemoryPool()); - } - - std::shared_ptr realtime_schema_layout; - if (ctx->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr layout, - RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, arrow_schema)); - realtime_schema_layout = std::move(layout); - } - - return std::make_unique( - file_store_path_factory, snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), - root_path_, schema, arrow_schema, realtime_schema_layout, partition_schema, - dv_maintainer_factory, primary_key_index_maintainer_factory, io_manager, key_comparator, - sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, - ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->EnableMultiThreadSpill(), - ctx->GetRealtimeContext(), ctx->GetExecutor(), ctx->GetMemoryPool()); - } + return CreateWrite(*ctx, schema, arrow_schema, partition_schema, options, + file_store_path_factory, snapshot_manager, io_manager, + ignore_previous_files); } Result> FileStoreWrite::Create(std::unique_ptr ctx) { @@ -853,110 +627,6 @@ Result> FileStoreWrite::Create(std::unique_ptr> CreateFileStoreScan( - const std::shared_ptr& path_factory, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, - const std::shared_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& executor, const std::shared_ptr& memory_pool, - const ScanContext* context, const std::string& root_path) { - auto fs = core_options.GetFileSystem(); - auto manifest_file_format = core_options.GetManifestFormat(); - // TODO(liancheng.lsz): support fallback branch in scan - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_list, - ManifestList::Create(fs, manifest_file_format, core_options.GetManifestCompression(), - path_factory, core_options.GetCache(), memory_pool)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_file, - ManifestFile::Create(fs, manifest_file_format, core_options.GetManifestCompression(), - path_factory, core_options.GetManifestTargetFileSize(), - memory_pool, core_options, partition_schema)); - std::unique_ptr scan; - if (table_schema->PrimaryKeys().empty()) { - if (core_options.DataEvolutionEnabled()) { - PAIMON_ASSIGN_OR_RAISE( - scan, DataEvolutionFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, - table_schema, arrow_schema, context->GetScanFilters(), core_options, - executor, memory_pool)); - } else { - PAIMON_ASSIGN_OR_RAISE( - scan, AppendOnlyFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, - table_schema, arrow_schema, context->GetScanFilters(), core_options, - executor, memory_pool)); - } - } else { - PAIMON_ASSIGN_OR_RAISE( - scan, KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, - context->GetScanFilters(), core_options, - executor, memory_pool)); - } - return WithTablePath(std::move(scan), root_path); - } - - static std::unique_ptr WithTablePath(std::unique_ptr&& scan, - const std::string& root_path) { - scan->WithTablePath(root_path); - return std::move(scan); - } - - static Result> CreateSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options, - const ScanContext* context) { - auto source_split_target_size = core_options.GetSourceSplitTargetSize(); - auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); - if (table_schema->PrimaryKeys().empty()) { - if (core_options.DataEvolutionEnabled()) { - return std::make_unique( - source_split_target_size, source_split_open_file_cost, - core_options.BlobSplitByFileSize()); - } - BucketMode bucket_mode = (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE - : BucketMode::HASH_FIXED); - return std::make_unique( - source_split_target_size, source_split_open_file_cost, bucket_mode); - } else { - // TODO(liancheng.lsz): support evolution - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - table_schema->TrimmedPrimaryKeys()); - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, - table_schema->GetFields(trimmed_primary_keys)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr key_comparator, - FieldsComparator::Create(trimmed_pk_fields, /*is_ascending_order=*/true)); - return std::make_unique( - source_split_target_size, source_split_open_file_cost, - core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), - key_comparator); - } - } - - static Result> CreateIndexFileHandler( - const CoreOptions& core_options, const std::shared_ptr& path_factory, - const std::shared_ptr& memory_pool) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_manifest_file, - IndexManifestFile::Create( - core_options.GetFileSystem(), core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, - core_options.GetBucket(), memory_pool, core_options)); - return std::make_unique( - core_options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(path_factory), - core_options.DeletionVectorsBitmap64(), memory_pool); - } -}; - Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options, const ScanContext& context, bool read_optimized) { if (!context.GetRealtimeContext()) { @@ -1064,18 +734,26 @@ Result> FileStoreTable::NewScan( core_options.GetFileFormat()->Identifier(), context->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_list, + store_->NewManifestList(core_options, path_factory, context->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_file, + store_->NewManifestFile(core_options, path_factory, partition_schema, + context->GetMemoryPool())); + // The managers the table carries, whose schema cache the dispatch already warmed. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_store_scan, - TableScanImpl::CreateFileStoreScan( - path_factory, arrow_schema, table_schema, core_options, - GetSnapshotManager(), GetSchemaManager(), context->GetExecutor(), - context->GetMemoryPool(), context.get(), root_path_)); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr split_generator, - TableScanImpl::CreateSplitGenerator(table_schema, core_options, context.get())); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_file_handler, - TableScanImpl::CreateIndexFileHandler(core_options, path_factory, - context->GetMemoryPool())); + std::shared_ptr file_store_scan, + NewFileStoreScan(manifest_list, manifest_file, arrow_schema, table_schema, core_options, + context->GetExecutor(), context->GetMemoryPool(), context.get())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr split_generator, + NewSplitGenerator(table_schema, core_options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_file_handler, + store_->NewIndexFileHandler(core_options, path_factory, context->GetMemoryPool())); auto snapshot_reader = std::make_shared( file_store_scan, path_factory, std::move(split_generator), std::move(index_file_handler)); const bool pk_table = !table_schema->PrimaryKeys().empty(); @@ -1171,9 +849,12 @@ Result> CreateInternalReadContext( return InternalReadContext::Create(context, table_schema, options, file_system, &root_path); } -Result> CreateTableRead( +} // namespace + +Result> FileStoreTable::CreateTableRead( const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor) { + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const { if (PredicateUtils::ContainsFullTextMatch(internal_context->GetPredicate())) { return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); } @@ -1189,19 +870,11 @@ Result> CreateTableRead( auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, - FileStore::MakePathFactory(internal_context->GetPath(), arrow_schema, - table_schema->PartitionKeys(), core_options, - core_options.GetFileFormat()->Identifier(), memory_pool)); - - if (internal_context->GetPrimaryKeys().empty()) { - return std::make_unique(path_factory, internal_context, memory_pool, - executor); - } - return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); + store_->NewPathFactory(arrow_schema, core_options, + core_options.GetFileFormat()->Identifier(), memory_pool)); + return NewTableRead(path_factory, internal_context, memory_pool, executor); } -} // namespace - Result> FileStoreTable::NewRead( std::unique_ptr context) const { std::shared_ptr read_context = std::move(context); @@ -1258,46 +931,6 @@ Result> TableRead::Create(std::unique_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& manifest_list, - const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, - const std::shared_ptr& arrow_schema, const CoreOptions& options, - const std::shared_ptr& executor, const std::shared_ptr& pool) { - return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) - -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan, - AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, scan_filter, - options, executor, pool)); - return std::unique_ptr(std::move(scan)); - }; -} - -CommitScanner::ScanSupplier CreatePkScanSupplier( - const std::shared_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& manifest_list, - const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, - const std::shared_ptr& arrow_schema, const CoreOptions& options, - const std::shared_ptr& executor, const std::shared_ptr& pool) { - return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) - -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan, - KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, scan_filter, - options, executor, pool)); - return std::unique_ptr(std::move(scan)); - }; -} - bool IsCatalogCommit(const CommitContext& ctx) { return AsVersionManaged(ctx.GetCatalog()) != nullptr; } @@ -1394,11 +1027,9 @@ Result> FileStoreTable::NewCommit( return Status::NotImplemented( "commit operation does not support object store file system for now"); } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr partition_computer, - BinaryRowPartitionComputer::Create(schema->PartitionKeys(), arrow_schema, - options.GetPartitionDefaultName(), - options.LegacyPartitionNameEnabled(), - ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr partition_computer, + store_->NewPartitionComputer(arrow_schema, options, ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, store_->NewPathFactory(arrow_schema, options, options.GetFileFormat()->Identifier(), @@ -1407,41 +1038,26 @@ Result> FileStoreTable::NewCommit( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr snapshot_manager, store_->NewSnapshotManager(options, ctx->GetCatalog(), catalog_identifier)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_list, - ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, options.GetCache(), - ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, + store_->NewManifestList(options, path_factory, ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_file, - ManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, - options.GetManifestTargetFileSize(), ctx->GetMemoryPool(), options, - partition_schema)); + store_->NewManifestFile(options, path_factory, partition_schema, ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); + store_->NewIndexManifestFile(options, path_factory, ctx->GetMemoryPool())); auto expire_snapshots = std::make_shared( snapshot_manager, path_factory, manifest_list, manifest_file, index_manifest_file, options.GetFileSystem(), options.GetExpireConfig(), options.RealtimeEnabled(), ctx->GetExecutor()); - CommitScanner::ScanSupplier scan_supplier; - if (schema->PrimaryKeys().empty()) { - scan_supplier = CreateAppendScanSupplier(snapshot_manager, GetSchemaManager(), - manifest_list, manifest_file, schema, arrow_schema, - options, ctx->GetExecutor(), ctx->GetMemoryPool()); - } else { - scan_supplier = CreatePkScanSupplier(snapshot_manager, GetSchemaManager(), manifest_list, - manifest_file, schema, arrow_schema, options, - ctx->GetExecutor(), ctx->GetMemoryPool()); - } + CommitScanner::ScanSupplier scan_supplier = + NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, schema, arrow_schema, + options, ctx->GetExecutor(), ctx->GetMemoryPool()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr snapshot_commit, diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 51c7c0b95..00959d257 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -25,6 +25,7 @@ #include "paimon/catalog/identifier.h" #include "paimon/catalog/table.h" +#include "paimon/core/operation/commit/commit_scanner.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/table/file_store.h" #include "paimon/core/utils/branch_manager.h" @@ -34,26 +35,42 @@ namespace paimon { +class InternalReadContext; +class IOManager; +class ManifestFile; +class ManifestList; +class RealtimeContext; class Schema; +class SplitGenerator; /// The managed table a catalog hands out: one with snapshots and manifests, which the schema under /// its root path describes. A format table - a directory of plain data files - is a `FormatTable`, /// the other kind of `Table`. /// +/// This is the flow the four operators share, Java Paimon's `AbstractFileStoreTable`: what the +/// table was constructed with - the schema, file system, root path, branch and managers - decides +/// what every operator serves, so it is read here once and handed down. The schema's primary keys +/// decide the kind of managed table it is - `AppendOnlyFileStoreTable` or +/// `PrimaryKeyFileStoreTable`, the `Create` factory below - and that kind builds the pieces the +/// flows hand down: the write itself, the scans, the read and the split generator. +/// /// Its `NewScan`/`NewRead`/`NewWrite`/`NewCommit` build the operators here, from the context plus -/// the schema, file system, root path, branch and managers the table was constructed with, so -/// they serve the table as it was loaded. The +/// the state the table was constructed with, so they serve the table as it was loaded. The /// context-only entry points - `TableScan::Create()`, `TableRead::Create()`, /// `FileStoreWrite::Create()` and `FileStoreCommit::Create()` - load the table the context names /// and hand the context to that table's `NewXxx`, which is how a context built without a table /// reaches the same operators. class FileStoreTable : public Table { public: - /// The constructor a catalog builds a managed table through, so a `shared_ptr` can hold one; - /// the base constructor is protected, which an inherited constructor would stay. - /// @param file_system File system the table's data is accessed through. - /// @param root_path Root path of the table in the file system. - /// @param uuid Metastore UUID, or empty if unavailable. + /// The managed table the schema describes, built as the kind its primary keys name: an + /// `AppendOnlyFileStoreTable` when there are none, a `PrimaryKeyFileStoreTable` when there + /// are, the way Java Paimon's `FileStoreTableFactory` does. Primary keys cannot be added or + /// dropped by a schema change, so the kind the table was built as is the kind every schema + /// its root path ever held describes. + /// @param schema The schema the table was loaded with, whose primary keys name the kind and + /// whose options the operators serve. It is null when the caller had no schema, and + /// the table is built as the append-only kind - its operators then report that no + /// schema was found. /// @param branch The branch the table serves, which the managers below read and write on. /// @param schema_manager A schema manager to carry instead of building one - the one a /// context dispatch read the schema through, whose cache the operators then serve. @@ -61,12 +78,13 @@ class FileStoreTable : public Table { /// `Copy` hands its table's over so both share one. /// A null manager is built over `root_path` on `branch`, so the operators always read /// through the managers the table carries. - FileStoreTable(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = "", - const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + static Result> Create( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid = "", + const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); /// The branch this table serves, as resolved when the table was built: the schema the table /// was loaded with came from it, and its managers point at it. @@ -120,6 +138,86 @@ class FileStoreTable : public Table { std::unique_ptr context) const override; private: + /// The read `NewRead` hands back, from the internal context it read the schema into: the + /// path factory comes from the store - a fallback branch's schema carries the table's + /// partition keys, which no schema change can alter - and the read itself is the kind's, + /// `NewTableRead`. + Result> CreateTableRead( + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const; + + protected: + /// The base constructor the kinds of managed table build through, `Create` above. The + /// parameters are the ones it takes; see there for what each one is. + FileStoreTable(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid, const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager); + + /// The committed progress a real-time write restores, from the latest snapshot the snapshot + /// manager reads: where the write picks up after, which both kinds of real-time write ask + /// for the same way. + static Status RestoreRealtimeCommittedProgress( + const std::shared_ptr& realtime_context, + const std::shared_ptr& snapshot_manager, const CoreOptions& options); + + /// The scan the table's root path names, which the kinds of file store scan read from: the + /// kinds hand their scans back through it. + static std::unique_ptr WithTablePath(std::unique_ptr&& scan, + const std::string& root_path); + + /// The write of this kind of table, which `NewWrite` hands the shared flow's pieces to: the + /// kind's writer, built over the path factory, snapshot manager and options that flow + /// resolved. The branch the write serves is the table's, `CurrentBranch()`. + /// @param io_manager The spill manager the context's temp directory names, or null when the + /// context named none. + /// @param ignore_previous_files Whether the write starts over rather than appending, which + /// the flow resolved with this kind's own rules. + virtual Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& schema, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const = 0; + + /// The scan of this kind of table, which `NewScan` hands the manifest list and file it built + /// to. The snapshot and schema managers are the table's own, which the scan serves. + virtual Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const = 0; + + /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. + virtual Result> NewSplitGenerator( + const std::shared_ptr& table_schema, + const CoreOptions& core_options) const = 0; + + /// The read of this kind of table, which `NewRead` hands the internal context it read the + /// schema into and the path factory it built to. + virtual Result> NewTableRead( + const std::shared_ptr& path_factory, + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const = 0; + + /// The manifest entry scan supplier of this kind of table, which `NewCommit` hands the + /// manifest list and file it built to. The schema manager the scans it hands out read through + /// is the table's own. + virtual CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const = 0; + /// The storage this table serves: the file system, root path, schema, branch and managers it /// was loaded with, and the factories that build the pieces every operator opens with. std::shared_ptr store_; diff --git a/src/paimon/core/table/primary_key_file_store_table.cpp b/src/paimon/core/table/primary_key_file_store_table.cpp new file mode 100644 index 000000000..564e015e3 --- /dev/null +++ b/src/paimon/core/table/primary_key_file_store_table.cpp @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/primary_key_file_store_table.h" + +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/global_index/btree/btree_defs.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/mergetree/compact/lookup_merge_function.h" +#include "paimon/core/mergetree/compact/merge_function.h" +#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_write.h" +#include "paimon/core/options/merge_engine.h" +#include "paimon/core/postpone/postpone_bucket_file_store_write.h" +#include "paimon/core/realtime/realtime_schema_layout.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/core/table/source/merge_tree_split_generator.h" +#include "paimon/core/table/source/split_generator.h" +#include "paimon/core/utils/primary_key_table_utils.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/write_context.h" + +namespace paimon { + +struct KeyValue; +template +class MergeFunctionWrapper; + +namespace { + +/// Whether a schema older than `current_schema_id` carried a primary key b-tree index, which a +/// write has to keep maintaining: the index files an earlier schema wrote are still there to be +/// read, and dropping the definition now would leave them orphaned. +Result HasHistoricalPrimaryKeyBTreeDefinition( + const std::shared_ptr& schema_manager, int64_t current_schema_id) { + if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { + return false; + } + PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager->ListAllIds()); + for (int64_t schema_id : schema_ids) { + if (schema_id >= current_schema_id) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical_schema, + schema_manager->ReadSchema(schema_id)); + const auto& historical_options = historical_schema->Options(); + auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS); + if (option != historical_options.end() && + !StringUtils::IsNullOrWhitespaceOnly(option->second)) { + return true; + } + } + return false; +} + +} // namespace + +PrimaryKeyFileStoreTable::PrimaryKeyFileStoreTable( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, + const std::string& branch, const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) + : FileStoreTable(file_system, root_path, schema, identifier, uuid, branch, schema_manager, + snapshot_manager) {} + +Result> PrimaryKeyFileStoreTable::CreateWrite( + const WriteContext& ctx, const std::shared_ptr& schema, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const { + // pk table + if (ctx.GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); + if (ignore_previous_files) { + return Status::NotImplemented("PK realtime requires restore from the latest snapshot"); + } + if (!ctx.GetWriteSchema().empty()) { + return Status::NotImplemented("PK realtime does not support a custom write schema"); + } + PAIMON_RETURN_NOT_OK( + RestoreRealtimeCommittedProgress(ctx.GetRealtimeContext(), snapshot_manager, options)); + } + if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { + return PostponeBucketFileStoreWrite::Create( + snapshot_manager, GetSchemaManager(), ctx.GetCommitUser(), root_path_, schema, + arrow_schema, partition_schema, io_manager, options, ctx.IsStreamingMode(), + ctx.IgnoreNumBucketCheck(), ctx.GetWriteId(), ctx.GetFileSystemSchemeToIdentifierMap(), + ctx.GetExecutor(), ctx.GetMemoryPool(), file_system_); + } + if (options.GetBucket() <= 0) { + return Status::Invalid( + fmt::format("not support bucket {} in key value table", options.GetBucket())); + } + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + schema->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, + schema->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key_comparator, + FieldsComparator::Create(trimmed_primary_key_fields, + options.SequenceFieldSortOrderIsAscending())); + auto primary_keys = schema->PrimaryKeys(); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, + PrimaryKeyTableUtils::CreateMergeFunction(arrow_schema, primary_keys, + options, ctx.GetMemoryPool())); + if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { + // don't wrap first row, it is already OK + merge_function = std::make_unique(std::move(merge_function)); + } + std::shared_ptr> merge_function_wrapper = + std::make_shared(std::move(merge_function)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr sequence_fields_comparator, + PrimaryKeyTableUtils::CreateSequenceFieldsComparator(schema->Fields(), options)); + + std::shared_ptr dv_maintainer_factory; + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions primary_key_index_definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + bool has_btree_index = false; + for (const PrimaryKeyIndexDefinition& definition : + primary_key_index_definitions.Definitions()) { + if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE) { + has_btree_index = true; + break; + } + } + std::optional latest_snapshot; + bool has_existing_index_manifest = false; + if (!has_btree_index) { + PAIMON_ASSIGN_OR_RAISE( + bool has_historical_btree_index, + HasHistoricalPrimaryKeyBTreeDefinition(GetSchemaManager(), schema->Id())); + if (has_historical_btree_index) { + PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); + has_existing_index_manifest = + latest_snapshot.has_value() && latest_snapshot->IndexManifest().has_value(); + } + } + std::shared_ptr index_file_handler; + if (options.DeletionVectorsEnabled() || has_btree_index || has_existing_index_manifest) { + PAIMON_ASSIGN_OR_RAISE(index_file_handler, store_->NewIndexFileHandler( + options, path_factory, ctx.GetMemoryPool())); + } + bool has_restored_btree_index = false; + if (!has_btree_index && has_existing_index_manifest) { + PAIMON_ASSIGN_OR_RAISE( + std::vector restored_btree_entries, + index_file_handler->Scan( + latest_snapshot.value(), [](const IndexManifestEntry& entry) -> bool { + return entry.index_file->IndexType() == BtreeDefs::kIdentifier && + IndexFileHandler::IsPrimaryKeySourceIndex(*entry.index_file); + })); + has_restored_btree_index = !restored_btree_entries.empty(); + } + if (options.DeletionVectorsEnabled()) { + dv_maintainer_factory = std::make_shared(index_file_handler); + } + std::shared_ptr + primary_key_index_maintainer_factory; + if (has_btree_index || has_restored_btree_index) { + primary_key_index_maintainer_factory = + std::make_shared( + root_path_, CurrentBranch(), schema, primary_key_index_definitions.Definitions(), + path_factory, index_file_handler, options, io_manager, ctx.EnableMultiThreadSpill(), + ctx.GetExecutor(), ctx.GetMemoryPool()); + } + + std::shared_ptr realtime_schema_layout; + if (ctx.GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr layout, + RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, arrow_schema)); + realtime_schema_layout = std::move(layout); + } + + return std::make_unique( + path_factory, snapshot_manager, GetSchemaManager(), ctx.GetCommitUser(), root_path_, schema, + arrow_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, + primary_key_index_maintainer_factory, io_manager, key_comparator, + sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, + ctx.IsStreamingMode(), ctx.IgnoreNumBucketCheck(), ctx.EnableMultiThreadSpill(), + ctx.GetRealtimeContext(), ctx.GetExecutor(), ctx.GetMemoryPool()); +} + +Result> PrimaryKeyFileStoreTable::NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const { + // TODO(liancheng.lsz): support fallback branch in scan + std::unique_ptr scan; + PAIMON_ASSIGN_OR_RAISE( + scan, KeyValueFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), manifest_list, + manifest_file, table_schema, arrow_schema, + context->GetScanFilters(), core_options, executor, + memory_pool)); + return WithTablePath(std::move(scan), root_path_); +} + +Result> PrimaryKeyFileStoreTable::NewSplitGenerator( + const std::shared_ptr& table_schema, const CoreOptions& core_options) const { + auto source_split_target_size = core_options.GetSourceSplitTargetSize(); + auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); + // TODO(liancheng.lsz): support evolution + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + table_schema->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, + table_schema->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr key_comparator, + FieldsComparator::Create(trimmed_pk_fields, /*is_ascending_order=*/true)); + return std::make_unique( + source_split_target_size, source_split_open_file_cost, + core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), key_comparator); +} + +Result> PrimaryKeyFileStoreTable::NewTableRead( + const std::shared_ptr& path_factory, + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const { + return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); +} + +CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) const { + // The schema manager the table carries, which the scans this supplier hands out read the + // schema through. + const std::shared_ptr& schema_manager = GetSchemaManager(); + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +} // namespace paimon diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h new file mode 100644 index 000000000..9c658758a --- /dev/null +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class CoreOptions; +class FileStoreScan; +class IOManager; +class SplitGenerator; +class TableSchema; + +/// The managed table a schema with primary keys describes, Java Paimon's +/// `PrimaryKeyFileStoreTable`. Its write merges by key, and its scan, read and commit read +/// through the merge. The operator flows are the ones `FileStoreTable` carries; the pieces +/// those flows hand down are built here. +class PrimaryKeyFileStoreTable : public FileStoreTable { + public: + /// The constructor `FileStoreTable::Create` builds a primary key table through; the + /// parameters are the ones it takes, and the base class carries them. + PrimaryKeyFileStoreTable(const std::shared_ptr& file_system, + const std::string& root_path, const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid = "", + const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); + + private: + Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& schema, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const override; + + Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const override; + + Result> NewSplitGenerator( + const std::shared_ptr& table_schema, + const CoreOptions& core_options) const override; + + Result> NewTableRead( + const std::shared_ptr& path_factory, + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const override; + + CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 01a7ce27e..fe99efe63 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -66,7 +66,7 @@ Result> CreateManagedTable( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr created, schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options)); - return std::make_shared( + return FileStoreTable::Create( file_system, path, checked_pointer_cast(std::shared_ptr(std::move(created))), identifier, /*uuid=*/"", branch); @@ -346,11 +346,13 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { {Identifier("db", "tbl"), Identifier("db", "tbl$branch_main"), Identifier("db", "tbl$branch_MAIN")}) { SCOPED_TRACE(main_identifier.ToString()); - // A catalog that loaded the schema itself calls the constructor directly, which a + // A catalog that loaded the schema itself builds the table through the factory, which a // rest catalog is shaped like. The alias resolves to the main branch as well. ASSERT_OK_AND_ASSIGN(std::string branch, main_identifier.GetBranchNameOrDefault()); - FileStoreTable table(fs, dir->Str(), schema, main_identifier, /*uuid=*/"", branch); - EXPECT_EQ("main", table.CurrentBranch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + FileStoreTable::Create(fs, dir->Str(), schema, main_identifier, + /*uuid=*/"", branch)); + EXPECT_EQ("main", table->CurrentBranch()); } } diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 32ebefc1e..b7dcfe04f 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -471,7 +471,7 @@ Result> RestCatalog::GetTable(const Identifier& identifie // The branch the identifier carries, which the schema and path above were read on, so the // table and its managers serve it. PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); - return std::make_shared( + return FileStoreTable::Create( table_file_system, table_path, schema, identifier, uuid, NormalizeBranch(std::move(branch)).value_or(Identifier::kDefaultMainBranch)); } diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index c4a200239..57a6e8d98 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -99,7 +99,7 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { // Serves the per-table file system when one was set, matching `GetTableFileSystem` // without recording the request, so a test can still tell the two apart. No location: // this mock's `GetTableLocation` is the unsupported stub. - return std::make_shared( + return FileStoreTable::Create( table_file_system_ != nullptr ? table_file_system_ : file_system_, /*root_path=*/"", table_schema_, identifier, table_uuid_); } From 09cb5875a46e9a7d6b2e72a46ab131c5a0a1e78d Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 13:39:56 +0800 Subject: [PATCH 06/19] tmp --- include/paimon/commit_context.h | 11 +- include/paimon/read_context.h | 11 +- include/paimon/scan_context.h | 12 +- include/paimon/write_context.h | 11 +- .../core/operation/internal_read_context.cpp | 24 +- .../core/operation/internal_read_context.h | 22 ++ .../table/append_only_file_store_table.cpp | 23 +- .../core/table/append_only_file_store_table.h | 22 +- src/paimon/core/table/file_store.cpp | 63 +++- src/paimon/core/table/file_store.h | 84 ++++- src/paimon/core/table/file_store_table.cpp | 312 +++++++++--------- src/paimon/core/table/file_store_table.h | 33 +- .../table/primary_key_file_store_table.cpp | 24 +- .../core/table/primary_key_file_store_table.h | 22 +- .../core/table/source/table_read_test.cpp | 17 +- .../table/system/audit_log_system_table.cpp | 14 +- .../system/read_optimized_system_table.cpp | 14 +- src/paimon/core/table/table_test.cpp | 145 +++++++- src/paimon/rest/rest_catalog.cpp | 18 + test/inte/scan_and_read_inte_test.cpp | 21 +- 20 files changed, 612 insertions(+), 291 deletions(-) diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index 1ec784f76..a70cacb7a 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -156,15 +156,18 @@ class PAIMON_EXPORT CommitContextBuilder { ~CommitContextBuilder(); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. + /// Set a configuration options map. The options take part in loading the table a commit + /// serves: the load merges them over the ones the table's schema carries and the table it + /// hands back serves the merged ones. A caller holding a table already loaded - a load + /// that is over - changes it through `Table::Copy()`; an option contradicting a table held + /// that way is refused when a commit is asked of it. A format table takes the options as + /// its own. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. CommitContextBuilder& SetOptions(const std::map& options); - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. + /// Add a single configuration option, served the way `SetOptions()` describes. /// /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` /// instead. diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index b9c534065..8cf565996 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -324,15 +324,18 @@ class PAIMON_EXPORT ReadContextBuilder { /// When set, read_field_ids and read_field_names are ignored. ReadContextBuilder& SetReadSchema(std::unique_ptr read_schema); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. + /// Set a configuration options map. The options take part in loading the table a read + /// serves: the load merges them over the ones the table's schema carries and the table it + /// hands back serves the merged ones. A caller holding a table already loaded - a load + /// that is over - changes it through `Table::Copy()`; an option contradicting a table held + /// that way is refused when a read is asked of it. A format table takes the options as its + /// own. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. ReadContextBuilder& SetOptions(const std::map& options); - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. + /// Add a single configuration option, served the way `SetOptions()` describes. /// /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` /// instead. diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 5be7c9c4e..fe57afb58 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -208,11 +208,15 @@ class PAIMON_EXPORT ScanContextBuilder { ScanContextBuilder& WithRealtimeContext( const std::shared_ptr& realtime_context); - /// The options added or set in `ScanContextBuilder` have high priority and will be merged with - /// the options in table schema. + /// Add a single configuration option. See `SetOptions()` for how the options a context + /// carries are served. ScanContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. + /// Set a configuration options map. The options take part in loading the table a scan + /// serves: the load merges them over the ones the table's schema carries and the table it + /// hands back serves the merged ones. A caller holding a table already loaded - a load + /// that is over - changes it through `Table::Copy()`; an option contradicting a table held + /// that way is refused when a scan is asked of it. A format table takes the options as its + /// own. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index 58c0b3f97..23195793f 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -183,15 +183,18 @@ class PAIMON_EXPORT WriteContextBuilder { ~WriteContextBuilder(); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. + /// Set a configuration options map. The options take part in loading the table a write + /// serves: the load merges them over the ones the table's schema carries and the table it + /// hands back serves the merged ones. A caller holding a table already loaded - a load + /// that is over - changes it through `Table::Copy()`; an option contradicting a table held + /// that way is refused when a write is asked of it. A format table takes the options as its + /// own. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. WriteContextBuilder& SetOptions(const std::map& options); - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. + /// Add a single configuration option, served the way `SetOptions()` describes. /// /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` /// instead. diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 6c0c43341..778976a46 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -273,18 +273,32 @@ std::optional InternalReadContext::TryResolveSpecialFieldByName( return std::nullopt; } +Result> InternalReadContext::Create( + const std::shared_ptr& context, const std::shared_ptr& table_schema, + CoreOptions core_options, const std::string* path) { + core_options.WithCache(context->GetCache()); + return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); +} + Result> InternalReadContext::Create( const std::shared_ptr& context, const std::shared_ptr& table_schema, const std::map& options, const std::shared_ptr& file_system, const std::string* path) { - // The file system a table reads through, or null for the read context's own, which callers - // without a table hand over. + // The file system the caller named, or null for the read context's own, which callers + // without a table hand over. The scheme map the context carries does not take part: a file + // system handed over wins and one missing falls to the resolving one, the same way a scan + // resolves its own. PAIMON_ASSIGN_OR_RAISE( CoreOptions core_options, - CoreOptions::FromMap( - options, file_system != nullptr ? file_system : context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); + CoreOptions::FromMap(options, file_system != nullptr ? file_system + : context->GetSpecificFileSystem())); core_options.WithCache(context->GetCache()); + return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); +} + +Result> InternalReadContext::CreateFromCoreOptions( + const std::shared_ptr& context, const std::shared_ptr& table_schema, + CoreOptions core_options, const std::string* path) { if (context->HasVectorOrFullTextSearch()) { if (!core_options.FileIndexReadEnabled()) { return Status::Invalid( diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index b1619c5bc..b7218a686 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -58,6 +58,21 @@ class InternalReadContext { std::string field_name; }; + /// Builds the context of a read a table serves: the options are the table's own, already + /// parsed - the store's `options()` - and the cache the context carries is armed on them + /// here. `path` names the root path the table was opened with, which the read reads from in + /// place of the one named by the context. + static Result> Create( + const std::shared_ptr& read_context, + const std::shared_ptr& table_schema, CoreOptions core_options, + const std::string* path = nullptr); + + /// Builds the context of a read whose options are composed on the spot rather than the + /// table's own - the fallback branch of a read, whose schema belongs to another branch, and + /// the reads a write or a lookup serve inside themselves. The options are parsed from + /// `options` here: the read's own file system is `file_system`, or the context's when null - + /// the scheme map the context carries does not take part. A cache the context carries is + /// armed on the options this parses. static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, @@ -171,6 +186,13 @@ class InternalReadContext { const std::shared_ptr& arrow_pool, std::optional path = std::nullopt); + /// The tail both `Create` entries share: the options arrive parsed and cached, and the + /// validations and the read schema below are the same whichever way they were resolved. + static Result> CreateFromCoreOptions( + const std::shared_ptr& read_context, + const std::shared_ptr& table_schema, CoreOptions core_options, + const std::string* path); + static std::optional TryResolveSpecialFieldById(int32_t field_id, const CoreOptions& core_options, bool has_vector_or_full_text_search); diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp index 0846aef90..cead79566 100644 --- a/src/paimon/core/table/append_only_file_store_table.cpp +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -48,22 +48,19 @@ namespace paimon { -AppendOnlyFileStoreTable::AppendOnlyFileStoreTable( - const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, - const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) - : FileStoreTable(file_system, root_path, schema, identifier, uuid, branch, schema_manager, - snapshot_manager) {} +AppendOnlyFileStoreTable::AppendOnlyFileStoreTable(const std::shared_ptr& store, + const Identifier& identifier, + const std::string& uuid) + : FileStoreTable(store, identifier, uuid) {} Result> AppendOnlyFileStoreTable::CreateWrite( - const WriteContext& ctx, const std::shared_ptr& schema, - const std::shared_ptr& arrow_schema, + const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { // append table + const std::shared_ptr& schema = store_->schema(); const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); if (options.GetBucket() == -1) { @@ -138,11 +135,11 @@ Result> AppendOnlyFileStoreTable::CreateWrite( Result> AppendOnlyFileStoreTable::NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const { // TODO(liancheng.lsz): support fallback branch in scan + const std::shared_ptr& table_schema = store_->schema(); std::unique_ptr scan; if (core_options.DataEvolutionEnabled()) { PAIMON_ASSIGN_OR_RAISE( @@ -161,7 +158,7 @@ Result> AppendOnlyFileStoreTable::NewFileStoreSca } Result> AppendOnlyFileStoreTable::NewSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options) const { + const CoreOptions& core_options) const { auto source_split_target_size = core_options.GetSourceSplitTargetSize(); auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); if (core_options.DataEvolutionEnabled()) { @@ -188,12 +185,12 @@ CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const { // The schema manager the table carries, which the scans this supplier hands out read the // schema through. const std::shared_ptr& schema_manager = GetSchemaManager(); + const std::shared_ptr& table_schema = store_->schema(); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) -> Result> { diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h index ea5374ba3..499431eb0 100644 --- a/src/paimon/core/table/append_only_file_store_table.h +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -24,13 +24,11 @@ #include "paimon/catalog/identifier.h" #include "paimon/core/table/file_store_table.h" -#include "paimon/core/utils/branch_manager.h" #include "paimon/result.h" #include "paimon/type_fwd.h" namespace paimon { -class CoreOptions; class FileStoreScan; class IOManager; class SplitGenerator; @@ -42,19 +40,14 @@ class TableSchema; /// carries; the pieces those flows hand down are built here. class AppendOnlyFileStoreTable : public FileStoreTable { public: - /// The constructor `FileStoreTable::Create` builds an append-only table through; the - /// parameters are the ones it takes, and the base class carries them. - AppendOnlyFileStoreTable(const std::shared_ptr& file_system, - const std::string& root_path, const std::shared_ptr& schema, - const Identifier& identifier, const std::string& uuid = "", - const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + /// The constructor `FileStoreTable::Create` builds an append-only table through: the store + /// carries every state, and the base class holds it. + AppendOnlyFileStoreTable(const std::shared_ptr& store, const Identifier& identifier, + const std::string& uuid = ""); private: Result> CreateWrite( - const WriteContext& ctx, const std::shared_ptr& schema, - const std::shared_ptr& arrow_schema, + const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, @@ -63,13 +56,11 @@ class AppendOnlyFileStoreTable : public FileStoreTable { Result> NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const override; Result> NewSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options) const override; Result> NewTableRead( @@ -82,7 +73,6 @@ class AppendOnlyFileStoreTable : public FileStoreTable { const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const override; diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index 80b6afb05..2c5a19d96 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -19,9 +19,11 @@ #include "paimon/core/table/file_store.h" +#include #include #include #include +#include #include #include "paimon/catalog/catalog.h" @@ -39,30 +41,71 @@ #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/index_file_path_factories.h" #include "paimon/core/utils/snapshot_manager.h" +#include "paimon/defs.h" #include "paimon/result.h" #include "paimon/status.h" namespace paimon { +Result> FileStore::Create( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) { + auto store = std::shared_ptr( + new FileStore(file_system, root_path, schema, branch, schema_manager, snapshot_manager)); + if (schema != nullptr) { + // The options the schema was loaded with, on the branch this store serves: what reads + // through them - whose cache keys are scoped by the branch - reads the same one the + // managers point at. + std::map options = schema->Options(); + if (BranchManager::IsMainBranch(store->branch_)) { + options.erase(Options::BRANCH); + } else { + options[Options::BRANCH] = store->branch_; + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(options, file_system, {})); + store->core_options_ = std::move(core_options); + } + // The file system the table is served through: the one the options resolve - a schema + // carrying access configuration names its own, which the file system the caller opened the + // table through wins over - and without a schema, the one handed over, whose absence leaves + // the default local one. + if (store->core_options_.has_value()) { + store->file_system_ = store->core_options_->GetFileSystem(); + } else if (file_system != nullptr) { + store->file_system_ = file_system; + } else { + PAIMON_ASSIGN_OR_RAISE(CoreOptions default_options, CoreOptions::FromMap({})); + store->file_system_ = default_options.GetFileSystem(); + } + // The managers read through the same file system: the metadata belongs to the data, and + // both were written under the options the schema carries. A manager handed over already + // points at `root_path` on `branch`; a missing one is built here, so the operators always + // read through the managers the store carries. + if (store->schema_manager_ == nullptr) { + store->schema_manager_ = + std::make_shared(store->file_system_, root_path, store->branch_); + } + if (store->snapshot_manager_ == nullptr) { + store->snapshot_manager_ = + std::make_shared(store->file_system_, root_path, store->branch_); + } + return store; +} + FileStore::FileStore(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager, const std::shared_ptr& snapshot_manager) : file_system_(file_system), + specified_file_system_(file_system), root_path_(root_path), schema_(schema), branch_(BranchManager::NormalizeBranch(branch)), schema_manager_(schema_manager), - snapshot_manager_(snapshot_manager) { - // A manager handed over points at `root_path` on `branch` already; a missing one is built - // here, so the operators always read through the managers the store carries. - if (schema_manager_ == nullptr) { - schema_manager_ = std::make_shared(file_system_, root_path_, branch_); - } - if (snapshot_manager_ == nullptr) { - snapshot_manager_ = std::make_shared(file_system_, root_path_, branch_); - } -} + snapshot_manager_(snapshot_manager) {} Result> FileStore::NewPathFactory( const std::shared_ptr& arrow_schema, const CoreOptions& options, diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index 83c34b1e7..6c2396bd6 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -24,6 +24,7 @@ #include #include "paimon/catalog/identifier.h" +#include "paimon/core/core_options.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" @@ -38,38 +39,53 @@ namespace paimon { class BinaryRowPartitionComputer; class Catalog; -class CoreOptions; class IndexFileHandler; class IndexManifestFile; class ManifestFile; class ManifestList; class TableSchema; -/// The storage a managed table is served through: the file system, root path, schema and branch -/// it was loaded with, and the schema and snapshot managers that read its root path on that -/// branch - the state Java Paimon's `AbstractFileStore` carries. `FileStoreTable` hands the -/// state it was loaded with to one of these and asks it back for the pieces every operator +/// The storage a managed table is served through: the file system, root path, schema, branch, +/// options and the schema and snapshot managers it was loaded with - the state Java Paimon's +/// `AbstractFileStore` carries, parsed once here rather than per operator. `FileStoreTable` hands +/// the state it was loaded with to one of these and asks it back for the pieces every operator /// opens with. /// +/// The options are the ones the table was loaded with on this branch - the schema's own, or +/// the ones a load merged over them - the base every operator serves: `NewXxx` takes no +/// options of its own beyond them. A cache is not part of them - `WithCache()` arms a copy an +/// operator makes - so the factories below take the options of the call, not the store's own. +/// /// The factory methods build a fresh instance on every call, never a cached one: a path factory /// names each file it builds, and a snapshot manager a write or a commit arms with a catalog /// loader is not one a scan should be served through afterwards. class FileStore { public: - /// @param file_system File system the table's data is accessed through. + /// @param file_system The file system the caller opened the table through, which wins over + /// the access configuration the schema's options may carry when it is not null. A + /// null one hands the resolution to that configuration: the table is then served by + /// the file system its options name - the way a load that only had a path and a + /// schema to read names it - and by the default local one when they name none. /// @param root_path Root path of the table in the file system. /// @param schema The schema the table was loaded with, whose partition keys name the - /// partitions the paths below are built for. + /// partitions the paths below are built for and whose options `options()` parses. It + /// is null when the caller had no schema; the operators then report that none was + /// found. /// @param branch The branch the store serves, normalized here. /// @param schema_manager A schema manager to carry instead of building one - the one a /// context dispatch read the schema through, whose cache the operators then serve. /// @param snapshot_manager A snapshot manager to carry instead of building one. /// A null manager is built over `root_path` on `branch`, so the operators always read /// through the managers the store carries. - FileStore(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const std::string& branch, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + /// + /// Building the store parses the schema's options, so it fails when they do not parse: the + /// error is the caller's to see, which is why a store is built through this factory rather + /// than the constructor. + static Result> Create( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const std::string& branch, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); /// The branch this store serves, as resolved when it was built. const std::string& branch() const { @@ -88,6 +104,40 @@ class FileStore { return snapshot_manager_; } + /// The schema the table was loaded with, which every operator serves rather than reading the + /// schema again. It is null when the caller had no schema. + const std::shared_ptr& schema() const { + return schema_; + } + + /// The file system the table's data is accessed through, as the table was opened with it: + /// the one the options resolve - a schema carrying access configuration names its own, + /// which the file system the caller opened the table through wins over - rather than the + /// one that call handed over, which `specified_file_system()` hands back untouched. + const std::shared_ptr& file_system() const { + return file_system_; + } + + /// The file system the caller opened the table through, as it was handed to `Create`: null + /// when the call had none, so a copy of the table resolves from its own options again and + /// the runtime ones it merges in - a file-system option among them - can speak. + const std::shared_ptr& specified_file_system() const { + return specified_file_system_; + } + + /// The root path of the table in its file system, as the table was opened with it. + const std::string& root_path() const { + return root_path_; + } + + /// The options the schema was loaded with on `branch()`, which every operator serves. They + /// exist only over a schema: an operator reads them after reporting that no schema was + /// found when there is none. + const CoreOptions& options() const { + assert(core_options_.has_value()); + return *core_options_; + } + /// The path factory an operator builds its metadata paths through: over this store's root /// path and the partition keys of its schema - which no schema change can alter, so a read /// serving a fallback branch's schema still names the same partitions. The file format @@ -146,12 +196,24 @@ class FileStore { const std::shared_ptr& memory_pool) const; private: + FileStore(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager); + std::shared_ptr file_system_; + /// The file system the caller opened the table through, as handed to `Create` - null when + /// the call had none - kept for a copy to hand over again: what `file_system_` holds is + /// resolved, and would pin the copy to this table's resolution. + std::shared_ptr specified_file_system_; std::string root_path_; std::shared_ptr schema_; std::string branch_; std::shared_ptr schema_manager_; std::shared_ptr snapshot_manager_; + /// The options of `schema_` on `branch_`, or unset when there is no schema. `Create` parses + /// them: the constructor cannot report a parse failure. + std::optional core_options_; }; } // namespace paimon diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 502556990..2e5f77af6 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -141,13 +141,45 @@ Result> TryLoadFormatTable( /*location_carries_paimon_metadata=*/true, options); } +/// The options a context carries, merged into the table the load hands back: a table is the +/// product of the load, so the options speak there - the way `Copy` merges them, over the ones +/// the schema carries - and `NewXxx` on the product serves them without merging again. A +/// caller holding a table already - a load that is over - changes it through `Copy` itself, +/// and an option that contradicts the table that way is refused. +Result> WithContextOptions( + const std::shared_ptr
& table, const std::shared_ptr& schema, + const std::map& options) { + if (table == nullptr || schema == nullptr || options.empty()) { + return table; + } + const std::map& schema_options = schema->Options(); + for (const auto& [key, value] : options) { + auto schema_option = schema_options.find(key); + // A key the schema does not carry, or carries differently: the merge has something to + // do. Every key agreeing with the schema leaves the table the load built as it is. + if (schema_option == schema_options.end() || schema_option->second != value) { + return table->Copy(options); + } + } + return table; +} + /// The tail every `TableOf` ends with: the schema one of them read says what the path names. A /// format table is handed back as the `FormatTable` it is, everything else as the /// `FileStoreTable` a managed table is, carrying the schema for its `NewXxx` to build from. A /// null schema is handed over as it is: what to say about a table that is not there is the /// managed path's to decide, and it says it in more detail. +/// +/// @param file_system The file system the schema above was read through, which a format table +/// is served by - it has no schema options of its own to name another one. +/// @param explicit_file_system The file system the context supplied, which a managed table is +/// opened through when there is one. A null one leaves the table to resolve from the +/// options its schema carries: access configuration the schema was stored with then +/// names the files it serves, rather than being pinned to the one this load happened to +/// read the schema through. Result> TableOfLoadedSchema( - const std::shared_ptr& file_system, const std::string& path, + const std::shared_ptr& file_system, + const std::shared_ptr& explicit_file_system, const std::string& path, const std::string& branch, const std::map& options, const std::shared_ptr& schema_manager, const std::shared_ptr& latest_schema, const std::optional& identifier, @@ -161,8 +193,10 @@ Result> TableOfLoadedSchema( // it sits in - the same convention `TryLoadFormatTable` follows above. Identifier table_identifier = identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); - return FileStoreTable::Create(file_system, path, latest_schema, table_identifier, uuid, branch, - schema_manager); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + FileStoreTable::Create(explicit_file_system, path, latest_schema, + table_identifier, uuid, branch, schema_manager)); + return WithContextOptions(table, latest_schema, options); } /// The managed table a context names behind a path: the schema is read through a manager of the @@ -172,12 +206,14 @@ Result> TableOfLoadedSchema( /// @param schema_json The schema of the table, when the caller already read it, or null when it /// has to be read here. It is honoured on the main branch only: a branch publishes its own /// under `branch/branch-`, which is where the read below goes. -Result> ManagedTableOf(const std::shared_ptr& file_system, - const std::string& path, const std::string& branch, - const std::map& options, - const std::optional& schema_json, - const std::optional& identifier, - const std::string& uuid = "") { +/// @param explicit_file_system The file system the context supplied, which the managed table the +/// schema describes is opened through - see `TableOfLoadedSchema`. +Result> ManagedTableOf( + const std::shared_ptr& file_system, + const std::shared_ptr& explicit_file_system, const std::string& path, + const std::string& branch, const std::map& options, + const std::optional& schema_json, const std::optional& identifier, + const std::string& uuid = "") { auto schema_manager = std::make_shared(file_system, path, branch); std::shared_ptr latest_schema; if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { @@ -191,8 +227,8 @@ Result> ManagedTableOf(const std::shared_ptr& latest_schema = *latest; } } - return TableOfLoadedSchema(file_system, path, branch, options, schema_manager, latest_schema, - identifier, uuid); + return TableOfLoadedSchema(file_system, explicit_file_system, path, branch, options, + schema_manager, latest_schema, identifier, uuid); } /// The identifier a catalog knows the table by, on `branch`: a non-main branch is folded into @@ -252,8 +288,9 @@ Result> TableOf(const ScanContext& ctx) { // below. One `TableScan` interface serves both, as Java Paimon serves both through one // `ReadBuilder`. const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), - ctx.GetSpecificTableSchema(), ctx.GetIdentifier()); + return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), + branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), + ctx.GetIdentifier()); } /// The table a read context names, read from what the context carries: a format table the @@ -288,8 +325,9 @@ Result> TableOf(const ReadContext& ctx) { // which files there are and how a row is put back together is its own. How a file is opened // is not: both paths go through `DataFileReaderFactory`. const std::string branch = ctx.GetBranch(); - return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), - ctx.GetSpecificTableSchema(), ctx.GetIdentifier()); + return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetSpecificFileSystem(), + ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), + ctx.GetIdentifier()); } /// The table a write context names, read from what the context carries: a format table the @@ -326,8 +364,8 @@ Result> TableOf(const WriteContext& ctx) { CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "write")); // A format table writes plain data files into a directory, so it never reaches the manifest // path below. - return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch, ctx.GetOptions(), - catalog_table_schema, ctx.GetIdentifier()); + return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, ctx.GetRootPath(), branch, + ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier()); } /// The table a commit context names, read from what the context carries: a format table the @@ -369,15 +407,18 @@ Result> TableOf(const CommitContext& ctx) { // A format table commits by renaming files into place, so it never reaches the snapshot path // below. The managed path here reads the commit's branch, so this reads the same one: the two // must not dispatch on different schemas. - return ManagedTableOf(tmp_options.GetFileSystem(), root_path, branch, ctx.GetOptions(), - catalog_table_schema, ctx.GetIdentifier(), ctx.GetTableId().value_or("")); + return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, root_path, branch, + ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), + ctx.GetTableId().value_or("")); } /// The options a context carries against the ones the table was loaded with. A table serves the /// options it was loaded with, so an option the context contradicts is refused rather than -/// honoured - a caller that means to change them copies the table. A key the table was not loaded -/// with is no contradiction: it names a runtime setting - a scan mode, a batch size - which is -/// none of the table's business, and is served as given. +/// honoured - a caller that means to change them copies the table. A table a load hands back +/// carries the context's own options merged in, so it agrees with the context by construction; +/// the refusal belongs to a table the caller held already, whose load is over. A key the table +/// was not loaded with is no contradiction: it names a runtime setting - a scan mode, a batch +/// size - which is none of the table's business, and is served as given. Status CheckOptionsAgreeWithTable(const TableSchema& table_schema, const std::map& context_options) { const std::map& table_options = table_schema.Options(); @@ -409,33 +450,6 @@ Status CheckOperatorContext(const ContextPointer& context, const char* kind) { return Status::OK(); } -/// The schema the table was constructed with, which a write or a commit serves as the table was -/// loaded rather than reading the schema again. -Result> LatestTableSchemaOfBranch( - const std::shared_ptr& schema, const std::string& branch) { - std::shared_ptr latest_schema = std::static_pointer_cast(schema); - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - return latest_schema; -} - -/// The options a write or a commit serves: the ones the table was loaded with - an option the -/// context contradicts is refused by `CheckOptionsAgreeWithTable` above - the ones the context -/// adds on top, and the table's branch, which the metadata paths below are built for: what reads -/// through these options, whose cache keys are scoped by it, has to read the same one. -Result> MergedOperatorOptions( - const TableSchema& table_schema, const std::map& context_options, - const std::string& branch) { - PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(table_schema, context_options)); - auto opts = table_schema.Options(); - for (const auto& [key, value] : context_options) { - opts[key] = value; - } - opts[Options::BRANCH] = branch; - return opts; -} - } // namespace Table::Table(const std::shared_ptr& file_system, const std::string& root_path, @@ -447,16 +461,10 @@ Table::Table(const std::shared_ptr& file_system, const std::string& identifier_(identifier), uuid_(uuid) {} -FileStoreTable::FileStoreTable(const std::shared_ptr& file_system, - const std::string& root_path, const std::shared_ptr& schema, - const Identifier& identifier, const std::string& uuid, - const std::string& branch, - const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) - : Table(file_system, root_path, schema, identifier, uuid), - store_(std::make_shared(file_system, root_path, - std::static_pointer_cast(schema), branch, - schema_manager, snapshot_manager)) {} +FileStoreTable::FileStoreTable(const std::shared_ptr& store, + const Identifier& identifier, const std::string& uuid) + : Table(store->file_system(), store->root_path(), store->schema(), identifier, uuid), + store_(store) {} Result> FileStoreTable::Create( const std::shared_ptr& file_system, const std::string& root_path, @@ -466,15 +474,25 @@ Result> FileStoreTable::Create( // The kind the schema's primary keys name, which no schema change can add or drop, so the // kind the table is built as is the kind every schema its root path ever held describes. A // caller without a schema gets the append-only kind, whose operators then report that no - // schema was found. - std::shared_ptr table_schema = std::static_pointer_cast(schema); + // schema was found. A schema that is not a table schema - the read-only one a system table + // exposes, say - is none to serve a managed table through, so it is refused rather than cast. + std::shared_ptr table_schema; + if (schema != nullptr) { + table_schema = std::dynamic_pointer_cast(schema); + if (table_schema == nullptr) { + return Status::Invalid( + fmt::format("{} is not a managed table schema", identifier.ToString())); + } + } + // The store parses the schema's options, so a schema whose options do not parse is refused + // here: the table it names cannot be served, not by one operator more than another. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr store, + FileStore::Create(file_system, root_path, table_schema, branch, + schema_manager, snapshot_manager)); if (table_schema != nullptr && !table_schema->PrimaryKeys().empty()) { - return std::make_shared(file_system, root_path, schema, - identifier, uuid, branch, schema_manager, - snapshot_manager); + return std::make_shared(store, identifier, uuid); } - return std::make_shared( - file_system, root_path, schema, identifier, uuid, branch, schema_manager, snapshot_manager); + return std::make_shared(store, identifier, uuid); } Result> FileStoreTable::Copy( @@ -490,8 +508,11 @@ Result> FileStoreTable::Copy( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); // The managers are handed over, not rebuilt: the copy serves the same root path and branch, - // so its operators read the schema cache this table already warmed. - return FileStoreTable::Create(file_system_, root_path_, + // so its operators read the schema cache this table already warmed. The file system is the + // one the caller opened this table through, not the one it resolved to: the copy resolves + // from its own merged options again, so a file-system option among the ones it was given + // speaks rather than being pinned to this table's resolution. + return FileStoreTable::Create(store_->specified_file_system(), root_path_, std::shared_ptr(std::move(merged_schema)), identifier_, CatalogUuid().value_or(""), CurrentBranch(), GetSchemaManager(), GetSnapshotManager()); @@ -502,8 +523,8 @@ Result> FileStoreTable::SwitchToBranch(const std::string& if (target_branch == CurrentBranch()) { // The table already serves this branch: a copy of it - schema and managers shared - is // the same table to the caller. - return FileStoreTable::Create(file_system_, root_path_, schema_, identifier_, - CatalogUuid().value_or(""), CurrentBranch(), + return FileStoreTable::Create(store_->specified_file_system(), root_path_, schema_, + identifier_, CatalogUuid().value_or(""), CurrentBranch(), GetSchemaManager(), GetSnapshotManager()); } // The schema of the branch switched to, read from its own directory, which a branch without @@ -522,9 +543,10 @@ Result> FileStoreTable::SwitchToBranch(const std::string& } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr branch_schema, (*latest)->MergeOptions(overridable)); - return FileStoreTable::Create( - file_system_, root_path_, std::shared_ptr(std::move(branch_schema)), - identifier_, CatalogUuid().value_or(""), target_branch, schema_manager); + return FileStoreTable::Create(store_->specified_file_system(), root_path_, + std::shared_ptr(std::move(branch_schema)), + identifier_, CatalogUuid().value_or(""), target_branch, + schema_manager); } Status FileStoreTable::RestoreRealtimeCommittedProgress( @@ -568,19 +590,18 @@ Result> FileStoreTable::NewWrite( MakeBranchCatalogIdentifier(ctx->GetIdentifier(), CurrentBranch())); // The schema the table was constructed with, so a write serves the table as it was loaded // rather than reading the schema again. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - LatestTableSchemaOfBranch(schema_, CurrentBranch())); + const std::shared_ptr& schema = store_->schema(); + if (schema == nullptr) { + return Status::Invalid( + fmt::format("not found latest schema in branch {}", CurrentBranch())); + } + // The options the store was built with are the ones this write serves: an option the + // context contradicts is refused rather than honoured, and a key the table was not loaded + // with names a runtime setting the write does not take - `Copy` derives the table that one + // belongs to. + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); + CoreOptions options = store_->options(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - // The options the table was loaded with, with a runtime setting the context adds on top: an - // option the context would have changed is refused rather than honoured, and the branch is - // the table's - the metadata paths below are built for it, so what reads through these - // options, whose cache keys are scoped by it, has to read the same one. - std::map opts; - PAIMON_ASSIGN_OR_RAISE(opts, - MergedOperatorOptions(*schema, ctx->GetOptions(), CurrentBranch())); - PAIMON_ASSIGN_OR_RAISE( - CoreOptions options, - CoreOptions::FromMap(opts, file_system_, ctx->GetFileSystemSchemeToIdentifierMap())); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); if (options.GetVectorFileFormat() && !ctx->GetWriteSchema().empty()) { @@ -613,9 +634,8 @@ Result> FileStoreTable::NewWrite( RealtimeContextImpl::Cast(ctx->GetRealtimeContext())); PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); } - return CreateWrite(*ctx, schema, arrow_schema, partition_schema, options, - file_store_path_factory, snapshot_manager, io_manager, - ignore_previous_files); + return CreateWrite(*ctx, arrow_schema, partition_schema, options, file_store_path_factory, + snapshot_manager, io_manager, ignore_previous_files); } Result> FileStoreWrite::Create(std::unique_ptr ctx) { @@ -676,34 +696,22 @@ Result> FileStoreTable::NewScan( PAIMON_RETURN_NOT_OK(CheckOperatorContext(context, "scan")); // The schema the table was constructed with, so a scan serves the table as it was loaded // rather than reading the schema again. - std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); - if (loaded_schema == nullptr) { + const std::shared_ptr& table_schema = store_->schema(); + if (table_schema == nullptr) { return Status::Invalid("not found latest schema"); } - const std::shared_ptr& table_schema = loaded_schema; PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*table_schema, context->GetOptions())); PAIMON_ASSIGN_OR_RAISE(bool read_optimized, OptionsUtils::GetValueFromMap(context->GetOptions(), kReadOptimizedScanOption, false)); - // The options the table was loaded with, with a runtime setting the context adds on top: an - // option the context would have changed is refused above, so the table's own stand as they - // are. The branch is the table's, not the context's: a table is the branch it was loaded - // for, so a branch the context carries is not honoured. - auto options = table_schema->Options(); - for (const auto& [key, value] : context->GetOptions()) { - if (key != Options::BRANCH) { - options[key] = value; - } - } - if (BranchManager::IsMainBranch(CurrentBranch())) { - options.erase(Options::BRANCH); - } else { - options[Options::BRANCH] = CurrentBranch(); - } - // The file system and root path the table was opened with, so a scan serves the table as it - // was loaded rather than resolving them from the context. - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, file_system_, {})); + // The options the store was built with are the ones this scan serves: an option the context + // contradicts is refused above, and a key the table was not loaded with names a runtime + // setting the scan does not take - `Copy` derives the table that one belongs to. The branch + // in them is the table's, not the context's: a table is the branch it was loaded for, so a + // branch the context carries is not honoured. + CoreOptions core_options = store_->options(); + // The cache is a runtime setting of this scan, not of the table: the copy the scan serves + // carries it. core_options.WithCache(context->GetCache()); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); @@ -747,10 +755,10 @@ Result> FileStoreTable::NewScan( // The managers the table carries, whose schema cache the dispatch already warmed. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr file_store_scan, - NewFileStoreScan(manifest_list, manifest_file, arrow_schema, table_schema, core_options, + NewFileStoreScan(manifest_list, manifest_file, arrow_schema, core_options, context->GetExecutor(), context->GetMemoryPool(), context.get())); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr split_generator, - NewSplitGenerator(table_schema, core_options)); + NewSplitGenerator(core_options)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr index_file_handler, store_->NewIndexFileHandler(core_options, path_factory, context->GetMemoryPool())); @@ -802,28 +810,24 @@ Result> TableScan::Create(std::unique_ptr> CreateInternalReadContext( +/// The read context of a fallback branch, whose schema is not the table's own: it is read +/// through a manager of that branch, and the options the read serves are the ones that schema +/// was loaded with - the same way the table's own read serves the store's - with the branch it +/// was read from in them. A branch the context carries is not honoured here. +/// @param file_system The file system the table was opened with, which the fallback reads +/// through in place of one resolved from the context. +/// @param root_path The root path the table was opened with, which the fallback reads from in +/// place of the one named by the context. +Result> CreateFallbackReadContext( const std::shared_ptr& context, const std::string& branch, - const std::shared_ptr& schema_manager, - const std::shared_ptr& loaded_schema, const std::shared_ptr& file_system, const std::string& root_path) { std::shared_ptr table_schema; const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (loaded_schema != nullptr) { - table_schema = loaded_schema; - } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { PAIMON_ASSIGN_OR_RAISE(table_schema, TableSchema::CreateFromJson(specific_table_schema.value())); } else { + auto schema_manager = std::make_shared(file_system, root_path, branch); PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, schema_manager->Latest()); if (!latest_schema) { @@ -833,16 +837,7 @@ Result> CreateInternalReadContext( table_schema = latest_schema.value(); } assert(table_schema); - - // The options the schema was loaded with, with a runtime setting the context adds on top - - // and the fallback below serves the ones its own schema was loaded with. The branch is the - // one named below, not the context's: a branch the context carries is not honoured here. auto options = table_schema->Options(); - for (const auto& [key, value] : context->GetOptions()) { - if (key != Options::BRANCH) { - options[key] = value; - } - } if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { options[Options::BRANCH] = branch; } @@ -881,18 +876,20 @@ Result> FileStoreTable::NewRead( PAIMON_RETURN_NOT_OK(CheckOperatorContext(read_context, "read")); // The schema the table was constructed with, so a read serves the table as it was loaded // rather than reading the schema again. Its options are the ones the read serves: an option - // the context would have changed is refused rather than honoured. - std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); + // the context would have changed is refused rather than honoured, and a key the table was + // not loaded with names a runtime setting the read does not take - `Copy` derives the table + // that one belongs to. + const std::shared_ptr& loaded_schema = store_->schema(); if (loaded_schema == nullptr) { return Status::Invalid("not found latest schema"); } PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); - // The branch the table serves, which the read reads on: a table is the branch it was loaded - // for, so the context's own branch is not consulted. + // The options the store was built with, which the read reads through on the branch the + // table serves: a table is the branch it was loaded for, so the context's own branch is + // not consulted. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr internal_context, - CreateInternalReadContext(read_context, CurrentBranch(), GetSchemaManager(), loaded_schema, - file_system_, root_path_)); + InternalReadContext::Create(read_context, loaded_schema, store_->options(), &root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, CreateTableRead(internal_context, read_context->GetMemoryPool(), @@ -909,10 +906,8 @@ Result> FileStoreTable::NewRead( // own rather than the table's. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr fallback_context, - CreateInternalReadContext( - read_context, /*branch=*/scan_fallback_branch.value(), - std::make_shared(file_system_, root_path_, scan_fallback_branch.value()), - /*loaded_schema=*/nullptr, file_system_, root_path_)); + CreateFallbackReadContext(read_context, /*branch=*/scan_fallback_branch.value(), + file_system_, root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, CreateTableRead(fallback_context, read_context->GetMemoryPool(), @@ -1002,18 +997,19 @@ Result> FileStoreTable::NewCommit( } // The schema the table was constructed with, so a commit serves the table as it was loaded // rather than reading the schema again. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - LatestTableSchemaOfBranch(schema_, CurrentBranch())); + const std::shared_ptr& schema = store_->schema(); + if (schema == nullptr) { + return Status::Invalid( + fmt::format("not found latest schema in branch {}", CurrentBranch())); + } std::shared_ptr arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - // The options the table was loaded with, with a runtime setting the context adds on top: an - // option the context would have changed is refused rather than honoured, and the branch is - // the table's - the metadata paths below are built for it, so what reads through these - // options, whose cache keys are scoped by it, has to read the same one. - std::map opts; - PAIMON_ASSIGN_OR_RAISE(opts, - MergedOperatorOptions(*schema, ctx->GetOptions(), CurrentBranch())); - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, file_system_)); + // The options the store was built with are the ones this commit serves: an option the + // context contradicts is refused rather than honoured, and a key the table was not loaded + // with names a runtime setting the commit does not take - `Copy` derives the table that one + // belongs to. + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); + CoreOptions options = store_->options(); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); assert(options.GetFileSystem()); assert(options.GetFileFormat()); @@ -1056,8 +1052,8 @@ Result> FileStoreTable::NewCommit( ctx->GetExecutor()); CommitScanner::ScanSupplier scan_supplier = - NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, schema, arrow_schema, - options, ctx->GetExecutor(), ctx->GetMemoryPool()); + NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, arrow_schema, options, + ctx->GetExecutor(), ctx->GetMemoryPool()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr snapshot_commit, diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 00959d257..8df9c4d86 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -148,13 +148,11 @@ class FileStoreTable : public Table { const std::shared_ptr& executor) const; protected: - /// The base constructor the kinds of managed table build through, `Create` above. The - /// parameters are the ones it takes; see there for what each one is. - FileStoreTable(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid, const std::string& branch, - const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager); + /// The base constructor the kinds of managed table build through, `Create` above: the store + /// the table was loaded with carries every state the kinds hand down - see `Create` for what + /// each piece of it is. + FileStoreTable(const std::shared_ptr& store, const Identifier& identifier, + const std::string& uuid); /// The committed progress a real-time write restores, from the latest snapshot the snapshot /// manager reads: where the write picks up after, which both kinds of real-time write ask @@ -170,32 +168,32 @@ class FileStoreTable : public Table { /// The write of this kind of table, which `NewWrite` hands the shared flow's pieces to: the /// kind's writer, built over the path factory, snapshot manager and options that flow - /// resolved. The branch the write serves is the table's, `CurrentBranch()`. + /// resolved. The schema is the store's own, the one the table was loaded with, and the + /// branch the write serves is the table's, `CurrentBranch()`. /// @param io_manager The spill manager the context's temp directory names, or null when the /// context named none. /// @param ignore_previous_files Whether the write starts over rather than appending, which /// the flow resolved with this kind's own rules. virtual Result> CreateWrite( - const WriteContext& ctx, const std::shared_ptr& schema, - const std::shared_ptr& arrow_schema, + const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const = 0; /// The scan of this kind of table, which `NewScan` hands the manifest list and file it built - /// to. The snapshot and schema managers are the table's own, which the scan serves. + /// to. The schema is the store's own and the snapshot and schema managers are the table's + /// own, which the scan serves. virtual Result> NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const = 0; - /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. + /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. The schema + /// it reads its kind's keys from is the store's own. virtual Result> NewSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options) const = 0; /// The read of this kind of table, which `NewRead` hands the internal context it read the @@ -207,13 +205,12 @@ class FileStoreTable : public Table { const std::shared_ptr& executor) const = 0; /// The manifest entry scan supplier of this kind of table, which `NewCommit` hands the - /// manifest list and file it built to. The schema manager the scans it hands out read through - /// is the table's own. + /// manifest list and file it built to. The schema is the store's own, and the schema manager + /// the scans it hands out read through is the table's own. virtual CommitScanner::ScanSupplier NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const = 0; diff --git a/src/paimon/core/table/primary_key_file_store_table.cpp b/src/paimon/core/table/primary_key_file_store_table.cpp index 564e015e3..06e27e0d1 100644 --- a/src/paimon/core/table/primary_key_file_store_table.cpp +++ b/src/paimon/core/table/primary_key_file_store_table.cpp @@ -96,22 +96,19 @@ Result HasHistoricalPrimaryKeyBTreeDefinition( } // namespace -PrimaryKeyFileStoreTable::PrimaryKeyFileStoreTable( - const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, - const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) - : FileStoreTable(file_system, root_path, schema, identifier, uuid, branch, schema_manager, - snapshot_manager) {} +PrimaryKeyFileStoreTable::PrimaryKeyFileStoreTable(const std::shared_ptr& store, + const Identifier& identifier, + const std::string& uuid) + : FileStoreTable(store, identifier, uuid) {} Result> PrimaryKeyFileStoreTable::CreateWrite( - const WriteContext& ctx, const std::shared_ptr& schema, - const std::shared_ptr& arrow_schema, + const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { // pk table + const std::shared_ptr& schema = store_->schema(); if (ctx.GetRealtimeContext()) { PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); if (ignore_previous_files) { @@ -227,11 +224,11 @@ Result> PrimaryKeyFileStoreTable::CreateWrite( Result> PrimaryKeyFileStoreTable::NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const { // TODO(liancheng.lsz): support fallback branch in scan + const std::shared_ptr& table_schema = store_->schema(); std::unique_ptr scan; PAIMON_ASSIGN_OR_RAISE( scan, KeyValueFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), manifest_list, @@ -242,10 +239,11 @@ Result> PrimaryKeyFileStoreTable::NewFileStoreSca } Result> PrimaryKeyFileStoreTable::NewSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options) const { + const CoreOptions& core_options) const { auto source_split_target_size = core_options.GetSourceSplitTargetSize(); auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); // TODO(liancheng.lsz): support evolution + const std::shared_ptr& table_schema = store_->schema(); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, table_schema->TrimmedPrimaryKeys()); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, @@ -270,12 +268,12 @@ CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const { // The schema manager the table carries, which the scans this supplier hands out read the // schema through. const std::shared_ptr& schema_manager = GetSchemaManager(); + const std::shared_ptr& table_schema = store_->schema(); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) -> Result> { diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h index 9c658758a..9b0ed5c9a 100644 --- a/src/paimon/core/table/primary_key_file_store_table.h +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -24,13 +24,11 @@ #include "paimon/catalog/identifier.h" #include "paimon/core/table/file_store_table.h" -#include "paimon/core/utils/branch_manager.h" #include "paimon/result.h" #include "paimon/type_fwd.h" namespace paimon { -class CoreOptions; class FileStoreScan; class IOManager; class SplitGenerator; @@ -42,19 +40,14 @@ class TableSchema; /// those flows hand down are built here. class PrimaryKeyFileStoreTable : public FileStoreTable { public: - /// The constructor `FileStoreTable::Create` builds a primary key table through; the - /// parameters are the ones it takes, and the base class carries them. - PrimaryKeyFileStoreTable(const std::shared_ptr& file_system, - const std::string& root_path, const std::shared_ptr& schema, - const Identifier& identifier, const std::string& uuid = "", - const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + /// The constructor `FileStoreTable::Create` builds a primary key table through: the store + /// carries every state, and the base class holds it. + PrimaryKeyFileStoreTable(const std::shared_ptr& store, const Identifier& identifier, + const std::string& uuid = ""); private: Result> CreateWrite( - const WriteContext& ctx, const std::shared_ptr& schema, - const std::shared_ptr& arrow_schema, + const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, @@ -63,13 +56,11 @@ class PrimaryKeyFileStoreTable : public FileStoreTable { Result> NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const override; Result> NewSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options) const override; Result> NewTableRead( @@ -82,7 +73,6 @@ class PrimaryKeyFileStoreTable : public FileStoreTable { const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const override; diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 3d580d64d..b6aef3992 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -251,7 +251,7 @@ TEST(TableReadTest, TestCreateAppendOnlyTableRead) { TEST(TableReadTest, TestMergeOptions) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; { - // The options the table was loaded with stand as they are, and a runtime setting the + // The options the schema carries stand in the table the load builds, and a setting the // context adds on top - the table was loaded with neither of these keys - is served as // given. ReadContextBuilder context_builder(path); @@ -276,14 +276,21 @@ TEST(TableReadTest, TestMergeOptions) { ASSERT_EQ(expected_options, core_options.ToMap()); } { - // An option that contradicts the table is refused: the read serves the options the table - // was loaded with, and a caller that means to change them copies the table. + // An option the schema also carries speaks at the load: the table the context is + // handed back is built with the context's value over the schema's, and the read serves + // the merged options. A caller holding a table already - a load that is over - is the + // one that gets a refusal instead, which `TableTest` covers. ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); context_builder.AddOption("bucket", "10"); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), - "option 'bucket' is '10' in the context but '2' on the table"); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + auto append_only_table_read = dynamic_cast(table_read.get()); + ASSERT_TRUE(append_only_table_read); + auto* typed_split_read = + dynamic_cast(append_only_table_read->split_reads_[0].get()); + ASSERT_TRUE(typed_split_read); + ASSERT_EQ(10, typed_split_read->options_.GetBucket()); } } diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 3d3f7b3e5..7254dff60 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -395,7 +395,12 @@ Result> AuditLogSystemTable::NewScan( .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + // The data table behind this view is served through the file system the view itself was + // opened with, so a context without one of its own falls back to it rather than the + // default backend. + .WithFileSystem(context->GetSpecificFileSystem() != nullptr + ? context->GetSpecificFileSystem() + : GetFileSystem()) .WithCache(context->GetCache()); if (scan_filter) { if (scan_filter->GetBucketFilter()) { @@ -458,7 +463,12 @@ Result> AuditLogSystemTable::CreateDataReadContext( .WithBranch(core_options.GetBranch()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + // The data table behind this view is served through the file system the view itself was + // opened with, so a context without one of its own falls back to it rather than the + // default backend. + .WithFileSystem(context->GetSpecificFileSystem() != nullptr + ? context->GetSpecificFileSystem() + : GetFileSystem()) .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .EnablePrefetch(context->EnablePrefetch()) .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 172fc964c..0d44b5989 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -71,7 +71,12 @@ Result> ReadOptimizedSystemTable::NewScan( .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + // The data table behind this view is served through the file system the view itself was + // opened with, so a context without one of its own falls back to it rather than the + // default backend. + .WithFileSystem(context->GetSpecificFileSystem() != nullptr + ? context->GetSpecificFileSystem() + : GetFileSystem()) .WithCache(context->GetCache()); if (context->GetLimit().has_value()) { builder.SetLimit(context->GetLimit().value()); @@ -114,7 +119,12 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + // The data table behind this view is served through the file system the view itself was + // opened with, so a context without one of its own falls back to it rather than the + // default backend. + .WithFileSystem(context->GetSpecificFileSystem() != nullptr + ? context->GetSpecificFileSystem() + : GetFileSystem()) .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .SetReadAheadCacheEnabled(context->ReadAheadCacheEnabled()) .WithCacheConfig(context->GetCacheConfig()) diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index fe99efe63..7fde349d9 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -28,6 +28,7 @@ #include "arrow/c/abi.h" #include "arrow/c/bridge.h" #include "gtest/gtest.h" +#include "paimon/cache/cache.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" #include "paimon/common/utils/checked_cast.h" @@ -72,6 +73,57 @@ Result> CreateManagedTable( /*uuid=*/"", branch); } +/// Counts the lookups a scan makes, which is enough to tell that the cache a context carries +/// reached the scan's manifest reading rather than being dropped on the way. +class CountingCache : public Cache { + public: + Result> Get( + const std::shared_ptr& key, + std::function>(const std::shared_ptr&)> + supplier) override { + gets++; + return supplier(key); + } + Status Put(const std::shared_ptr&, const std::shared_ptr&) override { + return Status::OK(); + } + void Invalidate(const std::shared_ptr&) override {} + void InvalidateAll() override {} + size_t Size() const override { + return 0; + } + + int32_t gets = 0; +}; + +/// Writes two rows through the table's own write and commit, so a scan below has data to find. +void WriteAndCommitTwoRows(const std::shared_ptr
& table, const std::string& root_path) { + WriteContextBuilder write_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); +} + } // namespace TEST(TableTest, TestTableCarriesManagersOfItsBranch) { @@ -387,10 +439,11 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { auto catalog = std::make_shared(); catalog->SetTableSchema(table->LatestSchema()); - // A catalog that hands out no per-table file system opens the table without one, and one - // that does opens it with the file system issued for that table. + // A catalog that hands out no per-table file system opens the table through the one its + // schema's options resolve - never through none, the default local one when they name no + // other - and one that does opens it with the file system issued for that table. ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog->GetTable(identifier)); - EXPECT_EQ(opened->GetFileSystem(), nullptr); + EXPECT_NE(opened->GetFileSystem(), nullptr); catalog->SetTableFileSystem(dir->GetFileSystem()); ASSERT_OK_AND_ASSIGN(opened, catalog->GetTable(identifier)); @@ -723,6 +776,92 @@ TEST(TableTest, TestOperatorsRefuseAnOptionThatContradictsTheTable) { copied->NewScan(std::move(ctx))); } +TEST(TableTest, TestOperatorsIgnoreARuntimeKeyTheTableWasNotLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, + /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + WriteAndCommitTwoRows(table, dir->Str()); + + // `scan.mode` and `scan.snapshot-id` name no option the table was loaded with, so they name + // a runtime setting the scan does not take: the scan serves the table's own options and + // reads its latest snapshot - a merged context would have gone looking for snapshot 999, + // which was never committed. + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.AddOption(Options::SCAN_MODE, "from-snapshot") + .AddOption(Options::SCAN_SNAPSHOT_ID, "999"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + ASSERT_EQ(plan->SnapshotId(), std::optional(1)); + + // A read takes a runtime key the same way: the key names nothing the table carries, so the + // read is served rather than refused. + ReadContextBuilder read_builder(dir->Str()); + read_builder.AddOption(Options::SCAN_MODE, "from-snapshot") + .AddOption(Options::SCAN_SNAPSHOT_ID, "999"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + table->NewRead(std::move(read_ctx))); + + // The runtime setting a caller means to serve is the copy's: `Table::Copy` derives the table + // whose options carry it, and that table's scan reads the snapshot it names - which is not + // there, as this table only ever committed snapshot 1. + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
copied, + table->Copy({{Options::SCAN_MODE, "from-snapshot"}, {Options::SCAN_SNAPSHOT_ID, "999"}})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_scan_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_scan, + copied->NewScan(std::move(copied_scan_ctx))); + ASSERT_NOK_WITH_MSG(copied_scan->CreatePlan(), "snapshot"); +} + +TEST(TableTest, TestCreateFailsWhenTheSchemaOptionsDoNotParse) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + // The schema itself carries no validation, so a schema written with options no + // `CoreOptions` accepts surfaces when the store parses them - at the load, not at the first + // operator built on the table. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + TableSchema::Create( + /*schema_id=*/0, + arrow::schema({arrow::field("id", arrow::int32(), + /*nullable=*/false)}), + /*partition_keys=*/{}, /*primary_keys=*/{}, {{Options::TARGET_FILE_ROW_NUM, "0"}})); + auto schema = checked_pointer_cast(std::shared_ptr(std::move(created))); + ASSERT_NOK_WITH_MSG( + FileStoreTable::Create(dir->GetFileSystem(), dir->Str(), schema, Identifier("db", "tbl")), + "target-file-row-num should be at least 1"); +} + +TEST(TableTest, TestTheCacheAScanContextCarriesReachesTheScan) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, + /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + WriteAndCommitTwoRows(table, dir->Str()); + + // The cache is a runtime setting the scan serves from the context rather than from the + // table's own options, so a scan built with one reads its manifests through it. + auto cache = std::make_shared(); + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.WithCache(cache); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + ASSERT_GT(cache->gets, 0) << "the cache the scan context carries never reached the scan"; +} + TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index b7dcfe04f..083fb62d5 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -434,6 +434,24 @@ Result> RestCatalog::LoadSystemTable( } Result> RestCatalog::GetTable(const Identifier& identifier) const { + // The global system tables of the "sys" database expose the catalog itself rather than any + // table the server stands behind, so they are served locally - the way `LoadTableSchema` + // serves them their schema - and never reach the managed-table load below, whose schema + // cast a read-only system table schema would not survive. + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, + GlobalSystemTableLoader::IsSupported(identifier.GetObjectName(), + api_->GetMergedOptions())); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + GlobalSystemTableContext context; + context.catalog = this; + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = api_->GetMergedOptions(); + return GlobalSystemTableLoader::Load(identifier.GetObjectName(), context); + } // A `T$system` identifier opens the read-only view it names, which reads the data table it // belongs to and answers for itself through the same `Table` interface. PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index 41d35de8d..4fa4e1da1 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -813,8 +813,17 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { auto lookup_predicate = PredicateBuilder::Equal( 0, "rowkey", field_type, Literal(field_type, lookup_key.data(), lookup_key.size())); + // The rescaled bucket count and, with it, the cache settings `FinishScanContext` + // also puts on the context: a table already held takes its runtime settings + // through the copy it is made of, and the context carries them only to agree + // with it. + std::map dynamic_options{ + {Options::BUCKET, current_bucket_count}}; + if (EnableSnapshotLiveManifestCache()) { + dynamic_options.emplace(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3"); + } ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, - table->Copy({{Options::BUCKET, current_bucket_count}})); + table->Copy(dynamic_options)); ScanContextBuilder scan_builder(table_path); scan_builder.SetPredicate(lookup_predicate) .AddOption(Options::BUCKET, current_bucket_count); @@ -4065,8 +4074,14 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { catalog->GetTable(Identifier("pk_table_scan_and_read_dv", "pk_table_scan_and_read_dv"))); for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, - table->Copy({{Options::BUCKET, current_bucket_count}})); + // The rescaled bucket count and, with it, the cache settings `FinishScanContext` also + // puts on the context: a table already held takes its runtime settings through the copy + // it is made of, and the context carries them only to agree with it. + std::map dynamic_options{{Options::BUCKET, current_bucket_count}}; + if (EnableSnapshotLiveManifestCache()) { + dynamic_options.emplace(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3"); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, table->Copy(dynamic_options)); ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); scan_context_builder.AddOption(Options::BUCKET, current_bucket_count); From 472c9c5f1c8b8df47a650103c52abef0f1e1ccf5 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 14:23:42 +0800 Subject: [PATCH 07/19] fix --- .../core/operation/internal_read_context.cpp | 4 +- src/paimon/core/table/file_store.cpp | 16 ++-- src/paimon/core/table/file_store.h | 21 ++++- src/paimon/core/table/file_store_table.cpp | 87 +++++++++++++++---- src/paimon/core/table/file_store_table.h | 6 +- .../table/system/audit_log_system_table.cpp | 15 ++-- .../system/read_optimized_system_table.cpp | 15 ++-- src/paimon/rest/rest_catalog.cpp | 14 +-- 8 files changed, 131 insertions(+), 47 deletions(-) diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 778976a46..784123828 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -290,8 +290,8 @@ Result> InternalReadContext::Create( // resolves its own. PAIMON_ASSIGN_OR_RAISE( CoreOptions core_options, - CoreOptions::FromMap(options, file_system != nullptr ? file_system - : context->GetSpecificFileSystem())); + CoreOptions::FromMap( + options, file_system != nullptr ? file_system : context->GetSpecificFileSystem())); core_options.WithCache(context->GetCache()); return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); } diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index 2c5a19d96..472626a01 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -51,13 +51,15 @@ Result> FileStore::Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) { - auto store = std::shared_ptr( - new FileStore(file_system, root_path, schema, branch, schema_manager, snapshot_manager)); + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map) { + auto store = std::shared_ptr(new FileStore( + file_system, root_path, schema, branch, schema_manager, snapshot_manager, fs_scheme_map)); if (schema != nullptr) { // The options the schema was loaded with, on the branch this store serves: what reads // through them - whose cache keys are scoped by the branch - reads the same one the - // managers point at. + // managers point at. The scheme map is the one the schema was read through, so a scheme + // it mapped resolves here the way it resolved there. std::map options = schema->Options(); if (BranchManager::IsMainBranch(store->branch_)) { options.erase(Options::BRANCH); @@ -65,7 +67,7 @@ Result> FileStore::Create( options[Options::BRANCH] = store->branch_; } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, file_system, {})); + CoreOptions::FromMap(options, file_system, fs_scheme_map)); store->core_options_ = std::move(core_options); } // The file system the table is served through: the one the options resolve - a schema @@ -98,9 +100,11 @@ Result> FileStore::Create( FileStore::FileStore(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map) : file_system_(file_system), specified_file_system_(file_system), + fs_scheme_map_(fs_scheme_map), root_path_(root_path), schema_(schema), branch_(BranchManager::NormalizeBranch(branch)), diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index 6c2396bd6..66ccc09c2 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -19,6 +19,7 @@ #pragma once +#include #include #include #include @@ -77,6 +78,11 @@ class FileStore { /// @param snapshot_manager A snapshot manager to carry instead of building one. /// A null manager is built over `root_path` on `branch`, so the operators always read /// through the managers the store carries. + /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom + /// schemes through. It takes part in building the store's options the way it took + /// part in reading the schema: without it a schema naming a custom scheme would + /// resolve to the default backend once its operators run. It is kept for a copy to + /// hand over again, so the copy resolves as this store does. /// /// Building the store parses the schema's options, so it fails when they do not parse: the /// error is the caller's to see, which is why a store is built through this factory rather @@ -85,7 +91,8 @@ class FileStore { const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + const std::shared_ptr& snapshot_manager = nullptr, + const std::map& fs_scheme_map = {}); /// The branch this store serves, as resolved when it was built. const std::string& branch() const { @@ -125,6 +132,12 @@ class FileStore { return specified_file_system_; } + /// The scheme-to-identifier map the options were resolved with, kept for a copy to resolve + /// through again: see `Create`'s `fs_scheme_map`. + const std::map& fs_scheme_map() const { + return fs_scheme_map_; + } + /// The root path of the table in its file system, as the table was opened with it. const std::string& root_path() const { return root_path_; @@ -199,13 +212,17 @@ class FileStore { FileStore(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager); + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map); std::shared_ptr file_system_; /// The file system the caller opened the table through, as handed to `Create` - null when /// the call had none - kept for a copy to hand over again: what `file_system_` holds is /// resolved, and would pin the copy to this table's resolution. std::shared_ptr specified_file_system_; + /// The scheme-to-identifier map the options were resolved with, kept for a copy to resolve + /// through again. + std::map fs_scheme_map_; std::string root_path_; std::shared_ptr schema_; std::string branch_; diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 2e5f77af6..e46259345 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -36,6 +36,7 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/sensitive_config_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_snapshot_commit.h" #include "paimon/core/catalog/catalog_utils.h" @@ -141,6 +142,26 @@ Result> TryLoadFormatTable( /*location_carries_paimon_metadata=*/true, options); } +/// Whether an option of a copy names the file system its store resolves: the backend +/// `file-system` picks, or the access configuration a `fs.` key carries. A copy whose dynamic +/// options change one of these resolves to another file system, so the managers of the table it +/// was copied from - built over that table's resolution - are not its to inherit: they would +/// keep reading metadata through the backend the copy just left. +bool FileSystemResolutionMayChange(const TableSchema& schema, + const std::map& dynamic_options) { + const std::map& schema_options = schema.Options(); + for (const auto& [key, value] : dynamic_options) { + if (key != Options::FILE_SYSTEM && key.rfind("fs.", 0) != 0) { + continue; + } + auto schema_option = schema_options.find(key); + if (schema_option == schema_options.end() || schema_option->second != value) { + return true; + } + } + return false; +} + /// The options a context carries, merged into the table the load hands back: a table is the /// product of the load, so the options speak there - the way `Copy` merges them, over the ones /// the schema carries - and `NewXxx` on the product serves them without merging again. A @@ -183,7 +204,7 @@ Result> TableOfLoadedSchema( const std::string& branch, const std::map& options, const std::shared_ptr& schema_manager, const std::shared_ptr& latest_schema, const std::optional& identifier, - const std::string& uuid = "") { + const std::string& uuid = "", const std::map& fs_scheme_map = {}) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, TryLoadFormatTable(file_system, path, branch, options, latest_schema)); if (format_table != nullptr) { @@ -195,7 +216,8 @@ Result> TableOfLoadedSchema( identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, FileStoreTable::Create(explicit_file_system, path, latest_schema, - table_identifier, uuid, branch, schema_manager)); + table_identifier, uuid, branch, schema_manager, + /*snapshot_manager=*/nullptr, fs_scheme_map)); return WithContextOptions(table, latest_schema, options); } @@ -213,7 +235,7 @@ Result> ManagedTableOf( const std::shared_ptr& explicit_file_system, const std::string& path, const std::string& branch, const std::map& options, const std::optional& schema_json, const std::optional& identifier, - const std::string& uuid = "") { + const std::string& uuid = "", const std::map& fs_scheme_map = {}) { auto schema_manager = std::make_shared(file_system, path, branch); std::shared_ptr latest_schema; if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { @@ -228,7 +250,7 @@ Result> ManagedTableOf( } } return TableOfLoadedSchema(file_system, explicit_file_system, path, branch, options, - schema_manager, latest_schema, identifier, uuid); + schema_manager, latest_schema, identifier, uuid, fs_scheme_map); } /// The identifier a catalog knows the table by, on `branch`: a non-main branch is folded into @@ -325,9 +347,12 @@ Result> TableOf(const ReadContext& ctx) { // which files there are and how a row is put back together is its own. How a file is opened // is not: both paths go through `DataFileReaderFactory`. const std::string branch = ctx.GetBranch(); + // The scheme map the schema was read through, so the table it describes resolves custom + // schemes the same way - the data belongs to the metadata, both sit behind the map. return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), - ctx.GetIdentifier()); + ctx.GetIdentifier(), /*uuid=*/"", + ctx.GetFileSystemSchemeToIdentifierMap()); } /// The table a write context names, read from what the context carries: a format table the @@ -365,7 +390,8 @@ Result> TableOf(const WriteContext& ctx) { // A format table writes plain data files into a directory, so it never reaches the manifest // path below. return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, ctx.GetRootPath(), branch, - ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier()); + ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), /*uuid=*/"", + ctx.GetFileSystemSchemeToIdentifierMap()); } /// The table a commit context names, read from what the context carries: a format table the @@ -425,9 +451,12 @@ Status CheckOptionsAgreeWithTable(const TableSchema& table_schema, for (const auto& [key, value] : context_options) { auto table_option = table_options.find(key); if (table_option != table_options.end() && table_option->second != value) { + // A value may be a credential, and the error travels into logs: it is redacted the + // way every surface that exposes options redacts them. return Status::Invalid( - fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, value, - table_option->second)); + fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, + SensitiveConfigUtils::RedactValue(key, value), + SensitiveConfigUtils::RedactValue(key, table_option->second))); } } return Status::OK(); @@ -470,7 +499,8 @@ Result> FileStoreTable::Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) { + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map) { // The kind the schema's primary keys name, which no schema change can add or drop, so the // kind the table is built as is the kind every schema its root path ever held describes. A // caller without a schema gets the append-only kind, whose operators then report that no @@ -488,7 +518,7 @@ Result> FileStoreTable::Create( // here: the table it names cannot be served, not by one operator more than another. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr store, FileStore::Create(file_system, root_path, table_schema, branch, - schema_manager, snapshot_manager)); + schema_manager, snapshot_manager, fs_scheme_map)); if (table_schema != nullptr && !table_schema->PrimaryKeys().empty()) { return std::make_shared(store, identifier, uuid); } @@ -507,15 +537,22 @@ Result> FileStoreTable::Copy( overridable.erase(Options::TYPE); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); - // The managers are handed over, not rebuilt: the copy serves the same root path and branch, - // so its operators read the schema cache this table already warmed. The file system is the - // one the caller opened this table through, not the one it resolved to: the copy resolves - // from its own merged options again, so a file-system option among the ones it was given - // speaks rather than being pinned to this table's resolution. + // The managers are handed over, not rebuilt, when the copy resolves the same file system + // this table does: its operators then read the schema cache this table already warmed. The + // file system is the one the caller opened this table through, not the one it resolved to: + // the copy resolves from its own merged options again, so a file-system option among the + // ones it was given speaks rather than being pinned to this table's resolution - and when + // one does change the resolution, fresh managers are built over it instead, so the metadata + // is read through the same backend the data is. The scheme map travels the same way, so the + // copy resolves custom schemes as this table does. + const bool fs_resolution_may_change = + FileSystemResolutionMayChange(*loaded_schema, overridable); return FileStoreTable::Create(store_->specified_file_system(), root_path_, std::shared_ptr(std::move(merged_schema)), identifier_, CatalogUuid().value_or(""), CurrentBranch(), - GetSchemaManager(), GetSnapshotManager()); + fs_resolution_may_change ? nullptr : GetSchemaManager(), + fs_resolution_may_change ? nullptr : GetSnapshotManager(), + store_->fs_scheme_map()); } Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { @@ -525,7 +562,8 @@ Result> FileStoreTable::SwitchToBranch(const std::string& // the same table to the caller. return FileStoreTable::Create(store_->specified_file_system(), root_path_, schema_, identifier_, CatalogUuid().value_or(""), CurrentBranch(), - GetSchemaManager(), GetSnapshotManager()); + GetSchemaManager(), GetSnapshotManager(), + store_->fs_scheme_map()); } // The schema of the branch switched to, read from its own directory, which a branch without // one does not have. @@ -546,7 +584,8 @@ Result> FileStoreTable::SwitchToBranch(const std::string& return FileStoreTable::Create(store_->specified_file_system(), root_path_, std::shared_ptr(std::move(branch_schema)), identifier_, CatalogUuid().value_or(""), target_branch, - schema_manager); + schema_manager, /*snapshot_manager=*/nullptr, + store_->fs_scheme_map()); } Status FileStoreTable::RestoreRealtimeCommittedProgress( @@ -838,8 +877,20 @@ Result> CreateFallbackReadContext( } assert(table_schema); auto options = table_schema->Options(); + // The dynamic options the load merged into the table this read serves - a sequence-number + // flag, a batch size - speak for the read, and the fallback branch serves the same read: + // they are merged over the fallback branch's own schema options the way the load merged + // them over the table's, so the fallback reader recognises what the main one does. `type` + // stays the branch's own: a copy never restyles the table, and neither does a fallback. + for (const auto& [key, value] : context->GetOptions()) { + if (key != Options::TYPE) { + options[key] = value; + } + } if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { options[Options::BRANCH] = branch; + } else { + options.erase(Options::BRANCH); } return InternalReadContext::Create(context, table_schema, options, file_system, &root_path); } diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 8df9c4d86..df841d548 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -78,13 +78,17 @@ class FileStoreTable : public Table { /// `Copy` hands its table's over so both share one. /// A null manager is built over `root_path` on `branch`, so the operators always read /// through the managers the table carries. + /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom + /// schemes through, which the store's options resolve through as well - see + /// `FileStore::Create`. `Copy` hands its table's map over. static Result> Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid = "", const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + const std::shared_ptr& snapshot_manager = nullptr, + const std::map& fs_scheme_map = {}); /// The branch this table serves, as resolved when the table was built: the schema the table /// was loaded with came from it, and its managers point at it. diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 7254dff60..e191b653a 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -458,17 +458,20 @@ Result> AuditLogSystemTable::CreateDataReadContext( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr base_read_schema, BaseReadSchema()); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); ReadContextBuilder builder(table_path_); + // The data table behind this view is served through the file system the view itself was + // opened with, but only when the context names neither a file system of its own nor a + // scheme map to resolve through - a file system and a scheme map cannot both be named, and + // the context's own answer always wins. + std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); + if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { + data_table_fs = GetFileSystem(); + } builder.SetOptions(read_options) .SetReadFieldNames(base_read_schema->field_names()) .WithBranch(core_options.GetBranch()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - // The data table behind this view is served through the file system the view itself was - // opened with, so a context without one of its own falls back to it rather than the - // default backend. - .WithFileSystem(context->GetSpecificFileSystem() != nullptr - ? context->GetSpecificFileSystem() - : GetFileSystem()) + .WithFileSystem(data_table_fs) .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .EnablePrefetch(context->EnablePrefetch()) .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 0d44b5989..d7193b5cb 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -108,6 +108,14 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon branch = branch_iter->second; } ReadContextBuilder builder(table_path_); + // The data table behind this view is served through the file system the view itself was + // opened with, but only when the context names neither a file system of its own nor a + // scheme map to resolve through - a file system and a scheme map cannot both be named, and + // the context's own answer always wins. + std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); + if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { + data_table_fs = GetFileSystem(); + } builder.SetOptions(options) .WithBranch(branch) .SetPredicate(context->GetPredicate()) @@ -119,12 +127,7 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - // The data table behind this view is served through the file system the view itself was - // opened with, so a context without one of its own falls back to it rather than the - // default backend. - .WithFileSystem(context->GetSpecificFileSystem() != nullptr - ? context->GetSpecificFileSystem() - : GetFileSystem()) + .WithFileSystem(data_table_fs) .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .SetReadAheadCacheEnabled(context->ReadAheadCacheEnabled()) .WithCacheConfig(context->GetCacheConfig()) diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 083fb62d5..07e4b000b 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -425,12 +425,14 @@ Result> RestCatalog::LoadSystemTable( if (branch) { dynamic_options[Options::BRANCH] = branch.value(); } - // The file system is only used to build the system table, whose schema does not - // read any file, so the catalog wide one is enough here. The credentials of the - // table are applied when its rows are read, through the file system of the read or - // scan context. - return SystemTableLoader::Load(system_table_name.value(), fs_, data_table_path, latest_schema, - dynamic_options); + // The system table reads the data table's own files - snapshots, manifests, data - so it is + // served by the file system that table's credentials open, which with data tokens is the + // only one that can: `GetTableFileSystem` folds the system identifier to the data table it + // belongs to and hands the catalog-wide file system out when no tokens are issued. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, + GetTableFileSystem(identifier)); + return SystemTableLoader::Load(system_table_name.value(), table_file_system, data_table_path, + latest_schema, dynamic_options); } Result> RestCatalog::GetTable(const Identifier& identifier) const { From c4f9d468e3a6664fe631f1f705eb674ac80071cb Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 14:42:48 +0800 Subject: [PATCH 08/19] fix --- src/paimon/core/table/file_store_table.cpp | 7 +++- src/paimon/core/table/table_test.cpp | 45 ++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index e46259345..11efaf24d 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -532,9 +532,14 @@ Result> FileStoreTable::Copy( return Status::Invalid(fmt::format("not found latest schema of {}", FullName())); } // `type` is structural and read from the schema alone, the promise `Copy` makes: an option - // given at the call must not restyle the table. + // given at the call must not restyle the table. `branch` is the same kind of promise: the + // copy is the table it was copied from, on the branch that table serves, so a branch option + // given at the call moves neither - it would only claim a branch the copy does not serve - + // and a caller that means to move it has `SwitchToBranch`. Both are dropped before the + // merge, so the copy's options never speak of a branch its operators do not. std::map overridable = dynamic_options; overridable.erase(Options::TYPE); + overridable.erase(Options::BRANCH); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); // The managers are handed over, not rebuilt, when the copy resolves the same file system diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 7fde349d9..b4478918b 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -153,6 +153,51 @@ TEST(TableTest, TestTableCarriesManagersOfItsBranch) { EXPECT_EQ(file_store_copy->GetSnapshotManager(), file_store_table->GetSnapshotManager()); } +TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // A copy handed a `branch` option is the table it was copied from: the branch is the + // table's own - `SwitchToBranch` is the way to change it - so the option moves neither the + // copy nor its options, and an operator asked for the branch the copy serves - `main` - is + // served rather than refused by options claiming a `dev` the copy does not serve. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({{Options::BRANCH, "dev"}})); + auto file_store_copy = std::dynamic_pointer_cast(copy); + ASSERT_TRUE(file_store_copy != nullptr); + EXPECT_EQ(file_store_copy->CurrentBranch(), "main"); + { + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.AddOption(Options::BRANCH, "main"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + file_store_copy->NewScan(std::move(ctx))); + } + + // The same the other way round: a copy of a branch table serves that branch still, so + // naming the branch it serves - `dev` - is agreed with rather than refused. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, + CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/branch_tbl", + identifier, /*branch=*/"dev", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_copy, + branch_table->Copy({{Options::BRANCH, "main"}})); + auto branch_file_store_copy = std::dynamic_pointer_cast(branch_copy); + ASSERT_TRUE(branch_file_store_copy != nullptr); + EXPECT_EQ(branch_file_store_copy->CurrentBranch(), "dev"); + { + ScanContextBuilder scan_builder(dir->Str() + "/branch_tbl"); + scan_builder.AddOption(Options::BRANCH, "dev"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + branch_file_store_copy->NewScan(std::move(ctx))); + } +} + TEST(TableTest, TestSwitchToBranch) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); From 7b3503290e1c5aee07c67fe8a1e3b4b00175d222 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 15:03:31 +0800 Subject: [PATCH 09/19] fix --- src/paimon/core/operation/internal_read_context.cpp | 2 +- src/paimon/core/table/file_store_table.cpp | 2 +- src/paimon/core/table/format/format_table.cpp | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 784123828..3eb169ac7 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -413,7 +413,7 @@ Result> InternalReadContext::CreateFromCore context, table_schema, read_schema, core_options, arrow_pool, path != nullptr ? std::optional(*path) : std::nullopt)); PAIMON_RETURN_NOT_OK(internal_context->BindIndexScores()); - return internal_context + return internal_context; } Status InternalReadContext::BindIndexScores() { diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 11efaf24d..c8e50eb62 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -329,7 +329,7 @@ Result> TableOf(const ReadContext& ctx) { PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, SystemTableLoader::TryParsePath(ctx.GetPath())); if (system_table_path) { - if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { + if (PredicateUtils::ContainsFullTextMatch(ctx.GetPredicate())) { return Status::NotImplemented( "FullTextMatch predicate execution is not implemented yet"); } diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index 6f2ac16a4..cb4141dde 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -43,6 +43,7 @@ #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" +#include "paimon/predicate/predicate_utils.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" #include "paimon/status.h" From d6cbfe387748e0512a02766183ed6f5f74183cbe Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 15:30:38 +0800 Subject: [PATCH 10/19] fix --- .../core/table/source/table_read_test.cpp | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index b6aef3992..072ba3e1d 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -36,6 +36,7 @@ #include "paimon/core/operation/split_read.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/key_value_table_read.h" @@ -134,20 +135,22 @@ TEST(TableReadTest, TestFullTextMatchExecutionIsNotImplemented) { auto fs = std::make_shared(); SchemaManager schema_manager(fs, table_path); ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); - Table table(fs, table_path, schema, Identifier("append_09.db", "append_09")); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, table_path, schema, Identifier("append_09.db", "append_09"))); for (bool table_entry : {false, true}) { ReadContextBuilder builder(table_path); builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); auto result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "FullTextMatch predicate execution is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); builder.SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(context, builder.Finish()); result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "Score-producing FullTextMatch cannot appear under OR"); } } @@ -159,13 +162,15 @@ TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) { auto fs = std::make_shared(); SchemaManager schema_manager(fs, path); ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); - Table table(fs, path, schema, Identifier("append_09.db", "append_09")); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, path, schema, Identifier("append_09.db", "append_09"))); for (bool table_entry : {false, true}) { ReadContextBuilder builder(path); builder.SetFullTextSearch(search).SetReadFieldNames({"f1", "_f0_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); auto result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "Per-field index score output is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); } @@ -323,7 +328,10 @@ TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { expected_options[read_via_option] = explicit_header ? "read-view" : "table-view"; ASSERT_EQ(dependency_context->root_path, "warehouse"); ASSERT_EQ(dependency_context->options, expected_options); - ASSERT_EQ(internal_context->GetTableSchema()->Options(), table_options); + // The load merged the read options into the table it built - a table is the product of + // the load - so the schema the read serves carries the explicit header over the one the + // schema was written with, the same value the dependency context sees. + ASSERT_EQ(internal_context->GetTableSchema()->Options(), expected_options); } } From 81d9a5ce908f3e117a66968aa3d3657443b58a27 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 15:43:57 +0800 Subject: [PATCH 11/19] fix --- src/paimon/core/table/source/table_read_test.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 072ba3e1d..3cd3b22de 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -142,15 +142,15 @@ TEST(TableReadTest, TestFullTextMatchExecutionIsNotImplemented) { ReadContextBuilder builder(table_path); builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - auto result = - table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); + auto result = table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "FullTextMatch predicate execution is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); builder.SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(context, builder.Finish()); - result = - table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); + result = table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "Score-producing FullTextMatch cannot appear under OR"); } } @@ -169,8 +169,8 @@ TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) { ReadContextBuilder builder(path); builder.SetFullTextSearch(search).SetReadFieldNames({"f1", "_f0_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - auto result = - table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); + auto result = table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "Per-field index score output is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); } From 231589e04b5c83b7713e6f351359c2ea18689e2b Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 17:32:08 +0800 Subject: [PATCH 12/19] fix --- src/paimon/core/table/file_store_table.cpp | 98 ++++++++++++++++++++-- src/paimon/core/table/table_test.cpp | 52 ++++++++++++ 2 files changed, 143 insertions(+), 7 deletions(-) diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index c8e50eb62..6a52903b1 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -19,6 +19,7 @@ #include "paimon/core/table/file_store_table.h" +#include #include #include #include @@ -103,6 +104,26 @@ class MemoryPool; namespace { +/// The identifier a table path names when no catalog name goes with it, the C++ reading of +/// the Java `SchemaManagerUtils#identifierFromPath` over `ignoreIfUnknownDatabase = true`: the +/// directory above the table's names the database when it carries the `.db` suffix, the way a +/// catalog lays its databases out, and a path that names no database - a table sitting directly +/// under the root, say - leaves the database unknown, which is all a caller that only knows the +/// directory can say. +Identifier IdentifierFromPath(const std::string& table_path, const std::string& branch) { + const std::vector paths = StringUtils::Split(table_path, "/"); + if (paths.size() < 2) { + return Identifier(Identifier::kUnknownDatabase, + paths.empty() ? std::string() : paths.front()); + } + const std::string& database = paths[paths.size() - 2]; + const std::string::size_type index = database.rfind(Catalog::DB_SUFFIX); + if (index == std::string::npos) { + return Identifier(Identifier::kUnknownDatabase, paths.back(), branch); + } + return Identifier(database.substr(0, index), paths.back(), branch); +} + /// Recognises a format table behind `table_path`, or returns null when it is not one. /// /// The scan, read, write and commit entry points below all take a path and have to answer the @@ -134,9 +155,10 @@ Result> TryLoadFormatTable( fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", table_path, branch)); } - // A context names a path, never a catalog identifier, so the table is known by the directory - // it sits in. - Identifier identifier(Identifier::kUnknownDatabase, PathUtil::GetName(table_path)); + // A context names a path, never a catalog identifier, so the table is known by the + // directory it sits in - which names the database too, when it carries the `.db` suffix a + // catalog lays its databases out with. + Identifier identifier = IdentifierFromPath(table_path, branch); return FormatTable::Create(file_system, table_path, identifier, checked_pointer_cast(table_schema), /*location_carries_paimon_metadata=*/true, options); @@ -162,6 +184,63 @@ bool FileSystemResolutionMayChange(const TableSchema& schema, return false; } +/// The options a copy is refused to change, mirroring the Java `SchemaManagerUtils# +/// checkAlterTableOption` over the immutable options the C++ side has constants for. These name +/// the physical layout and the file set the data on the table's path was written by: a +/// `bucket-key` the options rename, say, re-hashes the bucket pruning while the files stay +/// hashed by the old key, so the scans a copy serves would filter out the buckets its rows are +/// actually in. The check runs over every option the call gives, before the merge, so it holds +/// on every path that merges options into a table - `Copy` itself and the load a context +/// drives alike. The differences from the Java set: `merge-engine`, `bucket`, +/// `row-tracking.enabled` and `data-evolution.enabled` stay servable as dynamic options here - +/// the C++ tests load tables through context options that move them - and `type`/`branch` are +/// dropped by `Copy` before this point. +const char* const kImmutableOptions[] = { + Options::BUCKET_KEY, + Options::BUCKET_FUNCTION_TYPE, + Options::AGGREGATION_REMOVE_RECORD_ON_DELETE, + Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, + Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP, + Options::FORCE_LOOKUP, + Options::INDEX_FILE_IN_DATA_FILE_DIR, + Options::BLOB_FIELD, + Options::FALLBACK_BLOB_DESCRIPTOR_FIELD, + Options::BLOB_VIEW_FIELD, + Options::PK_CLUSTERING_OVERRIDE, +}; + +/// Whether the dynamic options of a copy stay within what the data on the table's path was +/// laid out with - the C++ reading of the Java `AbstractFileStoreTable#checkImmutability`: +/// every option that says what the schema already says moves nothing, an immutable one that +/// says anything else is refused, and an `ignore-delete` carries the directional rule of its +/// own the Java check carries too: it may only ever be turned on, never back off. +Status CheckImmutableOptions(const TableSchema& schema, + const std::map& dynamic_options) { + const std::map& schema_options = schema.Options(); + for (const auto& [key, value] : dynamic_options) { + auto schema_option = schema_options.find(key); + if (schema_option != schema_options.end() && schema_option->second == value) { + continue; + } + if (std::find(std::begin(kImmutableOptions), std::end(kImmutableOptions), key) != + std::end(kImmutableOptions)) { + return Status::Invalid(fmt::format("Change '{}' is not supported yet.", key)); + } + if (key == Options::IGNORE_DELETE) { + // An unset option is one that never ignored deletes, the same default the options + // parse to. + PAIMON_ASSIGN_OR_RAISE(bool old_ignore_delete, + OptionsUtils::GetValueFromMap(schema_options, key, false)); + PAIMON_ASSIGN_OR_RAISE(bool new_ignore_delete, + OptionsUtils::GetValueFromMap(dynamic_options, key)); + if (old_ignore_delete && !new_ignore_delete) { + return Status::Invalid("Cannot change ignore-delete from true to false."); + } + } + } + return Status::OK(); +} + /// The options a context carries, merged into the table the load hands back: a table is the /// product of the load, so the options speak there - the way `Copy` merges them, over the ones /// the schema carries - and `NewXxx` on the product serves them without merging again. A @@ -210,10 +289,11 @@ Result> TableOfLoadedSchema( if (format_table != nullptr) { return std::shared_ptr
(std::move(format_table)); } - // A context names a path, never a catalog identifier, so the table is known by the directory - // it sits in - the same convention `TryLoadFormatTable` follows above. - Identifier table_identifier = - identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); + // A context names a path, never a catalog identifier, so the table is known by the + // directory it sits in - which names the database too, when it carries the `.db` suffix a + // catalog lays its databases out with. The branch the schema was read on travels the same + // way, folded into the object name. + Identifier table_identifier = identifier.value_or(IdentifierFromPath(path, branch)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, FileStoreTable::Create(explicit_file_system, path, latest_schema, table_identifier, uuid, branch, schema_manager, @@ -540,6 +620,10 @@ Result> FileStoreTable::Copy( std::map overridable = dynamic_options; overridable.erase(Options::TYPE); overridable.erase(Options::BRANCH); + // The physical layout the data on the path was written by is as immovable as the style or + // the branch: the copy serves the same data, so it is refused an option that would lay new + // data out differently than the data it serves was laid out. + PAIMON_RETURN_NOT_OK(CheckImmutableOptions(*loaded_schema, overridable)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); // The managers are handed over, not rebuilt, when the copy resolves the same file system diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index b4478918b..3ea6cf54f 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -973,6 +973,58 @@ TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { ASSERT_FALSE(plan->Splits().empty()); } +TEST(TableTest, TestCopyRefusesThePhysicalOptionsTheDataWasLaidOutWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, + /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // The data on the path was hashed by the `bucket-key` it was written with: a copy that + // renamed the key would prune by a hash its rows never matched, dropping them silently, so + // the change is refused - at `Copy` and at the load a context drives alike, which merges + // through `Copy` - the way the Java `checkImmutability` refuses it. The same goes for the + // other options that name the physical layout, which a key the schema does not carry + // changes as much as one it carries differently. `merge-engine`, `bucket`, + // `row-tracking.enabled` and `data-evolution.enabled` are the options the Java immutable + // set holds that stay servable here. + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_KEY, "v"}}), + "Change 'bucket-key' is not supported yet."); + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_FUNCTION_TYPE, "hash"}}), + "Change 'bucket-function.type' is not supported yet."); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
restyled, + table->Copy({{Options::MERGE_ENGINE, "deduplicate"}})); + { + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.AddOption(Options::BUCKET_KEY, "v"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), + "Change 'bucket-key' is not supported yet."); + } + + // An option that says what the schema already says moves nothing, and a bucket count is + // the physical option a rescale legitimately rewrites, so a copy may move it. An + // `ignore-delete` may only be turned on, never back off. + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
unchanged, + table->Copy({{Options::BUCKET_KEY, "id"}, {Options::BUCKET, "1"}})); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
rescaled, + table->Copy({{Options::BUCKET, "2"}})); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
ignoring_deletes, + table->Copy({{Options::IGNORE_DELETE, "true"}})); + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr
ignore_delete_table, + CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/ignore_delete", + Identifier("db", "igd"), /*branch=*/"main", + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::IGNORE_DELETE, "true"}})); + ASSERT_NOK_WITH_MSG(ignore_delete_table->Copy({{Options::IGNORE_DELETE, "false"}}), + "Cannot change ignore-delete from true to false."); + } +} + TEST(TableTest, TestCreateBuildsTheOperator) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); From 043fbb64a7f093a9163bd08766d27c0333643f6b Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 17:46:07 +0800 Subject: [PATCH 13/19] update --- src/paimon/core/table/table_test.cpp | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 3ea6cf54f..b6cc22859 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -1025,6 +1025,40 @@ TEST(TableTest, TestCopyRefusesThePhysicalOptionsTheDataWasLaidOutWith) { } } +TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + + // A context that names no catalog knows the table by its directory: the directory above + // the table's names the database when it carries the `.db` suffix, the way the Java + // `identifierFromPath` reads it, so the table the load serves is named for both - which + // the load of a directory without a schema says in the error it fails with. The option + // the context carries is what drives the load to that error: it hands the table the merge + // would build, which is where the name is spoken. + const std::string table_dir = dir->Str() + "/my_db.db/my_table"; + ASSERT_OK(fs->Mkdirs(table_dir)); + { + ScanContextBuilder builder(table_dir); + builder.AddOption("read.batch-size", "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), + "not found latest schema of my_db.my_table"); + } + + // A table sitting directly under the root, with no `.db` directory above it, names no + // database: the table is known by its directory alone, the way it was before. + const std::string loose_table_dir = dir->Str() + "/loose_table"; + ASSERT_OK(fs->Mkdirs(loose_table_dir)); + { + ScanContextBuilder builder(loose_table_dir); + builder.AddOption("read.batch-size", "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), + "not found latest schema of loose_table"); + } +} + TEST(TableTest, TestCreateBuildsTheOperator) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); From cc12dd9afe00ef2d781f2e58558f5e100b36521e Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 26 Oct 2026 18:31:39 +0800 Subject: [PATCH 14/19] fix --- include/paimon/catalog/catalog.h | 14 +- include/paimon/catalog/table.h | 48 +- include/paimon/commit_context.h | 10 +- include/paimon/read_context.h | 10 +- include/paimon/scan_context.h | 13 +- include/paimon/table/format/format_table.h | 46 +- include/paimon/write_context.h | 10 +- src/paimon/CMakeLists.txt | 3 + src/paimon/core/catalog/catalog_utils.cpp | 27 + src/paimon/core/catalog/catalog_utils.h | 17 +- .../core/catalog/file_system_catalog.cpp | 15 +- src/paimon/core/catalog/snapshot_commit.h | 27 + ...eted_primary_key_index_maintainer_test.cpp | 26 +- .../append_only_file_store_write_test.cpp | 22 +- .../core/operation/internal_read_context.cpp | 48 + .../core/operation/internal_read_context.h | 31 +- .../core/operation/orphan_files_cleaner.cpp | 11 +- src/paimon/core/schema/schema_validation.cpp | 49 + src/paimon/core/schema/schema_validation.h | 9 + .../table/append_only_file_store_table.cpp | 16 +- .../core/table/append_only_file_store_table.h | 6 - src/paimon/core/table/file_store.cpp | 49 + src/paimon/core/table/file_store.h | 154 ++-- src/paimon/core/table/file_store_table.cpp | 857 +++--------------- src/paimon/core/table/file_store_table.h | 123 ++- src/paimon/core/table/format/format_table.cpp | 41 +- .../table/primary_key_file_store_table.cpp | 17 +- .../core/table/primary_key_file_store_table.h | 6 - .../core/table/source/realtime_table_scan.cpp | 47 + .../core/table/source/realtime_table_scan.h | 9 + .../table/system/audit_log_system_table.cpp | 14 +- .../core/table/system/binlog_system_table.cpp | 3 + .../core/table/system/global_system_tables.h | 9 +- .../table/system/in_memory_system_table.cpp | 5 + .../system/read_optimized_system_table.cpp | 14 +- src/paimon/core/table/system/system_table.cpp | 9 + src/paimon/core/table/system/system_table.h | 28 +- .../core/table/system/system_table_test.cpp | 59 ++ src/paimon/core/table/table.cpp | 33 + src/paimon/core/table/table_test.cpp | 315 ++++++- src/paimon/rest/rest_catalog.cpp | 31 +- src/paimon/rest/rest_catalog.h | 4 +- src/paimon/rest/rest_catalog_test.cpp | 41 + test/inte/realtime_write_inte_test.cpp | 57 +- test/inte/scan_and_read_inte_test.cpp | 79 +- test/inte/scan_inte_test.cpp | 27 +- .../write_and_read_inte_with_index_test.cpp | 3 +- 47 files changed, 1146 insertions(+), 1346 deletions(-) create mode 100644 src/paimon/core/table/table.cpp diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 0bb578383..033e7a822 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -147,12 +147,14 @@ class PAIMON_EXPORT Catalog { /// Gets a table. /// - /// The table's `type` option decides what is handed back: a `format-table` comes back as - /// the `FormatTable` it is - a directory of data files, with no snapshot and no manifest - - /// and every other implemented type as the managed table it is. A `T$system` identifier - /// names a read-only view instead, which comes back as the `SystemTable` it is. All go - /// through this one `Table` interface: `NewScan()` and `NewRead()` serve any of them, - /// `NewWrite()` and `NewCommit()` the writable ones. + /// The table's `type` option decides what is handed back: a `format-table` comes back as the + /// `FormatTable` it is - a directory of data files, with no snapshot and no manifest - and + /// every other implemented type as the managed table it is, while a `T$system` identifier names + /// a read-only `SystemTable`. All go through this one `Table` interface: `NewScan()` and + /// `NewRead()` serve any of them, `NewWrite()` and `NewCommit()` the writable ones. + /// + /// A global system table of the "sys" database reads its rows lazily through the catalog it + /// came from, so a catalog that serves one must be owned by a `std::shared_ptr`. /// /// @param identifier Identifier of the table to get. /// @return A result containing the table, or an error status. diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index a31d3100d..3364f6eff 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -44,36 +44,33 @@ class WriteContext; /// A table provides basic abstraction for table type. /// -/// The state a table of every kind carries - file system, root path, schema and identifier - lives -/// here, and each kind of table fills the operations in: the operators below create this table's -/// scan, read, write or commit, from a context the caller finishes through its builder. What the -/// context asks for that this table cannot honour is refused by name by the implementation rather +/// The state every kind of table carries - file system, root path, schema and identifier - lives +/// here, and each kind fills the operations in. The `NewXxx` operators below are the table-first +/// entry points: `TableScan::Create()` and friends load the table a context names and hand the +/// context to them. What a context asks for that the table cannot honour is refused by name rather /// than silently dropped. class PAIMON_EXPORT Table { public: virtual ~Table() = default; - /// Returns the file system the table was opened with, which serves the table's - /// files. A catalog that hands out per-table file systems opens a table with the - /// one for that table; it is null when the opener had none. The `NewXxx` - /// operations read the table through it. + /// Returns the file system the table was opened with, null when the opener had none. A + /// catalog that hands out per-table file systems opens a table with the one for that table. std::shared_ptr GetFileSystem() const { return file_system_; } - /// Returns the root path of the table in its file system. It is empty when the - /// opener had no location for the table, which a mock in a test is one case of. - /// The `NewXxx` operations serve the table from it. + /// Returns the root path of the table in its file system. It is empty when the opener had no + /// location for the table, which a mock in a test is one case of. const std::string& GetRootPath() const { return root_path_; } - /// A name to identify this table, which each kind of table answers for itself: a managed or - /// format table is named by its identifier, a system table by its own kind. + /// A name to identify this table: a managed or format table answers its identifier's object + /// name, a system table its own kind. virtual std::string Name() const = 0; - /// Full name of the table, which each kind of table answers for itself. A managed or format - /// table answers database.tableName, a system table the same name as `Name()`. + /// Full name of the table: `database.tableName` for a managed or format table, the same name + /// as `Name()` for a system table. virtual std::string FullName() const = 0; /// Returns the metastore UUID, falling back to `FullName()`. @@ -93,41 +90,34 @@ class PAIMON_EXPORT Table { return schema_; } - /// Copies this table with `dynamic_options` on top of the options it carries, which is - /// the precedence every context builder promises for a table it was handed rather than loaded - /// itself: an option given at the call wins over the one the schema stored. - /// - /// `type` is not overridable: it is structural and is read from the schema alone. + /// Copies this table with `dynamic_options` on top of the options it carries: an option given + /// at the call wins over the one the schema stored, the precedence every context builder + /// promises. `type` is not overridable - it is structural and read from the schema alone. /// /// @param dynamic_options Options given at the call. /// @return A result containing the copied table, or the error copying it reports. virtual Result> Copy( const std::map& dynamic_options) const = 0; - /// Creates a scan of this table from a finished scan context, the table-first entry point - /// that `TableScan::Create()` loads the table the context names and hands the context to. + /// Creates a scan of this table from a finished scan context. /// @param context The finished scan context. /// @return The scan, or the error building it reports. virtual Result> NewScan( std::unique_ptr context) const = 0; - /// Creates a read of this table from a finished read context, the table-first entry point - /// that `TableRead::Create()` loads the table the context names and hands the context to. + /// Creates a read of this table from a finished read context. /// @param context The finished read context. /// @return The read, or the error building it reports. virtual Result> NewRead( std::unique_ptr context) const = 0; - /// Creates a write of this table from a finished write context, the table-first entry point - /// that `FileStoreWrite::Create()` loads the table the context names and hands the context to. + /// Creates a write of this table from a finished write context. /// @param context The finished write context. /// @return The write, or the error building it reports. virtual Result> NewWrite( std::unique_ptr context) const = 0; - /// Creates a commit of this table from a finished commit context, the table-first entry point - /// that `FileStoreCommit::Create()` loads the table the context names and hands the context - /// to. + /// Creates a commit of this table from a finished commit context. /// @param context The finished commit context. /// @return The commit, or the error building it reports. virtual Result> NewCommit( diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index a70cacb7a..556db7bd2 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -156,12 +156,10 @@ class PAIMON_EXPORT CommitContextBuilder { ~CommitContextBuilder(); - /// Set a configuration options map. The options take part in loading the table a commit - /// serves: the load merges them over the ones the table's schema carries and the table it - /// hands back serves the merged ones. A caller holding a table already loaded - a load - /// that is over - changes it through `Table::Copy()`; an option contradicting a table held - /// that way is refused when a commit is asked of it. A format table takes the options as - /// its own. + /// Set a configuration options map to set some option entries which are not defined in the + /// table schema or whose values you want to overwrite. A load merges them into the table it + /// hands back; an option contradicting a table you loaded already is refused, so change that + /// one through `Table::Copy()`. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index 8cf565996..d01e28df5 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -324,12 +324,10 @@ class PAIMON_EXPORT ReadContextBuilder { /// When set, read_field_ids and read_field_names are ignored. ReadContextBuilder& SetReadSchema(std::unique_ptr read_schema); - /// Set a configuration options map. The options take part in loading the table a read - /// serves: the load merges them over the ones the table's schema carries and the table it - /// hands back serves the merged ones. A caller holding a table already loaded - a load - /// that is over - changes it through `Table::Copy()`; an option contradicting a table held - /// that way is refused when a read is asked of it. A format table takes the options as its - /// own. + /// Set a configuration options map to set some option entries which are not defined in the + /// table schema or whose values you want to overwrite. A load merges them into the table it + /// hands back; an option contradicting a table you loaded already is refused, so change that + /// one through `Table::Copy()`. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index fe57afb58..2e3f8d9a8 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -208,15 +208,12 @@ class PAIMON_EXPORT ScanContextBuilder { ScanContextBuilder& WithRealtimeContext( const std::shared_ptr& realtime_context); - /// Add a single configuration option. See `SetOptions()` for how the options a context - /// carries are served. + /// Add a single configuration option, served the way `SetOptions()` describes. ScanContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set a configuration options map. The options take part in loading the table a scan - /// serves: the load merges them over the ones the table's schema carries and the table it - /// hands back serves the merged ones. A caller holding a table already loaded - a load - /// that is over - changes it through `Table::Copy()`; an option contradicting a table held - /// that way is refused when a scan is asked of it. A format table takes the options as its - /// own. + /// Set a configuration options map to set some option entries which are not defined in the + /// table schema or whose values you want to overwrite. A load merges them into the table it + /// hands back; an option contradicting a table you loaded already is refused, so change that + /// one through `Table::Copy()`. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index 88444d360..c5bfe60d3 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -48,12 +48,11 @@ class FileSystem; /// Writes only insert - there is nowhere to record an update or a delete - so a read fills /// `_VALUE_KIND` with inserts throughout. /// -/// A format table is a `Table`, so the operators it inherits create its scan, read, write and -/// commit from this table, with the context's options applied on top of the ones it carries. The -/// entry points every other table uses remain open too - `TableScan::Create()` and friends - each -/// of which recognises a format table from the schema under the table path, or takes one already -/// loaded through the `FormatTable` constructor every context builder has, which is how a table -/// whose schema lives in a metastore is reached. +/// A format table is a `Table`, so it is reached both ways: through the `NewXxx` operators below, +/// and through the entry points every other table uses - `TableScan::Create()` and friends - which +/// recognise one from the schema under the table path, or take one already loaded through the +/// `FormatTable` constructor every context builder has, how a table whose schema lives in a +/// metastore is reached. /// /// Every instance lives in a `shared_ptr`: a copy that adds nothing hands the same one back. /// @@ -117,8 +116,7 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t return identifier_.GetFullName(); } - /// Copies the table with `dynamic_options` on top of the options it carries, the precedence - /// every context builder promises. The same table comes back when there is nothing to add. + /// The same table comes back when there is nothing to add. Result> Copy( const std::map& dynamic_options) const override; @@ -160,9 +158,8 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t return options_; } - /// Schema of the table, including its partition fields. It hides `Table::LatestSchema()`, - /// which returns the same schema as the base every table sees; this one hands it back as the - /// `DataSchema` a format table is always created with. + /// Schema of the table, including its partition fields. Hides `Table::LatestSchema()` to hand + /// back the same schema as the `DataSchema` a format table is always created with. std::shared_ptr LatestSchema() const { return std::static_pointer_cast(schema_); } @@ -179,25 +176,16 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t return location_carries_paimon_metadata_; } - /// Creates a scan of this table from a finished scan context, refusing by name what a format - /// table cannot honour. The context's options are applied on top of the ones this table - /// carries, the precedence every context builder promises. + /// The four operators below create this table's scan, read, write and commit, applying the + /// context's options on top of the ones this table carries and refusing by name what a + /// format table cannot honour. Result> NewScan(std::unique_ptr context) const override; - /// Creates a read of this table from a finished read context, refusing by name what a format - /// table cannot honour. The context's options are applied on top of the ones this table - /// carries, the precedence every context builder promises. Result> NewRead(std::unique_ptr context) const override; - /// Creates a write of this table from a finished write context, refusing by name what a format - /// table cannot honour. The context's options are applied on top of the ones this table - /// carries, the precedence every context builder promises. Result> NewWrite( std::unique_ptr context) const override; - /// Creates a commit of this table from a finished commit context, refusing by name what a - /// format table cannot honour. The context's options are applied on top of the ones this - /// table carries, the precedence every context builder promises. Result> NewCommit( std::unique_ptr context) const override; @@ -208,18 +196,14 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata); - /// This table as a `shared_ptr`, which a copy that adds nothing hands back. - /// - /// `shared_from_this()` on a const method hands back a `shared_ptr`, while - /// the copy below needs a mutable one; the cast is safe because the table is never written to. + /// This table as a `shared_ptr`. `shared_from_this()` on a const method hands back a + /// `shared_ptr`, while the copy below needs a mutable one; the cast is + /// safe because the table is never written to. std::shared_ptr SharedSelf() const { return std::const_pointer_cast(shared_from_this()); } - /// Copies this table with `dynamic_options` on top of the options it carries, as a format - /// table rather than the `Table` `Copy()` hands back, which is what its operators need. The - /// same table comes back when there is nothing to add; `type` is not overridable, and - /// `Create()` keeps the schema's own. + /// `Copy()` as a `FormatTable` rather than a `Table`, which is what its operators need. Result> CopySelf( const std::map& dynamic_options) const; diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index 23195793f..f1d3ac06e 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -183,12 +183,10 @@ class PAIMON_EXPORT WriteContextBuilder { ~WriteContextBuilder(); - /// Set a configuration options map. The options take part in loading the table a write - /// serves: the load merges them over the ones the table's schema carries and the table it - /// hands back serves the merged ones. A caller holding a table already loaded - a load - /// that is over - changes it through `Table::Copy()`; an option contradicting a table held - /// that way is refused when a write is asked of it. A format table takes the options as its - /// own. + /// Set a configuration options map to set some option entries which are not defined in the + /// table schema or whose values you want to overwrite. A load merges them into the table it + /// hands back; an option contradicting a table you loaded already is refused, so change that + /// one through `Table::Copy()`. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 8d1234c4d..87dffa570 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -267,6 +267,7 @@ set(PAIMON_CORE_SRCS core/catalog/catalog_utils.cpp core/catalog/file_system_catalog.cpp core/catalog/identifier.cpp + core/catalog/snapshot_commit.cpp core/core_options.cpp core/deletionvectors/deletion_vector.cpp core/deletionvectors/deletion_file_writer.cpp @@ -447,6 +448,7 @@ set(PAIMON_CORE_SRCS core/table/bucket_mode.cpp core/table/file_store.cpp core/table/file_store_table.cpp + core/table/file_store_table_factory.cpp core/table/format/format_file_listing.cpp core/table/format/format_file_naming.cpp core/table/format/format_path_validation.cpp @@ -492,6 +494,7 @@ set(PAIMON_CORE_SRCS core/table/system/system_table.cpp core/table/system/system_table_scan.cpp core/table/system/system_table_schema.cpp + core/table/table.cpp core/tag/tag.cpp core/utils/branch_manager.cpp core/utils/blob_view_lookup.cpp diff --git a/src/paimon/core/catalog/catalog_utils.cpp b/src/paimon/core/catalog/catalog_utils.cpp index 721942a88..4cf128a94 100644 --- a/src/paimon/core/catalog/catalog_utils.cpp +++ b/src/paimon/core/catalog/catalog_utils.cpp @@ -22,10 +22,12 @@ #include #include #include +#include #include "fmt/format.h" #include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/options/table_type.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/defs.h" @@ -43,6 +45,21 @@ Status SystemTableError(const Identifier& identifier, const std::string& action) } // namespace +Identifier CatalogUtils::IdentifierFromPath(const std::string& table_path, + const std::string& branch) { + const std::vector paths = StringUtils::Split(table_path, "/"); + if (paths.size() < 2) { + return Identifier(Identifier::kUnknownDatabase, + paths.empty() ? std::string() : paths.front()); + } + const std::string& database = paths[paths.size() - 2]; + const std::string::size_type index = database.rfind(Catalog::DB_SUFFIX); + if (index == std::string::npos) { + return Identifier(Identifier::kUnknownDatabase, paths.back(), branch); + } + return Identifier(database.substr(0, index), paths.back(), branch); +} + bool CatalogUtils::IsSystemDatabase(const std::string& db_name) { return db_name == Catalog::SYSTEM_DATABASE_NAME; } @@ -74,6 +91,16 @@ Result CatalogUtils::BranchIdentifier(const Identifier& identifier, return Identifier(identifier.GetDatabaseName(), data_table_name, branch); } +Result> CatalogUtils::BranchIdentifier( + const std::optional& identifier, const std::string& branch) { + if (!identifier) { + return std::optional(); + } + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + BranchIdentifier(identifier.value(), branch)); + return std::optional(std::move(branch_identifier)); +} + Status CatalogUtils::CheckNotBranch(const Identifier& identifier, const std::string& action) { PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); if (branch) { diff --git a/src/paimon/core/catalog/catalog_utils.h b/src/paimon/core/catalog/catalog_utils.h index 23cb77203..58775d4df 100644 --- a/src/paimon/core/catalog/catalog_utils.h +++ b/src/paimon/core/catalog/catalog_utils.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include "paimon/catalog/identifier.h" @@ -31,13 +32,19 @@ class Catalog; class FormatTable; class Schema; -/// Checks shared by the catalog implementations. Every check takes an `action` naming -/// the rejected operation in the error message, e.g. "dropTable". +/// The identifier tools and the checks shared by the catalog implementations. Every check takes an +/// `action` naming the rejected operation in the error message, e.g. "dropTable". class CatalogUtils { public: CatalogUtils() = delete; ~CatalogUtils() = delete; + /// The identifier a table path names when no catalog name goes with it, the C++ reading of the + /// Java `SchemaManagerUtils#identifierFromPath` over `ignoreIfUnknownDatabase = true`: the + /// directory above the table's names the database when it carries the `.db` suffix a catalog + /// lays its databases out with, and a path that names no database leaves it unknown. + static Identifier IdentifierFromPath(const std::string& table_path, const std::string& branch); + /// Returns whether `db_name` is the reserved system database "sys". static bool IsSystemDatabase(const std::string& db_name); @@ -58,6 +65,12 @@ class CatalogUtils { static Result BranchIdentifier(const Identifier& identifier, const std::string& branch); + /// The identifier a catalog knows the table by, on `branch`, when there is one to fold a + /// branch into: a caller without an identifier has none to ask the catalog about, and gets + /// none back rather than an error. + static Result> BranchIdentifier( + const std::optional& identifier, const std::string& branch); + /// Fails when `db_name` cannot be used as a single path component, which is required to /// keep the database path under the warehouse. static Status CheckValidDatabaseName(const std::string& db_name); diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index d3c6bd92e..fdcddf20e 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -287,6 +287,9 @@ Result> FileSystemCatalog::LoadTableSchema( if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } + // The table built here is transient - its schema is read within this call and nothing + // outlives it - so its context keeps no shared ownership of the catalog. A catalog that + // hands a global system table back out (`RestCatalog::GetTable`) retains it instead. GlobalSystemTableContext context; context.catalog = this; context.fs = fs_; @@ -316,8 +319,7 @@ Result> FileSystemCatalog::LoadTableSchema( Result> FileSystemCatalog::GetTable(const Identifier& identifier) const { PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { - // A system table reads the files of the data table it belongs to, through the file - // system of that table's catalog, and the schema it exposes is its own read-only one. + // A system table reads the data table's files through this catalog's file system. return LoadSystemTable(identifier); } // The schema is read on the table's own branch: `root/branch/branch-` below the @@ -330,8 +332,7 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide } PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); // The schema's `type` says what the table is. A format table is the same directory read as - // plain data, so the schema read above is all it takes to serve one: no manifest is looked - // for, and the checks a format table adds run in `FormatTable::Create`. + // plain data, so the schema read above is all it takes to serve one; no manifest is read. std::shared_ptr data_schema = checked_pointer_cast(*latest_schema); PAIMON_ASSIGN_OR_RAISE(bool is_format_table, TableTypeDefine::IsFormatTable(data_schema->Options())); @@ -347,8 +348,7 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide // table rather than at the first read. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); - // The branch the schema above was read on, which the table carries so its managers read - // the same branch: `GetBranchNameOrDefault` folds any spelling of `main`. + // The branch the schema above was read on, so the table's managers read the same one. PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); return FileStoreTable::Create(fs_, table_path, checked_pointer_cast(*latest_schema), identifier, /*uuid=*/"", branch); @@ -374,8 +374,7 @@ Result> FileSystemCatalog::LoadSystemTable( if (branch) { dynamic_options[Options::BRANCH] = branch.value(); } - // A system table reads the files of the data table it belongs to, so its root path is - // that table's. + // A system table reads the files of the data table it belongs to, so its root path is that one. PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); return SystemTableLoader::Load(system_table_name.value(), fs_, table_path, latest_schema.value(), dynamic_options); diff --git a/src/paimon/core/catalog/snapshot_commit.h b/src/paimon/core/catalog/snapshot_commit.h index 9500205cb..fa451c204 100644 --- a/src/paimon/core/catalog/snapshot_commit.h +++ b/src/paimon/core/catalog/snapshot_commit.h @@ -18,20 +18,47 @@ #pragma once +#include #include #include #include +#include "paimon/catalog/identifier.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/snapshot.h" namespace paimon { +class Catalog; +class FileSystem; +class SnapshotManager; + /// Interface to commit snapshot atomically. class SnapshotCommit { public: virtual ~SnapshotCommit() = default; + /// The commit a table publishes its snapshots through, the C++ reading of the Java + /// `CatalogEnvironment#snapshotCommit`: a catalog that version-manages the table commits + /// through it, a caller that only wants the request built gets one with no catalog attached, + /// and everything else publishes by renaming the snapshot file into place. + /// + /// @param catalog The catalog the commit was handed, or null for one that has none. + /// @param catalog_identifier The identifier the catalog knows the table by, on the branch the + /// commit commits to. A version-managing catalog needs one and is refused without it. + /// @param table_id The catalog's UUID for the table, captured before the changes were + /// prepared; null when there is none. + /// @param use_rest_catalog_commit Whether the caller asks for the request rather than for the + /// catalog to send it. + /// @param file_system The file system the snapshot file is renamed through. + /// @param snapshot_manager The manager that names the snapshot file renamed. + static Result> Create( + const std::shared_ptr& catalog, + const std::optional& catalog_identifier, + const std::optional& table_id, bool use_rest_catalog_commit, + const std::shared_ptr& file_system, + const std::shared_ptr& snapshot_manager); + /// @param base_snapshot_uuid Base snapshot UUID; null for an absent or legacy snapshot. /// @param snapshot Snapshot to be committed. /// @param branch Branch to publish the snapshot on; an empty name is the main branch. diff --git a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp index 769bbf326..81803ed90 100644 --- a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp +++ b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp @@ -33,7 +33,6 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/global_index/indexed_split_impl.h" @@ -474,21 +473,15 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsRestoresAndReplacesCompacted ASSERT_EQ(0, data_split_count); ReadContextBuilder read_builder(table_path_); + // The source build above keeps lazy decoding enabled. Disable it only for the final data-row + // assertion, whose predicate reader is outside the maintenance path covered by this test. read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) + .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - // The source build above keeps lazy decoding enabled. Disable it only for the final data-row - // assertion, whose predicate reader is outside the maintenance path covered by this test. - // The table is copied with it off because a read through the loaded table refuses an option - // that contradicts the one it carries. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(directory_->Str(), {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded_table, - catalog->GetTable(Identifier("db", "table"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied_table, - loaded_table->Copy({{"orc.read.enable-lazy-decoding", "false"}})); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - copied_table->NewRead(std::move(read_context))); + TableRead::Create(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -714,18 +707,11 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, ContinuesCompactionWhenIndexBuildF ReadContextBuilder read_builder(table_path_); read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) + .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - // Like the test above, the assertion reads with lazy decoding disabled, so it reads through - // a copy of the table carrying it off: a read through the loaded table refuses an option that - // contradicts the one it carries. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(directory_->Str(), {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded_table, - catalog->GetTable(Identifier("db", "table"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied_table, - loaded_table->Copy({{"orc.read.enable-lazy-decoding", "false"}})); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - copied_table->NewRead(std::move(read_context))); + TableRead::Create(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp index 3ce48ee2c..bfeaefced 100644 --- a/src/paimon/core/operation/append_only_file_store_write_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp @@ -36,7 +36,6 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" @@ -514,25 +513,10 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingNewWriterIgnoresExisting /*commit_identifier=*/0); Commit(table_path, avro_options, avro_commit_msgs); - // The second batch is written through a copy of the table carrying the shredding options: - // an option on the context would contradict the table the catalog loaded, which a write - // through that table refuses. - ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir->Str(), avro_options)); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
avro_table, - catalog->GetTable(Identifier("foo", "bar"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
shredded_table, - avro_table->Copy(shredding_options)); - WriteContextBuilder shredded_builder(table_path, commit_user_); - ASSERT_OK_AND_ASSIGN(std::unique_ptr shredded_context, shredded_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto shredded_write, - shredded_table->NewWrite(std::move(shredded_context))); - ASSERT_OK(shredded_write->Write(MakeBatch(logical_schema, R"([ + auto second_commit_msgs = WriteAndPrepare(table_path, logical_schema, shredding_options, R"([ [2, [["c", 3], ["d", 4], ["e", 5]]] - ])"))); - ASSERT_OK_AND_ASSIGN(auto second_commit_msgs, - shredded_write->PrepareCommit(/*wait_compaction=*/false, - /*commit_identifier=*/1)); - ASSERT_OK(shredded_write->Close()); + ])", + /*commit_identifier=*/1); auto second_file_schema = ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), shredding_options); auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/1); diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 3eb169ac7..8866eae9f 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -19,6 +19,7 @@ #include "paimon/core/operation/internal_read_context.h" #include +#include #include #include #include @@ -44,7 +45,10 @@ #include "paimon/common/utils/url_utils.h" #include "paimon/core/options/map_storage_layout.h" #include "paimon/core/schema/arrow_schema_validator.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/nested_projection_utils.h" +#include "paimon/defs.h" #include "paimon/predicate/compound_predicate.h" #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/predicate_utils.h" @@ -296,6 +300,50 @@ Result> InternalReadContext::Create( return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); } +Result> InternalReadContext::CreateForFallbackBranch( + const std::shared_ptr& context, const std::string& branch, + const std::shared_ptr& file_system, const std::string& root_path, + const std::map& table_dynamic_options) { + std::shared_ptr table_schema; + const auto& specific_table_schema = context->GetSpecificTableSchema(); + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { + PAIMON_ASSIGN_OR_RAISE(table_schema, + TableSchema::CreateFromJson(specific_table_schema.value())); + } else { + auto schema_manager = std::make_shared(file_system, root_path, branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, + schema_manager->Latest()); + if (!latest_schema) { + return Status::Invalid( + fmt::format("schema file not found in path {}, branch {}", root_path, branch)); + } + table_schema = latest_schema.value(); + } + assert(table_schema); + auto options = table_schema->Options(); + // The dynamic options the load merged into this table - and the ones its copies accumulated - + // speak for the read, and the fallback branch serves the same read: they are merged over the + // fallback's own schema options the way the load merged them over the table's, so the fallback + // reader recognises what the main one does. `type` stays the branch's own: neither a copy nor + // a fallback restyles the table. + for (const auto& [key, value] : table_dynamic_options) { + if (key != Options::TYPE) { + options[key] = value; + } + } + for (const auto& [key, value] : context->GetOptions()) { + if (key != Options::TYPE) { + options[key] = value; + } + } + if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { + options[Options::BRANCH] = branch; + } else { + options.erase(Options::BRANCH); + } + return Create(context, table_schema, options, file_system, &root_path); +} + Result> InternalReadContext::CreateFromCoreOptions( const std::shared_ptr& context, const std::shared_ptr& table_schema, CoreOptions core_options, const std::string* path) { diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index b7218a686..43783e950 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -59,20 +59,17 @@ class InternalReadContext { }; /// Builds the context of a read a table serves: the options are the table's own, already - /// parsed - the store's `options()` - and the cache the context carries is armed on them - /// here. `path` names the root path the table was opened with, which the read reads from in - /// place of the one named by the context. + /// parsed, and the cache the context carries is armed on them here. `path` names the root path + /// the table was opened with, which the read reads from in place of the context's own. static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, CoreOptions core_options, const std::string* path = nullptr); - /// Builds the context of a read whose options are composed on the spot rather than the - /// table's own - the fallback branch of a read, whose schema belongs to another branch, and - /// the reads a write or a lookup serve inside themselves. The options are parsed from - /// `options` here: the read's own file system is `file_system`, or the context's when null - - /// the scheme map the context carries does not take part. A cache the context carries is - /// armed on the options this parses. + /// Builds the context of a read whose options are composed on the spot rather than the table's + /// own - the fallback branch of a read, whose schema belongs to another branch, and the reads a + /// write or a lookup serve inside themselves. The read's own file system is `file_system`, or + /// the context's when null; the scheme map the context carries does not take part. static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, @@ -80,6 +77,22 @@ class InternalReadContext { const std::shared_ptr& file_system = nullptr, const std::string* path = nullptr); + /// The read context of a fallback branch, whose schema is not the table's own: it is read + /// through a manager of that branch, and the read serves the options that schema was loaded + /// with, the branch it was read from among them. A branch the context carries is not honoured + /// here. The C++ reading of the Java `FileStoreTableFactory#createOtherBranchTable`, which + /// loads the other branch's table the same way for the fallback reader to serve. + /// @param file_system The file system the table was opened with, in place of one resolved from + /// the context. + /// @param root_path The root path the table was opened with, in place of the one the context + /// names. + /// @param table_dynamic_options The dynamic options the load merged into the table - and the + /// ones its copies accumulated. + static Result> CreateForFallbackBranch( + const std::shared_ptr& context, const std::string& branch, + const std::shared_ptr& file_system, const std::string& root_path, + const std::map& table_dynamic_options); + /// Maps score output positions in the read schema to their logical search sources. const std::map& GetIndexScoreBindings() const { return index_score_bindings_; diff --git a/src/paimon/core/operation/orphan_files_cleaner.cpp b/src/paimon/core/operation/orphan_files_cleaner.cpp index 397ba68ea..8fae06e2c 100644 --- a/src/paimon/core/operation/orphan_files_cleaner.cpp +++ b/src/paimon/core/operation/orphan_files_cleaner.cpp @@ -33,6 +33,7 @@ #include "paimon/core/operation/orphan_files_cleaner_impl.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" @@ -150,15 +151,7 @@ Result> CleanContextBuilder::Finish() { Result> OrphanFilesCleaner::Create( std::unique_ptr&& ctx) { - if (ctx == nullptr) { - return Status::Invalid("clean context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "clean")); if (ctx->GetOlderThanMs() < 0) { return Status::Invalid("older than time needs to be greater than or equal to 0."); } diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 3a73838d9..5d112417f 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -76,6 +76,29 @@ namespace { constexpr char kDeletionVectorsMergeOnRead[] = "deletion-vectors.merge-on-read"; +/// The options a copy is refused to change, mirroring the Java `SchemaManagerUtils# +/// checkAlterTableOption` over the immutable options the C++ side has constants for. They name +/// the physical layout the data on the table's path was written by: a `bucket-key` the options +/// rename, say, re-hashes the bucket pruning while the files stay hashed by the old key, so the +/// scans a copy serves would filter out the buckets its rows are actually in. The check runs over +/// every option the call gives, before the merge, so it holds on every path that merges options +/// into a table. Unlike the Java set, `merge-engine`, `bucket`, `row-tracking.enabled` and +/// `data-evolution.enabled` stay servable as dynamic options here, and `type`/`branch` are dropped +/// by `Copy` before this point. +const char* const kImmutableOptions[] = { + Options::BUCKET_KEY, + Options::BUCKET_FUNCTION_TYPE, + Options::AGGREGATION_REMOVE_RECORD_ON_DELETE, + Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, + Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP, + Options::FORCE_LOOKUP, + Options::INDEX_FILE_IN_DATA_FILE_DIR, + Options::BLOB_FIELD, + Options::FALLBACK_BLOB_DESCRIPTOR_FIELD, + Options::BLOB_VIEW_FIELD, + Options::PK_CLUSTERING_OVERRIDE, +}; + bool ContainsBlobField(const std::shared_ptr& field) { if (BlobUtils::IsBlobField(field)) { return true; @@ -402,6 +425,32 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) { return Status::OK(); } +Status SchemaValidation::CheckImmutableOptions( + const TableSchema& schema, const std::map& dynamic_options) { + const std::map& schema_options = schema.Options(); + for (const auto& [key, value] : dynamic_options) { + auto schema_option = schema_options.find(key); + if (schema_option != schema_options.end() && schema_option->second == value) { + continue; + } + if (std::find(std::begin(kImmutableOptions), std::end(kImmutableOptions), key) != + std::end(kImmutableOptions)) { + return Status::Invalid(fmt::format("Change '{}' is not supported yet.", key)); + } + if (key == Options::IGNORE_DELETE) { + // An unset option is one that never ignored deletes, the same default they parse to. + PAIMON_ASSIGN_OR_RAISE(bool old_ignore_delete, + OptionsUtils::GetValueFromMap(schema_options, key, false)); + PAIMON_ASSIGN_OR_RAISE(bool new_ignore_delete, + OptionsUtils::GetValueFromMap(dynamic_options, key)); + if (old_ignore_delete && !new_ignore_delete) { + return Status::Invalid("Cannot change ignore-delete from true to false."); + } + } + } + return Status::OK(); +} + Status SchemaValidation::ValidateNoDuplicateField(const std::vector& field_names, const std::string& error_message_intro) { auto duplicate_field_names = ObjectUtils::DuplicateItems(field_names); diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h index 813aaebb4..ba169c057 100644 --- a/src/paimon/core/schema/schema_validation.h +++ b/src/paimon/core/schema/schema_validation.h @@ -51,6 +51,15 @@ class SchemaValidation { static Status ValidateTableSchema(const TableSchema& schema); + /// Whether a copy's dynamic options stay within what the data on the table's path was laid out + /// with - the C++ reading of the Java `AbstractFileStoreTable#checkImmutability`. An option + /// that says what the schema already says moves nothing, an immutable one that says anything + /// else is refused, and `ignore-delete` carries the Java check's directional rule of its own: + /// it may only ever be turned on, never back off. The immutable options themselves are the + /// Java `CoreOptions.IMMUTABLE_OPTIONS`, as far as the C++ side has constants for them. + static Status CheckImmutableOptions(const TableSchema& schema, + const std::map& dynamic_options); + /// Validates real-time options after operation-specific options have been merged with the /// table schema options. static Status ValidateRealtime(const TableSchema& schema, const CoreOptions& options); diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp index cead79566..fb504ee4c 100644 --- a/src/paimon/core/table/append_only_file_store_table.cpp +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -38,7 +38,6 @@ #include "paimon/core/table/bucket_mode.h" #include "paimon/core/table/file_store.h" #include "paimon/core/table/source/append_only_split_generator.h" -#include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_evolution_split_generator.h" #include "paimon/core/table/source/split_generator.h" #include "paimon/core/utils/snapshot_manager.h" @@ -60,7 +59,7 @@ Result> AppendOnlyFileStoreTable::CreateWrite( const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { // append table - const std::shared_ptr& schema = store_->schema(); + const std::shared_ptr& schema = store_->GetSchema(); const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); if (options.GetBucket() == -1) { @@ -139,7 +138,7 @@ Result> AppendOnlyFileStoreTable::NewFileStoreSca const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const { // TODO(liancheng.lsz): support fallback branch in scan - const std::shared_ptr& table_schema = store_->schema(); + const std::shared_ptr& table_schema = store_->GetSchema(); std::unique_ptr scan; if (core_options.DataEvolutionEnabled()) { PAIMON_ASSIGN_OR_RAISE( @@ -172,15 +171,6 @@ Result> AppendOnlyFileStoreTable::NewSplitGenera source_split_open_file_cost, bucket_mode); } -Result> AppendOnlyFileStoreTable::NewTableRead( - const std::shared_ptr& path_factory, - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor) const { - return std::make_unique(path_factory, internal_context, memory_pool, - executor); -} - CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, @@ -190,7 +180,7 @@ CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( // The schema manager the table carries, which the scans this supplier hands out read the // schema through. const std::shared_ptr& schema_manager = GetSchemaManager(); - const std::shared_ptr& table_schema = store_->schema(); + const std::shared_ptr& table_schema = store_->GetSchema(); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) -> Result> { diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h index 499431eb0..9b4af46ae 100644 --- a/src/paimon/core/table/append_only_file_store_table.h +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -63,12 +63,6 @@ class AppendOnlyFileStoreTable : public FileStoreTable { Result> NewSplitGenerator( const CoreOptions& core_options) const override; - Result> NewTableRead( - const std::shared_ptr& path_factory, - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor) const override; - CommitScanner::ScanSupplier NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index 472626a01..fe8b092c7 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -47,6 +47,19 @@ namespace paimon { +namespace { + +/// Whether an option key takes part in file-system resolution: the backend `file-system` picks, +/// an `fs.` access key carries, or a native S3 key names - `s3.` and its `s3a.` spelling, the +/// prefixes the S3 file system consumes alongside `fs.s3.`/`fs.s3a.`. The scheme-map keys a +/// context carries are the same set. +bool IsFileSystemResolutionKey(const std::string& key) { + return key == Options::FILE_SYSTEM || key.rfind("fs.", 0) == 0 || key.rfind("s3.", 0) == 0 || + key.rfind("s3a.", 0) == 0; +} + +} // namespace + Result> FileStore::Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, @@ -97,6 +110,42 @@ Result> FileStore::Create( return store; } +bool FileStore::FileSystemResolutionMayChange( + const TableSchema& schema, const std::map& dynamic_options) { + const std::map& schema_options = schema.Options(); + for (const auto& [key, value] : dynamic_options) { + if (!IsFileSystemResolutionKey(key)) { + continue; + } + auto schema_option = schema_options.find(key); + if (schema_option == schema_options.end() || schema_option->second != value) { + return true; + } + } + return false; +} + +bool FileStore::FileSystemResolutionAgrees(const std::map& left, + const std::map& right) { + for (const auto& [key, value] : left) { + if (IsFileSystemResolutionKey(key)) { + auto right_option = right.find(key); + if (right_option == right.end() || right_option->second != value) { + return false; + } + } + } + for (const auto& [key, value] : right) { + if (IsFileSystemResolutionKey(key)) { + auto left_option = left.find(key); + if (left_option == left.end() || left_option->second != value) { + return false; + } + } + } + return true; +} + FileStore::FileStore(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager, diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index 66ccc09c2..e343ec2c4 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -47,46 +47,34 @@ class ManifestList; class TableSchema; /// The storage a managed table is served through: the file system, root path, schema, branch, -/// options and the schema and snapshot managers it was loaded with - the state Java Paimon's -/// `AbstractFileStore` carries, parsed once here rather than per operator. `FileStoreTable` hands -/// the state it was loaded with to one of these and asks it back for the pieces every operator -/// opens with. +/// options and managers it was loaded with - the state Java Paimon's `AbstractFileStore` carries, +/// parsed once here rather than per operator. `FileStoreTable` hands the state it was loaded with +/// to one of these and asks it back for the pieces every operator opens with. /// -/// The options are the ones the table was loaded with on this branch - the schema's own, or -/// the ones a load merged over them - the base every operator serves: `NewXxx` takes no -/// options of its own beyond them. A cache is not part of them - `WithCache()` arms a copy an -/// operator makes - so the factories below take the options of the call, not the store's own. -/// -/// The factory methods build a fresh instance on every call, never a cached one: a path factory -/// names each file it builds, and a snapshot manager a write or a commit arms with a catalog -/// loader is not one a scan should be served through afterwards. +/// Every operator serves the options the table was loaded with, so the factories below take the +/// options of the call: a cache is not part of the table's, `WithCache()` arms a copy an operator +/// makes. They build a fresh instance on every call, never a cached one - a snapshot manager a +/// write arms with a catalog loader is not one a scan should be served through afterwards. class FileStore { public: /// @param file_system The file system the caller opened the table through, which wins over - /// the access configuration the schema's options may carry when it is not null. A - /// null one hands the resolution to that configuration: the table is then served by - /// the file system its options name - the way a load that only had a path and a - /// schema to read names it - and by the default local one when they name none. + /// the access configuration the schema's options may carry when it is not null. A null + /// one hands the resolution to that configuration, and to the default local file system + /// when the options name none. /// @param root_path Root path of the table in the file system. - /// @param schema The schema the table was loaded with, whose partition keys name the - /// partitions the paths below are built for and whose options `options()` parses. It - /// is null when the caller had no schema; the operators then report that none was - /// found. + /// @param schema The schema the table was loaded with, whose partition keys name the partitions + /// the paths below are built for and whose options `GetOptions()` parses. Null when the + /// caller had none, which the operators then report. /// @param branch The branch the store serves, normalized here. - /// @param schema_manager A schema manager to carry instead of building one - the one a - /// context dispatch read the schema through, whose cache the operators then serve. - /// @param snapshot_manager A snapshot manager to carry instead of building one. - /// A null manager is built over `root_path` on `branch`, so the operators always read - /// through the managers the store carries. + /// @param schema_manager A schema manager to carry instead of building one - the one a context + /// dispatch read the schema through, whose cache the operators then serve. + /// @param snapshot_manager A snapshot manager to carry instead of building one; a null one is + /// built over `root_path` on `branch`. /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom - /// schemes through. It takes part in building the store's options the way it took - /// part in reading the schema: without it a schema naming a custom scheme would - /// resolve to the default backend once its operators run. It is kept for a copy to - /// hand over again, so the copy resolves as this store does. + /// schemes through, kept so a copy resolves as this store does. /// - /// Building the store parses the schema's options, so it fails when they do not parse: the - /// error is the caller's to see, which is why a store is built through this factory rather - /// than the constructor. + /// Building the store parses the schema's options, so it fails when they do not parse - which + /// is why a store is built through this factory rather than the constructor. static Result> Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, @@ -94,69 +82,74 @@ class FileStore { const std::shared_ptr& snapshot_manager = nullptr, const std::map& fs_scheme_map = {}); - /// The branch this store serves, as resolved when it was built. - const std::string& branch() const { + /// Whether a copy's dynamic options change the file system its store resolves to. If they do, + /// the managers of the table it was copied from are not its to inherit: built over that + /// table's resolution, they would keep reading metadata through the backend the copy just + /// left. + static bool FileSystemResolutionMayChange( + const TableSchema& schema, const std::map& dynamic_options); + + /// Whether two options maps resolve to the same file system: every resolution key has to + /// agree, a missing one on both sides included. + static bool FileSystemResolutionAgrees(const std::map& left, + const std::map& right); + + const std::string& CurrentBranch() const { return branch_; } - /// The schema manager the operators read through. It carries the schema cache of `root_path` - /// on `branch()`. - const std::shared_ptr& schema_manager() const { + /// The schema manager the operators read through, carrying the schema cache of `root_path` on + /// `CurrentBranch()`. + const std::shared_ptr& GetSchemaManager() const { return schema_manager_; } /// The snapshot manager a scan reads through. A write or a commit builds its own: see /// `NewSnapshotManager`. - const std::shared_ptr& snapshot_manager() const { + const std::shared_ptr& GetSnapshotManager() const { return snapshot_manager_; } - /// The schema the table was loaded with, which every operator serves rather than reading the - /// schema again. It is null when the caller had no schema. - const std::shared_ptr& schema() const { + /// The schema the table was loaded with, null when the caller had none. + const std::shared_ptr& GetSchema() const { return schema_; } - /// The file system the table's data is accessed through, as the table was opened with it: - /// the one the options resolve - a schema carrying access configuration names its own, - /// which the file system the caller opened the table through wins over - rather than the - /// one that call handed over, which `specified_file_system()` hands back untouched. - const std::shared_ptr& file_system() const { + /// The file system the table's data is accessed through: the one the options resolve, which + /// the file system the caller opened the table through wins over. + const std::shared_ptr& GetFileSystem() const { return file_system_; } - /// The file system the caller opened the table through, as it was handed to `Create`: null - /// when the call had none, so a copy of the table resolves from its own options again and - /// the runtime ones it merges in - a file-system option among them - can speak. - const std::shared_ptr& specified_file_system() const { + /// The file system the caller opened the table through, as handed to `Create`: null when the + /// call had none, so a copy resolves from its own options again. + const std::shared_ptr& GetSpecifiedFileSystem() const { return specified_file_system_; } /// The scheme-to-identifier map the options were resolved with, kept for a copy to resolve - /// through again: see `Create`'s `fs_scheme_map`. - const std::map& fs_scheme_map() const { + /// through again. + const std::map& GetFsSchemeMap() const { return fs_scheme_map_; } - /// The root path of the table in its file system, as the table was opened with it. - const std::string& root_path() const { + const std::string& GetRootPath() const { return root_path_; } - /// The options the schema was loaded with on `branch()`, which every operator serves. They - /// exist only over a schema: an operator reads them after reporting that no schema was - /// found when there is none. - const CoreOptions& options() const { + /// The options the schema was loaded with on `CurrentBranch()`. They exist only over a schema: + /// an operator reads them after reporting that none was found when there is not. + const CoreOptions& GetOptions() const { assert(core_options_.has_value()); return *core_options_; } /// The path factory an operator builds its metadata paths through: over this store's root - /// path and the partition keys of its schema - which no schema change can alter, so a read - /// serving a fallback branch's schema still names the same partitions. The file format - /// identifier is the caller's to name: a write names the format its level-0 files are - /// written in, which the options may set per level, while a scan, a read and a commit name - /// the table's format. + /// path and the partition keys of its schema, which no schema change can alter, so an + /// operator serving a schema read after the table was loaded still names the same partitions. + /// The file format identifier is the caller's to name: a write names the format its level-0 + /// files are written in, which the options may set per level, while a scan, a read and a + /// commit name the table's format. Result> NewPathFactory( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::string& file_format_identifier, @@ -164,46 +157,39 @@ class FileStore { /// The snapshot manager a write or a commit reads snapshots through, over this store's root /// path and branch. The file system is the caller's to name, from the options its operator - /// serves: a table opened without one has its options build the resolving file system. A - /// catalog that version-manages the table hands its snapshots out itself, so its loader is - /// attached, the catalog captured to keep the versioned interface - and the loader with it - /// - alive. Such a catalog names the table by an identifier, which the context must have - /// supplied. + /// serves. A catalog that version-manages the table hands its snapshots out itself, so its + /// loader is attached and the catalog captured to keep it alive; such a catalog names the + /// table by an identifier, which the context must have supplied. Result> NewSnapshotManager( const CoreOptions& options, const std::shared_ptr& catalog, const std::optional& catalog_identifier) const; - /// The manifest list a scan or a commit reads the table's manifest entries through, over - /// the caller's path factory: an operator's manifests sit in the paths its path factory - /// names. + /// The manifest list a scan or a commit reads the table's manifest entries through. Result> NewManifestList( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// The manifest file a scan or a commit reads a single manifest's entries through, over - /// the caller's path factory and the partition schema its entries are pruned by. + /// The manifest file a scan or a commit reads a single manifest's entries through, pruned by + /// `partition_schema`. Result> NewManifestFile( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& partition_schema, const std::shared_ptr& memory_pool) const; /// The index manifest file a write, a scan or a commit reads the table's index manifest - /// through, over the caller's path factory. + /// through. Result> NewIndexManifestFile( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// The index file handler a write's deletion vectors and indexes are maintained through, or - /// a scan reads them through: the index manifest file above and the path factories over the - /// caller's path factory, in one handler. + /// The index file handler a write maintains its deletion vectors and indexes through, or a + /// scan reads them through. Result> NewIndexFileHandler( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; /// The partition computer a commit names the partitions it writes with, over the partition - /// keys of this store's schema - which no schema change can alter, so a commit serving a - /// schema read after the table was loaded still names the same partitions - and the - /// caller's arrow schema and options. + /// keys of this store's schema and the caller's arrow schema. Result> NewPartitionComputer( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& memory_pool) const; @@ -216,20 +202,14 @@ class FileStore { const std::map& fs_scheme_map); std::shared_ptr file_system_; - /// The file system the caller opened the table through, as handed to `Create` - null when - /// the call had none - kept for a copy to hand over again: what `file_system_` holds is - /// resolved, and would pin the copy to this table's resolution. std::shared_ptr specified_file_system_; - /// The scheme-to-identifier map the options were resolved with, kept for a copy to resolve - /// through again. std::map fs_scheme_map_; std::string root_path_; std::shared_ptr schema_; std::string branch_; std::shared_ptr schema_manager_; std::shared_ptr snapshot_manager_; - /// The options of `schema_` on `branch_`, or unset when there is no schema. `Create` parses - /// them: the constructor cannot report a parse failure. + /// The options of `schema_` on `branch_`, unset when there is no schema. std::optional core_options_; }; diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 6a52903b1..7e68b3567 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -19,13 +19,11 @@ #include "paimon/core/table/file_store_table.h" -#include #include #include #include #include #include -#include #include "fmt/format.h" #include "paimon/catalog/catalog.h" @@ -34,14 +32,10 @@ #include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/binary_row_partition_computer.h" -#include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/options_utils.h" -#include "paimon/common/utils/path_util.h" #include "paimon/common/utils/sensitive_config_utils.h" #include "paimon/common/utils/string_utils.h" -#include "paimon/core/catalog/catalog_snapshot_commit.h" #include "paimon/core/catalog/catalog_utils.h" -#include "paimon/core/catalog/renaming_snapshot_commit.h" #include "paimon/core/catalog/snapshot_commit.h" #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/core_options.h" @@ -54,7 +48,6 @@ #include "paimon/core/operation/file_store_commit_impl.h" #include "paimon/core/operation/file_store_scan.h" #include "paimon/core/operation/internal_read_context.h" -#include "paimon/core/options/table_type.h" #include "paimon/core/realtime/realtime_context_impl.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/schema_validation.h" @@ -62,20 +55,22 @@ #include "paimon/core/snapshot.h" #include "paimon/core/table/append_only_file_store_table.h" #include "paimon/core/table/file_store.h" +#include "paimon/core/table/file_store_table_factory.h" #include "paimon/core/table/primary_key_file_store_table.h" +#include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_evolution_batch_scan.h" #include "paimon/core/table/source/data_table_batch_scan.h" #include "paimon/core/table/source/data_table_stream_scan.h" #include "paimon/core/table/source/fallback_table_read.h" +#include "paimon/core/table/source/key_value_table_read.h" #include "paimon/core/table/source/primary_key_index_batch_scan.h" #include "paimon/core/table/source/read_optimized_scan_options.h" #include "paimon/core/table/source/realtime_table_scan.h" #include "paimon/core/table/source/snapshot/snapshot_reader.h" -#include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/file_store_path_factory.h" -#include "paimon/core/utils/primary_key_table_utils.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" @@ -88,7 +83,6 @@ #include "paimon/scan_context.h" #include "paimon/schema/schema.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/write_context.h" @@ -102,477 +96,9 @@ namespace paimon { class Executor; class MemoryPool; -namespace { - -/// The identifier a table path names when no catalog name goes with it, the C++ reading of -/// the Java `SchemaManagerUtils#identifierFromPath` over `ignoreIfUnknownDatabase = true`: the -/// directory above the table's names the database when it carries the `.db` suffix, the way a -/// catalog lays its databases out, and a path that names no database - a table sitting directly -/// under the root, say - leaves the database unknown, which is all a caller that only knows the -/// directory can say. -Identifier IdentifierFromPath(const std::string& table_path, const std::string& branch) { - const std::vector paths = StringUtils::Split(table_path, "/"); - if (paths.size() < 2) { - return Identifier(Identifier::kUnknownDatabase, - paths.empty() ? std::string() : paths.front()); - } - const std::string& database = paths[paths.size() - 2]; - const std::string::size_type index = database.rfind(Catalog::DB_SUFFIX); - if (index == std::string::npos) { - return Identifier(Identifier::kUnknownDatabase, paths.back(), branch); - } - return Identifier(database.substr(0, index), paths.back(), branch); -} - -/// Recognises a format table behind `table_path`, or returns null when it is not one. -/// -/// The scan, read, write and commit entry points below all take a path and have to answer the -/// same question before they can dispatch, so one place answers it and they cannot disagree. -/// Null is also the answer when no schema was found under the path at all: what to say about a -/// table that is not there is the managed path's to decide, and it says it in more detail. -/// -/// The table type is read from the schema, never from `options`: `type` is structural, and one -/// call's options must not decide what kind of table this is. `branch` names the branch the -/// schema was read from: anything but the main branch is refused once the schema turns out to be -/// a format table's, which has no branches to read. -Result> TryLoadFormatTable( - const std::shared_ptr& file_system, const std::string& table_path, - const std::string& branch, const std::map& options, - const std::shared_ptr& table_schema) { - if (table_schema == nullptr) { - return std::shared_ptr(); - } - // From the schema alone: see `FormatTable::Create()`. - PAIMON_ASSIGN_OR_RAISE(bool is_format_table, - TableTypeDefine::IsFormatTable(table_schema->Options())); - if (!is_format_table) { - return std::shared_ptr(); - } - // A format table has no branches, so it is refused rather than served from the table path - // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. - if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { - return Status::Invalid( - fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", - table_path, branch)); - } - // A context names a path, never a catalog identifier, so the table is known by the - // directory it sits in - which names the database too, when it carries the `.db` suffix a - // catalog lays its databases out with. - Identifier identifier = IdentifierFromPath(table_path, branch); - return FormatTable::Create(file_system, table_path, identifier, - checked_pointer_cast(table_schema), - /*location_carries_paimon_metadata=*/true, options); -} - -/// Whether an option of a copy names the file system its store resolves: the backend -/// `file-system` picks, or the access configuration a `fs.` key carries. A copy whose dynamic -/// options change one of these resolves to another file system, so the managers of the table it -/// was copied from - built over that table's resolution - are not its to inherit: they would -/// keep reading metadata through the backend the copy just left. -bool FileSystemResolutionMayChange(const TableSchema& schema, - const std::map& dynamic_options) { - const std::map& schema_options = schema.Options(); - for (const auto& [key, value] : dynamic_options) { - if (key != Options::FILE_SYSTEM && key.rfind("fs.", 0) != 0) { - continue; - } - auto schema_option = schema_options.find(key); - if (schema_option == schema_options.end() || schema_option->second != value) { - return true; - } - } - return false; -} - -/// The options a copy is refused to change, mirroring the Java `SchemaManagerUtils# -/// checkAlterTableOption` over the immutable options the C++ side has constants for. These name -/// the physical layout and the file set the data on the table's path was written by: a -/// `bucket-key` the options rename, say, re-hashes the bucket pruning while the files stay -/// hashed by the old key, so the scans a copy serves would filter out the buckets its rows are -/// actually in. The check runs over every option the call gives, before the merge, so it holds -/// on every path that merges options into a table - `Copy` itself and the load a context -/// drives alike. The differences from the Java set: `merge-engine`, `bucket`, -/// `row-tracking.enabled` and `data-evolution.enabled` stay servable as dynamic options here - -/// the C++ tests load tables through context options that move them - and `type`/`branch` are -/// dropped by `Copy` before this point. -const char* const kImmutableOptions[] = { - Options::BUCKET_KEY, - Options::BUCKET_FUNCTION_TYPE, - Options::AGGREGATION_REMOVE_RECORD_ON_DELETE, - Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, - Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP, - Options::FORCE_LOOKUP, - Options::INDEX_FILE_IN_DATA_FILE_DIR, - Options::BLOB_FIELD, - Options::FALLBACK_BLOB_DESCRIPTOR_FIELD, - Options::BLOB_VIEW_FIELD, - Options::PK_CLUSTERING_OVERRIDE, -}; - -/// Whether the dynamic options of a copy stay within what the data on the table's path was -/// laid out with - the C++ reading of the Java `AbstractFileStoreTable#checkImmutability`: -/// every option that says what the schema already says moves nothing, an immutable one that -/// says anything else is refused, and an `ignore-delete` carries the directional rule of its -/// own the Java check carries too: it may only ever be turned on, never back off. -Status CheckImmutableOptions(const TableSchema& schema, - const std::map& dynamic_options) { - const std::map& schema_options = schema.Options(); - for (const auto& [key, value] : dynamic_options) { - auto schema_option = schema_options.find(key); - if (schema_option != schema_options.end() && schema_option->second == value) { - continue; - } - if (std::find(std::begin(kImmutableOptions), std::end(kImmutableOptions), key) != - std::end(kImmutableOptions)) { - return Status::Invalid(fmt::format("Change '{}' is not supported yet.", key)); - } - if (key == Options::IGNORE_DELETE) { - // An unset option is one that never ignored deletes, the same default the options - // parse to. - PAIMON_ASSIGN_OR_RAISE(bool old_ignore_delete, - OptionsUtils::GetValueFromMap(schema_options, key, false)); - PAIMON_ASSIGN_OR_RAISE(bool new_ignore_delete, - OptionsUtils::GetValueFromMap(dynamic_options, key)); - if (old_ignore_delete && !new_ignore_delete) { - return Status::Invalid("Cannot change ignore-delete from true to false."); - } - } - } - return Status::OK(); -} - -/// The options a context carries, merged into the table the load hands back: a table is the -/// product of the load, so the options speak there - the way `Copy` merges them, over the ones -/// the schema carries - and `NewXxx` on the product serves them without merging again. A -/// caller holding a table already - a load that is over - changes it through `Copy` itself, -/// and an option that contradicts the table that way is refused. -Result> WithContextOptions( - const std::shared_ptr
& table, const std::shared_ptr& schema, - const std::map& options) { - if (table == nullptr || schema == nullptr || options.empty()) { - return table; - } - const std::map& schema_options = schema->Options(); - for (const auto& [key, value] : options) { - auto schema_option = schema_options.find(key); - // A key the schema does not carry, or carries differently: the merge has something to - // do. Every key agreeing with the schema leaves the table the load built as it is. - if (schema_option == schema_options.end() || schema_option->second != value) { - return table->Copy(options); - } - } - return table; -} - -/// The tail every `TableOf` ends with: the schema one of them read says what the path names. A -/// format table is handed back as the `FormatTable` it is, everything else as the -/// `FileStoreTable` a managed table is, carrying the schema for its `NewXxx` to build from. A -/// null schema is handed over as it is: what to say about a table that is not there is the -/// managed path's to decide, and it says it in more detail. -/// -/// @param file_system The file system the schema above was read through, which a format table -/// is served by - it has no schema options of its own to name another one. -/// @param explicit_file_system The file system the context supplied, which a managed table is -/// opened through when there is one. A null one leaves the table to resolve from the -/// options its schema carries: access configuration the schema was stored with then -/// names the files it serves, rather than being pinned to the one this load happened to -/// read the schema through. -Result> TableOfLoadedSchema( - const std::shared_ptr& file_system, - const std::shared_ptr& explicit_file_system, const std::string& path, - const std::string& branch, const std::map& options, - const std::shared_ptr& schema_manager, - const std::shared_ptr& latest_schema, const std::optional& identifier, - const std::string& uuid = "", const std::map& fs_scheme_map = {}) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - TryLoadFormatTable(file_system, path, branch, options, latest_schema)); - if (format_table != nullptr) { - return std::shared_ptr
(std::move(format_table)); - } - // A context names a path, never a catalog identifier, so the table is known by the - // directory it sits in - which names the database too, when it carries the `.db` suffix a - // catalog lays its databases out with. The branch the schema was read on travels the same - // way, folded into the object name. - Identifier table_identifier = identifier.value_or(IdentifierFromPath(path, branch)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - FileStoreTable::Create(explicit_file_system, path, latest_schema, - table_identifier, uuid, branch, schema_manager, - /*snapshot_manager=*/nullptr, fs_scheme_map)); - return WithContextOptions(table, latest_schema, options); -} - -/// The managed table a context names behind a path: the schema is read through a manager of the -/// context's branch - handed over instead when the caller already read it - and what the schema -/// says is there decides the table `TableOfLoadedSchema` hands back. -/// -/// @param schema_json The schema of the table, when the caller already read it, or null when it -/// has to be read here. It is honoured on the main branch only: a branch publishes its own -/// under `branch/branch-`, which is where the read below goes. -/// @param explicit_file_system The file system the context supplied, which the managed table the -/// schema describes is opened through - see `TableOfLoadedSchema`. -Result> ManagedTableOf( - const std::shared_ptr& file_system, - const std::shared_ptr& explicit_file_system, const std::string& path, - const std::string& branch, const std::map& options, - const std::optional& schema_json, const std::optional& identifier, - const std::string& uuid = "", const std::map& fs_scheme_map = {}) { - auto schema_manager = std::make_shared(file_system, path, branch); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, TableSchema::CreateFromJson(schema_json.value())); - } else { - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager->Latest()); - if (latest) { - latest_schema = *latest; - } - } - return TableOfLoadedSchema(file_system, explicit_file_system, path, branch, options, - schema_manager, latest_schema, identifier, uuid, fs_scheme_map); -} - -/// The identifier a catalog knows the table by, on `branch`: a non-main branch is folded into -/// the identifier, so the catalog is asked for the table on that branch rather than the main one. -Result> MakeBranchCatalogIdentifier( - const std::optional& identifier, const std::string& branch) { - if (!identifier) { - return std::optional(); - } - PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, - CatalogUtils::BranchIdentifier(identifier.value(), branch)); - return std::optional(std::move(branch_identifier)); -} - -/// The schema a catalog may hold for the table, read from it on the main branch only and null -/// otherwise. A branch publishes its own schema under `branch/branch-`, which is where -/// `ManagedTableOf` reads instead, so a catalog answering for no branch schema does not fail the -/// `kind` operation to one. `kind` also names the operator the missing-identifier message speaks -/// of. -Result> CatalogMainBranchSchemaJson( - const std::shared_ptr& catalog, const std::optional& catalog_identifier, - const std::string& branch, const char* kind) { - if (catalog == nullptr) { - return std::optional(); - } - if (!catalog_identifier) { - return Status::Invalid(fmt::format("a catalog {} requires a table identifier", kind)); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - catalog->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(std::string schema_json, schema->GetJsonSchema()); - return std::optional(std::move(schema_json)); - } - return std::optional(); -} - -/// The table a scan context names, read from what the context carries: a format table the -/// builder was handed, a system table behind a `T$system` path, or the table - format or -/// managed - the schema under the context's path says is there. -Result> TableOf(const ScanContext& ctx) { - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), {})); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(ctx.GetPath())); - if (system_table_path) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), - ctx.GetPath(), ctx.GetOptions())); - return std::shared_ptr
(std::move(system_table)); - } - // A format table is planned by listing directories, so it never reaches the manifest path - // below. One `TableScan` interface serves both, as Java Paimon serves both through one - // `ReadBuilder`. - const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), - branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), - ctx.GetIdentifier()); -} - -/// The table a read context names, read from what the context carries: a format table the -/// builder was handed, a system table behind a `T$system` path, or the table - format or -/// managed - the schema under the context's path says is there. -Result> TableOf(const ReadContext& ctx) { - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, - CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), - ctx.GetFileSystemSchemeToIdentifierMap())); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(ctx.GetPath())); - if (system_table_path) { - if (PredicateUtils::ContainsFullTextMatch(ctx.GetPredicate())) { - return Status::NotImplemented( - "FullTextMatch predicate execution is not implemented yet"); - } - if (ctx.HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for system tables"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), - ctx.GetPath(), ctx.GetOptions())); - return std::shared_ptr
(std::move(system_table)); - } - - // A format table has no manifest and a file another engine wrote carries no field ids, so - // which files there are and how a row is put back together is its own. How a file is opened - // is not: both paths go through `DataFileReaderFactory`. - const std::string branch = ctx.GetBranch(); - // The scheme map the schema was read through, so the table it describes resolves custom - // schemes the same way - the data belongs to the metadata, both sit behind the map. - return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetSpecificFileSystem(), - ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), - ctx.GetIdentifier(), /*uuid=*/"", - ctx.GetFileSystemSchemeToIdentifierMap()); -} - -/// The table a write context names, read from what the context carries: a format table the -/// builder was handed, or the table - format or managed - the schema under the context's root -/// path says is there. -Result> TableOf(const WriteContext& ctx) { - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); - std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx.GetFileSystemSchemeToIdentifierMap().empty() && - ctx.GetCatalog() != nullptr) { - if (!ctx.GetIdentifier()) { - return Status::Invalid("a catalog write requires a table identifier"); - } - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the write uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx.GetOptions(), specific_fs, - ctx.GetFileSystemSchemeToIdentifierMap())); - std::string branch = ctx.GetBranch(); - PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, - MakeBranchCatalogIdentifier(ctx.GetIdentifier(), branch)); - // Only the main branch may have a current schema the catalog alone holds. A branch publishes - // its own under `branch/branch-`, which is where the load below and a read of the branch - // both read it, so a catalog answering for no branch schema does not fail a write to one. - PAIMON_ASSIGN_OR_RAISE( - std::optional catalog_table_schema, - CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "write")); - // A format table writes plain data files into a directory, so it never reaches the manifest - // path below. - return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, ctx.GetRootPath(), branch, - ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), /*uuid=*/"", - ctx.GetFileSystemSchemeToIdentifierMap()); -} - -/// The table a commit context names, read from what the context carries: a format table the -/// builder was handed, or the table - format or managed - the schema under the context's root -/// path says is there. -Result> TableOf(const CommitContext& ctx) { - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); - // Use catalog credentials unless the caller supplied a file system. - std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx.GetCatalog() != nullptr) { - if (!ctx.GetIdentifier()) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - // Use the credentials of this table, which a catalog issuing per-table temporary - // ones only hands out through GetTableFileSystem. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx.GetOptions(), specific_fs)); - const std::string& root_path = ctx.GetRootPath(); - // The branch the context was built for, already validated by `CommitContextBuilder`. - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - ctx.GetIdentifier(), ctx.GetOptions(), - /*explicit_branch=*/std::nullopt, "commit")); - PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, - MakeBranchCatalogIdentifier(ctx.GetIdentifier(), branch)); - // Catalog-managed schemas may be absent from the table directory, but only on the main branch: - // a branch publishes its own under `branch/branch-`, which is where the load below and a - // read of the branch both read it, so a catalog answering for no branch schema does not fail a - // commit to one. - PAIMON_ASSIGN_OR_RAISE( - std::optional catalog_table_schema, - CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "commit")); - // A format table commits by renaming files into place, so it never reaches the snapshot path - // below. The managed path here reads the commit's branch, so this reads the same one: the two - // must not dispatch on different schemas. - return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, root_path, branch, - ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), - ctx.GetTableId().value_or("")); -} - -/// The options a context carries against the ones the table was loaded with. A table serves the -/// options it was loaded with, so an option the context contradicts is refused rather than -/// honoured - a caller that means to change them copies the table. A table a load hands back -/// carries the context's own options merged in, so it agrees with the context by construction; -/// the refusal belongs to a table the caller held already, whose load is over. A key the table -/// was not loaded with is no contradiction: it names a runtime setting - a scan mode, a batch -/// size - which is none of the table's business, and is served as given. -Status CheckOptionsAgreeWithTable(const TableSchema& table_schema, - const std::map& context_options) { - const std::map& table_options = table_schema.Options(); - for (const auto& [key, value] : context_options) { - auto table_option = table_options.find(key); - if (table_option != table_options.end() && table_option->second != value) { - // A value may be a credential, and the error travels into logs: it is redacted the - // way every surface that exposes options redacts them. - return Status::Invalid( - fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, - SensitiveConfigUtils::RedactValue(key, value), - SensitiveConfigUtils::RedactValue(key, table_option->second))); - } - } - return Status::OK(); -} - -/// The checks every operator entry point opens with: a context, a memory pool and an executor -/// must all be there for the operator to be built at all. `kind` names the operator the first -/// message speaks of, which is all that sets the entry points' messages apart. -template -Status CheckOperatorContext(const ContextPointer& context, const char* kind) { - if (context == nullptr) { - return Status::Invalid(fmt::format("{} context is null pointer", kind)); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - return Status::OK(); -} - -} // namespace - -Table::Table(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid) - : file_system_(file_system), - root_path_(root_path), - schema_(schema), - identifier_(identifier), - uuid_(uuid) {} - FileStoreTable::FileStoreTable(const std::shared_ptr& store, const Identifier& identifier, const std::string& uuid) - : Table(store->file_system(), store->root_path(), store->schema(), identifier, uuid), + : Table(store->GetFileSystem(), store->GetRootPath(), store->GetSchema(), identifier, uuid), store_(store) {} Result> FileStoreTable::Create( @@ -580,12 +106,12 @@ Result> FileStoreTable::Create( const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, const std::string& branch, const std::shared_ptr& schema_manager, const std::shared_ptr& snapshot_manager, - const std::map& fs_scheme_map) { - // The kind the schema's primary keys name, which no schema change can add or drop, so the - // kind the table is built as is the kind every schema its root path ever held describes. A - // caller without a schema gets the append-only kind, whose operators then report that no - // schema was found. A schema that is not a table schema - the read-only one a system table - // exposes, say - is none to serve a managed table through, so it is refused rather than cast. + const std::map& fs_scheme_map, + const std::map& dynamic_options) { + // The kind the schema's primary keys name, which no schema change can add or drop, so the kind + // the table is built as is the kind every schema its root path ever held describes. A caller + // without a schema gets the append-only kind, whose operators report that none was found; a + // schema that is not a table schema - a system table's read-only one - is refused, not cast. std::shared_ptr table_schema; if (schema != nullptr) { table_schema = std::dynamic_pointer_cast(schema); @@ -594,15 +120,18 @@ Result> FileStoreTable::Create( fmt::format("{} is not a managed table schema", identifier.ToString())); } } - // The store parses the schema's options, so a schema whose options do not parse is refused - // here: the table it names cannot be served, not by one operator more than another. + // The store parses the schema's options, so one that does not parse is refused here. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr store, FileStore::Create(file_system, root_path, table_schema, branch, schema_manager, snapshot_manager, fs_scheme_map)); + std::shared_ptr table; if (table_schema != nullptr && !table_schema->PrimaryKeys().empty()) { - return std::make_shared(store, identifier, uuid); + table = std::make_shared(store, identifier, uuid); + } else { + table = std::make_shared(store, identifier, uuid); } - return std::make_shared(store, identifier, uuid); + table->dynamic_options_ = dynamic_options; + return table; } Result> FileStoreTable::Copy( @@ -612,69 +141,80 @@ Result> FileStoreTable::Copy( return Status::Invalid(fmt::format("not found latest schema of {}", FullName())); } // `type` is structural and read from the schema alone, the promise `Copy` makes: an option - // given at the call must not restyle the table. `branch` is the same kind of promise: the - // copy is the table it was copied from, on the branch that table serves, so a branch option - // given at the call moves neither - it would only claim a branch the copy does not serve - - // and a caller that means to move it has `SwitchToBranch`. Both are dropped before the - // merge, so the copy's options never speak of a branch its operators do not. + // given at the call must not restyle the table. `branch` is the same promise - the copy serves + // the branch this table does, and a caller that means to move has `SwitchToBranch`. Both are + // dropped before the merge, so the copy's options never speak of a branch it does not serve. std::map overridable = dynamic_options; overridable.erase(Options::TYPE); overridable.erase(Options::BRANCH); - // The physical layout the data on the path was written by is as immovable as the style or - // the branch: the copy serves the same data, so it is refused an option that would lay new - // data out differently than the data it serves was laid out. - PAIMON_RETURN_NOT_OK(CheckImmutableOptions(*loaded_schema, overridable)); + // The physical layout the data on the path was written by is as immovable as the style or the + // branch: the copy serves the same data. + PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*loaded_schema, overridable)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); - // The managers are handed over, not rebuilt, when the copy resolves the same file system - // this table does: its operators then read the schema cache this table already warmed. The - // file system is the one the caller opened this table through, not the one it resolved to: - // the copy resolves from its own merged options again, so a file-system option among the - // ones it was given speaks rather than being pinned to this table's resolution - and when - // one does change the resolution, fresh managers are built over it instead, so the metadata - // is read through the same backend the data is. The scheme map travels the same way, so the - // copy resolves custom schemes as this table does. + // The managers are handed over, not rebuilt, when the copy resolves the same file system this + // table does: its operators then read the schema cache this table already warmed. The copy + // resolves from the file system the caller opened this table through, not the one it resolved + // to, so a file-system option it was given speaks; when one does change the resolution, fresh + // managers are built over it instead. The scheme map travels the same way. const bool fs_resolution_may_change = - FileSystemResolutionMayChange(*loaded_schema, overridable); - return FileStoreTable::Create(store_->specified_file_system(), root_path_, + FileStore::FileSystemResolutionMayChange(*loaded_schema, overridable); + // The dynamic options the copy was given join the ones this table was loaded with - which a + // copy of a copy carries the same way - so a read's fallback reader serves what the copy's own + // reader does however many copies deep. The branch was dropped above, so none accumulates, and + // an option given again replaces the one carried, as the merge above does. + std::map accumulated = dynamic_options_; + for (const auto& [key, value] : overridable) { + accumulated.insert_or_assign(key, value); + } + return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, std::shared_ptr(std::move(merged_schema)), identifier_, CatalogUuid().value_or(""), CurrentBranch(), fs_resolution_may_change ? nullptr : GetSchemaManager(), fs_resolution_may_change ? nullptr : GetSnapshotManager(), - store_->fs_scheme_map()); + store_->GetFsSchemeMap(), accumulated); } Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { const std::string target_branch = BranchManager::NormalizeBranch(branch); if (target_branch == CurrentBranch()) { - // The table already serves this branch: a copy of it - schema and managers shared - is - // the same table to the caller. - return FileStoreTable::Create(store_->specified_file_system(), root_path_, schema_, + // The table already serves this branch, so a copy of it is the same table to the caller. + return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, schema_, identifier_, CatalogUuid().value_or(""), CurrentBranch(), GetSchemaManager(), GetSnapshotManager(), - store_->fs_scheme_map()); + store_->GetFsSchemeMap(), dynamic_options_); } - // The schema of the branch switched to, read from its own directory, which a branch without - // one does not have. + // The schema of the branch switched to, read from its own directory. auto schema_manager = std::make_shared(file_system_, root_path_, target_branch); PAIMON_ASSIGN_OR_RAISE(std::optional> latest, schema_manager->Latest()); if (!latest) { return Status::Invalid(fmt::format("branch {} does not exist", target_branch)); } - // A branch other than main is named in the options the schema carries - the way a read - // context names it - so what reads through them serves the branch the managers point at. + // A branch other than main is named in the options the schema carries - the way a read context + // names it - so what reads through them serves the branch the managers point at. std::map overridable; if (target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { overridable[Options::BRANCH] = target_branch; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr branch_schema, (*latest)->MergeOptions(overridable)); - return FileStoreTable::Create(store_->specified_file_system(), root_path_, + // The branch's schema may name another file system than this table's options resolved - a + // recreation that changed the access configuration - and a manager kept through that + // disagreement would read the branch's metadata through the backend this table left, so the + // branch builds its own. The schema is in hand already, so only the cache is lost. + const std::shared_ptr branch_schema_manager = + (store_->GetSpecifiedFileSystem() == nullptr && + (!store_->GetFsSchemeMap().empty() || + !FileStore::FileSystemResolutionAgrees( + branch_schema->Options(), std::static_pointer_cast(schema_)->Options()))) + ? nullptr + : schema_manager; + return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, std::shared_ptr(std::move(branch_schema)), identifier_, CatalogUuid().value_or(""), target_branch, - schema_manager, /*snapshot_manager=*/nullptr, - store_->fs_scheme_map()); + branch_schema_manager, /*snapshot_manager=*/nullptr, + store_->GetFsSchemeMap()); } Status FileStoreTable::RestoreRealtimeCommittedProgress( @@ -695,6 +235,22 @@ Status FileStoreTable::RestoreRealtimeCommittedProgress( return Status::OK(); } +Status FileStoreTable::CheckOptionsAgreeWithTable( + const TableSchema& table_schema, const std::map& context_options) { + const std::map& table_options = table_schema.Options(); + for (const auto& [key, value] : context_options) { + auto table_option = table_options.find(key); + if (table_option != table_options.end() && table_option->second != value) { + // A value may be a credential and the error travels into logs, so it is redacted. + return Status::Invalid( + fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, + SensitiveConfigUtils::RedactValue(key, value), + SensitiveConfigUtils::RedactValue(key, table_option->second))); + } + } + return Status::OK(); +} + std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr&& scan, const std::string& root_path) { scan->WithTablePath(root_path); @@ -703,32 +259,24 @@ std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr> FileStoreTable::NewWrite( std::unique_ptr ctx) const { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "write")); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - // A catalog write names its table; the file system itself comes from the table, which its - // catalog's credentials already opened. + // A catalog write names its table; the file system comes from the table's own credentials. if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { return Status::Invalid("a catalog write requires a table identifier"); } - // The write goes to the branch the table serves: a table is the branch it was loaded for, - // so the context's own branch is not consulted. + // The write goes to the branch the table serves, not the context's own. PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, - MakeBranchCatalogIdentifier(ctx->GetIdentifier(), CurrentBranch())); - // The schema the table was constructed with, so a write serves the table as it was loaded - // rather than reading the schema again. - const std::shared_ptr& schema = store_->schema(); + CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); + const std::shared_ptr& schema = store_->GetSchema(); if (schema == nullptr) { return Status::Invalid( fmt::format("not found latest schema in branch {}", CurrentBranch())); } - // The options the store was built with are the ones this write serves: an option the - // context contradicts is refused rather than honoured, and a key the table was not loaded - // with names a runtime setting the write does not take - `Copy` derives the table that one - // belongs to. PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); - CoreOptions options = store_->options(); + CoreOptions options = store_->GetOptions(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); @@ -767,64 +315,15 @@ Result> FileStoreTable::NewWrite( } Result> FileStoreWrite::Create(std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "write")); - // First the table the context names, then the context handed to that table's `NewWrite`. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); return table->NewWrite(std::move(ctx)); } -namespace { - -Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options, - const ScanContext& context, bool read_optimized) { - if (!context.GetRealtimeContext()) { - return Status::OK(); - } - if (!core_options.RealtimeEnabled()) { - return Status::Invalid("real-time scan requires realtime.enabled=true"); - } - if (core_options.GetBucket() <= 0) { - return Status::Invalid("real-time union read requires fixed bucket mode"); - } - if (core_options.DataEvolutionEnabled()) { - return Status::Invalid("real-time union read does not support data evolution"); - } - if (!table_schema.PrimaryKeys().empty()) { - if (read_optimized) { - return Status::NotImplemented( - "PK real-time union read does not support read-optimized scans"); - } - PAIMON_RETURN_NOT_OK( - PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema)); - } - if (context.IsStreamingMode()) { - return Status::Invalid("real-time union read currently supports batch scans only"); - } - if (context.GetLimit()) { - return Status::Invalid("real-time union read does not support scan limit pushdown"); - } - if (context.GetGlobalIndexResult()) { - return Status::Invalid("real-time union read does not support global index splits"); - } - StartupMode startup_mode = core_options.GetStartupMode(); - const bool specified_snapshot = - startup_mode == StartupMode::FromSnapshot() && core_options.GetScanSnapshotId().has_value(); - if (!(startup_mode == StartupMode::LatestFull() || startup_mode == StartupMode::Latest() || - specified_snapshot)) { - return Status::Invalid( - "real-time union read requires the latest snapshot or an explicit snapshot id"); - } - return Status::OK(); -} - -} // namespace - Result> FileStoreTable::NewScan( std::unique_ptr context) const { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(context, "scan")); - // The schema the table was constructed with, so a scan serves the table as it was loaded - // rather than reading the schema again. - const std::shared_ptr& table_schema = store_->schema(); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + const std::shared_ptr& table_schema = store_->GetSchema(); if (table_schema == nullptr) { return Status::Invalid("not found latest schema"); } @@ -832,20 +331,15 @@ Result> FileStoreTable::NewScan( PAIMON_ASSIGN_OR_RAISE(bool read_optimized, OptionsUtils::GetValueFromMap(context->GetOptions(), kReadOptimizedScanOption, false)); - // The options the store was built with are the ones this scan serves: an option the context - // contradicts is refused above, and a key the table was not loaded with names a runtime - // setting the scan does not take - `Copy` derives the table that one belongs to. The branch - // in them is the table's, not the context's: a table is the branch it was loaded for, so a - // branch the context carries is not honoured. - CoreOptions core_options = store_->options(); - // The cache is a runtime setting of this scan, not of the table: the copy the scan serves - // carries it. + // The branch in the store's options is the table's, so one the context carries is not honoured. + CoreOptions core_options = store_->GetOptions(); + // The cache is a runtime setting of this scan, not of the table. core_options.WithCache(context->GetCache()); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); PAIMON_RETURN_NOT_OK( - ValidateRealtimeScan(*table_schema, core_options, *context, read_optimized)); + RealtimeTableScan::ValidateScan(*table_schema, core_options, *context, read_optimized)); // validate options if (core_options.GetBucket() == -1) { if (!table_schema->PrimaryKeys().empty()) { @@ -880,7 +374,6 @@ Result> FileStoreTable::NewScan( store_->NewManifestFile(core_options, path_factory, partition_schema, context->GetMemoryPool())); - // The managers the table carries, whose schema cache the dispatch already warmed. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr file_store_scan, NewFileStoreScan(manifest_list, manifest_file, arrow_schema, core_options, @@ -930,62 +423,11 @@ Result> FileStoreTable::NewScan( } Result> TableScan::Create(std::unique_ptr context) { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(context, "scan")); - // First the table the context names, then the context handed to that table's `NewScan`. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*context)); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*context)); return table->NewScan(std::move(context)); } -namespace { - -/// The read context of a fallback branch, whose schema is not the table's own: it is read -/// through a manager of that branch, and the options the read serves are the ones that schema -/// was loaded with - the same way the table's own read serves the store's - with the branch it -/// was read from in them. A branch the context carries is not honoured here. -/// @param file_system The file system the table was opened with, which the fallback reads -/// through in place of one resolved from the context. -/// @param root_path The root path the table was opened with, which the fallback reads from in -/// place of the one named by the context. -Result> CreateFallbackReadContext( - const std::shared_ptr& context, const std::string& branch, - const std::shared_ptr& file_system, const std::string& root_path) { - std::shared_ptr table_schema; - const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - auto schema_manager = std::make_shared(file_system, root_path, branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - schema_manager->Latest()); - if (!latest_schema) { - return Status::Invalid( - fmt::format("schema file not found in path {}, branch {}", root_path, branch)); - } - table_schema = latest_schema.value(); - } - assert(table_schema); - auto options = table_schema->Options(); - // The dynamic options the load merged into the table this read serves - a sequence-number - // flag, a batch size - speak for the read, and the fallback branch serves the same read: - // they are merged over the fallback branch's own schema options the way the load merged - // them over the table's, so the fallback reader recognises what the main one does. `type` - // stays the branch's own: a copy never restyles the table, and neither does a fallback. - for (const auto& [key, value] : context->GetOptions()) { - if (key != Options::TYPE) { - options[key] = value; - } - } - if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { - options[Options::BRANCH] = branch; - } else { - options.erase(Options::BRANCH); - } - return InternalReadContext::Create(context, table_schema, options, file_system, &root_path); -} - -} // namespace - Result> FileStoreTable::CreateTableRead( const std::shared_ptr& internal_context, const std::shared_ptr& memory_pool, @@ -1007,29 +449,29 @@ Result> FileStoreTable::CreateTableRead( std::shared_ptr path_factory, store_->NewPathFactory(arrow_schema, core_options, core_options.GetFileFormat()->Identifier(), memory_pool)); - return NewTableRead(path_factory, internal_context, memory_pool, executor); + // The reader is selected by the schema the read serves, not by the table's kind: an + // append-only table can have a primary-key fallback branch and vice versa, so a reader typed + // after the main table would fail the fallback's splits. + if (internal_context->GetPrimaryKeys().empty()) { + return std::make_unique(path_factory, internal_context, memory_pool, + executor); + } + return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); } Result> FileStoreTable::NewRead( std::unique_ptr context) const { std::shared_ptr read_context = std::move(context); - PAIMON_RETURN_NOT_OK(CheckOperatorContext(read_context, "read")); - // The schema the table was constructed with, so a read serves the table as it was loaded - // rather than reading the schema again. Its options are the ones the read serves: an option - // the context would have changed is refused rather than honoured, and a key the table was - // not loaded with names a runtime setting the read does not take - `Copy` derives the table - // that one belongs to. - const std::shared_ptr& loaded_schema = store_->schema(); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); + const std::shared_ptr& loaded_schema = store_->GetSchema(); if (loaded_schema == nullptr) { return Status::Invalid("not found latest schema"); } PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); - // The options the store was built with, which the read reads through on the branch the - // table serves: a table is the branch it was loaded for, so the context's own branch is - // not consulted. - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr internal_context, - InternalReadContext::Create(read_context, loaded_schema, store_->options(), &root_path_)); + // The read reads through the branch the table serves, not the context's own. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, + InternalReadContext::Create(read_context, loaded_schema, + store_->GetOptions(), &root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, CreateTableRead(internal_context, read_context->GetMemoryPool(), @@ -1042,12 +484,11 @@ Result> FileStoreTable::NewRead( return std::move(table_read); } - // The fallback branch is not the table's, so its schema is read through a manager of its - // own rather than the table's. - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr fallback_context, - CreateFallbackReadContext(read_context, /*branch=*/scan_fallback_branch.value(), - file_system_, root_path_)); + // The fallback branch is not the table's, so its schema is read through its own manager. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fallback_context, + InternalReadContext::CreateForFallbackBranch( + read_context, /*branch=*/scan_fallback_branch.value(), file_system_, + root_path_, dynamic_options_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, CreateTableRead(fallback_context, read_context->GetMemoryPool(), @@ -1058,62 +499,23 @@ Result> FileStoreTable::NewRead( } Result> TableRead::Create(std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "read")); - // First the table the context names, then the context handed to that table's `NewRead`. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); return table->NewRead(std::move(ctx)); } -namespace { - -bool IsCatalogCommit(const CommitContext& ctx) { - return AsVersionManaged(ctx.GetCatalog()) != nullptr; -} - -Result> NewSnapshotCommit( - const CommitContext& ctx, const std::optional& catalog_identifier, - const std::shared_ptr& fs, - const std::shared_ptr& snapshot_manager) { - if (IsCatalogCommit(ctx)) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - // Reuse the caller's UUID so table recreation cannot redirect this commit. - return std::shared_ptr(std::make_shared( - ctx.GetCatalog(), catalog_identifier.value(), ctx.GetTableId())); - } - if (ctx.UseRESTCatalogCommit()) { - return std::shared_ptr( - std::make_shared(ctx.GetTableId())); - } - // Catalogs without version management may still supply a table ID. - if (ctx.GetTableId() && ctx.GetCatalog() == nullptr) { - return Status::Invalid( - "a commit which publishes the snapshot by writing it to the table directory sends no " - "commit table request, so there is nowhere to put a table id; commit through a " - "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); - } - return std::shared_ptr( - std::make_shared(fs, snapshot_manager)); -} - -} // namespace - Result> FileStoreTable::NewCommit( std::unique_ptr ctx) const { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "commit")); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - // A catalog commit names its table; the file system itself comes from the table, which its - // catalog's credentials already opened. + // A catalog commit names its table; the file system comes from the table's own credentials. if (specific_fs == nullptr && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { return Status::Invalid("a catalog commit requires a table identifier"); } - // A commit commits to the root path the table was opened with and the branch the table - // serves, in place of the ones named by the context: a table is the branch it was loaded - // for, so the context's own branch is not consulted. + // A commit commits to the root path and branch the table was loaded with, not the context's. PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, - MakeBranchCatalogIdentifier(ctx->GetIdentifier(), CurrentBranch())); + CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); // The id a snapshot records the catalog's current schema by, read through the loader below // rather than trusted from a schema this commit will not re-read. FileStoreCommitImpl::SchemaIdLoader schema_id_loader; @@ -1135,21 +537,15 @@ Result> FileStoreTable::NewCommit( return data_schema->Id(); }; } - // The schema the table was constructed with, so a commit serves the table as it was loaded - // rather than reading the schema again. - const std::shared_ptr& schema = store_->schema(); + const std::shared_ptr& schema = store_->GetSchema(); if (schema == nullptr) { return Status::Invalid( fmt::format("not found latest schema in branch {}", CurrentBranch())); } std::shared_ptr arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - // The options the store was built with are the ones this commit serves: an option the - // context contradicts is refused rather than honoured, and a key the table was not loaded - // with names a runtime setting the commit does not take - `Copy` derives the table that one - // belongs to. PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); - CoreOptions options = store_->options(); + CoreOptions options = store_->GetOptions(); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); assert(options.GetFileSystem()); assert(options.GetFileFormat()); @@ -1157,7 +553,8 @@ Result> FileStoreTable::NewCommit( PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path_)); // Catalog commits do not require atomic rename on the object store. The inte-test marker is // not a table option, so it is read from the context the commit was handed. - if (is_object_store && !ctx->UseRESTCatalogCommit() && !IsCatalogCommit(*ctx) && + if (is_object_store && !ctx->UseRESTCatalogCommit() && + AsVersionManaged(ctx->GetCatalog()) == nullptr && ctx->GetOptions().find("enable-object-store-commit-in-inte-test") == ctx->GetOptions().end()) { return Status::NotImplemented( @@ -1195,9 +592,10 @@ Result> FileStoreTable::NewCommit( NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, arrow_schema, options, ctx->GetExecutor(), ctx->GetMemoryPool()); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr snapshot_commit, - NewSnapshotCommit(*ctx, catalog_identifier, options.GetFileSystem(), snapshot_manager)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_commit, + SnapshotCommit::Create(ctx->GetCatalog(), catalog_identifier, + ctx->GetTableId(), ctx->UseRESTCatalogCommit(), + options.GetFileSystem(), snapshot_manager)); return std::make_unique( ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path_, ctx->GetCommitUser(), @@ -1209,9 +607,8 @@ Result> FileStoreTable::NewCommit( Result> FileStoreCommit::Create( std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "commit")); - // First the table the context names, then the context handed to that table's `NewCommit`. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); return table->NewCommit(std::move(ctx)); } diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index df841d548..1fb307495 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -53,34 +53,25 @@ class SplitGenerator; /// decide the kind of managed table it is - `AppendOnlyFileStoreTable` or /// `PrimaryKeyFileStoreTable`, the `Create` factory below - and that kind builds the pieces the /// flows hand down: the write itself, the scans, the read and the split generator. -/// -/// Its `NewScan`/`NewRead`/`NewWrite`/`NewCommit` build the operators here, from the context plus -/// the state the table was constructed with, so they serve the table as it was loaded. The -/// context-only entry points - `TableScan::Create()`, `TableRead::Create()`, -/// `FileStoreWrite::Create()` and `FileStoreCommit::Create()` - load the table the context names -/// and hand the context to that table's `NewXxx`, which is how a context built without a table -/// reaches the same operators. class FileStoreTable : public Table { public: /// The managed table the schema describes, built as the kind its primary keys name: an /// `AppendOnlyFileStoreTable` when there are none, a `PrimaryKeyFileStoreTable` when there /// are, the way Java Paimon's `FileStoreTableFactory` does. Primary keys cannot be added or - /// dropped by a schema change, so the kind the table was built as is the kind every schema - /// its root path ever held describes. + /// dropped by a schema change, so the kind is the same for every schema the root path held. /// @param schema The schema the table was loaded with, whose primary keys name the kind and - /// whose options the operators serve. It is null when the caller had no schema, and - /// the table is built as the append-only kind - its operators then report that no - /// schema was found. + /// whose options the operators serve. Null when the caller had none, in which case the + /// table is built as the append-only kind and its operators report that none was found. /// @param branch The branch the table serves, which the managers below read and write on. - /// @param schema_manager A schema manager to carry instead of building one - the one a - /// context dispatch read the schema through, whose cache the operators then serve. - /// @param snapshot_manager A snapshot manager to carry instead of building one, the way - /// `Copy` hands its table's over so both share one. - /// A null manager is built over `root_path` on `branch`, so the operators always read - /// through the managers the table carries. + /// @param schema_manager A schema manager to carry instead of building one - see + /// `FileStore::Create`. + /// @param snapshot_manager A snapshot manager to carry instead of building one, the way `Copy` + /// hands its table's over so both share one - see `FileStore::Create`. /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom - /// schemes through, which the store's options resolve through as well - see - /// `FileStore::Create`. `Copy` hands its table's map over. + /// schemes through, which the store's options resolve through as well. `Copy` hands its + /// table's map over. + /// @param dynamic_options The options a copy was given over the ones its schema stored, which + /// `Copy` accumulates - see the member of the same name. static Result> Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, @@ -88,26 +79,19 @@ class FileStoreTable : public Table { const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, const std::shared_ptr& schema_manager = nullptr, const std::shared_ptr& snapshot_manager = nullptr, - const std::map& fs_scheme_map = {}); + const std::map& fs_scheme_map = {}, + const std::map& dynamic_options = {}); - /// The branch this table serves, as resolved when the table was built: the schema the table - /// was loaded with came from it, and its managers point at it. const std::string& CurrentBranch() const { - return store_->branch(); + return store_->CurrentBranch(); } - /// The schema manager the operators read through. It carries the schema cache of `root_path` - /// on `CurrentBranch()`, which every operator serves rather than each reading the schema - /// files again. const std::shared_ptr& GetSchemaManager() const { - return store_->schema_manager(); + return store_->GetSchemaManager(); } - /// The snapshot manager a scan reads through. A write or a commit builds its own: it arms - /// the manager with a catalog snapshot loader, which a shared one would keep serving to - /// scans afterwards. const std::shared_ptr& GetSnapshotManager() const { - return store_->snapshot_manager(); + return store_->GetSnapshotManager(); } std::string Name() const override { @@ -118,17 +102,15 @@ class FileStoreTable : public Table { return identifier_.GetFullName(); } - /// Copies the table with `dynamic_options` on top of the ones its schema stored, the - /// precedence `Copy` promises. The file system, root path and identifier stay as they are: - /// they name this table, which an option does not rename. + /// The file system, root path and identifier stay as they are: they name this table, which an + /// option does not rename. Result> Copy( const std::map& dynamic_options) const override; /// Returns the table of `branch`, as Java Paimon's `switchToBranch` does. The branch switched - /// to is served as it is: its schema is read from its own directory, so the dynamic options - /// of this table are not carried over. Switching to the branch already served hands back a - /// copy of this table, schema and managers shared, and a branch with no schema does not - /// exist. + /// to is served as it is: its schema is read from its own directory, so this table's dynamic + /// options are not carried over. Switching to the branch already served hands back a copy of + /// this table, schema and managers shared, and a branch with no schema does not exist. Result> SwitchToBranch(const std::string& branch) const; Result> NewScan(std::unique_ptr context) const override; @@ -142,38 +124,43 @@ class FileStoreTable : public Table { std::unique_ptr context) const override; private: - /// The read `NewRead` hands back, from the internal context it read the schema into: the - /// path factory comes from the store - a fallback branch's schema carries the table's - /// partition keys, which no schema change can alter - and the read itself is the kind's, - /// `NewTableRead`. + /// The options a context carries against the ones the table was loaded with. A table serves + /// the options it was loaded with, so an option the context contradicts is refused rather than + /// honoured - a caller that means to change them copies the table. A table a load hands back + /// carries the context's own options merged in - `FileStoreTableFactory` does the merging - so + /// it agrees by construction; the refusal belongs to a table the caller held already, whose + /// load is over. A key the table was not loaded with is no contradiction: it names a runtime + /// setting - a scan mode, a batch size - which is none of the table's business, and is served + /// as given. + static Status CheckOptionsAgreeWithTable( + const TableSchema& table_schema, const std::map& context_options); + + /// The read `NewRead` hands back, from the internal context it read the schema into: the path + /// factory comes from the store, and the reader is selected by the schema the read serves - the + /// primary keys it carries - rather than by the table's own kind. Result> CreateTableRead( const std::shared_ptr& internal_context, const std::shared_ptr& memory_pool, const std::shared_ptr& executor) const; protected: - /// The base constructor the kinds of managed table build through, `Create` above: the store - /// the table was loaded with carries every state the kinds hand down - see `Create` for what - /// each piece of it is. + /// The base constructor the kinds of managed table build through: the store carries every state + /// the kinds hand down - see `Create` for what each piece of it is. FileStoreTable(const std::shared_ptr& store, const Identifier& identifier, const std::string& uuid); /// The committed progress a real-time write restores, from the latest snapshot the snapshot - /// manager reads: where the write picks up after, which both kinds of real-time write ask - /// for the same way. + /// manager reads: where the write picks up after, which both kinds ask for the same way. static Status RestoreRealtimeCommittedProgress( const std::shared_ptr& realtime_context, const std::shared_ptr& snapshot_manager, const CoreOptions& options); - /// The scan the table's root path names, which the kinds of file store scan read from: the - /// kinds hand their scans back through it. + /// The scan the table's root path names, which the kinds of file store scan read from. static std::unique_ptr WithTablePath(std::unique_ptr&& scan, const std::string& root_path); - /// The write of this kind of table, which `NewWrite` hands the shared flow's pieces to: the - /// kind's writer, built over the path factory, snapshot manager and options that flow - /// resolved. The schema is the store's own, the one the table was loaded with, and the - /// branch the write serves is the table's, `CurrentBranch()`. + /// The write of this kind of table, which `NewWrite` hands the shared flow's pieces to. The + /// schema and the branch the write serves are the table's own. /// @param io_manager The spill manager the context's temp directory names, or null when the /// context named none. /// @param ignore_previous_files Whether the write starts over rather than appending, which @@ -186,8 +173,7 @@ class FileStoreTable : public Table { const std::shared_ptr& io_manager, bool ignore_previous_files) const = 0; /// The scan of this kind of table, which `NewScan` hands the manifest list and file it built - /// to. The schema is the store's own and the snapshot and schema managers are the table's - /// own, which the scan serves. + /// to, over the table's own schema and managers. virtual Result> NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, @@ -195,22 +181,12 @@ class FileStoreTable : public Table { const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const = 0; - /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. The schema - /// it reads its kind's keys from is the store's own. + /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. virtual Result> NewSplitGenerator( const CoreOptions& core_options) const = 0; - /// The read of this kind of table, which `NewRead` hands the internal context it read the - /// schema into and the path factory it built to. - virtual Result> NewTableRead( - const std::shared_ptr& path_factory, - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor) const = 0; - - /// The manifest entry scan supplier of this kind of table, which `NewCommit` hands the - /// manifest list and file it built to. The schema is the store's own, and the schema manager - /// the scans it hands out read through is the table's own. + /// The manifest entry scan supplier of this kind of table, which `NewCommit` hands the manifest + /// list and file it built to. virtual CommitScanner::ScanSupplier NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, @@ -219,9 +195,14 @@ class FileStoreTable : public Table { const std::shared_ptr& executor, const std::shared_ptr& pool) const = 0; - /// The storage this table serves: the file system, root path, schema, branch and managers it - /// was loaded with, and the factories that build the pieces every operator opens with. + /// The storage this table serves - see `FileStore`. std::shared_ptr store_; + + /// The dynamic options the table was loaded with over the ones its schema stored, which `Copy` + /// accumulates into the copy it derives. They stay separate from the schema they were merged + /// into because the fallback reader of a read needs them alone: it serves another branch's + /// schema, over which the same options are merged again. + std::map dynamic_options_; }; } // namespace paimon diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index cb4141dde..d4b611eac 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -39,6 +39,7 @@ #include "paimon/core/table/format/format_table_file_store_write.h" #include "paimon/core/table/format/format_table_read.h" #include "paimon/core/table/format/format_table_scan.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" @@ -226,15 +227,7 @@ Result> FormatTable::GetArrowSchema() const { Result> FormatTable::NewScan( std::unique_ptr context) const { std::shared_ptr shared_context = std::move(context); - if (shared_context == nullptr) { - return Status::Invalid("scan context is null pointer"); - } - if (shared_context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (shared_context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(shared_context, "scan")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, CopySelf(shared_context->GetOptions())); // Anything a `ScanContext` carries that a format table cannot honour is refused rather than @@ -280,15 +273,7 @@ Result> FormatTable::NewScan( Result> FormatTable::NewRead( std::unique_ptr context) const { std::shared_ptr read_context = std::move(context); - if (read_context == nullptr) { - return Status::Invalid("read context is null pointer"); - } - if (read_context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (read_context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); if (read_context->HasVectorOrFullTextSearch()) { return Status::NotImplemented( "VectorSearch or FullTextSearch is not supported for format tables"); @@ -305,15 +290,7 @@ Result> FormatTable::NewRead( Result> FormatTable::NewWrite( std::unique_ptr context) const { - if (context == nullptr) { - return Status::Invalid("write context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "write")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, CopySelf(context->GetOptions())); if (context->GetCatalog() != nullptr) { return Status::Invalid( @@ -346,15 +323,7 @@ Result> FormatTable::NewWrite( Result> FormatTable::NewCommit( std::unique_ptr context) const { - if (context == nullptr) { - return Status::Invalid("commit context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "commit")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, CopySelf(context->GetOptions())); if (!context->IgnoreEmptyCommit()) { diff --git a/src/paimon/core/table/primary_key_file_store_table.cpp b/src/paimon/core/table/primary_key_file_store_table.cpp index 06e27e0d1..9a5875c9f 100644 --- a/src/paimon/core/table/primary_key_file_store_table.cpp +++ b/src/paimon/core/table/primary_key_file_store_table.cpp @@ -52,7 +52,6 @@ #include "paimon/core/snapshot.h" #include "paimon/core/table/bucket_mode.h" #include "paimon/core/table/file_store.h" -#include "paimon/core/table/source/key_value_table_read.h" #include "paimon/core/table/source/merge_tree_split_generator.h" #include "paimon/core/table/source/split_generator.h" #include "paimon/core/utils/primary_key_table_utils.h" @@ -108,7 +107,7 @@ Result> PrimaryKeyFileStoreTable::CreateWrite( const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { // pk table - const std::shared_ptr& schema = store_->schema(); + const std::shared_ptr& schema = store_->GetSchema(); if (ctx.GetRealtimeContext()) { PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); if (ignore_previous_files) { @@ -228,7 +227,7 @@ Result> PrimaryKeyFileStoreTable::NewFileStoreSca const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const { // TODO(liancheng.lsz): support fallback branch in scan - const std::shared_ptr& table_schema = store_->schema(); + const std::shared_ptr& table_schema = store_->GetSchema(); std::unique_ptr scan; PAIMON_ASSIGN_OR_RAISE( scan, KeyValueFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), manifest_list, @@ -243,7 +242,7 @@ Result> PrimaryKeyFileStoreTable::NewSplitGenera auto source_split_target_size = core_options.GetSourceSplitTargetSize(); auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); // TODO(liancheng.lsz): support evolution - const std::shared_ptr& table_schema = store_->schema(); + const std::shared_ptr& table_schema = store_->GetSchema(); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, table_schema->TrimmedPrimaryKeys()); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, @@ -256,14 +255,6 @@ Result> PrimaryKeyFileStoreTable::NewSplitGenera core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), key_comparator); } -Result> PrimaryKeyFileStoreTable::NewTableRead( - const std::shared_ptr& path_factory, - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor) const { - return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); -} - CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, @@ -273,7 +264,7 @@ CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( // The schema manager the table carries, which the scans this supplier hands out read the // schema through. const std::shared_ptr& schema_manager = GetSchemaManager(); - const std::shared_ptr& table_schema = store_->schema(); + const std::shared_ptr& table_schema = store_->GetSchema(); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) -> Result> { diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h index 9b0ed5c9a..ce7547a71 100644 --- a/src/paimon/core/table/primary_key_file_store_table.h +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -63,12 +63,6 @@ class PrimaryKeyFileStoreTable : public FileStoreTable { Result> NewSplitGenerator( const CoreOptions& core_options) const override; - Result> NewTableRead( - const std::shared_ptr& path_factory, - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor) const override; - CommitScanner::ScanSupplier NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, diff --git a/src/paimon/core/table/source/realtime_table_scan.cpp b/src/paimon/core/table/source/realtime_table_scan.cpp index 42a7171b3..cfb61c17d 100644 --- a/src/paimon/core/table/source/realtime_table_scan.cpp +++ b/src/paimon/core/table/source/realtime_table_scan.cpp @@ -27,17 +27,21 @@ #include "fmt/format.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/core_options.h" #include "paimon/core/operation/commit/realtime_commit_properties.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/snapshot.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/plan_impl.h" #include "paimon/core/table/source/realtime_split.h" #include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/primary_key_table_utils.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/realtime/realtime_context.h" #include "paimon/realtime/realtime_store.h" #include "paimon/scan_context.h" #include "paimon/status.h" +#include "paimon/table/source/startup_mode.h" namespace paimon { @@ -203,6 +207,49 @@ Result>> RealtimeTableScan::CreateRealtimeSpl return result; } +Status RealtimeTableScan::ValidateScan(const TableSchema& table_schema, + const CoreOptions& core_options, const ScanContext& context, + bool read_optimized) { + if (!context.GetRealtimeContext()) { + return Status::OK(); + } + if (!core_options.RealtimeEnabled()) { + return Status::Invalid("real-time scan requires realtime.enabled=true"); + } + if (core_options.GetBucket() <= 0) { + return Status::Invalid("real-time union read requires fixed bucket mode"); + } + if (core_options.DataEvolutionEnabled()) { + return Status::Invalid("real-time union read does not support data evolution"); + } + if (!table_schema.PrimaryKeys().empty()) { + if (read_optimized) { + return Status::NotImplemented( + "PK real-time union read does not support read-optimized scans"); + } + PAIMON_RETURN_NOT_OK( + PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema)); + } + if (context.IsStreamingMode()) { + return Status::Invalid("real-time union read currently supports batch scans only"); + } + if (context.GetLimit()) { + return Status::Invalid("real-time union read does not support scan limit pushdown"); + } + if (context.GetGlobalIndexResult()) { + return Status::Invalid("real-time union read does not support global index splits"); + } + StartupMode startup_mode = core_options.GetStartupMode(); + const bool specified_snapshot = + startup_mode == StartupMode::FromSnapshot() && core_options.GetScanSnapshotId().has_value(); + if (!(startup_mode == StartupMode::LatestFull() || startup_mode == StartupMode::Latest() || + specified_snapshot)) { + return Status::Invalid( + "real-time union read requires the latest snapshot or an explicit snapshot id"); + } + return Status::OK(); +} + RealtimeTableScan::RealtimeTableScan(std::unique_ptr&& disk_scan, bool pk_table, const std::shared_ptr& realtime_context, const std::shared_ptr& path_factory, diff --git a/src/paimon/core/table/source/realtime_table_scan.h b/src/paimon/core/table/source/realtime_table_scan.h index 821e67fd7..318319139 100644 --- a/src/paimon/core/table/source/realtime_table_scan.h +++ b/src/paimon/core/table/source/realtime_table_scan.h @@ -30,14 +30,23 @@ namespace paimon { +class CoreOptions; class FileStorePathFactory; class FileSystem; +class ScanContext; class ScanFilter; class SnapshotManager; +class TableSchema; /// Adds process-local memory splits to a normal data-table batch scan. class RealtimeTableScan : public TableScan { public: + /// The checks a scan that asks for a real-time union read opens with, run before one of these + /// is built: the table has to be able to serve it, and the options have to name a scan one + /// can be served over. A context that asks for none of it passes straight through. + static Status ValidateScan(const TableSchema& table_schema, const CoreOptions& core_options, + const ScanContext& context, bool read_optimized); + RealtimeTableScan(std::unique_ptr&& disk_scan, bool pk_table, const std::shared_ptr& realtime_context, const std::shared_ptr& path_factory, diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index e191b653a..09fd6328f 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -44,6 +44,7 @@ #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" @@ -386,6 +387,7 @@ Result> AuditLogSystemTable::ArrowSchema() const Result> AuditLogSystemTable::NewScan( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { return Status::NotImplemented("audit_log system table predicate pushdown is not supported"); } @@ -395,9 +397,7 @@ Result> AuditLogSystemTable::NewScan( .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - // The data table behind this view is served through the file system the view itself was - // opened with, so a context without one of its own falls back to it rather than the - // default backend. + // The view's own file system when the context names none, rather than the default backend. .WithFileSystem(context->GetSpecificFileSystem() != nullptr ? context->GetSpecificFileSystem() : GetFileSystem()) @@ -420,6 +420,8 @@ Result> AuditLogSystemTable::NewScan( Result> AuditLogSystemTable::NewRead( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return NewChangelogRead(context, std::make_shared()); } @@ -458,10 +460,8 @@ Result> AuditLogSystemTable::CreateDataReadContext( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr base_read_schema, BaseReadSchema()); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); ReadContextBuilder builder(table_path_); - // The data table behind this view is served through the file system the view itself was - // opened with, but only when the context names neither a file system of its own nor a - // scheme map to resolve through - a file system and a scheme map cannot both be named, and - // the context's own answer always wins. + // The view's own file system, but only when the context names neither one of its own nor a + // scheme map to resolve through - a file system and a scheme map cannot both be named. std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { data_table_fs = GetFileSystem(); diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index 8a821d45f..a3c39078d 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -31,6 +31,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/table/source/table_read.h" @@ -100,6 +101,8 @@ Result> BinlogSystemTable::ArrowSchema() const { Result> BinlogSystemTable::NewRead( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return NewChangelogRead(context, CreateBinlogBatchConverter()); } diff --git a/src/paimon/core/table/system/global_system_tables.h b/src/paimon/core/table/system/global_system_tables.h index 3d6936262..eadc83af1 100644 --- a/src/paimon/core/table/system/global_system_tables.h +++ b/src/paimon/core/table/system/global_system_tables.h @@ -33,7 +33,14 @@ class FileSystem; /// Context passed to global system table constructors, providing catalog-level /// access for enumerating databases, tables, and reading metadata. struct GlobalSystemTableContext { - const Catalog* catalog = nullptr; // non-owning pointer + /// The catalog the table reads its rows through, non-owning: the rows are built lazily, + /// so a table that outlives the call that built it must be handed the shared ownership + /// below too, or this pointer dangles at the first `BuildRows`. + const Catalog* catalog = nullptr; + /// The shared ownership a returned table keeps of its catalog, so its lazy `BuildRows` + /// cannot read through one already destroyed. Empty where the table is transient - used + /// within the call, before the catalog can go away. + std::shared_ptr catalog_retainer; std::shared_ptr fs; std::string warehouse; std::map catalog_options; diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index 773f621b8..d21d6fb4d 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -29,6 +29,7 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/core/io/generic_row_to_arrow_array_converter.h" #include "paimon/core/table/system/system_table_scan.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/memory/memory_pool.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" @@ -117,6 +118,10 @@ Result> InMemorySystemTable::NewScan( Result> InMemorySystemTable::NewRead( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + // The rows this view serves are its own, read straight out of memory: there is no search + // over them, so a search the caller asked for would be quietly answered with plain rows. + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return std::make_unique( checked_pointer_cast(shared_from_this()), context->GetMemoryPool()); diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index d7193b5cb..65a48c048 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -27,6 +27,7 @@ #include "paimon/common/types/data_field.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/read_optimized_scan_options.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" @@ -61,6 +62,7 @@ std::map ReadOptimizedSystemTable::ReadOptimizedOption Result> ReadOptimizedSystemTable::NewScan( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetRealtimeContext() && !table_schema_->PrimaryKeys().empty()) { return Status::NotImplemented( "PK real-time union read does not support read-optimized scans"); @@ -71,9 +73,7 @@ Result> ReadOptimizedSystemTable::NewScan( .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - // The data table behind this view is served through the file system the view itself was - // opened with, so a context without one of its own falls back to it rather than the - // default backend. + // The view's own file system when the context names none, rather than the default backend. .WithFileSystem(context->GetSpecificFileSystem() != nullptr ? context->GetSpecificFileSystem() : GetFileSystem()) @@ -108,10 +108,8 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon branch = branch_iter->second; } ReadContextBuilder builder(table_path_); - // The data table behind this view is served through the file system the view itself was - // opened with, but only when the context names neither a file system of its own nor a - // scheme map to resolve through - a file system and a scheme map cannot both be named, and - // the context's own answer always wins. + // The view's own file system, but only when the context names neither one of its own nor a + // scheme map to resolve through - a file system and a scheme map cannot both be named. std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { data_table_fs = GetFileSystem(); @@ -151,6 +149,8 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon Result> ReadOptimizedSystemTable::NewRead( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, CreateDataReadContext(context)); return TableRead::Create(std::move(base_context)); diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index cf4864071..0161ad6b7 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -42,6 +42,7 @@ #include "paimon/core/utils/branch_manager.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" +#include "paimon/read_context.h" #include "paimon/status.h" #include "paimon/write_context.h" @@ -185,6 +186,14 @@ SystemTable::SystemTable(const std::shared_ptr& fs, const std::strin : Table(fs, table_path, /*schema=*/nullptr, Identifier(Identifier::kUnknownDatabase, "system_table"), /*uuid=*/"") {} +Status SystemTable::CheckNoSearch(const std::shared_ptr& context) { + if (context->HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch does not support system-table reads yet"); + } + return Status::OK(); +} + Status SystemTable::AdoptOwnSchema() { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, ArrowSchema()); schema_ = std::make_shared(std::move(arrow_schema)); diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 96eead153..4c235d3cf 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -50,13 +50,10 @@ struct SystemTablePath { bool is_global = false; }; -/// Base class for table-scoped system tables such as `T$options` and `T$snapshots`. -/// -/// A system table is the read-only view a `T$system` identifier names: it exposes a schema of -/// its own and serves scans and reads from the data table it belongs to, through the same -/// `Table` interface as every other kind of table. `Name()` - which each kind of system table -/// answers for itself - is its full name as well: the identifier it is opened by belongs to the -/// data table underneath and would name that one. +/// Base class for table-scoped system tables such as `T$options` and `T$snapshots`: the read-only +/// view a `T$system` identifier names, which exposes a schema of its own and serves scans and +/// reads from the data table it belongs to. `Name()` - which each kind answers for itself - is +/// its full name as well: the identifier it is opened by belongs to the data table underneath. class SystemTable : public Table, public std::enable_shared_from_this { public: std::string FullName() const override { @@ -66,14 +63,11 @@ class SystemTable : public Table, public std::enable_shared_from_this> ArrowSchema() const = 0; - /// A system table has no options of its own to override: its schema is the read-only view it - /// exposes, which is not reconfigured by a copy. The data table underneath is copied through - /// its own `Table`. + /// A system table is read-only: copying it, writing it and committing for it are refused + /// rather than routed to the data table underneath, which its own `Table` serves. Result> Copy( const std::map& dynamic_options) const override; - /// A system table is read-only: writing it or committing for it is refused rather than - /// routed to the data table underneath. Result> NewWrite( std::unique_ptr context) const override; @@ -85,9 +79,13 @@ class SystemTable : public Table, public std::enable_shared_from_this& fs, const std::string& table_path); - /// Loaders call this once the object exists, because `ArrowSchema()` is virtual and cannot - /// run from a constructor: it fills the schema the base hands out with this table's own - /// read-only `SystemTableSchema`. + /// Refuses a read whose context asks for a search: each system table builds a context of its + /// own for the data table underneath, which carries no search, so answering one would return + /// plain rows and quietly drop the search and its top K. + static Status CheckNoSearch(const std::shared_ptr& context); + + /// Loaders call this once the object exists, because `ArrowSchema()` is virtual and cannot run + /// from a constructor: it fills in this table's own read-only `SystemTableSchema`. Status AdoptOwnSchema(); friend class SystemTableLoader; diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 0a9f85a2e..f1f0cbfb1 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -45,10 +45,13 @@ #include "paimon/fs/file_system_factory.h" #include "paimon/memory/memory_pool.h" #include "paimon/metrics.h" +#include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" #include "paimon/result.h" +#include "paimon/scan_context.h" #include "paimon/status.h" +#include "paimon/table/source/table_read.h" #include "paimon/testing/mock/mock_file_batch_reader.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" @@ -301,6 +304,62 @@ TEST(SystemTableTest, TestGlobalSystemTableWithoutCatalogReturnsNotImplemented) "global system table requires catalog context: tables"); } +// A system table reached through `NewXxx` directly - one a catalog handed out - is checked the +// way the static factories check the contexts they take: a null context is refused rather than +// dereferenced. +TEST(SystemTableTest, TestOperatorEntriesRefuseANullContext) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); + + ASSERT_NOK_WITH_MSG(audit_log.NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(binlog.NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(read_optimized.NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(audit_log.NewRead(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(binlog.NewRead(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(read_optimized.NewRead(nullptr), "read context is null pointer"); +} + +// The reads these views serve build a context of their own for the data table underneath, which +// carries no search: a search the caller asked for is refused rather than quietly answered with +// plain rows. +TEST(SystemTableTest, TestNewReadRefusesASearch) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); + + auto make_context = [] { + auto vector_search = std::make_shared( + "v", /*limit=*/3, std::vector{1.0f}, /*pre_filter=*/nullptr, + /*predicate=*/nullptr, VectorSearch::DistanceType::COSINE, + std::map{}); + ReadContextBuilder builder("/tmp/table"); + builder.SetVectorSearch(vector_search); + return builder.Finish(); + }; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr audit_log_context, make_context()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr binlog_context, make_context()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_optimized_context, make_context()); + + ASSERT_NOK_WITH_MSG(audit_log.NewRead(std::move(audit_log_context)), + "VectorSearch or FullTextSearch does not support system-table reads yet"); + ASSERT_NOK_WITH_MSG(binlog.NewRead(std::move(binlog_context)), + "VectorSearch or FullTextSearch does not support system-table reads yet"); + ASSERT_NOK_WITH_MSG(read_optimized.NewRead(std::move(read_optimized_context)), + "VectorSearch or FullTextSearch does not support system-table reads yet"); +} + namespace { /// A catalog that serves the files of every table through `GetTableFileSystem`, like one diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp new file mode 100644 index 000000000..3180a48fe --- /dev/null +++ b/src/paimon/core/table/table.cpp @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/catalog/table.h" + +namespace paimon { + +Table::Table(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid) + : file_system_(file_system), + root_path_(root_path), + schema_(schema), + identifier_(identifier), + uuid_(uuid) {} + +} // namespace paimon diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index b6cc22859..055fc5995 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -36,10 +36,13 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/table/source/fallback_data_split.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" #include "paimon/record_batch.h" #include "paimon/scan_context.h" #include "paimon/schema/schema.h" @@ -49,6 +52,8 @@ #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_catalog.h" #include "paimon/testing/mock/mock_file_system.h" +#include "paimon/testing/utils/counting_cache_test_utils.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" @@ -73,29 +78,6 @@ Result> CreateManagedTable( /*uuid=*/"", branch); } -/// Counts the lookups a scan makes, which is enough to tell that the cache a context carries -/// reached the scan's manifest reading rather than being dropped on the way. -class CountingCache : public Cache { - public: - Result> Get( - const std::shared_ptr& key, - std::function>(const std::shared_ptr&)> - supplier) override { - gets++; - return supplier(key); - } - Status Put(const std::shared_ptr&, const std::shared_ptr&) override { - return Status::OK(); - } - void Invalidate(const std::shared_ptr&) override {} - void InvalidateAll() override {} - size_t Size() const override { - return 0; - } - - int32_t gets = 0; -}; - /// Writes two rows through the table's own write and commit, so a scan below has data to find. void WriteAndCommitTwoRows(const std::shared_ptr
& table, const std::string& root_path) { WriteContextBuilder write_builder(root_path, "commit-user"); @@ -124,6 +106,42 @@ void WriteAndCommitTwoRows(const std::shared_ptr
& table, const std::strin ASSERT_OK(write->Close()); } +/// Writes one `id`/`v` row through the table's own write and commit, so a scan below has data +/// to find. +void WriteAndCommitRow(const std::shared_ptr
& table, const std::string& root_path, + int32_t id, int32_t v) { + WriteContextBuilder write_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + arrow::Int32Builder vs_builder; + ASSERT_TRUE(ids_builder.AppendValues({id}).ok()); + ASSERT_TRUE(vs_builder.AppendValues({v}).ok()); + std::shared_ptr ids_array; + std::shared_ptr vs_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + ASSERT_TRUE(vs_builder.Finish(&vs_array).ok()); + auto struct_array = + arrow::StructArray::Make({ids_array, vs_array}, + {arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32())}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); +} + } // namespace TEST(TableTest, TestTableCarriesManagersOfItsBranch) { @@ -198,6 +216,219 @@ TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { } } +TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + // A primary-key table whose reads fall back to the `fb` branch, and that branch's schema. + SchemaManager schema_manager(fs, path, "main"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::SCAN_FALLBACK_BRANCH, "fb"}})); + SchemaManager fallback_schema_manager(fs, path, "fb"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr fallback_schema, + fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(created))), + identifier, /*uuid=*/"", /*branch=*/"main")); + + // The copy turns the sequence-number column on, and a read projects it through a context + // that carries no options of its own. The main reader serves the copy's merged options, so + // it knows the column; the fallback serves the `fb` branch's schema, whose options do not + // name it, so the copy's own dynamic options are what let the fallback reader know it too - + // without them the read as a whole fails to build. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, + table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "_SEQUENCE_NUMBER"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + copy->NewRead(std::move(read_context))); +} + +TEST(TableTest, TestACopyOfACopyServesTheLatestValueOfARepeatedDynamicOption) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + // A primary-key table whose reads fall back to the `fb` branch, and that branch's schema. + SchemaManager schema_manager(fs, path, "main"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::SCAN_FALLBACK_BRANCH, "fb"}})); + SchemaManager fallback_schema_manager(fs, path, "fb"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr fallback_schema, + fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(created))), + identifier, /*uuid=*/"", /*branch=*/"main")); + + // The option is turned off and then on again by the copy the read serves, so the copy's + // schema carries the latest value. The fallback branch's schema does not carry the option + // at all, so what lets its reader know the column is the copy's accumulated dynamic + // options - which must speak the same latest value the schema does, not the older one a + // first copy left, or the fallback reader fails to build while the main one succeeds. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
off, + table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "false"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, + off->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "_SEQUENCE_NUMBER"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + copy->NewRead(std::move(read_context))); +} + +TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + // The main branch is an append-only table whose reads fall back to the `fb` branch, and + // that branch is laid out as a primary-key table: a second write of the key the first + // wrote is a merge, not a duplicate. + SchemaManager main_schema_manager(fs, path, "main"); + auto row_schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32())}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_created, + main_schema_manager.CreateTable(row_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::SCAN_FALLBACK_BRANCH, "fb"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
main, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(main_created))), + identifier, /*uuid=*/"", /*branch=*/"main")); + + SchemaManager fallback_schema_manager(fs, path, "fb"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr fallback_created, + fallback_schema_manager.CreateTable(row_schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
fallback, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(fallback_created))), + identifier, /*uuid=*/"", /*branch=*/"fb")); + WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/10); + WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); + + // The read of the main table carries its fallback reader, which reads the `fb` branch's + // data through the `fb` branch's own schema - so it is the primary-key read that schema's + // layout asks for, not the append-only one the main table's kind would pick. Its splits + // are the `fb` branch's own, scanned through a table that serves `fb` and read as + // fallback ones, which is how a fallback reader is asked for its rows. + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "v"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, main->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(path).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fallback_scan, + fallback->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fallback_plan, fallback_scan->CreatePlan()); + ASSERT_FALSE(fallback_plan->Splits().empty()); + std::vector> fallback_splits; + for (const std::shared_ptr& split : fallback_plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split != nullptr); + fallback_splits.push_back( + std::make_shared(data_split, /*is_fallback=*/true)); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(fallback_splits)); + + // The one merged row the `fb` branch's primary-key layout leaves behind - at the value + // the second write carried - is what the fallback reader answers with; the append-only + // reader the main table's kind would have picked answers with both rows, or refuses the + // split outright. + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_EQ(array->length(), 1); + auto struct_array = std::dynamic_pointer_cast(array); + ASSERT_TRUE(struct_array != nullptr); + EXPECT_EQ(struct_array->num_fields(), 3); + auto value_kind_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto id_array = std::dynamic_pointer_cast(struct_array->field(1)); + auto v_array = std::dynamic_pointer_cast(struct_array->field(2)); + ASSERT_TRUE(value_kind_array != nullptr); + ASSERT_TRUE(id_array != nullptr); + ASSERT_TRUE(v_array != nullptr); + EXPECT_EQ(value_kind_array->Value(0), 0); + EXPECT_EQ(id_array->Value(0), 1); + EXPECT_EQ(v_array->Value(0), 20); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof_batch, batch_reader->NextBatch()); + EXPECT_TRUE(BatchReader::IsEofBatch(eof_batch)); +} + +TEST(TableTest, TestACopyThatMovesAnS3OptionRebuildsItsManagers) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable( + fs, path, identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}, {"s3.endpoint", "http://old"}})); + auto file_store_table = std::dynamic_pointer_cast(table); + ASSERT_TRUE(file_store_table != nullptr); + + // A native `s3.` option names the file system the S3 backend resolves from, the way an + // `fs.` one does: a copy that moves it re-resolves, so the managers are rebuilt rather + // than inherited - otherwise the copy would read its metadata through the old endpoint + // while its data goes through the new one. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
moved, + table->Copy({{"s3.endpoint", "http://new"}})); + auto moved_file_store_table = std::dynamic_pointer_cast(moved); + ASSERT_TRUE(moved_file_store_table != nullptr); + EXPECT_NE(moved_file_store_table->GetSchemaManager(), file_store_table->GetSchemaManager()); + EXPECT_NE(moved_file_store_table->GetSnapshotManager(), file_store_table->GetSnapshotManager()); + + // A copy whose `s3.` option says what the schema already says resolves the same file + // system, so the managers are the ones this table already holds. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
kept, table->Copy({{"s3.endpoint", "http://old"}})); + auto kept_file_store_table = std::dynamic_pointer_cast(kept); + ASSERT_TRUE(kept_file_store_table != nullptr); + EXPECT_EQ(kept_file_store_table->GetSchemaManager(), file_store_table->GetSchemaManager()); + EXPECT_EQ(kept_file_store_table->GetSnapshotManager(), file_store_table->GetSnapshotManager()); +} + TEST(TableTest, TestSwitchToBranch) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -896,15 +1127,20 @@ TEST(TableTest, TestTheCacheAScanContextCarriesReachesTheScan) { WriteAndCommitTwoRows(table, dir->Str()); // The cache is a runtime setting the scan serves from the context rather than from the - // table's own options, so a scan built with one reads its manifests through it. - auto cache = std::make_shared(); + // table's own options, so a scan built with one reads its manifests through it. A scan reads + // the manifest list of a snapshot on its executor's threads, so the counter the cache keeps + // is guarded rather than a plain field. + auto cache = std::make_shared( + std::map{{CacheKind::MANIFEST, 64 * 1024 * 1024}, + {CacheKind::SNAPSHOT_LIVE_MANIFEST, 64 * 1024 * 1024}}); ScanContextBuilder scan_builder(dir->Str()); scan_builder.WithCache(cache); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_FALSE(plan->Splits().empty()); - ASSERT_GT(cache->gets, 0) << "the cache the scan context carries never reached the scan"; + ASSERT_GT(cache->GetCount(CacheKind::MANIFEST), 0) + << "the cache the scan context carries never reached the scan"; } TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { @@ -1029,33 +1265,44 @@ TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); const std::shared_ptr fs = dir->GetFileSystem(); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("dt", arrow::utf8())}); // A context that names no catalog knows the table by its directory: the directory above // the table's names the database when it carries the `.db` suffix, the way the Java - // `identifierFromPath` reads it, so the table the load serves is named for both - which - // the load of a directory without a schema says in the error it fails with. The option - // the context carries is what drives the load to that error: it hands the table the merge - // would build, which is where the name is spoken. + // `identifierFromPath` reads it, so the table the load serves is named for both. The + // refusal a format table open speaks names the table - and the option the context + // carries is what drives the open to it, since the schema alone opens cleanly. const std::string table_dir = dir->Str() + "/my_db.db/my_table"; ASSERT_OK(fs->Mkdirs(table_dir)); + SchemaManager schema_manager(fs, table_dir); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr format_schema, + schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, + {{"type", "format-table"}})); { ScanContextBuilder builder(table_dir); - builder.AddOption("read.batch-size", "2"); + builder.AddOption(Options::REALTIME_ENABLED, "true"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), - "not found latest schema of my_db.my_table"); + "cannot open format table my_db.my_table"); } // A table sitting directly under the root, with no `.db` directory above it, names no // database: the table is known by its directory alone, the way it was before. const std::string loose_table_dir = dir->Str() + "/loose_table"; ASSERT_OK(fs->Mkdirs(loose_table_dir)); + SchemaManager loose_schema_manager(fs, loose_table_dir); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr loose_schema, + loose_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, + {{"type", "format-table"}})); { ScanContextBuilder builder(loose_table_dir); - builder.AddOption("read.batch-size", "2"); + builder.AddOption(Options::REALTIME_ENABLED, "true"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), - "not found latest schema of loose_table"); + "cannot open format table loose_table"); } } diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 07e4b000b..fec764db3 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -425,10 +425,8 @@ Result> RestCatalog::LoadSystemTable( if (branch) { dynamic_options[Options::BRANCH] = branch.value(); } - // The system table reads the data table's own files - snapshots, manifests, data - so it is - // served by the file system that table's credentials open, which with data tokens is the - // only one that can: `GetTableFileSystem` folds the system identifier to the data table it - // belongs to and hands the catalog-wide file system out when no tokens are issued. + // The system table reads the data table's own files, so it is served by that table's own + // credentials - with data tokens the only file system that can open them. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); return SystemTableLoader::Load(system_table_name.value(), table_file_system, data_table_path, @@ -437,9 +435,8 @@ Result> RestCatalog::LoadSystemTable( Result> RestCatalog::GetTable(const Identifier& identifier) const { // The global system tables of the "sys" database expose the catalog itself rather than any - // table the server stands behind, so they are served locally - the way `LoadTableSchema` - // serves them their schema - and never reach the managed-table load below, whose schema - // cast a read-only system table schema would not survive. + // table the server stands behind, so they are served locally and never reach the managed-table + // load below, whose schema cast a read-only system table schema would not survive. if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { PAIMON_ASSIGN_OR_RAISE(bool supported, GlobalSystemTableLoader::IsSupported(identifier.GetObjectName(), @@ -449,13 +446,21 @@ Result> RestCatalog::GetTable(const Identifier& identifie } GlobalSystemTableContext context; context.catalog = this; + // A global system table builds its rows lazily through the catalog above, so the table + // handed back has to keep it alive or the pointer dangles at the first `BuildRows`. A + // catalog not owned by a `std::shared_ptr` cannot be kept, so it is refused by name. + std::shared_ptr self = weak_from_this().lock(); + if (self == nullptr) { + return Status::Invalid( + "a global system table retains its catalog, so the catalog must be owned by " + "a std::shared_ptr before it is asked for one"); + } + context.catalog_retainer = std::move(self); context.fs = fs_; context.warehouse = warehouse_; context.catalog_options = api_->GetMergedOptions(); return GlobalSystemTableLoader::Load(identifier.GetObjectName(), context); } - // A `T$system` identifier opens the read-only view it names, which reads the data table it - // belongs to and answers for itself through the same `Table` interface. PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); @@ -467,9 +472,8 @@ Result> RestCatalog::GetTable(const Identifier& identifie std::string table_path; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier, &uuid, &table_path)); - // The schema the same response carried says what the table is: a format table is handed - // back as the `FormatTable` it is, built from the location of the same response, rather - // than refused as the managed table it is not. + // The schema the same response carried says what the table is: a format table is handed back + // as the `FormatTable` it is rather than refused as the managed table it is not. std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); if (data_schema != nullptr) { PAIMON_ASSIGN_OR_RAISE(bool is_format_table, @@ -488,8 +492,7 @@ Result> RestCatalog::GetTable(const Identifier& identifie // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); - // The branch the identifier carries, which the schema and path above were read on, so the - // table and its managers serve it. + // The branch the schema and path above were read on, which the table's managers then serve. PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); return FileStoreTable::Create( table_file_system, table_path, schema, identifier, uuid, diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 9a4e6eb06..469096e6e 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -42,7 +42,9 @@ class TableSchema; /// A catalog backed by a REST catalog server. Metadata operations are delegated to /// `RestApi`; table data is accessed through the file system configured by the /// server-merged options. -class RestCatalog : public Catalog, public VersionManagedCatalog { +class RestCatalog : public Catalog, + public VersionManagedCatalog, + public std::enable_shared_from_this { public: /// Creates the catalog: fetches and merges "/v1/config" from the server configured /// by `CatalogOptions::URI`, then builds the file system from the merged options. diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 68081328e..5d418fc63 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -51,10 +51,17 @@ #include "paimon/file_store_commit.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" #include "paimon/rest/mock_rest_server.h" #include "paimon/rest/rest_api.h" +#include "paimon/scan_context.h" #include "paimon/schema/schema.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/plan.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/snapshot_test_helper.h" #include "paimon/testing/utils/testharness.h" @@ -1470,6 +1477,40 @@ TEST_F(RestCatalogTest, SystemDatabase) { ASSERT_TRUE(missing.IsNotExist()) << missing.ToString(); } +TEST_F(RestCatalogTest, GlobalSystemTableRetainsItsCatalog) { + // A global system table builds its rows lazily through the catalog that served it, so the + // table handed back keeps that catalog alive: reading it after the caller let its own + // reference go still answers, rather than reading through a catalog already destroyed. + ASSERT_OK_AND_ASSIGN(std::unique_ptr created, CreateRestCatalog()); + std::shared_ptr catalog = std::move(created); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("sys", "tables"))); + std::weak_ptr weak = catalog; + catalog.reset(); + ASSERT_FALSE(weak.expired()) << "the global system table does not retain its catalog"; + + // The rows are built through the retained catalog, so they still answer once the caller's + // own reference is gone: the one table the catalog stands behind shows up in them. + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_GT(array->length(), 0); +} + TEST_F(RestCatalogTest, ListSnapshots) { ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index fd9b49650..5de6c9fa7 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -43,7 +43,6 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/table/special_fields.h" @@ -560,41 +559,6 @@ class RealtimeWriteInteTest : public ::testing::Test { return FileStoreWrite::Create(std::move(context)); } - // A writer that means to change an option the table was created with writes through a copy - // carrying it, which the context then agrees with. - Result> CopiedTable( - const std::map& overrides) const { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, Catalog::Create(dir_->Str(), {})); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, - catalog->GetTable(Identifier("foo", "bar"))); - return table->Copy(overrides); - } - - // Evolves the table schema to the current options with its fields unchanged, the way an - // ALTER TABLE changing an option would: the table itself then serves the changed option. - // The fields are carried over from the latest schema so their attributes survive: rebuilt - // raw fields would lose the non-nullable primary keys the table schema enforces. - Status EvolveSchemaWithOptions() const { - SchemaManager schema_manager(dir_->GetFileSystem(), table_path_); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (!latest.has_value()) { - return Status::Invalid("table schema does not exist"); - } - return TestHelper::WriteNextSchema(dir_->GetFileSystem(), table_path_, - latest.value()->Fields(), - latest.value()->HighestFieldId(), options_); - } - - Result> CreateRealtimeWriter( - const std::shared_ptr& realtime_context, - const std::shared_ptr
& table) const { - WriteContextBuilder builder(table_path_, commit_user_); - builder.SetOptions(options_).WithStreamingMode(true).WithRealtimeContext(realtime_context); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return table->NewWrite(std::move(context)); - } - Result> CreateRealtimeWriter() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context, RealtimeContext::Create()); @@ -1285,36 +1249,32 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeOperationsRequireEnabledOption) { disabled_options[Options::REALTIME_ENABLED] = "false"; ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - // The table was created enabled, so each operation below goes through a copy carrying it - // disabled, whose contexts then agree: each reports that real-time needs the option itself. - ASSERT_OK_AND_ASSIGN(std::shared_ptr
disabled_table, - CopiedTable({{Options::REALTIME_ENABLED, "false"}})); WriteContextBuilder write_builder(table_path_, commit_user_); write_builder.SetOptions(disabled_options) .WithStreamingMode(true) .WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(disabled_table->NewWrite(std::move(write_context)), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), "real-time write requires realtime.enabled=true"); ScanContextBuilder scan_builder(table_path_); scan_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(disabled_table->NewScan(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "real-time scan requires realtime.enabled=true"); ReadContextBuilder read_builder(table_path_); read_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(disabled_table->NewRead(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "real-time read requires realtime.enabled=true"); CommitContextBuilder commit_builder(table_path_, commit_user_); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.SetOptions(disabled_options).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - disabled_table->NewCommit(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context))); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(/*realtime_commits=*/{}, /*commit_identifier=*/0, /*watermark=*/std::nullopt), @@ -2124,9 +2084,6 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateAcrossMultipleFileIndexStor ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; - // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: - // the schema is evolved to it, so every later write, commit, and read serves it. - ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_batch, @@ -3809,9 +3766,6 @@ TEST_F(RealtimeWriteInteTest, TestAppendRealtimeReadUsesEmbeddedAndExternalBitma ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; - // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: - // the schema is evolved to it, so every later write, commit, and read serves it. - ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN( @@ -4129,9 +4083,6 @@ TEST_F(RealtimeWriteInteTest, TestPkDvRealtimeReadUsesEmbeddedAndExternalBitmapI ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; - // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: - // the schema is evolved to it, so every later write, commit, and read serves it. - ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_batch, diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index 4fa4e1da1..b7cc73b96 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -36,9 +36,6 @@ #include "fmt/ranges.h" #include "gtest/gtest.h" #include "paimon/bucket/bucket_id_calculator.h" -#include "paimon/catalog/catalog.h" -#include "paimon/catalog/identifier.h" -#include "paimon/catalog/table.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/table/special_fields.h" @@ -799,36 +796,19 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { } ASSERT_GE(other_index, 0); // These keys share a current two-bucket selection but need different historical buckets. - // The data was written with four buckets, so reading a rescaled layout reads through a - // copy of the table carrying the rescaled bucket count: an option on the context would - // contradict the table the catalog loaded. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, - Catalog::Create(test_dir->Str(), {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - catalog->GetTable(Identifier("foo", "bar"))); for (int32_t lookup_index : {32, other_index}) { + // The data was written with four buckets. Read it with rescaled table options too. for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); const std::string lookup_key = fmt::format("key{:03}", lookup_index); auto lookup_predicate = PredicateBuilder::Equal( 0, "rowkey", field_type, Literal(field_type, lookup_key.data(), lookup_key.size())); - // The rescaled bucket count and, with it, the cache settings `FinishScanContext` - // also puts on the context: a table already held takes its runtime settings - // through the copy it is made of, and the context carries them only to agree - // with it. - std::map dynamic_options{ - {Options::BUCKET, current_bucket_count}}; - if (EnableSnapshotLiveManifestCache()) { - dynamic_options.emplace(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3"); - } - ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, - table->Copy(dynamic_options)); ScanContextBuilder scan_builder(table_path); scan_builder.SetPredicate(lookup_predicate) .AddOption(Options::BUCKET, current_bucket_count); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_builder)); - ASSERT_OK_AND_ASSIGN(auto scan, rescaled_table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); if (EnableSnapshotLiveManifestCache()) { ASSERT_OK_AND_ASSIGN( @@ -840,7 +820,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { .AddOption(Options::BUCKET, current_bucket_count); ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); ASSERT_OK_AND_ASSIGN(auto cached_scan, - rescaled_table->NewScan(std::move(cached_context))); + TableScan::Create(std::move(cached_context))); ASSERT_OK_AND_ASSIGN(plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); @@ -3106,21 +3086,12 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice0", 6)); - // The table was loaded with its index enabled, so the index-disabled pass below goes through - // a copy carrying it disabled: an option on the context would contradict the table. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, - Catalog::Create(paimon::test::GetDataDir() + file_format, {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - catalog->GetTable(Identifier("append_with_bitmap", "append_with_bitmap"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
disabled_table, - table->Copy({{"file-index.read.enabled", "false"}})); - // scan ScanContextBuilder scan_context_builder(table_path); ASSERT_OK_AND_ASSIGN( auto scan_context, scan_context_builder.AddOption("file-index.read.enabled", "false").Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, disabled_table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -3131,7 +3102,7 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { read_context_builder.AddOption("file-index.read.enabled", "false"); AddReadOptionsForPrefetch(&read_context_builder); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, disabled_table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3187,13 +3158,6 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, std::nullopt)); ASSERT_FALSE(commit_messages.empty()); - // The statistics-only baseline below disables the file index the table carries, so it scans - // through a copy with it disabled: an option on the context would contradict the table. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(dir->Str(), {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("foo", "bar"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
stats_only_table, - table->Copy({{Options::FILE_INDEX_READ_ENABLED, "false"}})); - // 15 falls inside the file's [10, 30] min/max range, so statistics alone must retain the // file. Disabling file-index reads verifies that baseline explicitly. const std::shared_ptr absent_predicate = PredicateBuilder::Equal( @@ -3205,7 +3169,7 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - stats_only_table->NewScan(std::move(stats_only_scan_context))); + TableScan::Create(std::move(stats_only_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); @@ -3298,13 +3262,6 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc const std::shared_ptr predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"renamed_value", FieldType::INT, Literal(15)); - // The statistics-only baseline below disables the file index the table carries, so it scans - // through a copy with it disabled: an option on the context would contradict the table. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(dir->Str(), {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("foo", "bar"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
stats_only_table, - table->Copy({{Options::FILE_INDEX_READ_ENABLED, "false"}})); - // The evolved min/max range is still [10, 30], so statistics alone retain the old file. ScanContextBuilder stats_only_scan_builder(table_path); stats_only_scan_builder.SetOptions(options) @@ -3313,7 +3270,7 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - stats_only_table->NewScan(std::move(stats_only_scan_context))); + TableScan::Create(std::move(stats_only_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); @@ -4064,31 +4021,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT, Literal(static_cast(0))); - // Historical files use two buckets even when the current option has changed, so the scan - // goes through a copy of the table carrying the bucket count it selects by: an option on - // the context would contradict the table the catalog loaded. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, - Catalog::Create(paimon::test::GetDataDir() + file_format, {})); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr
table, - catalog->GetTable(Identifier("pk_table_scan_and_read_dv", "pk_table_scan_and_read_dv"))); + // Historical files use two buckets even when the current option has changed. for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); - // The rescaled bucket count and, with it, the cache settings `FinishScanContext` also - // puts on the context: a table already held takes its runtime settings through the copy - // it is made of, and the context carries them only to agree with it. - std::map dynamic_options{{Options::BUCKET, current_bucket_count}}; - if (EnableSnapshotLiveManifestCache()) { - dynamic_options.emplace(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3"); - } - ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, table->Copy(dynamic_options)); ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); scan_context_builder.AddOption(Options::BUCKET, current_bucket_count); scan_context_builder.SetPartitionFilter({{{"f1", "10"}}}); scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, rescaled_table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); AddReadOptionsForPrefetch(&read_context_builder); @@ -4107,8 +4049,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { .SetPartitionFilter({{{"f1", "10"}}}) .SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); - ASSERT_OK_AND_ASSIGN(auto cached_scan, - rescaled_table->NewScan(std::move(cached_context))); + ASSERT_OK_AND_ASSIGN(auto cached_scan, TableScan::Create(std::move(cached_context))); ASSERT_OK_AND_ASSIGN(result_plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index 1502c39a7..3f949484d 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -32,9 +32,6 @@ #include "arrow/api.h" #include "gtest/gtest.h" -#include "paimon/catalog/catalog.h" -#include "paimon/catalog/identifier.h" -#include "paimon/catalog/table.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/data_define.h" #include "paimon/common/utils/decimal_utils.h" @@ -1164,18 +1161,10 @@ TEST_P(ScanInteTest, TestScanAppendWithInvalidOptions) { "scan.snapshot-id or scan.tag-name must be set when startup mode is FROM_SNAPSHOT"); } { - // `bucket` = -2 is refused by the scan itself, which a copy of the table carrying it - // reaches: an option on the context would contradict the table the catalog loaded, - // which a scan through that table refuses. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, - Catalog::Create(paimon::test::GetDataDir() + "orc", {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - catalog->GetTable(Identifier("append_09", "append_09"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "-2"}})); ScanContextBuilder context_builder(table_path); - context_builder.WithStreamingMode(true); + context_builder.AddOption(Options::BUCKET, "-2").WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(copied->NewScan(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "do not support bucket=-2 in scan process"); } } @@ -1737,18 +1726,10 @@ TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { TEST_P(ScanInteTest, TestScanPKWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; - // Like the append case above, the invalid bucket count rides on a copy of the table: the - // scan refuses it where the bucket count is validated. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, - Catalog::Create(paimon::test::GetDataDir() + "orc", {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - catalog->GetTable(Identifier("pk_table_with_total_buckets", - "pk_table_with_total_buckets"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "-1"}})); ScanContextBuilder context_builder(table_path); - context_builder.WithStreamingMode(true); + context_builder.AddOption(Options::BUCKET, "-1").WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(copied->NewScan(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "do not support pk table bucket=-1 in scan process"); } diff --git a/test/inte/write_and_read_inte_with_index_test.cpp b/test/inte/write_and_read_inte_with_index_test.cpp index 9123a3385..efda3de03 100644 --- a/test/inte/write_and_read_inte_with_index_test.cpp +++ b/test/inte/write_and_read_inte_with_index_test.cpp @@ -1250,7 +1250,8 @@ TEST_P(WriteAndReadInteWithIndexTest, TestReadWithLimits) { PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice", 5)); ReadContextBuilder context_builder(path); - context_builder.AddOption("test.enable-adaptive-prefetch-strategy", "false") + context_builder.AddOption(Options::FILE_FORMAT, "orc") + .AddOption("test.enable-adaptive-prefetch-strategy", "false") .AddOption(Options::READ_BATCH_SIZE, "1") .AddOption("orc.read.enable-metrics", "true") .SetPredicate(predicate); From 13fba3780476ce4177d2197e0d7f62816bdd3047 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 27 Oct 2026 09:42:57 +0800 Subject: [PATCH 15/19] fix --- src/paimon/core/catalog/snapshot_commit.cpp | 59 +++ .../core/table/file_store_table_factory.cpp | 346 ++++++++++++++++++ .../core/table/file_store_table_factory.h | 51 +++ src/paimon/core/utils/context_utils.h | 51 +++ 4 files changed, 507 insertions(+) create mode 100644 src/paimon/core/catalog/snapshot_commit.cpp create mode 100644 src/paimon/core/table/file_store_table_factory.cpp create mode 100644 src/paimon/core/table/file_store_table_factory.h create mode 100644 src/paimon/core/utils/context_utils.h diff --git a/src/paimon/core/catalog/snapshot_commit.cpp b/src/paimon/core/catalog/snapshot_commit.cpp new file mode 100644 index 000000000..35efe767c --- /dev/null +++ b/src/paimon/core/catalog/snapshot_commit.cpp @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/catalog/snapshot_commit.h" + +#include +#include + +#include "paimon/core/catalog/catalog_snapshot_commit.h" +#include "paimon/core/catalog/renaming_snapshot_commit.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +Result> SnapshotCommit::Create( + const std::shared_ptr& catalog, const std::optional& catalog_identifier, + const std::optional& table_id, bool use_rest_catalog_commit, + const std::shared_ptr& file_system, + const std::shared_ptr& snapshot_manager) { + if (AsVersionManaged(catalog) != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + // Reuse the caller's UUID so table recreation cannot redirect this commit. + return std::shared_ptr( + std::make_shared(catalog, catalog_identifier.value(), table_id)); + } + if (use_rest_catalog_commit) { + return std::shared_ptr(std::make_shared(table_id)); + } + // Catalogs without version management may still supply a table ID. + if (table_id && catalog == nullptr) { + return Status::Invalid( + "a commit which publishes the snapshot by writing it to the table directory sends no " + "commit table request, so there is nowhere to put a table id; commit through a " + "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); + } + return std::shared_ptr( + std::make_shared(file_system, snapshot_manager)); +} + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp new file mode 100644 index 000000000..2b5c9a40c --- /dev/null +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/table/file_store_table_factory.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" +#include "paimon/commit_context.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/fs/file_system.h" +#include "paimon/predicate/predicate_utils.h" +#include "paimon/read_context.h" +#include "paimon/result.h" +#include "paimon/scan_context.h" +#include "paimon/schema/schema.h" +#include "paimon/status.h" +#include "paimon/table/format/format_table.h" +#include "paimon/write_context.h" + +namespace paimon { + +namespace { + +/// Recognises a format table behind `table_path`, or returns null when it is not one. The scan, +/// read, write and commit entry points below all dispatch on the same question, so one place +/// answers it and they cannot disagree. Null is also the answer when no schema was found under +/// the path at all: what to say about a table that is not there is the managed path's to decide. +/// +/// The table type is read from the schema, never from `options`: `type` is structural, and one +/// call's options must not decide what kind of table this is. +Result> TryLoadFormatTable( + const std::shared_ptr& file_system, const std::string& table_path, + const std::string& branch, const std::map& options, + const std::shared_ptr& table_schema) { + if (table_schema == nullptr) { + return std::shared_ptr(); + } + // From the schema alone: see `FormatTable::Create()`. + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(table_schema->Options())); + if (!is_format_table) { + return std::shared_ptr(); + } + // A format table has no branches, so it is refused rather than served from the table path + // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. + if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { + return Status::Invalid( + fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", + table_path, branch)); + } + Identifier identifier = CatalogUtils::IdentifierFromPath(table_path, branch); + return FormatTable::Create(file_system, table_path, identifier, + checked_pointer_cast(table_schema), + /*location_carries_paimon_metadata=*/true, options); +} + +/// The options a context carries, merged into the table the load hands back: a table is the +/// product of the load, so the options speak there - the way `Copy` merges them - and `NewXxx` on +/// the product serves them without merging again. A caller holding a table already changes it +/// through `Copy` itself, and an option that contradicts the table that way is refused. +Result> WithContextOptions( + const std::shared_ptr
& table, const std::shared_ptr& schema, + const std::map& options) { + if (table == nullptr || schema == nullptr || options.empty()) { + return table; + } + const std::map& schema_options = schema->Options(); + for (const auto& [key, value] : options) { + auto schema_option = schema_options.find(key); + // A key the schema does not carry, or carries differently: the merge has something to do. + if (schema_option == schema_options.end() || schema_option->second != value) { + return table->Copy(options); + } + } + return table; +} + +/// The tail every `Create` ends with: the schema one of them read says what the path names. A +/// format table is handed back as the `FormatTable` it is, everything else as the `FileStoreTable` +/// a managed table is, carrying the schema for its `NewXxx` to build from. A null schema is handed +/// over as it is - see `TryLoadFormatTable`. +/// +/// @param file_system The file system the schema above was read through, which a format table +/// is served by - it has no schema options of its own to name another one. +/// @param explicit_file_system The file system the context supplied, which a managed table is +/// opened through when there is one. A null one leaves the table to resolve from its +/// schema's own access configuration rather than being pinned to the file system this load +/// happened to read the schema through. +Result> TableOfLoadedSchema( + const std::shared_ptr& file_system, + const std::shared_ptr& explicit_file_system, const std::string& path, + const std::string& branch, const std::map& options, + const std::shared_ptr& schema_manager, + const std::shared_ptr& latest_schema, const std::optional& identifier, + const std::string& uuid = "", const std::map& fs_scheme_map = {}) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + TryLoadFormatTable(file_system, path, branch, options, latest_schema)); + if (format_table != nullptr) { + return std::shared_ptr
(std::move(format_table)); + } + // The branch the schema was read on is folded into the object name. + Identifier table_identifier = + identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)); + // The manager was built over the file system the context's own options resolve, while the + // store resolves the schema's: a schema carrying access configuration names a backend the + // context's options never did, and a manager kept through that disagreement would read the + // table's metadata through the wrong one. The schema is in hand already, so dropping the + // manager costs its cache alone. A context that supplied a file system wins on both sides and + // keeps it; a scheme map only the store's resolution takes part in is the same disagreement. + std::shared_ptr manager = schema_manager; + if (manager != nullptr && latest_schema != nullptr && explicit_file_system == nullptr && + (!fs_scheme_map.empty() || + !FileStore::FileSystemResolutionAgrees(latest_schema->Options(), options))) { + manager = nullptr; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + FileStoreTable::Create(explicit_file_system, path, latest_schema, + table_identifier, uuid, branch, manager, + /*snapshot_manager=*/nullptr, fs_scheme_map)); + return WithContextOptions(table, latest_schema, options); +} + +/// The managed table a context names behind a path: the schema is read through a manager of the +/// context's branch - handed over instead when the caller already read it - and what the schema +/// says is there decides the table `TableOfLoadedSchema` hands back. +/// +/// @param schema_json The schema of the table, when the caller already read it, or null when it +/// has to be read here. It is honoured on the main branch only; a branch publishes its own +/// under `branch/branch-`, which is where the read below goes. +/// @param explicit_file_system The file system the context supplied, which the managed table the +/// schema describes is opened through - see `TableOfLoadedSchema`. +Result> ManagedTableOf( + const std::shared_ptr& file_system, + const std::shared_ptr& explicit_file_system, const std::string& path, + const std::string& branch, const std::map& options, + const std::optional& schema_json, const std::optional& identifier, + const std::string& uuid = "", const std::map& fs_scheme_map = {}) { + auto schema_manager = std::make_shared(file_system, path, branch); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, TableSchema::CreateFromJson(schema_json.value())); + } else { + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager->Latest()); + if (latest) { + latest_schema = *latest; + } + } + return TableOfLoadedSchema(file_system, explicit_file_system, path, branch, options, + schema_manager, latest_schema, identifier, uuid, fs_scheme_map); +} + +/// The schema a catalog may hold for the table, read from it on the main branch only and null +/// otherwise: a branch publishes its own under `branch/branch-`, which is where +/// `ManagedTableOf` reads instead, so a catalog answering for no branch schema does not fail the +/// `kind` operation to one. `kind` also names the operator the messages speak of. +Result> CatalogMainBranchSchemaJson( + const std::shared_ptr& catalog, const std::optional& catalog_identifier, + const std::string& branch, const char* kind) { + if (catalog == nullptr) { + return std::optional(); + } + if (!catalog_identifier) { + return Status::Invalid(fmt::format("a catalog {} requires a table identifier", kind)); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + catalog->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(std::string schema_json, schema->GetJsonSchema()); + return std::optional(std::move(schema_json)); + } + return std::optional(); +} + +} // namespace + +/// The table a scan context names: the format table the builder was handed, a system table behind +/// a `T$system` path, or the table the schema under the context's path says is there. +Result> FileStoreTableFactory::Create(const ScanContext& ctx) { + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), {})); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(ctx.GetPath())); + if (system_table_path) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), + ctx.GetPath(), ctx.GetOptions())); + return std::shared_ptr
(std::move(system_table)); + } + // One `TableScan` interface serves a format table too, the way Java Paimon serves both + // through one `ReadBuilder`; a format table is planned by listing directories instead. + const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); + return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), + branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), + ctx.GetIdentifier()); +} + +/// The table a read context names: the format table the builder was handed, a system table behind +/// a `T$system` path, or the table the schema under the context's path says is there. +Result> FileStoreTableFactory::Create(const ReadContext& ctx) { + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, + CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), + ctx.GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(ctx.GetPath())); + if (system_table_path) { + if (PredicateUtils::ContainsFullTextMatch(ctx.GetPredicate())) { + return Status::NotImplemented( + "FullTextMatch predicate execution is not implemented yet"); + } + if (ctx.HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch is not supported for system tables"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), + ctx.GetPath(), ctx.GetOptions())); + return std::shared_ptr
(std::move(system_table)); + } + + // A format table has no manifest and a file another engine wrote carries no field ids, so how + // a row is put back together is its own; how a file is opened is not - both paths go through + // `DataFileReaderFactory`. + const std::string branch = ctx.GetBranch(); + // The scheme map the schema was read through, so the table's data resolves custom schemes the + // same way its metadata does. + return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetSpecificFileSystem(), + ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), + ctx.GetIdentifier(), /*uuid=*/"", + ctx.GetFileSystemSchemeToIdentifierMap()); +} + +/// The table a write context names: the format table the builder was handed, or the table the +/// schema under the context's root path says is there. +Result> FileStoreTableFactory::Create(const WriteContext& ctx) { + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); + std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx.GetFileSystemSchemeToIdentifierMap().empty() && + ctx.GetCatalog() != nullptr) { + if (!ctx.GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); + } + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the write uses this table's. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), specific_fs, + ctx.GetFileSystemSchemeToIdentifierMap())); + std::string branch = ctx.GetBranch(); + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), branch)); + // Only the main branch may have a current schema the catalog alone holds: a branch publishes + // its own under `branch/branch-`, which is where the load below reads it instead. + PAIMON_ASSIGN_OR_RAISE( + std::optional catalog_table_schema, + CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "write")); + return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, ctx.GetRootPath(), branch, + ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), /*uuid=*/"", + ctx.GetFileSystemSchemeToIdentifierMap()); +} + +/// The table a commit context names: the format table the builder was handed, or the table the +/// schema under the context's root path says is there. +Result> FileStoreTableFactory::Create(const CommitContext& ctx) { + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); + // Use this table's catalog credentials unless the caller supplied a file system: a catalog + // issuing per-table temporary ones only hands them out through GetTableFileSystem. + std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx.GetCatalog() != nullptr) { + if (!ctx.GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), specific_fs)); + const std::string& root_path = ctx.GetRootPath(); + // The branch the context was built for, already validated by `CommitContextBuilder`. + PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( + ctx.GetIdentifier(), ctx.GetOptions(), + /*explicit_branch=*/std::nullopt, "commit")); + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), branch)); + // Only the main branch may have a current schema the catalog alone holds: a branch publishes + // its own under `branch/branch-`, which is where the load below reads it instead. + PAIMON_ASSIGN_OR_RAISE( + std::optional catalog_table_schema, + CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "commit")); + // The managed path below reads the commit's branch, so this reads the same one: the two must + // not dispatch on different schemas. + return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, root_path, branch, + ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), + ctx.GetTableId().value_or("")); +} + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table_factory.h b/src/paimon/core/table/file_store_table_factory.h new file mode 100644 index 000000000..175834ff6 --- /dev/null +++ b/src/paimon/core/table/file_store_table_factory.h @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "paimon/result.h" + +namespace paimon { + +class CommitContext; +class ReadContext; +class ScanContext; +class Table; +class WriteContext; + +/// The table a context names, Java Paimon's `FileStoreTableFactory`: the format table the builder +/// was handed, a system table behind a `T$system` path, or the table the schema under the +/// context's path says is there. The four operators' static `Create` entries all dispatch on the +/// same question, so one place answers it and they cannot disagree. +class FileStoreTableFactory { + public: + FileStoreTableFactory() = delete; + ~FileStoreTableFactory() = delete; + + static Result> Create(const ScanContext& context); + + static Result> Create(const ReadContext& context); + + static Result> Create(const WriteContext& context); + + static Result> Create(const CommitContext& context); +}; + +} // namespace paimon diff --git a/src/paimon/core/utils/context_utils.h b/src/paimon/core/utils/context_utils.h new file mode 100644 index 000000000..57e337904 --- /dev/null +++ b/src/paimon/core/utils/context_utils.h @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "fmt/format.h" +#include "paimon/status.h" + +namespace paimon { + +/// The checks every kind of table's operator entry points share. +class ContextUtils { + public: + ContextUtils() = delete; + ~ContextUtils() = delete; + + /// The checks every operator entry point opens with: a context, a memory pool and an executor + /// must all be there for the operator to be built at all, so a null one is refused rather than + /// dereferenced. `kind` names the operator the first message speaks of, which is all that sets + /// the entry points' messages apart. + template + static Status CheckOperatorContext(const ContextPointer& context, const char* kind) { + if (context == nullptr) { + return Status::Invalid(fmt::format("{} context is null pointer", kind)); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + return Status::OK(); + } +}; + +} // namespace paimon From f4437a7dca367d0908f93b9b7ecfe20c02faebad Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 27 Oct 2026 16:10:48 +0800 Subject: [PATCH 16/19] fix --- benchmark/benchmark_suite.cpp | 2 +- docs/source/user_guide/catalog.rst | 20 +-- docs/source/user_guide/format_table.rst | 8 +- examples/read_write_demo.cpp | 2 +- include/paimon/catalog/catalog.h | 11 +- include/paimon/catalog/table.h | 16 +-- include/paimon/file_store_commit.h | 4 +- include/paimon/file_store_write.h | 4 +- include/paimon/table/source/table_read.h | 4 +- include/paimon/table/source/table_scan.h | 4 +- src/paimon/core/catalog/catalog.cpp | 15 ++- .../core/catalog/file_system_catalog.cpp | 15 ++- .../core/catalog/file_system_catalog_test.cpp | 16 ++- ...eted_primary_key_index_maintainer_test.cpp | 4 +- .../core/operation/file_store_write_test.cpp | 5 +- .../core/operation/internal_read_context.cpp | 30 +++-- .../core/operation/internal_read_context.h | 41 +++--- .../core/table/append_only_file_store_table.h | 2 +- src/paimon/core/table/file_store.h | 114 ++++++++--------- src/paimon/core/table/file_store_table.cpp | 6 +- src/paimon/core/table/file_store_table.h | 120 +++++++++--------- .../core/table/file_store_table_factory.cpp | 18 +-- .../core/table/file_store_table_factory.h | 8 +- src/paimon/core/table/format/format_table.cpp | 30 +++-- .../core/table/format/format_table_test.cpp | 29 +++++ .../core/table/primary_key_file_store_table.h | 2 +- .../core/table/source/table_read_test.cpp | 10 +- .../table/system/audit_log_system_table.cpp | 15 ++- .../table/system/audit_log_system_table.h | 7 +- .../core/table/system/binlog_system_table.cpp | 8 +- .../core/table/system/binlog_system_table.h | 1 - .../table/system/global_system_tables.cpp | 70 +++++----- .../core/table/system/global_system_tables.h | 25 ++-- .../table/system/in_memory_system_table.cpp | 4 +- .../table/system/in_memory_system_table.h | 3 +- .../table/system/metadata_system_tables.cpp | 49 ++----- .../table/system/metadata_system_tables.h | 8 -- .../system/read_optimized_system_table.cpp | 8 +- .../system/read_optimized_system_table.h | 1 - src/paimon/core/table/system/system_table.cpp | 19 ++- src/paimon/core/table/system/system_table.h | 48 ++++--- .../core/table/system/system_table_test.cpp | 63 +++++++-- src/paimon/core/table/table_test.cpp | 26 +++- src/paimon/rest/rest_catalog.cpp | 23 ++-- src/paimon/rest/rest_catalog_test.cpp | 38 +++++- src/paimon/testing/mock/mock_catalog.h | 4 +- test/inte/read_inte_test.cpp | 29 +++-- test/inte/realtime_write_inte_test.cpp | 4 +- 48 files changed, 571 insertions(+), 422 deletions(-) diff --git a/benchmark/benchmark_suite.cpp b/benchmark/benchmark_suite.cpp index 0a48a9173..6ec44f0d2 100644 --- a/benchmark/benchmark_suite.cpp +++ b/benchmark/benchmark_suite.cpp @@ -368,7 +368,7 @@ Status EnsureTable(const std::string& root_path, const std::string& db_name, const std::shared_ptr& schema, const std::vector& primary_keys = {}) { PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr catalog, + std::shared_ptr catalog, AddContext(paimon::Catalog::Create(root_path, options), "create catalog")); PAIMON_RETURN_NOT_OK( AddContext(catalog->CreateDatabase(db_name, options, true), "create database")); diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index d881968c4..599b4ffce 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -83,7 +83,7 @@ registered on the REST server. The catalog is configured through the {"token.provider", "bear"}, {"token", ""}, }; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr catalog, paimon::Catalog::Create(/*root_path=*/"my_instance", options)); For DLF, configure one credential source. Static AK/SK credentials, an optional @@ -122,15 +122,12 @@ without an atomic snapshot-file rename: .. code-block:: cpp - // Writers and committers share ownership of the catalog. - std::shared_ptr shared_catalog(std::move(catalog)); - // Load the table's identity before preparing changes. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - shared_catalog->GetTable(paimon::Identifier("db", "tbl"))); + catalog->GetTable(paimon::Identifier("db", "tbl"))); paimon::CommitContextBuilder builder(table_path, "commit-user"); - builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); // CatalogUuid() stays null when the server supplies no id. if (std::optional table_uuid = table->CatalogUuid()) { builder.WithTableId(table_uuid.value()); @@ -147,7 +144,7 @@ offsets when a real-time writer is created: .. code-block:: cpp paimon::WriteContextBuilder write_builder(table_path, "commit-user"); - write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + write_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, @@ -183,7 +180,7 @@ returns credentials that are still valid and reloads them before they expire. "oss://bucket/tbl", options)); // MyFileSystem merges provider->GetCredentials() over its own options per access. auto my_fs = std::make_shared(provider); - write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")) + write_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")) .WithFileSystem(my_fs); The snapshot being committed carries a uuid generated on the client, and the @@ -262,18 +259,15 @@ for that table's file system and passing it in through ``WithFileSystem``: .. code-block:: cpp - // Readers and scanners share ownership of the catalog. - std::shared_ptr shared_catalog(std::move(catalog)); - paimon::ScanContextBuilder scan_builder(table_path); - scan_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + scan_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, paimon::TableScan::Create(std::move(scan_context))); paimon::ReadContextBuilder read_builder(table_path); - read_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + read_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 4be21a1dd..7ee90976d 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -53,9 +53,9 @@ That path needs a schema under the table's own location, which is what every tab catalog serves has. A table loaded from a catalog that keeps the schema elsewhere, such as the REST catalog, has none: nothing under the location says that it is a format table, nor that ``schema`` and ``branch`` below it are data rather than metadata. Such a table is read and written -by handing the loaded ``FormatTable`` to the context instead of a path - every context builder has -a constructor taking one - which is what Java does through ``FormatTable.newReadBuilder()`` and -``newBatchWriteBuilder()``: +by handing the loaded ``FormatTable`` to the context builder instead of a path - every builder has +a constructor taking one - and calling the table's own operator, which is what Java does through +``FormatTable.newReadBuilder()`` and ``newBatchWriteBuilder()``: .. code-block:: cpp @@ -68,7 +68,7 @@ a constructor taking one - which is what Java does through ``FormatTable.newRead PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - paimon::TableScan::Create(std::move(scan_context))); + table->NewScan(std::move(scan_context))); The table already carries its schema and the file system it was loaded through, so a context built from one refuses ``SetTableSchema()``, ``WithFileSystem()``, diff --git a/examples/read_write_demo.cpp b/examples/read_write_demo.cpp index df46fcc54..e911bedb1 100644 --- a/examples/read_write_demo.cpp +++ b/examples/read_write_demo.cpp @@ -61,7 +61,7 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, {paimon::Options::FILE_SYSTEM, "local"}}; // create table - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr catalog, paimon::Catalog::Create(root_path, options)); PAIMON_RETURN_NOT_OK(catalog->CreateDatabase(db_name, options, /*ignore_if_exists=*/false)); arrow::FieldVector fields = { diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 033e7a822..3898a1967 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -68,8 +68,11 @@ class PAIMON_EXPORT Catalog { /// schemes, including the file systems it builds from per-table data /// tokens, keeps routing each scheme to its backend. Ignored when /// `file_system` is supplied, which serves every scheme itself. - /// @return A result containing a unique pointer to a `Catalog` instance, or an error status. - static Result> Create( + /// @return A result containing a shared pointer to a `Catalog` instance, or an error status. + /// @note Shared rather than unique ownership: a global system table of the `sys` database + /// builds its rows lazily through the catalog it was handed, so the table keeps the + /// catalog alive. A catalog built by this factory always satisfies that. + static Result> Create( const std::string& root_path, const std::map& options, const std::shared_ptr& file_system = nullptr, const std::map& fs_scheme_to_identifier_map = {}); @@ -154,7 +157,9 @@ class PAIMON_EXPORT Catalog { /// `NewRead()` serve any of them, `NewWrite()` and `NewCommit()` the writable ones. /// /// A global system table of the "sys" database reads its rows lazily through the catalog it - /// came from, so a catalog that serves one must be owned by a `std::shared_ptr`. + /// came from, so the table keeps shared ownership of that catalog. `Catalog::Create()` hands + /// back a `std::shared_ptr`, which satisfies this. A catalog held some other way, such as + /// through a `std::unique_ptr`, cannot be kept and is refused by name. /// /// @param identifier Identifier of the table to get. /// @return A result containing the table, or an error status. diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index 3364f6eff..9164623c0 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -45,10 +45,10 @@ class WriteContext; /// A table provides basic abstraction for table type. /// /// The state every kind of table carries - file system, root path, schema and identifier - lives -/// here, and each kind fills the operations in. The `NewXxx` operators below are the table-first -/// entry points: `TableScan::Create()` and friends load the table a context names and hand the -/// context to them. What a context asks for that the table cannot honour is refused by name rather -/// than silently dropped. +/// here, and each kind fills in the operations. The `NewXxx` operators below are the table-first +/// entry points: `TableScan::Create()` and its friends load the table a context names and hand the +/// context to it. A table refuses by name whatever a context asks for that it cannot honour, +/// rather than dropping it silently. class PAIMON_EXPORT Table { public: virtual ~Table() = default; @@ -65,8 +65,8 @@ class PAIMON_EXPORT Table { return root_path_; } - /// A name to identify this table: a managed or format table answers its identifier's object - /// name, a system table its own kind. + /// A name to identify this table. A managed or format table answers the object name of its + /// identifier, a system table its own name. virtual std::string Name() const = 0; /// Full name of the table: `database.tableName` for a managed or format table, the same name @@ -90,9 +90,9 @@ class PAIMON_EXPORT Table { return schema_; } - /// Copies this table with `dynamic_options` on top of the options it carries: an option given + /// Copies this table with `dynamic_options` on top of the options it carries. An option given /// at the call wins over the one the schema stored, the precedence every context builder - /// promises. `type` is not overridable - it is structural and read from the schema alone. + /// promises. `type` is not overridable: it is structural and read from the schema alone. /// /// @param dynamic_options Options given at the call. /// @return A result containing the copied table, or the error copying it reports. diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index 581cadf2d..8a6eaad5c 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -50,8 +50,8 @@ class CommitMessage; /// manages versions. `UseRESTCatalogCommit()` only prepares a request for the caller to send. class PAIMON_EXPORT FileStoreCommit { public: - /// Create an instance of `FileStoreCommit`: the table the context names is loaded first, then - /// the context is handed to that table's `NewCommit`. + /// Create an instance of `FileStoreCommit`. A context that already carries a loaded table goes + /// straight to that table's `NewCommit`; otherwise the table the context names is loaded first. /// /// @param context A unique pointer to the `CommitContext` used for commit operations. /// diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index 5f74d6536..0778e5bb7 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -42,8 +42,8 @@ class WriteContext; /// Interface for write operations in a file store. class PAIMON_EXPORT FileStoreWrite { public: - /// Create an instance of `FileStoreWrite`: the table the context names is loaded first, then - /// the context is handed to that table's `NewWrite`. + /// Create an instance of `FileStoreWrite`. A context that already carries a loaded table goes + /// straight to that table's `NewWrite`; otherwise the table the context names is loaded first. /// /// @param context A unique pointer to the `WriteContext` used for write operations. /// diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index 2bfd84fff..5f8303fe3 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -42,8 +42,8 @@ class PAIMON_EXPORT TableRead { public: virtual ~TableRead() = default; - /// Create an instance of `TableRead`: the table the context names is loaded first, then the - /// context is handed to that table's `NewRead`. + /// Create an instance of `TableRead`. A context that already carries a loaded table goes + /// straight to that table's `NewRead`; otherwise the table the context names is loaded first. /// /// @param context A unique pointer to the `ReadContext` used for read operations. /// @return A Result containing a unique pointer to the `TableRead` instance. diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h index 39f0453c8..4944dafd2 100644 --- a/include/paimon/table/source/table_scan.h +++ b/include/paimon/table/source/table_scan.h @@ -36,8 +36,8 @@ class ScanContext; /// A scanner interface for reading table's meta and create a plan. class PAIMON_EXPORT TableScan { public: - /// Create an instance of `TableScan`: the table the context names is loaded first, then the - /// context is handed to that table's `NewScan`. + /// Create an instance of `TableScan`. A context that already carries a loaded table goes + /// straight to that table's `NewScan`; otherwise the table the context names is loaded first. /// /// @param context A unique pointer to the `ScanContext` used for scan operations. /// @return A Result containing a unique pointer to the `TableScan` instance. diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index 318125ec6..b14a53eab 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -40,7 +40,7 @@ const char Catalog::SYSTEM_TABLE_SPLITTER[] = "$"; const char Catalog::DB_SUFFIX[] = ".db"; const char Catalog::DB_LOCATION_PROP[] = "location"; -Result> Catalog::Create( +Result> Catalog::Create( const std::string& root_path, const std::map& options, const std::shared_ptr& file_system, const std::map& fs_scheme_to_identifier_map) { @@ -53,8 +53,14 @@ Result> Catalog::Create( } if (metastore == "rest") { #ifdef PAIMON_ENABLE_REST - return RestCatalog::Create(root_path, options, file_system, RestHttpClient::Config(), - fs_scheme_to_identifier_map); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr rest_catalog, + RestCatalog::Create(root_path, options, file_system, RestHttpClient::Config(), + fs_scheme_to_identifier_map)); + // Built from the derived type so that the `enable_shared_from_this` base a + // `RestCatalog` carries is initialized: a `sys` table keeps its catalog alive through + // `weak_from_this()`, which stays empty when the pointer is widened first. + return std::shared_ptr(std::move(rest_catalog)); #else return Status::NotImplemented( "the rest catalog requires building paimon with PAIMON_ENABLE_REST=ON"); @@ -65,7 +71,8 @@ Result> Catalog::Create( } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system, fs_scheme_to_identifier_map)); - return std::make_unique(core_options.GetFileSystem(), root_path, options); + return std::shared_ptr( + std::make_unique(core_options.GetFileSystem(), root_path, options)); } bool Catalog::SupportsVersionManagement() const { diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index fdcddf20e..d061acbf2 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -287,16 +287,15 @@ Result> FileSystemCatalog::LoadTableSchema( if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } - // The table built here is transient - its schema is read within this call and nothing - // outlives it - so its context keeps no shared ownership of the catalog. A catalog that - // hands a global system table back out (`RestCatalog::GetTable`) retains it instead. + // The table built here is transient: only its schema is read and nothing outlives the + // call. It builds no rows, so the context carries no catalog. GlobalSystemTableContext context; - context.catalog = this; context.fs = fs_; context.warehouse = warehouse_; context.catalog_options = catalog_options_; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + GlobalSystemTableLoader::LoadForSchema(identifier.GetObjectName(), context)); return system_table->LatestSchema(); } // The system table suffix is dropped when the data table identifier is rebuilt below, so the @@ -351,7 +350,9 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide // The branch the schema above was read on, so the table's managers read the same one. PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); return FileStoreTable::Create(fs_, table_path, checked_pointer_cast(*latest_schema), - identifier, /*uuid=*/"", branch); + identifier, /*uuid=*/"", branch, /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, + /*dynamic_options=*/{}); } Result> FileSystemCatalog::LoadSystemTable( diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index f31c0391d..294b7e9b9 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -231,11 +231,12 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(options_identifier)); // A system table comes back as the `SystemTable` it is: it answers for itself with its own - // read-only schema and the root path of the data table it reads. + // read-only schema and the root path of the data table it reads. Its name carries the data + // table it belongs to. ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); ASSERT_TRUE(std::dynamic_pointer_cast(system_table) == nullptr); - ASSERT_EQ("options", system_table->Name()); - ASSERT_EQ("options", system_table->FullName()); + ASSERT_EQ("tbl1$options", system_table->Name()); + ASSERT_EQ("tbl1$options", system_table->FullName()); ASSERT_EQ(system_table->GetRootPath(), table_path); ASSERT_EQ((std::vector{"key", "value"}), system_table->LatestSchema()->FieldNames()); @@ -281,7 +282,7 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(Identifier("db1", "tbl1$options"))); ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); - ASSERT_EQ("options", system_table->Name()); + ASSERT_EQ("tbl1$options", system_table->Name()); // The scan and the read work from a context of the caller's, like every other table. ScanContextBuilder scan_builder(system_table->GetRootPath()); @@ -310,13 +311,14 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { write_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); ASSERT_NOK_WITH_MSG(system_table->NewWrite(std::move(write_context)), - "options system table is read-only: writing is not supported"); + "tbl1$options system table is read-only: writing is not supported"); CommitContextBuilder commit_builder(system_table->GetRootPath(), "commit_user"); commit_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(system_table->NewCommit(std::move(commit_context)), - "options system table is read-only: committing for it is not supported"); + ASSERT_NOK_WITH_MSG( + system_table->NewCommit(std::move(commit_context)), + "tbl1$options system table is read-only: committing for it is not supported"); } TEST(FileSystemCatalogTest, TestBranchTableLoadsBranchSchema) { diff --git a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp index 81803ed90..85690b25f 100644 --- a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp +++ b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp @@ -298,7 +298,7 @@ class BucketedPrimaryKeyIndexMaintainerTest : public ::testing::TestWithParam catalog, + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(directory_->Str(), {})); ASSERT_OK(catalog->CreateDatabase("db", {}, /*ignore_if_exists=*/false)); ASSERT_OK(catalog->CreateTable(Identifier("db", "table"), &c_schema, @@ -373,7 +373,7 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsIndexForColumnNamedLikeInter schema_ = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), arrow::field(kColumnName, arrow::utf8(), /*nullable=*/false)}); options_[Options::PK_BTREE_INDEX_COLUMNS] = kColumnName; - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(directory_->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(directory_->Str(), {})); for (bool spillable : {false, true}) { SCOPED_TRACE(spillable); const std::string table_name = spillable ? "row_id_spill" : "row_id_memory"; diff --git a/src/paimon/core/operation/file_store_write_test.cpp b/src/paimon/core/operation/file_store_write_test.cpp index 520e850e5..ffee33161 100644 --- a/src/paimon/core/operation/file_store_write_test.cpp +++ b/src/paimon/core/operation/file_store_write_test.cpp @@ -170,14 +170,13 @@ TEST(FileStoreWriteTest, TestCreateAppendTable) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{"f0", "f3"}, /*primary_keys=*/{}, /*options=*/{}, /*ignore_if_exists=*/false)); - std::shared_ptr shared_catalog(std::move(catalog)); - ASSERT_FALSE(shared_catalog->SupportsVersionManagement()); + ASSERT_FALSE(catalog->SupportsVersionManagement()); for (bool use_catalog : {false, true}) { SCOPED_TRACE(use_catalog); WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "commit_user_1"); if (use_catalog) { - context_builder.WithCatalog(shared_catalog, Identifier("foo", "bar")); + context_builder.WithCatalog(catalog, Identifier("foo", "bar")); } ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 8866eae9f..3ee0eb755 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -277,17 +277,24 @@ std::optional InternalReadContext::TryResolveSpecialFieldByName( return std::nullopt; } -Result> InternalReadContext::Create( +Result> InternalReadContext::CreateForTable( const std::shared_ptr& context, const std::shared_ptr& table_schema, - CoreOptions core_options, const std::string* path) { + CoreOptions core_options, const std::string& root_path) { core_options.WithCache(context->GetCache()); - return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); + return CreateFromCoreOptions(context, table_schema, std::move(core_options), root_path); } Result> InternalReadContext::Create( + const std::shared_ptr& context, const std::shared_ptr& table_schema, + const std::map& options) { + return CreateFromOptions(context, table_schema, options, /*file_system=*/nullptr, + /*path=*/std::nullopt); +} + +Result> InternalReadContext::CreateFromOptions( const std::shared_ptr& context, const std::shared_ptr& table_schema, const std::map& options, - const std::shared_ptr& file_system, const std::string* path) { + const std::shared_ptr& file_system, std::optional path) { // The file system the caller named, or null for the read context's own, which callers // without a table hand over. The scheme map the context carries does not take part: a file // system handed over wins and one missing falls to the resolving one, the same way a scan @@ -297,7 +304,7 @@ Result> InternalReadContext::Create( CoreOptions::FromMap( options, file_system != nullptr ? file_system : context->GetSpecificFileSystem())); core_options.WithCache(context->GetCache()); - return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); + return CreateFromCoreOptions(context, table_schema, std::move(core_options), std::move(path)); } Result> InternalReadContext::CreateForFallbackBranch( @@ -341,12 +348,12 @@ Result> InternalReadContext::CreateForFallb } else { options.erase(Options::BRANCH); } - return Create(context, table_schema, options, file_system, &root_path); + return CreateFromOptions(context, table_schema, options, file_system, root_path); } Result> InternalReadContext::CreateFromCoreOptions( const std::shared_ptr& context, const std::shared_ptr& table_schema, - CoreOptions core_options, const std::string* path) { + CoreOptions core_options, std::optional path) { if (context->HasVectorOrFullTextSearch()) { if (!core_options.FileIndexReadEnabled()) { return Status::Invalid( @@ -458,8 +465,7 @@ Result> InternalReadContext::CreateFromCore } std::shared_ptr arrow_pool = GetArrowPool(context->GetMemoryPool()); auto internal_context = std::unique_ptr(new InternalReadContext( - context, table_schema, read_schema, core_options, arrow_pool, - path != nullptr ? std::optional(*path) : std::nullopt)); + context, table_schema, read_schema, core_options, std::move(path), arrow_pool)); PAIMON_RETURN_NOT_OK(internal_context->BindIndexScores()); return internal_context; } @@ -539,8 +545,8 @@ InternalReadContext::InternalReadContext(const std::shared_ptr& rea const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, const CoreOptions& options, - const std::shared_ptr& arrow_pool, - std::optional path) + std::optional path, + const std::shared_ptr& arrow_pool) : read_context_(read_context), table_schema_(table_schema), read_schema_(read_schema), @@ -560,7 +566,7 @@ Result> InternalReadContext::CreateWithSche // The new read_schema is the minimal column set for COUNT(*). return std::shared_ptr( new InternalReadContext(original->read_context_, original->table_schema_, new_read_schema, - original->options_, original->arrow_pool_, original->path_)); + original->options_, original->path_, original->arrow_pool_)); } } // namespace paimon diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index 43783e950..9e5d91ce0 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -58,24 +58,23 @@ class InternalReadContext { std::string field_name; }; - /// Builds the context of a read a table serves: the options are the table's own, already - /// parsed, and the cache the context carries is armed on them here. `path` names the root path - /// the table was opened with, which the read reads from in place of the context's own. - static Result> Create( + /// Builds the context of the read a table serves: the options are the table's own, already + /// parsed, and the cache the context carries is armed on them here. + /// @param root_path The root path the table was opened with. The read reads from it in place of + /// the path the context names. + static Result> CreateForTable( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, CoreOptions core_options, - const std::string* path = nullptr); + const std::string& root_path); /// Builds the context of a read whose options are composed on the spot rather than the table's - /// own - the fallback branch of a read, whose schema belongs to another branch, and the reads a - /// write or a lookup serve inside themselves. The read's own file system is `file_system`, or - /// the context's when null; the scheme map the context carries does not take part. + /// own: the reads a write or a lookup serve inside themselves. Such a read serves the file + /// system the context resolved and reads from the path the context names. A fallback branch + /// reads through `CreateForFallbackBranch` instead, whose schema belongs to another branch. static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, - const std::map& options, - const std::shared_ptr& file_system = nullptr, - const std::string* path = nullptr); + const std::map& options); /// The read context of a fallback branch, whose schema is not the table's own: it is read /// through a manager of that branch, and the read serves the options that schema was loaded @@ -195,16 +194,24 @@ class InternalReadContext { InternalReadContext(const std::shared_ptr& read_context, const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, - const CoreOptions& options, - const std::shared_ptr& arrow_pool, - std::optional path = std::nullopt); + const CoreOptions& options, std::optional path, + const std::shared_ptr& arrow_pool); + + /// The tail of a read whose options are a map: they resolve to `CoreOptions` against + /// `file_system`, or against the context's own when it is null, and then go where + /// `CreateFromCoreOptions` takes them. + static Result> CreateFromOptions( + const std::shared_ptr& read_context, + const std::shared_ptr& table_schema, + const std::map& options, + const std::shared_ptr& file_system, std::optional path); /// The tail both `Create` entries share: the options arrive parsed and cached, and the /// validations and the read schema below are the same whichever way they were resolved. static Result> CreateFromCoreOptions( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, CoreOptions core_options, - const std::string* path); + std::optional path); static std::optional TryResolveSpecialFieldById(int32_t field_id, const CoreOptions& core_options, @@ -224,8 +231,8 @@ class InternalReadContext { std::shared_ptr arrow_pool_; CoreOptions options_; std::map index_score_bindings_; - /// The root path a table reads through, or null for the read context's own, which callers - /// without a table hand over. + /// The root path a table reads through, or empty for the read context's own, which a caller + /// without a table leaves it as. std::optional path_; }; diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h index 9b4af46ae..1f94d580a 100644 --- a/src/paimon/core/table/append_only_file_store_table.h +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -43,7 +43,7 @@ class AppendOnlyFileStoreTable : public FileStoreTable { /// The constructor `FileStoreTable::Create` builds an append-only table through: the store /// carries every state, and the base class holds it. AppendOnlyFileStoreTable(const std::shared_ptr& store, const Identifier& identifier, - const std::string& uuid = ""); + const std::string& uuid); private: Result> CreateWrite( diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index e343ec2c4..339dd3810 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -46,51 +46,47 @@ class ManifestFile; class ManifestList; class TableSchema; -/// The storage a managed table is served through: the file system, root path, schema, branch, -/// options and managers it was loaded with - the state Java Paimon's `AbstractFileStore` carries, -/// parsed once here rather than per operator. `FileStoreTable` hands the state it was loaded with -/// to one of these and asks it back for the pieces every operator opens with. +/// Holds the storage a managed table reads and writes through: the file system, root path, +/// schema, branch, options and managers. Java Paimon carries the same state in `AbstractFileStore`. +/// `FileStoreTable` builds one when it loads and asks it for the pieces each operator opens with. /// -/// Every operator serves the options the table was loaded with, so the factories below take the -/// options of the call: a cache is not part of the table's, `WithCache()` arms a copy an operator -/// makes. They build a fresh instance on every call, never a cached one - a snapshot manager a -/// write arms with a catalog loader is not one a scan should be served through afterwards. +/// The factories below take the options of the call, so an operator can serve a cache of its own +/// over the options the table was loaded with. Each call builds a fresh instance and never a +/// cached one: a snapshot manager a write armed with a catalog loader must not serve a scan. class FileStore { public: - /// @param file_system The file system the caller opened the table through, which wins over - /// the access configuration the schema's options may carry when it is not null. A null - /// one hands the resolution to that configuration, and to the default local file system - /// when the options name none. + /// Builds the store. It parses the schema's options and fails when they do not parse, so the + /// constructor is private and callers come here. + /// @param file_system The file system the caller opened the table through. It wins over the + /// access configuration in the schema's options. Null leaves the resolution to that + /// configuration, and to the default local file system when the options name none. /// @param root_path Root path of the table in the file system. - /// @param schema The schema the table was loaded with, whose partition keys name the partitions - /// the paths below are built for and whose options `GetOptions()` parses. Null when the + /// @param schema The schema the table was loaded with. Its partition keys name the partitions + /// the paths below are built for, and `GetOptions()` parses its options. Null when the /// caller had none, which the operators then report. /// @param branch The branch the store serves, normalized here. - /// @param schema_manager A schema manager to carry instead of building one - the one a context + /// @param schema_manager A schema manager to carry instead of building one: the one a context /// dispatch read the schema through, whose cache the operators then serve. - /// @param snapshot_manager A snapshot manager to carry instead of building one; a null one is - /// built over `root_path` on `branch`. + /// @param snapshot_manager A snapshot manager to carry instead of building one. Null builds one + /// over `root_path` on `branch`. /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom /// schemes through, kept so a copy resolves as this store does. - /// - /// Building the store parses the schema's options, so it fails when they do not parse - which - /// is why a store is built through this factory rather than the constructor. static Result> Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr, - const std::map& fs_scheme_map = {}); - - /// Whether a copy's dynamic options change the file system its store resolves to. If they do, - /// the managers of the table it was copied from are not its to inherit: built over that - /// table's resolution, they would keep reading metadata through the backend the copy just - /// left. + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map); + + /// Returns whether a copy's dynamic options change the file system its store resolves to. When + /// they do, the copy must not inherit the managers of the table it was copied from: they were + /// built over that table's resolution and would keep reading metadata through the backend the + /// copy left. static bool FileSystemResolutionMayChange( const TableSchema& schema, const std::map& dynamic_options); - /// Whether two options maps resolve to the same file system: every resolution key has to - /// agree, a missing one on both sides included. + /// Returns whether two options maps resolve to the same file system. Every resolution key has + /// to agree, including a key missing on both sides. static bool FileSystemResolutionAgrees(const std::map& left, const std::map& right); @@ -98,8 +94,8 @@ class FileStore { return branch_; } - /// The schema manager the operators read through, carrying the schema cache of `root_path` on - /// `CurrentBranch()`. + /// The schema manager the operators read through. It carries the schema cache of `root_path` + /// on `CurrentBranch()`. const std::shared_ptr& GetSchemaManager() const { return schema_manager_; } @@ -115,20 +111,20 @@ class FileStore { return schema_; } - /// The file system the table's data is accessed through: the one the options resolve, which - /// the file system the caller opened the table through wins over. + /// The file system the table's data is accessed through: the one the options resolve, unless + /// the caller opened the table through one of its own. const std::shared_ptr& GetFileSystem() const { return file_system_; } - /// The file system the caller opened the table through, as handed to `Create`: null when the + /// The file system the caller opened the table through, as handed to `Create`. Null when the /// call had none, so a copy resolves from its own options again. const std::shared_ptr& GetSpecifiedFileSystem() const { return specified_file_system_; } - /// The scheme-to-identifier map the options were resolved with, kept for a copy to resolve - /// through again. + /// The scheme-to-identifier map the options were resolved with, kept so a copy can resolve + /// through it again. const std::map& GetFsSchemeMap() const { return fs_scheme_map_; } @@ -137,59 +133,59 @@ class FileStore { return root_path_; } - /// The options the schema was loaded with on `CurrentBranch()`. They exist only over a schema: - /// an operator reads them after reporting that none was found when there is not. + /// The options of the schema loaded on `CurrentBranch()`. They exist only over a schema, so an + /// operator reports that no schema was found before it ever reads them. const CoreOptions& GetOptions() const { assert(core_options_.has_value()); return *core_options_; } - /// The path factory an operator builds its metadata paths through: over this store's root - /// path and the partition keys of its schema, which no schema change can alter, so an - /// operator serving a schema read after the table was loaded still names the same partitions. - /// The file format identifier is the caller's to name: a write names the format its level-0 - /// files are written in, which the options may set per level, while a scan, a read and a - /// commit name the table's format. + /// Builds the path factory an operator names its metadata paths through, over this store's + /// root path and the partition keys of its schema. No schema change can alter the partition + /// keys, so an operator serving a schema read after the table was loaded still names the same + /// partitions. The caller names the file format identifier: a write names the format of its + /// level-0 files, which the options may set per level, while a scan, a read and a commit name + /// the table's format. Result> NewPathFactory( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::string& file_format_identifier, const std::shared_ptr& memory_pool) const; - /// The snapshot manager a write or a commit reads snapshots through, over this store's root - /// path and branch. The file system is the caller's to name, from the options its operator - /// serves. A catalog that version-manages the table hands its snapshots out itself, so its - /// loader is attached and the catalog captured to keep it alive; such a catalog names the - /// table by an identifier, which the context must have supplied. + /// Builds the snapshot manager a write or a commit reads snapshots through, over this store's + /// root path and branch. The caller names the file system, from the options its operator + /// serves. A catalog that version-manages the table hands out its own snapshots, so its loader + /// is attached and the catalog is captured to keep it alive. Such a catalog names the table by + /// an identifier, which the context must have supplied. Result> NewSnapshotManager( const CoreOptions& options, const std::shared_ptr& catalog, const std::optional& catalog_identifier) const; - /// The manifest list a scan or a commit reads the table's manifest entries through. + /// Builds the manifest list a scan or a commit reads the table's manifest entries through. Result> NewManifestList( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// The manifest file a scan or a commit reads a single manifest's entries through, pruned by - /// `partition_schema`. + /// Builds the manifest file a scan or a commit reads a single manifest's entries through, + /// pruned by `partition_schema`. Result> NewManifestFile( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& partition_schema, const std::shared_ptr& memory_pool) const; - /// The index manifest file a write, a scan or a commit reads the table's index manifest - /// through. + /// Builds the index manifest file a write, a scan or a commit reads the table's index + /// manifest through. Result> NewIndexManifestFile( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// The index file handler a write maintains its deletion vectors and indexes through, or a - /// scan reads them through. + /// Builds the index file handler a write maintains its deletion vectors and indexes through, + /// or a scan reads them through. Result> NewIndexFileHandler( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// The partition computer a commit names the partitions it writes with, over the partition - /// keys of this store's schema and the caller's arrow schema. + /// Builds the partition computer a commit names the partitions it writes with, over the + /// partition keys of this store's schema and the caller's arrow schema. Result> NewPartitionComputer( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& memory_pool) const; diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 7e68b3567..3945de5b7 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -214,7 +214,7 @@ Result> FileStoreTable::SwitchToBranch(const std::string& std::shared_ptr(std::move(branch_schema)), identifier_, CatalogUuid().value_or(""), target_branch, branch_schema_manager, /*snapshot_manager=*/nullptr, - store_->GetFsSchemeMap()); + store_->GetFsSchemeMap(), /*dynamic_options=*/{}); } Status FileStoreTable::RestoreRealtimeCommittedProgress( @@ -470,8 +470,8 @@ Result> FileStoreTable::NewRead( PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); // The read reads through the branch the table serves, not the context's own. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, - InternalReadContext::Create(read_context, loaded_schema, - store_->GetOptions(), &root_path_)); + InternalReadContext::CreateForTable(read_context, loaded_schema, + store_->GetOptions(), root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, CreateTableRead(internal_context, read_context->GetMemoryPool(), diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 1fb307495..04b088ab8 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -43,44 +43,44 @@ class RealtimeContext; class Schema; class SplitGenerator; -/// The managed table a catalog hands out: one with snapshots and manifests, which the schema under -/// its root path describes. A format table - a directory of plain data files - is a `FormatTable`, -/// the other kind of `Table`. +/// The managed table a catalog hands out: one with snapshots and manifests, described by the +/// schema under its root path. A format table - a directory of plain data files - is a +/// `FormatTable`, the other kind of `Table`. /// -/// This is the flow the four operators share, Java Paimon's `AbstractFileStoreTable`: what the -/// table was constructed with - the schema, file system, root path, branch and managers - decides -/// what every operator serves, so it is read here once and handed down. The schema's primary keys -/// decide the kind of managed table it is - `AppendOnlyFileStoreTable` or -/// `PrimaryKeyFileStoreTable`, the `Create` factory below - and that kind builds the pieces the -/// flows hand down: the write itself, the scans, the read and the split generator. +/// The four operators share one flow, the one Java Paimon's `AbstractFileStoreTable` carries. What +/// the table was loaded with - schema, file system, root path, branch and managers - decides what +/// every operator serves, so it is read once here and handed down. The schema's primary keys decide +/// the kind of managed table it is, `AppendOnlyFileStoreTable` or `PrimaryKeyFileStoreTable`, which +/// the `Create` factory below picks. That kind then builds the pieces the flow hands down: the +/// write itself, the scans, the read and the split generator. class FileStoreTable : public Table { public: - /// The managed table the schema describes, built as the kind its primary keys name: an - /// `AppendOnlyFileStoreTable` when there are none, a `PrimaryKeyFileStoreTable` when there - /// are, the way Java Paimon's `FileStoreTableFactory` does. Primary keys cannot be added or - /// dropped by a schema change, so the kind is the same for every schema the root path held. - /// @param schema The schema the table was loaded with, whose primary keys name the kind and - /// whose options the operators serve. Null when the caller had none, in which case the - /// table is built as the append-only kind and its operators report that none was found. + /// Builds the managed table the schema describes, as the kind its primary keys name: an + /// `AppendOnlyFileStoreTable` when there are none, a `PrimaryKeyFileStoreTable` when there are. + /// Java Paimon's `FileStoreTableFactory` picks the kind the same way. A schema change can + /// neither add nor drop primary keys, so every schema the root path ever held names one kind. + /// @param schema The schema the table was loaded with. Its primary keys name the kind and its + /// options are what the operators serve. Null when the caller had none, in which case + /// the table is built as the append-only kind and its operators report that none was + /// found. /// @param branch The branch the table serves, which the managers below read and write on. - /// @param schema_manager A schema manager to carry instead of building one - see + /// @param schema_manager A schema manager to carry instead of building one. See /// `FileStore::Create`. - /// @param snapshot_manager A snapshot manager to carry instead of building one, the way `Copy` - /// hands its table's over so both share one - see `FileStore::Create`. + /// @param snapshot_manager A snapshot manager to carry instead of building one. `Copy` hands + /// its table's over so both share one. See `FileStore::Create`. /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom /// schemes through, which the store's options resolve through as well. `Copy` hands its /// table's map over. /// @param dynamic_options The options a copy was given over the ones its schema stored, which - /// `Copy` accumulates - see the member of the same name. + /// `Copy` accumulates. See the member of the same name. static Result> Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = "", - const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr, - const std::map& fs_scheme_map = {}, - const std::map& dynamic_options = {}); + const std::string& uuid, const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map, + const std::map& dynamic_options); const std::string& CurrentBranch() const { return store_->CurrentBranch(); @@ -102,15 +102,15 @@ class FileStoreTable : public Table { return identifier_.GetFullName(); } - /// The file system, root path and identifier stay as they are: they name this table, which an - /// option does not rename. + /// Returns a copy that serves `dynamic_options`. The file system, root path and identifier + /// stay as they are: they name this table, and an option does not rename it. Result> Copy( const std::map& dynamic_options) const override; /// Returns the table of `branch`, as Java Paimon's `switchToBranch` does. The branch switched /// to is served as it is: its schema is read from its own directory, so this table's dynamic - /// options are not carried over. Switching to the branch already served hands back a copy of - /// this table, schema and managers shared, and a branch with no schema does not exist. + /// options are not carried over. Switching to the branch this table already serves hands back a + /// copy of it with the schema and the managers shared. A branch with no schema does not exist. Result> SwitchToBranch(const std::string& branch) const; Result> NewScan(std::unique_ptr context) const override; @@ -124,47 +124,47 @@ class FileStoreTable : public Table { std::unique_ptr context) const override; private: - /// The options a context carries against the ones the table was loaded with. A table serves - /// the options it was loaded with, so an option the context contradicts is refused rather than - /// honoured - a caller that means to change them copies the table. A table a load hands back - /// carries the context's own options merged in - `FileStoreTableFactory` does the merging - so - /// it agrees by construction; the refusal belongs to a table the caller held already, whose - /// load is over. A key the table was not loaded with is no contradiction: it names a runtime - /// setting - a scan mode, a batch size - which is none of the table's business, and is served - /// as given. + /// Checks the options a context carries against the ones the table was loaded with. A table + /// serves the options it was loaded with, so an option the context contradicts is refused: a + /// caller that means to change them copies the table. A table a load hands back carries the + /// context's own options merged in - `FileStoreTableFactory` merges them - so it agrees by + /// construction. The refusal belongs to a table the caller already held, whose load is over. A + /// key the table was not loaded with is no contradiction: it names a runtime setting, a scan + /// mode or a batch size, which is none of the table's business and is served as given. static Status CheckOptionsAgreeWithTable( const TableSchema& table_schema, const std::map& context_options); - /// The read `NewRead` hands back, from the internal context it read the schema into: the path - /// factory comes from the store, and the reader is selected by the schema the read serves - the - /// primary keys it carries - rather than by the table's own kind. + /// Builds the read `NewRead` hands back, from the internal context it read the schema into. The + /// path factory comes from the store. The reader is picked by the schema the read serves - the + /// primary keys it carries - not by the table's own kind. Result> CreateTableRead( const std::shared_ptr& internal_context, const std::shared_ptr& memory_pool, const std::shared_ptr& executor) const; protected: - /// The base constructor the kinds of managed table build through: the store carries every state - /// the kinds hand down - see `Create` for what each piece of it is. + /// The base constructor the kinds of managed table build through. The store carries every state + /// they hand down; see `Create` for what each piece of it is. FileStoreTable(const std::shared_ptr& store, const Identifier& identifier, const std::string& uuid); - /// The committed progress a real-time write restores, from the latest snapshot the snapshot - /// manager reads: where the write picks up after, which both kinds ask for the same way. + /// Restores the committed progress of a real-time write from the latest snapshot the snapshot + /// manager reads. It tells the write where to pick up, and both kinds ask for it the same way. static Status RestoreRealtimeCommittedProgress( const std::shared_ptr& realtime_context, const std::shared_ptr& snapshot_manager, const CoreOptions& options); - /// The scan the table's root path names, which the kinds of file store scan read from. + /// Returns the scan restricted to the table's root path, which the kinds of file store scan + /// read from. static std::unique_ptr WithTablePath(std::unique_ptr&& scan, const std::string& root_path); - /// The write of this kind of table, which `NewWrite` hands the shared flow's pieces to. The - /// schema and the branch the write serves are the table's own. + /// Builds the write of this kind of table. `NewWrite` hands the pieces of the shared flow to + /// it. The schema and the branch the write serves are the table's own. /// @param io_manager The spill manager the context's temp directory names, or null when the /// context named none. - /// @param ignore_previous_files Whether the write starts over rather than appending, which - /// the flow resolved with this kind's own rules. + /// @param ignore_previous_files Whether the write starts over rather than appends, which the + /// flow resolved with this kind's own rules. virtual Result> CreateWrite( const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, @@ -172,8 +172,8 @@ class FileStoreTable : public Table { const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const = 0; - /// The scan of this kind of table, which `NewScan` hands the manifest list and file it built - /// to, over the table's own schema and managers. + /// Builds the scan of this kind of table. `NewScan` hands it the manifest list and the manifest + /// file it built, over the table's own schema and managers. virtual Result> NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, @@ -181,12 +181,12 @@ class FileStoreTable : public Table { const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const = 0; - /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. + /// Builds the split generator of this kind of table, Java Paimon's `splitGenerator()`. virtual Result> NewSplitGenerator( const CoreOptions& core_options) const = 0; - /// The manifest entry scan supplier of this kind of table, which `NewCommit` hands the manifest - /// list and file it built to. + /// Builds the manifest entry scan supplier of this kind of table. `NewCommit` hands it the + /// manifest list and the manifest file it built. virtual CommitScanner::ScanSupplier NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, @@ -195,13 +195,13 @@ class FileStoreTable : public Table { const std::shared_ptr& executor, const std::shared_ptr& pool) const = 0; - /// The storage this table serves - see `FileStore`. + /// The storage this table reads and writes through. See `FileStore`. std::shared_ptr store_; - /// The dynamic options the table was loaded with over the ones its schema stored, which `Copy` - /// accumulates into the copy it derives. They stay separate from the schema they were merged - /// into because the fallback reader of a read needs them alone: it serves another branch's - /// schema, over which the same options are merged again. + /// The dynamic options the table was loaded with, over the ones its schema stored. `Copy` + /// accumulates them into the copy it derives. They stay separate from the schema they were + /// merged into because a fallback reader needs them on their own: it serves another branch's + /// schema and merges the same options over it again. std::map dynamic_options_; }; diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp index 2b5c9a40c..d5082ef73 100644 --- a/src/paimon/core/table/file_store_table_factory.cpp +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -125,7 +125,7 @@ Result> TableOfLoadedSchema( const std::string& branch, const std::map& options, const std::shared_ptr& schema_manager, const std::shared_ptr& latest_schema, const std::optional& identifier, - const std::string& uuid = "", const std::map& fs_scheme_map = {}) { + const std::string& uuid, const std::map& fs_scheme_map) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, TryLoadFormatTable(file_system, path, branch, options, latest_schema)); if (format_table != nullptr) { @@ -146,10 +146,11 @@ Result> TableOfLoadedSchema( !FileStore::FileSystemResolutionAgrees(latest_schema->Options(), options))) { manager = nullptr; } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - FileStoreTable::Create(explicit_file_system, path, latest_schema, - table_identifier, uuid, branch, manager, - /*snapshot_manager=*/nullptr, fs_scheme_map)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + FileStoreTable::Create(explicit_file_system, path, latest_schema, table_identifier, uuid, + branch, manager, /*snapshot_manager=*/nullptr, fs_scheme_map, + /*dynamic_options=*/{})); return WithContextOptions(table, latest_schema, options); } @@ -167,7 +168,7 @@ Result> ManagedTableOf( const std::shared_ptr& explicit_file_system, const std::string& path, const std::string& branch, const std::map& options, const std::optional& schema_json, const std::optional& identifier, - const std::string& uuid = "", const std::map& fs_scheme_map = {}) { + const std::string& uuid, const std::map& fs_scheme_map) { auto schema_manager = std::make_shared(file_system, path, branch); std::shared_ptr latest_schema; if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { @@ -230,7 +231,8 @@ Result> FileStoreTableFactory::Create(const ScanContext& const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), - ctx.GetIdentifier()); + ctx.GetIdentifier(), + /*uuid=*/"", /*fs_scheme_map=*/{}); } /// The table a read context names: the format table the builder was handed, a system table behind @@ -340,7 +342,7 @@ Result> FileStoreTableFactory::Create(const CommitContext // not dispatch on different schemas. return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, root_path, branch, ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), - ctx.GetTableId().value_or("")); + ctx.GetTableId().value_or(""), /*fs_scheme_map=*/{}); } } // namespace paimon diff --git a/src/paimon/core/table/file_store_table_factory.h b/src/paimon/core/table/file_store_table_factory.h index 175834ff6..a1bb77062 100644 --- a/src/paimon/core/table/file_store_table_factory.h +++ b/src/paimon/core/table/file_store_table_factory.h @@ -30,10 +30,10 @@ class ScanContext; class Table; class WriteContext; -/// The table a context names, Java Paimon's `FileStoreTableFactory`: the format table the builder -/// was handed, a system table behind a `T$system` path, or the table the schema under the -/// context's path says is there. The four operators' static `Create` entries all dispatch on the -/// same question, so one place answers it and they cannot disagree. +/// Loads the table a context names, as Java Paimon's `FileStoreTableFactory` does: the format +/// table the builder was handed, the system table behind a `T$system` path, or the managed table +/// the schema under the context's path describes. The static `Create` of all four operators asks +/// the same question, so one place answers it and they cannot disagree. class FileStoreTableFactory { public: FileStoreTableFactory() = delete; diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index d4b611eac..f14ac66d1 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -224,6 +224,9 @@ Result> FormatTable::GetArrowSchema() const { return schema_->GetArrowSchema(); } +// The refusals below use two codes. `Invalid` is a setting that clashes with what a format table +// is, and will never work: no snapshots, no buckets, no real-time store, no manifests, no +// catalog-side version. `NotImplemented` is a capability this code lacks today and could gain. Result> FormatTable::NewScan( std::unique_ptr context) const { std::shared_ptr shared_context = std::move(context); @@ -233,15 +236,15 @@ Result> FormatTable::NewScan( // Anything a `ScanContext` carries that a format table cannot honour is refused rather than // silently dropped. if (shared_context->IsStreamingMode()) { - return Status::NotImplemented( + return Status::Invalid( "a format table has no snapshots, so there is nothing for a streaming scan to follow"); } if (shared_context->GetRealtimeContext() != nullptr) { - return Status::NotImplemented( + return Status::Invalid( "a format table has no real-time store to union with what is on disk"); } if (shared_context->GetGlobalIndexResult() != nullptr) { - return Status::NotImplemented("a format table carries no global index"); + return Status::Invalid("a format table carries no global index"); } std::map partition_filter; std::shared_ptr filters = shared_context->GetScanFilters(); @@ -253,12 +256,12 @@ Result> FormatTable::NewScan( "implemented yet; give the partition values as a partition filter instead"); } if (filters->GetBucketFilter()) { - return Status::NotImplemented("a format table has no buckets to filter by"); + return Status::Invalid("a format table has no buckets to filter by"); } const std::vector>& partition_filters = filters->GetPartitionFilters(); if (partition_filters.size() > 1) { - return Status::NotImplemented("a format table scan takes at most one partition filter"); + return Status::Invalid("a format table scan takes at most one partition filter"); } if (partition_filters.size() == 1) { partition_filter = partition_filters.front(); @@ -298,12 +301,12 @@ Result> FormatTable::NewWrite( "format table"); } if (context->IsStreamingMode()) { - return Status::NotImplemented( + return Status::Invalid( "a format table has no snapshots, so there is nothing a streaming write could " "commit against"); } if (context->GetRealtimeContext() != nullptr) { - return Status::NotImplemented("a format table has no real-time store to write into"); + return Status::Invalid("a format table has no real-time store to write into"); } if (!context->GetWriteSchema().empty()) { return Status::NotImplemented( @@ -313,8 +316,7 @@ Result> FormatTable::NewWrite( // A write id prefixes a postpone-bucket writer's files so that one compaction reader can // put them back in order. A format table has no buckets, so it would identify nothing. if (context->GetWriteId().has_value()) { - return Status::NotImplemented( - "a format table has no buckets, so a write id would name nothing"); + return Status::Invalid("a format table has no buckets, so a write id would name nothing"); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, FormatTableFileStoreWrite::Create(table, context->GetMemoryPool())); @@ -327,26 +329,26 @@ Result> FormatTable::NewCommit( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, CopySelf(context->GetOptions())); if (!context->IgnoreEmptyCommit()) { - return Status::NotImplemented( + return Status::Invalid( "a format table cannot record an empty commit: keeping one means writing a " "snapshot that adds no files, and there are no snapshots here"); } if (context->UseRESTCatalogCommit()) { - return Status::NotImplemented( + return Status::Invalid( "a format table commits by renaming files into place, not by sending a snapshot " "to a rest catalog"); } if (context->GetTableId()) { - return Status::NotImplemented( + return Status::Invalid( "a format table sends no commit table request, so there is nowhere to put a table id"); } if (context->GetCatalog() != nullptr) { - return Status::NotImplemented( + return Status::Invalid( "a format table keeps no snapshot for a catalog to take, so it commits by renaming " "files into place rather than through the catalog it was loaded from"); } if (context->AppendCommitCheckConflict()) { - return Status::NotImplemented( + return Status::Invalid( "a format table has no manifests to check a concurrent commit against"); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index 2dac6c102..84fbce351 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -2766,6 +2766,35 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { reader->Close(); } +TEST(FormatTableTest, TestACatalogInTheContextIsRefusedAlikeByWriteAndCommit) { + // `WithCatalog()` asks for a catalog-side version of the commit, which a format table has no + // way to produce. Both halves of the write path answer with the same code, so a caller reads + // one kind of failure out of them rather than "cannot be done" from one and "not yet" from + // the other. + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + + auto catalog = std::make_shared(); + catalog->SetTableSchema(table->LatestSchema()); + Identifier identifier("db", "fmt"); + + WriteContextBuilder write_builder(dir->Str(), "test-user"); + write_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + Result> write = table->NewWrite(std::move(write_context)); + ASSERT_FALSE(write.ok()); + ASSERT_TRUE(write.status().IsInvalid()); + + CommitContextBuilder commit_builder(dir->Str(), "test-user"); + commit_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + Result> commit = table->NewCommit(std::move(commit_context)); + ASSERT_FALSE(commit.ok()); + ASSERT_TRUE(commit.status().IsInvalid()); +} + TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { // A table a catalog keeps the schema for, the way a rest catalog does. Nothing under its // location says that it is a format table, nor that everything below the location is data, so diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h index ce7547a71..2d3390e86 100644 --- a/src/paimon/core/table/primary_key_file_store_table.h +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -43,7 +43,7 @@ class PrimaryKeyFileStoreTable : public FileStoreTable { /// The constructor `FileStoreTable::Create` builds a primary key table through: the store /// carries every state, and the base class holds it. PrimaryKeyFileStoreTable(const std::shared_ptr& store, const Identifier& identifier, - const std::string& uuid = ""); + const std::string& uuid); private: Result> CreateWrite( diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 3cd3b22de..3e79c537d 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -137,7 +137,10 @@ TEST(TableReadTest, TestFullTextMatchExecutionIsNotImplemented) { ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( std::shared_ptr table, - FileStoreTable::Create(fs, table_path, schema, Identifier("append_09.db", "append_09"))); + FileStoreTable::Create(fs, table_path, schema, Identifier("append_09.db", "append_09"), + /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); for (bool table_entry : {false, true}) { ReadContextBuilder builder(table_path); builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); @@ -164,7 +167,10 @@ TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) { ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( std::shared_ptr table, - FileStoreTable::Create(fs, path, schema, Identifier("append_09.db", "append_09"))); + FileStoreTable::Create(fs, path, schema, Identifier("append_09.db", "append_09"), + /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); for (bool table_entry : {false, true}) { ReadContextBuilder builder(path); builder.SetFullTextSearch(search).SetReadFieldNames({"f1", "_f0_INDEX_SCORE"}); diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 09fd6328f..0bf82ead3 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -359,16 +359,19 @@ std::unique_ptr CreateChangelogBatchReader( AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, std::map options) - : SystemTable(fs, table_path), + : AuditLogSystemTable(std::move(fs), table_path, std::move(table_schema), std::move(options), + ComposeName(table_path, kName)) {} + +AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, + std::shared_ptr table_schema, + std::map options, + std::string name) + : SystemTable(fs, table_path, std::move(name)), fs_(std::move(fs)), table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} -std::string AuditLogSystemTable::Name() const { - return kName; -} - Result> AuditLogSystemTable::ArrowSchema() const { std::shared_ptr row_kind_field = DataField::ConvertDataFieldToArrowField(SpecialFields::RowKind()); @@ -398,6 +401,8 @@ Result> AuditLogSystemTable::NewScan( .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) // The view's own file system when the context names none, rather than the default backend. + // A `ScanContext` carries no scheme-to-identifier map, so unlike `CreateDataReadContext` + // below there is no map here to resolve through instead. .WithFileSystem(context->GetSpecificFileSystem() != nullptr ? context->GetSpecificFileSystem() : GetFileSystem()) diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h index 7395c9797..4cc324b1d 100644 --- a/src/paimon/core/table/system/audit_log_system_table.h +++ b/src/paimon/core/table/system/audit_log_system_table.h @@ -58,12 +58,17 @@ class AuditLogSystemTable : public SystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> NewScan(std::unique_ptr context) const override; Result> NewRead(std::unique_ptr context) const override; protected: + /// For a derived changelog view that answers with a kind of its own: `BinlogSystemTable` names + /// itself `T$binlog`, not the `T$audit_log` the public constructor composes. + AuditLogSystemTable(std::shared_ptr fs, std::string table_path, + std::shared_ptr table_schema, + std::map options, std::string name); + Result> NewChangelogRead( const std::shared_ptr& context, std::shared_ptr converter) const; diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index a3c39078d..8299f17f5 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -74,12 +74,8 @@ std::shared_ptr CreateBinlogBatchConverter() { BinlogSystemTable::BinlogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, std::map options) - : AuditLogSystemTable(std::move(fs), std::move(table_path), std::move(table_schema), - std::move(options)) {} - -std::string BinlogSystemTable::Name() const { - return kName; -} + : AuditLogSystemTable(std::move(fs), table_path, std::move(table_schema), std::move(options), + ComposeName(table_path, kName)) {} Result> BinlogSystemTable::ArrowSchema() const { std::shared_ptr rowkind_field = diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h index 3a939fcef..3a28ffa93 100644 --- a/src/paimon/core/table/system/binlog_system_table.h +++ b/src/paimon/core/table/system/binlog_system_table.h @@ -40,7 +40,6 @@ class BinlogSystemTable : public AuditLogSystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> NewRead(std::unique_ptr context) const override; }; diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index a66362b39..92f10c49d 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -126,6 +126,18 @@ Result IsEnabled(const GlobalSystemTableRegistryEntry& entry, CatalogOptionsSystemTable::kEnabledOption, false); } +// Whether a table reads its rows by enumerating what the catalog holds, and so cannot be built +// without one. +bool RequiresCatalog(const std::string& table_name) { + std::string normalized = StringUtils::ToLowerCase(table_name); + for (const auto& entry : GlobalSystemTableRegistry()) { + if (entry.name == normalized) { + return entry.requires_catalog; + } + } + return false; +} + struct CatalogTableInfo { std::string database_name; std::string table_name; @@ -134,11 +146,16 @@ struct CatalogTableInfo { // Match Java CatalogUtils::listAllTables: tolerate databases or tables removed concurrently, but // propagate all other catalog and schema errors instead of returning incomplete system-table rows. -Result> LoadAllDataTables(const Catalog& catalog) { +// `table_name` names the table asking, for the refusal a missing catalog reports. +Result> LoadAllDataTables( + const std::shared_ptr& catalog, const std::string& table_name) { + if (catalog == nullptr) { + return Status::NotImplemented("global system table requires catalog context: ", table_name); + } std::vector result; - PAIMON_ASSIGN_OR_RAISE(std::vector databases, catalog.ListDatabases()); + PAIMON_ASSIGN_OR_RAISE(std::vector databases, catalog->ListDatabases()); for (const std::string& database : databases) { - Result> tables_result = catalog.ListTables(database); + Result> tables_result = catalog->ListTables(database); if (!tables_result.ok()) { if (tables_result.status().IsNotExist()) { continue; @@ -147,7 +164,7 @@ Result> LoadAllDataTables(const Catalog& catalog) } for (const std::string& table : tables_result.value()) { Identifier identifier(database, table); - Result> schema_result = catalog.LoadTableSchema(identifier); + Result> schema_result = catalog->LoadTableSchema(identifier); if (!schema_result.ok()) { if (schema_result.status().IsNotExist()) { continue; @@ -288,6 +305,16 @@ Result GlobalSystemTableLoader::IsSupported( } Result> GlobalSystemTableLoader::Load( + const std::string& table_name, const GlobalSystemTableContext& context) { + // Refused here rather than at the first read: a table handed back to a caller that can never + // build its rows is a failure the caller finds out about late and far from its cause. + if (context.catalog == nullptr && RequiresCatalog(table_name)) { + return Status::NotImplemented("global system table requires catalog context: ", table_name); + } + return LoadForSchema(table_name, context); +} + +Result> GlobalSystemTableLoader::LoadForSchema( const std::string& table_name, const GlobalSystemTableContext& context) { std::string normalized = StringUtils::ToLowerCase(table_name); for (const auto& entry : GlobalSystemTableRegistry()) { @@ -296,10 +323,6 @@ Result> GlobalSystemTableLoader::Load( if (!enabled) { return Status::NotExist("global system table is disabled: ", table_name); } - if (entry.requires_catalog && context.catalog == nullptr) { - return Status::NotImplemented("global system table requires catalog context: ", - table_name); - } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, entry.factory(context)); // The table is handed out through the `Table` interface, which serves the schema @@ -330,11 +353,7 @@ Result> GlobalSystemTableLoader::GetSupportedTableNames // ============================================================================= CatalogOptionsSystemTable::CatalogOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable(context.fs, "sys/catalog_options"), context_(std::move(context)) {} - -std::string CatalogOptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/catalog_options", kName), context_(std::move(context)) {} Result> CatalogOptionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -366,11 +385,8 @@ Result> CatalogOptionsSystemTable::BuildRows() const { // ============================================================================= AllTableOptionsSystemTable::AllTableOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable(context.fs, "sys/all_table_options"), context_(std::move(context)) {} - -std::string AllTableOptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/all_table_options", kName), + context_(std::move(context)) {} Result> AllTableOptionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -386,7 +402,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { for (const auto& [key, value] : table.schema->Options()) { GenericRow row(schema->num_fields()); @@ -405,11 +421,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { // ============================================================================= TablesSystemTable::TablesSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable(context.fs, "sys/tables"), context_(std::move(context)) {} - -std::string TablesSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/tables", kName), context_(std::move(context)) {} Result> TablesSystemTable::ArrowSchema() const { return arrow::schema({ @@ -435,7 +447,7 @@ Result> TablesSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { const std::shared_ptr& data_schema = table.schema; @@ -477,11 +489,7 @@ Result> TablesSystemTable::BuildRows() const { // ============================================================================= PartitionsSystemTable::PartitionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable(context.fs, "sys/partitions"), context_(std::move(context)) {} - -std::string PartitionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/partitions", kName), context_(std::move(context)) {} Result> PartitionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -501,7 +509,7 @@ Result> PartitionsSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { const std::shared_ptr& data_schema = table.schema; Identifier id(table.database_name, table.table_name); diff --git a/src/paimon/core/table/system/global_system_tables.h b/src/paimon/core/table/system/global_system_tables.h index eadc83af1..280f57479 100644 --- a/src/paimon/core/table/system/global_system_tables.h +++ b/src/paimon/core/table/system/global_system_tables.h @@ -33,14 +33,11 @@ class FileSystem; /// Context passed to global system table constructors, providing catalog-level /// access for enumerating databases, tables, and reading metadata. struct GlobalSystemTableContext { - /// The catalog the table reads its rows through, non-owning: the rows are built lazily, - /// so a table that outlives the call that built it must be handed the shared ownership - /// below too, or this pointer dangles at the first `BuildRows`. - const Catalog* catalog = nullptr; - /// The shared ownership a returned table keeps of its catalog, so its lazy `BuildRows` - /// cannot read through one already destroyed. Empty where the table is transient - used - /// within the call, before the catalog can go away. - std::shared_ptr catalog_retainer; + /// The catalog this table reads its rows through. Rows are built lazily, so a table handed back + /// to a caller keeps shared ownership of it. Empty when the table is transient and only its + /// schema is read. `GlobalSystemTableLoader::Load` refuses a table that enumerates data tables + /// when this is empty. + std::shared_ptr catalog; std::shared_ptr fs; std::string warehouse; std::map catalog_options; @@ -55,7 +52,6 @@ class CatalogOptionsSystemTable : public InMemorySystemTable { explicit CatalogOptionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -71,7 +67,6 @@ class AllTableOptionsSystemTable : public InMemorySystemTable { explicit AllTableOptionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -87,7 +82,6 @@ class TablesSystemTable : public InMemorySystemTable { explicit TablesSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -103,7 +97,6 @@ class PartitionsSystemTable : public InMemorySystemTable { explicit PartitionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -120,9 +113,17 @@ class GlobalSystemTableLoader { static Result IsSupported(const std::string& table_name, const std::map& catalog_options); + /// Loads a global system table for a caller to read. A table that enumerates data tables needs + /// the catalog in the context, so it is refused here when the context carries none. static Result> Load(const std::string& table_name, const GlobalSystemTableContext& context); + /// Loads a global system table for the schema it exposes alone, the way + /// `Catalog::LoadTableSchema` does. No rows are built, so a context that carries no catalog is + /// accepted here and refused by `Load` instead. + static Result> LoadForSchema( + const std::string& table_name, const GlobalSystemTableContext& context); + static Result> GetSupportedTableNames( const std::map& catalog_options); }; diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index d21d6fb4d..87fdd1d29 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -107,8 +107,8 @@ class InMemorySystemTableRead : public TableRead { } // namespace InMemorySystemTable::InMemorySystemTable(const std::shared_ptr& fs, - std::string table_path) - : SystemTable(fs, table_path), table_path_(std::move(table_path)) {} + std::string table_path, std::string name) + : SystemTable(fs, table_path, std::move(name)), table_path_(std::move(table_path)) {} Result> InMemorySystemTable::NewScan( std::unique_ptr /*context*/) const { diff --git a/src/paimon/core/table/system/in_memory_system_table.h b/src/paimon/core/table/system/in_memory_system_table.h index 90cd46d84..c72d7389e 100644 --- a/src/paimon/core/table/system/in_memory_system_table.h +++ b/src/paimon/core/table/system/in_memory_system_table.h @@ -36,7 +36,8 @@ class FileSystem; /// It provides the common singleton split scan and one-shot batch reader. class InMemorySystemTable : public SystemTable { public: - InMemorySystemTable(const std::shared_ptr& fs, std::string table_path); + InMemorySystemTable(const std::shared_ptr& fs, std::string table_path, + std::string name); Result> NewScan( std::unique_ptr /*context*/) const override; diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index e9e00ea42..1b5927004 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -507,11 +507,8 @@ Result> WriteColsValue( OptionsSystemTable::OptionsSystemTable(const std::shared_ptr& fs, std::string table_path, std::shared_ptr table_schema) - : InMemorySystemTable(fs, table_path), table_schema_(std::move(table_schema)) {} - -std::string OptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), + table_schema_(std::move(table_schema)) {} Result> OptionsSystemTable::ArrowSchema() const { return arrow::schema({arrow::field("key", arrow::utf8(), /*nullable=*/false), @@ -533,13 +530,9 @@ Result> OptionsSystemTable::BuildRows() const { SnapshotsSystemTable::SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string SnapshotsSystemTable::Name() const { - return kName; -} - Result> SnapshotsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("snapshot_id", arrow::int64(), /*nullable=*/false), @@ -595,13 +588,9 @@ Result> SnapshotsSystemTable::BuildRows() const { SchemasSystemTable::SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string SchemasSystemTable::Name() const { - return kName; -} - Result> SchemasSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("schema_id", arrow::int64(), /*nullable=*/false), @@ -651,13 +640,9 @@ Result> SchemasSystemTable::BuildRows() const { TagsSystemTable::TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string TagsSystemTable::Name() const { - return kName; -} - Result> TagsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("tag_name", arrow::utf8(), /*nullable=*/false), @@ -701,13 +686,9 @@ Result> TagsSystemTable::BuildRows() const { BranchesSystemTable::BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string BranchesSystemTable::Name() const { - return kName; -} - Result> BranchesSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("branch_name", arrow::utf8(), /*nullable=*/false), @@ -740,13 +721,9 @@ Result> BranchesSystemTable::BuildRows() const { ConsumersSystemTable::ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string ConsumersSystemTable::Name() const { - return kName; -} - Result> ConsumersSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("consumer_id", arrow::utf8(), /*nullable=*/false), @@ -775,14 +752,10 @@ ManifestsSystemTable::ManifestsSystemTable(std::shared_ptr fs, std:: std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} -std::string ManifestsSystemTable::Name() const { - return kName; -} - Result> ManifestsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("file_name", arrow::utf8(), /*nullable=*/false), @@ -844,14 +817,10 @@ Result> ManifestsSystemTable::BuildRows() const { FilesSystemTable::FilesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} -std::string FilesSystemTable::Name() const { - return kName; -} - Result> FilesSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("partition", arrow::utf8(), /*nullable=*/true), diff --git a/src/paimon/core/table/system/metadata_system_tables.h b/src/paimon/core/table/system/metadata_system_tables.h index ae2e5155b..1127e37ed 100644 --- a/src/paimon/core/table/system/metadata_system_tables.h +++ b/src/paimon/core/table/system/metadata_system_tables.h @@ -38,7 +38,6 @@ class OptionsSystemTable : public InMemorySystemTable { OptionsSystemTable(const std::shared_ptr& fs, std::string table_path, std::shared_ptr table_schema); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -63,7 +62,6 @@ class SnapshotsSystemTable : public InMemorySystemTable { SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -78,7 +76,6 @@ class SchemasSystemTable : public InMemorySystemTable { SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -93,7 +90,6 @@ class TagsSystemTable : public InMemorySystemTable { TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -108,7 +104,6 @@ class BranchesSystemTable : public InMemorySystemTable { BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -124,7 +119,6 @@ class ConsumersSystemTable : public InMemorySystemTable { ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -141,7 +135,6 @@ class ManifestsSystemTable : public InMemorySystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -158,7 +151,6 @@ class FilesSystemTable : public InMemorySystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 65a48c048..2c4123f2f 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -40,15 +40,11 @@ ReadOptimizedSystemTable::ReadOptimizedSystemTable(const std::shared_ptr table_schema, std::map options) - : SystemTable(fs, table_path), + : SystemTable(fs, table_path, ComposeName(table_path, kName)), table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} -std::string ReadOptimizedSystemTable::Name() const { - return kName; -} - Result> ReadOptimizedSystemTable::ArrowSchema() const { return DataField::ConvertDataFieldsToArrowSchema(table_schema_->Fields()); } @@ -74,6 +70,8 @@ Result> ReadOptimizedSystemTable::NewScan( .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) // The view's own file system when the context names none, rather than the default backend. + // A `ScanContext` carries no scheme-to-identifier map, so unlike `CreateDataReadContext` + // below there is no map here to resolve through instead. .WithFileSystem(context->GetSpecificFileSystem() != nullptr ? context->GetSpecificFileSystem() : GetFileSystem()) diff --git a/src/paimon/core/table/system/read_optimized_system_table.h b/src/paimon/core/table/system/read_optimized_system_table.h index 7f92e04e2..82fec3ef4 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.h +++ b/src/paimon/core/table/system/read_optimized_system_table.h @@ -38,7 +38,6 @@ class ReadOptimizedSystemTable : public SystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> NewScan(std::unique_ptr context) const override; Result> NewRead(std::unique_ptr context) const override; diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 0161ad6b7..7a8146534 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -42,6 +42,7 @@ #include "paimon/core/utils/branch_manager.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" +#include "paimon/predicate/predicate_utils.h" #include "paimon/read_context.h" #include "paimon/status.h" #include "paimon/write_context.h" @@ -182,15 +183,27 @@ std::optional FindSystemTableFactory(const std::string& syst } // namespace -SystemTable::SystemTable(const std::shared_ptr& fs, const std::string& table_path) +SystemTable::SystemTable(const std::shared_ptr& fs, const std::string& table_path, + std::string name) : Table(fs, table_path, /*schema=*/nullptr, - Identifier(Identifier::kUnknownDatabase, "system_table"), /*uuid=*/"") {} + Identifier(Identifier::kUnknownDatabase, "system_table"), /*uuid=*/""), + name_(std::move(name)) {} + +std::string SystemTable::ComposeName(const std::string& table_path, const std::string& kind) { + return PathUtil::GetName(table_path) + Catalog::SYSTEM_TABLE_SPLITTER + kind; +} Status SystemTable::CheckNoSearch(const std::shared_ptr& context) { if (context->HasVectorOrFullTextSearch()) { return Status::NotImplemented( "VectorSearch or FullTextSearch does not support system-table reads yet"); } + // A `FullTextMatch` predicate is not a TopK request, so the check above does not see it. The + // factory refuses one for a system table with the same message. An in-memory view serves rows + // of its own and never evaluates a predicate, so without this it answers with all of them. + if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { + return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + } return Status::OK(); } @@ -282,7 +295,7 @@ Result> SystemTableLoader::LoadFromPath( // The warehouse is the grandparent of the sys/
path context.warehouse = PathUtil::GetParentDirPath(PathUtil::GetParentDirPath(path)); context.catalog_options = dynamic_options; - // Note: context.catalog is intentionally left as nullptr here. + // Note: context.catalog is intentionally left empty here. // Global tables loaded from path do not have a Catalog reference and // cannot enumerate databases/tables. Only tables that don't require // catalog enumeration (e.g. catalog_options) will work in this path. diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 4c235d3cf..640672236 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -50,21 +50,29 @@ struct SystemTablePath { bool is_global = false; }; -/// Base class for table-scoped system tables such as `T$options` and `T$snapshots`: the read-only -/// view a `T$system` identifier names, which exposes a schema of its own and serves scans and -/// reads from the data table it belongs to. `Name()` - which each kind answers for itself - is -/// its full name as well: the identifier it is opened by belongs to the data table underneath. +/// Base class for the table-scoped system tables such as `T$options` and `T$snapshots`. A +/// `T$system` identifier names a read-only view that exposes a schema of its own and serves scans +/// and reads from the data table it belongs to. +/// +/// A system table of a data table answers `$` for `Name()`. One of the `sys` +/// database answers the bare kind name. `FullName()` returns the same as `Name()` in both cases, +/// which is what Java Paimon's `Table.fullName()` defaults to. class SystemTable : public Table, public std::enable_shared_from_this { public: + std::string Name() const override { + return name_; + } + std::string FullName() const override { - return Name(); + return name_; } - /// The read-only schema this system table exposes, which is not the data table's schema. + /// Returns the read-only schema this system table exposes. It is not the data table's schema. virtual Result> ArrowSchema() const = 0; - /// A system table is read-only: copying it, writing it and committing for it are refused - /// rather than routed to the data table underneath, which its own `Table` serves. + /// A system table is read-only, so copying it, writing it and committing for it are refused + /// instead of routed to the data table underneath. That table serves them through its own + /// `Table`. Result> Copy( const std::map& dynamic_options) const override; @@ -77,19 +85,29 @@ class SystemTable : public Table, public std::enable_shared_from_this& fs, const std::string& table_path); - - /// Refuses a read whose context asks for a search: each system table builds a context of its - /// own for the data table underneath, which carries no search, so answering one would return - /// plain rows and quietly drop the search and its top K. + /// @param name The name this table answers with: `ComposeName(table_path, kind)` for a + /// table-scoped system table, the bare kind name for one of the `sys` database. + SystemTable(const std::shared_ptr& fs, const std::string& table_path, + std::string name); + + /// Composes the name of a table-scoped system table. Java Paimon's `OptionsTable.name()` + /// composes it the same way: `location.getName() + SYSTEM_TABLE_SPLITTER + OPTIONS`. + static std::string ComposeName(const std::string& table_path, const std::string& kind); + + /// Refuses a read whose context asks for a search: a root TopK request or a `FullTextMatch` + /// predicate. Neither is evaluated on the rows a system table serves, so answering one + /// would return plain rows and quietly drop the search and its top K. static Status CheckNoSearch(const std::shared_ptr& context); - /// Loaders call this once the object exists, because `ArrowSchema()` is virtual and cannot run - /// from a constructor: it fills in this table's own read-only `SystemTableSchema`. + /// Fills in this table's own read-only `SystemTableSchema`. A loader calls it once the object + /// exists, because `ArrowSchema()` is virtual and cannot run from a constructor. Status AdoptOwnSchema(); friend class SystemTableLoader; friend class GlobalSystemTableLoader; + + private: + std::string name_; }; /// Loads system table implementations from parsed table identifiers or table paths. diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index f1f0cbfb1..d6333475b 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -38,6 +38,7 @@ #include "paimon/core/table/system/audit_log_system_table.h" #include "paimon/core/table/system/binlog_system_table.h" #include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/metadata_system_tables.h" #include "paimon/core/table/system/read_optimized_system_table.h" #include "paimon/core/table/system/system_table_scan.h" #include "paimon/defs.h" @@ -45,6 +46,9 @@ #include "paimon/fs/file_system_factory.h" #include "paimon/memory/memory_pool.h" #include "paimon/metrics.h" +#include "paimon/predicate/full_text_match.h" +#include "paimon/predicate/predicate.h" +#include "paimon/predicate/predicate_builder.h" #include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" @@ -222,7 +226,9 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, "/tmp/table", table_schema, /*dynamic_options=*/{})); - ASSERT_EQ(system_table->Name(), ReadOptimizedSystemTable::kName); + // The name carries the data table it belongs to, the way Java Paimon's + // `ReadOptimizedTable.name()` does. + ASSERT_EQ(system_table->Name(), "table$ro"); ASSERT_OK_AND_ASSIGN(std::shared_ptr arrow_schema, system_table->ArrowSchema()); ASSERT_EQ(arrow_schema->field_names(), (std::vector{"pk", "v"})); @@ -360,6 +366,39 @@ TEST(SystemTableTest, TestNewReadRefusesASearch) { "VectorSearch or FullTextSearch does not support system-table reads yet"); } +// `SetPredicate` carries a search too, as a `FullTextMatch` leaf. An in-memory view reads rows +// of its own and never evaluates a predicate, so without the check it would answer with all of +// them where `TableRead::Create` refuses the same request. +TEST(SystemTableTest, TestNewReadRefusesAFullTextMatchPredicate) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr in_memory, + SystemTableLoader::Load(OptionsSystemTable::kName, /*fs=*/nullptr, + "/tmp/table", table_schema, + /*dynamic_options=*/{})); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); + + auto match = std::make_shared("pk", R"({"match":{"query":"apple"}})"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::FullTextMatch(/*field_index=*/0, match)); + + auto make_context = [&predicate] { + ReadContextBuilder builder("/tmp/table"); + builder.SetPredicate(predicate); + return builder.Finish(); + }; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr in_memory_context, make_context()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_optimized_context, make_context()); + + ASSERT_NOK_WITH_MSG(in_memory->NewRead(std::move(in_memory_context)), + "FullTextMatch predicate execution is not implemented yet"); + ASSERT_NOK_WITH_MSG(read_optimized.NewRead(std::move(read_optimized_context)), + "FullTextMatch predicate execution is not implemented yet"); +} + namespace { /// A catalog that serves the files of every table through `GetTableFileSystem`, like one @@ -392,21 +431,25 @@ TEST(SystemTableTest, TestGlobalPartitionsUsesTheFileSystemOfEachTable) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - PerTableFileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); - ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); + // Held by a `shared_ptr` because a global system table keeps shared ownership of the catalog + // it reads its rows through. + std::shared_ptr catalog = + std::make_shared(core_options.GetFileSystem(), dir->Str(), + options); + ASSERT_OK(catalog->CreateDatabase("db1", options, /*ignore_if_exists=*/true)); arrow::Schema typed_schema( {arrow::field("dt", arrow::utf8()), arrow::field("v", arrow::int32(), /*nullable=*/true)}); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(typed_schema, &c_schema).ok()); - Status created = catalog.CreateTable(Identifier("db1", "t1"), &c_schema, - /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, options, - /*ignore_if_exists=*/false); + Status created = catalog->CreateTable(Identifier("db1", "t1"), &c_schema, + /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false); ArrowSchemaRelease(&c_schema); ASSERT_OK(created); GlobalSystemTableContext context; - context.catalog = &catalog; + context.catalog = catalog; context.fs = core_options.GetFileSystem(); context.warehouse = dir->Str(); context.catalog_options = options; @@ -416,16 +459,16 @@ TEST(SystemTableTest, TestGlobalPartitionsUsesTheFileSystemOfEachTable) { // still the ones asked for before its files are listed. ASSERT_OK_AND_ASSIGN(std::vector rows, partitions.BuildRows()); ASSERT_TRUE(rows.empty()); - ASSERT_EQ((std::vector{"db1.t1"}), catalog.requested); + ASSERT_EQ((std::vector{"db1.t1"}), catalog->requested); // A table whose credentials cannot be issued fails the query, so that a partial result // is never mistaken for the whole one. - catalog.failure = Status::Invalid("no credentials for the table"); + catalog->failure = Status::Invalid("no credentials for the table"); ASSERT_NOK_WITH_MSG(partitions.BuildRows(), "no credentials for the table"); // A table that went away between being listed and being read is skipped instead, like // one whose location has already been removed. - catalog.failure = Status::NotExist("table dropped"); + catalog->failure = Status::NotExist("table dropped"); ASSERT_OK_AND_ASSIGN(std::vector skipped, partitions.BuildRows()); ASSERT_TRUE(skipped.empty()); } diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 055fc5995..5100dfd78 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -75,7 +75,8 @@ Result> CreateManagedTable( return FileStoreTable::Create( file_system, path, checked_pointer_cast(std::shared_ptr(std::move(created))), identifier, - /*uuid=*/"", branch); + /*uuid=*/"", branch, /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); } /// Writes two rows through the table's own write and commit, so a scan below has data to find. @@ -243,7 +244,8 @@ TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(created))), - identifier, /*uuid=*/"", /*branch=*/"main")); + identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); // The copy turns the sequence-number column on, and a read projects it through a context // that carries no options of its own. The main reader serves the copy's merged options, so @@ -286,7 +288,8 @@ TEST(TableTest, TestACopyOfACopyServesTheLatestValueOfARepeatedDynamicOption) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(created))), - identifier, /*uuid=*/"", /*branch=*/"main")); + identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); // The option is turned off and then on again by the copy the read serves, so the copy's // schema carries the latest value. The fallback branch's schema does not carry the option @@ -328,7 +331,8 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(main_created))), - identifier, /*uuid=*/"", /*branch=*/"main")); + identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); SchemaManager fallback_schema_manager(fs, path, "fb"); ASSERT_OK_AND_ASSIGN( @@ -341,7 +345,8 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(fallback_created))), - identifier, /*uuid=*/"", /*branch=*/"fb")); + identifier, /*uuid=*/"", /*branch=*/"fb", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/10); WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); @@ -679,7 +684,11 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { ASSERT_OK_AND_ASSIGN(std::string branch, main_identifier.GetBranchNameOrDefault()); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, FileStoreTable::Create(fs, dir->Str(), schema, main_identifier, - /*uuid=*/"", branch)); + /*uuid=*/"", branch, + /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, + /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); EXPECT_EQ("main", table->CurrentBranch()); } } @@ -1112,7 +1121,10 @@ TEST(TableTest, TestCreateFailsWhenTheSchemaOptionsDoNotParse) { /*partition_keys=*/{}, /*primary_keys=*/{}, {{Options::TARGET_FILE_ROW_NUM, "0"}})); auto schema = checked_pointer_cast(std::shared_ptr(std::move(created))); ASSERT_NOK_WITH_MSG( - FileStoreTable::Create(dir->GetFileSystem(), dir->Str(), schema, Identifier("db", "tbl")), + FileStoreTable::Create(dir->GetFileSystem(), dir->Str(), schema, Identifier("db", "tbl"), + /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{}), "target-file-row-num should be at least 1"); } diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index fec764db3..0752af6f7 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -383,13 +383,15 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } + // Only the schema is read here and the table does not outlive the call, so it needs no + // catalog and this one hands over no ownership of itself. GlobalSystemTableContext context; - context.catalog = this; context.fs = fs_; context.warehouse = warehouse_; context.catalog_options = api_->GetMergedOptions(); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + GlobalSystemTableLoader::LoadForSchema(identifier.GetObjectName(), context)); return system_table->LatestSchema(); } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); @@ -444,18 +446,17 @@ Result> RestCatalog::GetTable(const Identifier& identifie if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } - GlobalSystemTableContext context; - context.catalog = this; - // A global system table builds its rows lazily through the catalog above, so the table - // handed back has to keep it alive or the pointer dangles at the first `BuildRows`. A - // catalog not owned by a `std::shared_ptr` cannot be kept, so it is refused by name. + // A global system table builds its rows lazily through this catalog, so the table handed + // back has to keep it alive or the pointer dangles at the first `BuildRows`. A catalog + // not owned by a `std::shared_ptr` cannot be kept, so it is refused by name. std::shared_ptr self = weak_from_this().lock(); if (self == nullptr) { return Status::Invalid( "a global system table retains its catalog, so the catalog must be owned by " "a std::shared_ptr before it is asked for one"); } - context.catalog_retainer = std::move(self); + GlobalSystemTableContext context; + context.catalog = std::move(self); context.fs = fs_; context.warehouse = warehouse_; context.catalog_options = api_->GetMergedOptions(); @@ -496,7 +497,9 @@ Result> RestCatalog::GetTable(const Identifier& identifie PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); return FileStoreTable::Create( table_file_system, table_path, schema, identifier, uuid, - NormalizeBranch(std::move(branch)).value_or(Identifier::kDefaultMainBranch)); + NormalizeBranch(std::move(branch)).value_or(Identifier::kDefaultMainBranch), + /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, + /*dynamic_options=*/{}); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 5d418fc63..cc197b45d 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -743,7 +743,7 @@ TEST_F(RestCatalogTest, RecreatedTableLoadsNewCredentialsBeforeOldTokenExpires) } TEST_F(RestCatalogTest, CatalogFactoryMetastoreDispatch) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(kWarehouse, options_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(kWarehouse, options_)); ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); ASSERT_TRUE(databases.empty()); options_[CatalogOptions::METASTORE] = "something-else"; @@ -1003,13 +1003,13 @@ TEST_F(RestCatalogTest, SystemTableSchema) { requests_before = state_->get_table_requests; } ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(system_identifier)); - // A system table comes back as the read-only `SystemTable` it is, answering for itself - // with the kind it serves rather than the identifier it was opened by. + // A system table comes back as the read-only `SystemTable` it is, naming itself after the + // data table it belongs to. std::shared_ptr system_table = std::dynamic_pointer_cast(table); ASSERT_NE(nullptr, system_table); - ASSERT_EQ("options", system_table->Name()); + ASSERT_EQ("t1$options", system_table->Name()); ASSERT_EQ((std::vector{"key", "value"}), table->LatestSchema()->FieldNames()); - ASSERT_EQ("options", table->Uuid()); + ASSERT_EQ("t1$options", table->Uuid()); { std::lock_guard lock(state_->mutex); ASSERT_EQ(requests_before + 1, state_->get_table_requests); @@ -1511,6 +1511,34 @@ TEST_F(RestCatalogTest, GlobalSystemTableRetainsItsCatalog) { ASSERT_GT(array->length(), 0); } +TEST_F(RestCatalogTest, GlobalSystemTableWorksThroughThePublicCatalogFactory) { + // `Catalog::Create()` is the factory a caller outside paimon uses, and it hands back a + // `Catalog` rather than a `RestCatalog`. A global system table keeps its catalog alive + // through that pointer, so this path has to work too - a catalog whose shared ownership + // was not set up would fail here with the refusal naming the missing ownership. + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(kWarehouse, options_)); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("sys", "tables"))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_GT(array->length(), 0); +} + TEST_F(RestCatalogTest, ListSnapshots) { ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index 57a6e8d98..c94284eb2 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -101,7 +101,9 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { // this mock's `GetTableLocation` is the unsupported stub. return FileStoreTable::Create( table_file_system_ != nullptr ? table_file_system_ : file_system_, - /*root_path=*/"", table_schema_, identifier, table_uuid_); + /*root_path=*/"", table_schema_, identifier, table_uuid_, + /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); } void SetTableSchema(const std::shared_ptr& table_schema) { diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 2c24607a2..ef8e5c853 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -4173,8 +4173,9 @@ TEST_P(ReadInteTest, TestSpecificFs) { namespace { Result> ReadGlobalSystemTable( - const std::string& table_name, Catalog* catalog, const std::shared_ptr& fs, - const std::string& warehouse, const std::map& options) { + const std::string& table_name, const std::shared_ptr& catalog, + const std::shared_ptr& fs, const std::string& warehouse, + const std::map& options) { GlobalSystemTableContext ctx; ctx.catalog = catalog; ctx.fs = fs; @@ -4229,8 +4230,8 @@ TEST(SystemTableReadInteTest, TestReadGlobalCatalogOptions) { std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); ASSERT_OK_AND_ASSIGN(auto result, - ReadGlobalSystemTable("catalog_options", catalog.get(), - catalog->GetFileSystem(), warehouse, options)); + ReadGlobalSystemTable("catalog_options", catalog, catalog->GetFileSystem(), + warehouse, options)); auto struct_array = SingleStructChunk(result); ASSERT_TRUE(struct_array); auto key_array = std::dynamic_pointer_cast(struct_array->field(0)); @@ -4301,7 +4302,7 @@ TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) { // Directly test BuildRows { GlobalSystemTableContext ctx; - ctx.catalog = catalog.get(); + ctx.catalog = catalog; ctx.fs = catalog->GetFileSystem(); ctx.warehouse = warehouse; ctx.catalog_options = options; @@ -4310,9 +4311,9 @@ TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) { ASSERT_GT(rows.size(), 0) << "BuildRows returned empty, expected at least 1 row"; } - ASSERT_OK_AND_ASSIGN(auto result, - ReadGlobalSystemTable("all_table_options", catalog.get(), - catalog->GetFileSystem(), warehouse, options)); + ASSERT_OK_AND_ASSIGN( + auto result, ReadGlobalSystemTable("all_table_options", catalog, catalog->GetFileSystem(), + warehouse, options)); auto struct_array = SingleStructChunk(result); ASSERT_TRUE(struct_array); ASSERT_GE(struct_array->length(), 1) << "result has " << struct_array->length() << " rows"; @@ -4392,7 +4393,7 @@ TEST(SystemTableReadInteTest, TestReadGlobalTables) { ArrowSchemaRelease(&no_pk_schema); ASSERT_OK_AND_ASSIGN(auto result, - ReadGlobalSystemTable("tables", catalog.get(), fs, warehouse, options)); + ReadGlobalSystemTable("tables", catalog, fs, warehouse, options)); auto struct_array = SingleStructChunk(result); ASSERT_TRUE(struct_array); ASSERT_GE(struct_array->length(), 1); @@ -4490,8 +4491,8 @@ TEST(SystemTableReadInteTest, TestReadGlobalPartitions) { ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); auto fs = catalog->GetFileSystem(); - ASSERT_OK_AND_ASSIGN(auto part_result, ReadGlobalSystemTable("partitions", catalog.get(), fs, - warehouse, options)); + ASSERT_OK_AND_ASSIGN(auto part_result, + ReadGlobalSystemTable("partitions", catalog, fs, warehouse, options)); auto struct_array = SingleStructChunk(part_result); ASSERT_TRUE(struct_array); ASSERT_EQ(StructFieldNames(struct_array), @@ -4542,7 +4543,7 @@ TEST(SystemTableReadInteTest, TestGlobalSystemTablesPropagateCorruptSchema) { /*overwrite=*/true)); for (const std::string system_table : {"all_table_options", "tables", "partitions"}) { - ASSERT_NOK(ReadGlobalSystemTable(system_table, catalog.get(), fs, warehouse, options)); + ASSERT_NOK(ReadGlobalSystemTable(system_table, catalog, fs, warehouse, options)); } } @@ -4578,7 +4579,7 @@ TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptSnapshot ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(table_path, "snapshot/snapshot-1"), "{invalid-json", /*overwrite=*/true)); - ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options)); + ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog, fs, warehouse, options)); } TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptManifest) { @@ -4617,7 +4618,7 @@ TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptManifest PathUtil::JoinPath(table_path, "manifest/" + snapshot->BaseManifestList()), "corrupt", /*overwrite=*/true)); - ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options)); + ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog, fs, warehouse, options)); } } // namespace paimon::test diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 5de6c9fa7..7111f5302 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -504,7 +504,7 @@ class RealtimeWriteInteTest : public ::testing::Test { void CreateTable(const std::vector& partition_keys) const { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*schema_, c_schema.get()).ok()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(dir_->Str(), options_)); ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false)); ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), c_schema.get(), partition_keys, @@ -516,7 +516,7 @@ class RealtimeWriteInteTest : public ::testing::Test { const std::vector& primary_keys = {"id"}) const { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*schema_, c_schema.get()).ok()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(dir_->Str(), options_)); ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false)); std::vector table_primary_keys = partition_keys; From 303751242db76a9c876fcf5aafd366a4c6befcdb Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 27 Oct 2026 16:51:21 +0800 Subject: [PATCH 17/19] fix --- docs/source/user_guide/catalog.rst | 2 +- .../core/operation/file_store_commit.cpp | 16 ++++++++++ .../core/operation/file_store_write.cpp | 11 +++++++ src/paimon/core/schema/schema_validation.cpp | 5 +++ src/paimon/core/table/file_store_table.cpp | 32 +++---------------- src/paimon/core/table/source/table_read.cpp | 10 ++++++ src/paimon/core/table/source/table_scan.cpp | 10 ++++++ src/paimon/core/table/table_test.cpp | 7 ++++ 8 files changed, 65 insertions(+), 28 deletions(-) diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 599b4ffce..ca593e38e 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -134,7 +134,7 @@ without an atomic snapshot-file rename: } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - paimon::FileStoreCommit::Create(std::move(context))); + table->NewCommit(std::move(context))); PAIMON_RETURN_NOT_OK(commit->Commit(commit_messages)); Configure native writers with the same catalog and identifier. They load the diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index 4e0747437..9b6c3975c 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -18,3 +18,19 @@ */ #include "paimon/file_store_commit.h" + +#include "paimon/catalog/table.h" +#include "paimon/commit_context.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" + +namespace paimon { + +Result> FileStoreCommit::Create( + std::unique_ptr ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + return table->NewCommit(std::move(ctx)); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index c1b889473..ea73b2246 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -18,8 +18,19 @@ #include "paimon/file_store_write.h" +#include "paimon/catalog/table.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/write_context.h" + namespace paimon { +Result> FileStoreWrite::Create(std::unique_ptr ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + return table->NewWrite(std::move(ctx)); +} + Result> FileStoreWrite::PrepareCommitWithProgress(int64_t) { return Status::Invalid("prepare commit with progress requires a real-time writer"); } diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 5d112417f..c799c24f8 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -85,6 +85,11 @@ constexpr char kDeletionVectorsMergeOnRead[] = "deletion-vectors.merge-on-read"; /// into a table. Unlike the Java set, `merge-engine`, `bucket`, `row-tracking.enabled` and /// `data-evolution.enabled` stay servable as dynamic options here, and `type`/`branch` are dropped /// by `Copy` before this point. +// TODO(yonghao.fyh): refuse the rest of Java's immutable set, which this list does not cover yet: +// `blob-descriptor-field` (only its deprecated `blob.stored-descriptor-fields` fallback is above), +// `data-file.path-directory`, `sequence.snapshot-ordering`, `rowkind.field`, `primary-key`, +// `primary-key.nullable`, `partition`, `dynamic-bucket.initial-buckets`, `video-frame-field`, and +// the four servable keys named above, should the C++ side stop serving them as dynamic options. const char* const kImmutableOptions[] = { Options::BUCKET_KEY, Options::BUCKET_FUNCTION_TYPE, diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 3945de5b7..da86a590e 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -55,7 +55,6 @@ #include "paimon/core/snapshot.h" #include "paimon/core/table/append_only_file_store_table.h" #include "paimon/core/table/file_store.h" -#include "paimon/core/table/file_store_table_factory.h" #include "paimon/core/table/primary_key_file_store_table.h" #include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_evolution_batch_scan.h" @@ -210,9 +209,13 @@ Result> FileStoreTable::SwitchToBranch(const std::string& branch_schema->Options(), std::static_pointer_cast(schema_)->Options()))) ? nullptr : schema_manager; + // The identifier names the branch switched to, as Java's `newCatalogEnvironment` rebuilds it, + // so `Name()` and `FullName()` answer for the table served rather than the one left behind. + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + CatalogUtils::BranchIdentifier(identifier_, target_branch)); return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, std::shared_ptr(std::move(branch_schema)), - identifier_, CatalogUuid().value_or(""), target_branch, + branch_identifier, CatalogUuid().value_or(""), target_branch, branch_schema_manager, /*snapshot_manager=*/nullptr, store_->GetFsSchemeMap(), /*dynamic_options=*/{}); } @@ -314,12 +317,6 @@ Result> FileStoreTable::NewWrite( snapshot_manager, io_manager, ignore_previous_files); } -Result> FileStoreWrite::Create(std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); - return table->NewWrite(std::move(ctx)); -} - Result> FileStoreTable::NewScan( std::unique_ptr context) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); @@ -422,12 +419,6 @@ Result> FileStoreTable::NewScan( return batch_scan; } -Result> TableScan::Create(std::unique_ptr context) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*context)); - return table->NewScan(std::move(context)); -} - Result> FileStoreTable::CreateTableRead( const std::shared_ptr& internal_context, const std::shared_ptr& memory_pool, @@ -498,12 +489,6 @@ Result> FileStoreTable::NewRead( std::move(fallback_table_read), arrow_pool); } -Result> TableRead::Create(std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); - return table->NewRead(std::move(ctx)); -} - Result> FileStoreTable::NewCommit( std::unique_ptr ctx) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); @@ -605,11 +590,4 @@ Result> FileStoreTable::NewCommit( expire_snapshots, GetSchemaManager(), std::move(scan_supplier)); } -Result> FileStoreCommit::Create( - std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); - return table->NewCommit(std::move(ctx)); -} - } // namespace paimon diff --git a/src/paimon/core/table/source/table_read.cpp b/src/paimon/core/table/source/table_read.cpp index e723c54bb..5688c7650 100644 --- a/src/paimon/core/table/source/table_read.cpp +++ b/src/paimon/core/table/source/table_read.cpp @@ -19,8 +19,18 @@ #include "paimon/table/source/table_read.h" +#include "paimon/catalog/table.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" + namespace paimon { +Result> TableRead::Create(std::unique_ptr ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + return table->NewRead(std::move(ctx)); +} + Result> TableRead::CreateCountReader( const std::vector>& splits) { (void)splits; diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index be5d5b955..d3dd794b4 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -19,10 +19,20 @@ #include "paimon/table/source/table_scan.h" +#include "paimon/catalog/table.h" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/scan_context.h" namespace paimon { +Result> TableScan::Create(std::unique_ptr context) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*context)); + return table->NewScan(std::move(context)); +} + std::shared_ptr TableScan::GetMetrics() const { return std::make_shared(); } diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 5100dfd78..8b7b2da45 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -456,6 +456,10 @@ TEST(TableTest, TestSwitchToBranch) { auto dev_file_store_table = std::dynamic_pointer_cast(dev_table); ASSERT_TRUE(dev_file_store_table != nullptr); EXPECT_EQ("dev", dev_file_store_table->CurrentBranch()); + // The identifier names the branch switched to, so `Name()` and `FullName()` answer for the + // table served, and a caller that reloads through `FullName()` lands on the same branch. + EXPECT_EQ("tbl$branch_dev", dev_table->Name()); + EXPECT_EQ("tbl$branch_dev", dev_table->FullName()); ASSERT_EQ((std::vector{"c0"}), dev_table->LatestSchema()->FieldNames()); auto dev_table_schema = std::dynamic_pointer_cast(dev_table->LatestSchema()); ASSERT_TRUE(dev_table_schema != nullptr); @@ -480,6 +484,9 @@ TEST(TableTest, TestSwitchToBranch) { auto main_again_file_store_table = std::dynamic_pointer_cast(main_again); ASSERT_TRUE(main_again_file_store_table != nullptr); EXPECT_EQ("main", main_again_file_store_table->CurrentBranch()); + // Switching back leaves no branch in the name either, the way the main branch is addressed. + EXPECT_EQ("tbl", main_again->Name()); + EXPECT_EQ("tbl", main_again->FullName()); ASSERT_EQ((std::vector{"id"}), main_again->LatestSchema()->FieldNames()); auto main_again_schema = std::dynamic_pointer_cast(main_again->LatestSchema()); ASSERT_TRUE(main_again_schema != nullptr); From f18de29673b9882b119a04815a96ca766753cfaf Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 27 Oct 2026 18:24:11 +0800 Subject: [PATCH 18/19] fix --- docs/source/api/format_table.rst | 28 +- docs/source/user_guide/catalog.rst | 11 +- docs/source/user_guide/format_table.rst | 38 ++- include/paimon/commit_context.h | 24 +- include/paimon/file_store_commit.h | 5 +- include/paimon/file_store_write.h | 5 +- include/paimon/read_context.h | 53 ++-- include/paimon/scan_context.h | 20 +- include/paimon/table/source/table_read.h | 5 +- include/paimon/table/source/table_scan.h | 5 +- include/paimon/write_context.h | 20 -- src/paimon/core/operation/commit_context.cpp | 37 +-- .../core/operation/commit_context_test.cpp | 2 +- .../core/operation/file_store_commit_test.cpp | 3 +- src/paimon/core/operation/read_context.cpp | 50 +--- src/paimon/core/operation/scan_context.cpp | 42 +-- src/paimon/core/operation/write_context.cpp | 42 +-- .../core/table/file_store_table_factory.cpp | 126 +++----- .../core/table/file_store_table_factory.h | 9 +- src/paimon/core/table/format/format_table.cpp | 4 +- .../core/table/format/format_table_test.cpp | 277 +++++++----------- .../core/table/source/table_scan_test.cpp | 12 +- .../core/table/system/system_table_test.cpp | 16 + src/paimon/core/table/table_test.cpp | 24 +- test/inte/read_inte_test.cpp | 48 ++- test/inte/realtime_write_inte_test.cpp | 9 +- 26 files changed, 327 insertions(+), 588 deletions(-) diff --git a/docs/source/api/format_table.rst b/docs/source/api/format_table.rst index 1ec964375..36295493f 100644 --- a/docs/source/api/format_table.rst +++ b/docs/source/api/format_table.rst @@ -35,18 +35,16 @@ and :cpp:class:`paimon::FileStoreCommit`. See :doc:`../user_guide/format_table`. Contexts ======== -Each of those entry points builds its context from a table path, and the schema under that path -says what kind of table it is. A caller that already holds a ``FormatTable`` hands it over instead, -through the constructor each context builder has for one: - -* :cpp:class:`paimon::ScanContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::ReadContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::WriteContextBuilder`\ ``(const std::shared_ptr&)`` -* :cpp:class:`paimon::CommitContextBuilder`\ ``(const std::shared_ptr&)`` - -This is the only way to reach a format table whose schema does not live under its own location, -such as one a REST catalog serves: nothing under the location says that it is a format table, nor -that what sits below it is data rather than metadata. The table knows both, so a setting that would -answer either question again - ``SetTableSchema()``, ``WithFileSystem()`` and a branch among them - -is refused by these builders rather than quietly ignored. Options given at the call still win over -the ones the table carries, as they do everywhere else. +Each of those entry points builds its context the way it does for a data table: from the table's +location, through the builder's usual constructor: + +* :cpp:class:`paimon::ScanContextBuilder`\ ``(const std::string&)`` +* :cpp:class:`paimon::ReadContextBuilder`\ ``(const std::string&)`` +* :cpp:class:`paimon::WriteContextBuilder`\ ``(const std::string&)`` +* :cpp:class:`paimon::CommitContextBuilder`\ ``(const std::string&)`` + +The context then goes to the table's own operator - ``NewScan()``, ``NewRead()``, ``NewWrite()`` or +``NewCommit()`` - which serves a table whose schema does not live under its own location as well, +such as one a REST catalog serves: the table carries its schema and the file system it was loaded +through, so nothing under the location has to answer for them. Options given at the call still win +over the ones the table carries, as they do everywhere else. diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index ca593e38e..8c0dd4d22 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -163,8 +163,8 @@ Both builders use the file system of the table being written or committed for manifests and data, so a catalog that issues temporary credentials per table serves them through ``Catalog::GetTableFileSystem``. ``WithFileSystem`` overrides it; writers also allow ``WithFileSystemSchemeToIdentifierMap`` to override -file-system selection. For format tables, use ``WriteContextBuilder(FormatTable)`` -instead. +file-system selection. A format table is written through the table's own +``NewWrite`` instead; see :doc:`format_table`. A file system passed to ``WithFileSystem`` is used exactly as it is: neither the built-in file systems nor the table's own credentials are involved, so a custom @@ -279,10 +279,9 @@ snapshots, manifests and data under the table path, so it signs every access wit the table's credentials and reloads them as they expire. An explicit ``WithFileSystem`` takes precedence, so the catalog is not asked. A catalog that issues no per-table credentials returns its catalog-level file system, so this is -also how you read with the catalog's own object-store credentials. For a format -table, use ``ReadContextBuilder(FormatTable)`` or ``ScanContextBuilder(FormatTable)`` -instead; ``WithCatalog`` is rejected because the table already carries the file -system it was loaded through. +also how you read with the catalog's own object-store credentials. A format table +is read through the table's own ``NewRead`` or ``NewScan`` instead; see +:doc:`format_table`. The C++ REST catalog covers the database, table, snapshot and commit operations of the ``Catalog`` API. The parts of the Java REST catalog that have no C++ diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 7ee90976d..7579b9dff 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -44,18 +44,10 @@ A format table is served through the same :cpp:func:`Catalog::GetTable` as every its ``type`` option says what it is, and the call hands back the ``FormatTable`` it describes, through the ``Table`` interface the two kinds of table share. -Reading and writing go through the entry points every other table uses: ``TableScan::Create``, -``TableRead::Create``, ``FileStoreWrite::Create`` and ``FileStoreCommit::Create``, each from its -usual context builder. Each reads the table's schema - from under the table path, or from the one -the context carries - and dispatches to a format table when its ``type`` says so. - -That path needs a schema under the table's own location, which is what every table a file system -catalog serves has. A table loaded from a catalog that keeps the schema elsewhere, such as the -REST catalog, has none: nothing under the location says that it is a format table, nor that -``schema`` and ``branch`` below it are data rather than metadata. Such a table is read and written -by handing the loaded ``FormatTable`` to the context builder instead of a path - every builder has -a constructor taking one - and calling the table's own operator, which is what Java does through -``FormatTable.newReadBuilder()`` and ``newBatchWriteBuilder()``: +Reading and writing go through the table itself: a caller holding one calls ``NewScan()``, +``NewRead()``, ``NewWrite()`` or ``NewCommit()`` on it, with a context its usual builder builds from +the table's location. This is what Java serves through ``FormatTable.newReadBuilder()`` and +``newBatchWriteBuilder()``: .. code-block:: cpp @@ -64,22 +56,26 @@ a constructor taking one - and calling the table's own operator, which is what J // The table's `type` option is `format-table`, so this is a `FormatTable`. std::shared_ptr table = std::dynamic_pointer_cast(loaded); - paimon::ScanContextBuilder scan_builder(table); + paimon::ScanContextBuilder scan_builder(table->Location()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, table->NewScan(std::move(scan_context))); -The table already carries its schema and the file system it was loaded through, so a context built -from one refuses ``SetTableSchema()``, ``WithFileSystem()``, -``WithFileSystemSchemeToIdentifierMap()``, ``WithCatalog()`` and a branch. Options +Because the operator comes from the table itself, this serves a table whose schema does not live +under its own location as well, such as one a REST catalog serves: the table carries its schema and +the file system it was loaded through, so nothing under the location has to answer for them. Options given at the call still win over the ones the schema stored, as they do everywhere else. -``FormatTable`` is a derived class of ``Table``, so the table serves the same calls itself: a -caller holding one through a ``std::shared_ptr`` calls ``NewScan()``, -``NewRead()``, ``NewWrite()`` or ``NewCommit()`` on it, and the operators come back for the table -it is, whether or not the context carries one. This is the C++ side of what Java serves through -one ``Table.newReadBuilder()`` for both kinds of table. +The generic entry points - ``TableScan::Create``, ``TableRead::Create``, ``FileStoreWrite::Create`` +and ``FileStoreCommit::Create`` - serve a data table only: a load from a path is the data table's +way in, and a format table the schema under the path names is refused with a message pointing at +the catalog and the table's own operators above. + +``FormatTable`` is a derived class of ``Table``, so a caller holding one through a +``std::shared_ptr`` calls the same operators on it without the cast: they come back +for the table it is, whatever kind it turns out to be. This is the C++ side of what Java serves +through one ``Table.newReadBuilder()`` for both kinds of table. A batch comes back in the table's column order, or the projection's when the read names one, behind the ``_VALUE_KIND`` field every ``BatchReader`` carries. Which columns a *file* is asked for diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index 556db7bd2..c38cdca4c 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -32,7 +32,6 @@ namespace paimon { class Catalog; class Executor; -class FormatTable; class MemoryPool; /// `CommitContext` is some configuration for commit operations. @@ -48,8 +47,7 @@ class PAIMON_EXPORT CommitContext { bool append_commit_check_conflict, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table); + const std::map& options); CommitContext(const std::string& root_path, const std::string& commit_user, bool ignore_empty_commit, bool use_rest_catalog_commit, @@ -59,8 +57,7 @@ class PAIMON_EXPORT CommitContext { const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table); + const std::map& options); ~CommitContext(); const std::string& GetRootPath() const { @@ -114,12 +111,6 @@ class PAIMON_EXPORT CommitContext { return options_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - private: std::string root_path_; std::string commit_user_; @@ -133,7 +124,6 @@ class PAIMON_EXPORT CommitContext { std::shared_ptr executor_; std::shared_ptr specific_file_system_; std::map options_; - std::shared_ptr format_table_; }; /// `CommitContextBuilder` used to build a `CommitContext`, has input validation. @@ -144,16 +134,6 @@ class PAIMON_EXPORT CommitContextBuilder { /// @param commit_user The user identifier for the commit operation. CommitContextBuilder(const std::string& root_path, const std::string& commit_user); - /// Constructs a `CommitContextBuilder` for a format table that is already loaded: the only way - /// to commit to one whose schema lives in a metastore rather than under its location, such as - /// a table a REST catalog serves. The table carries what such a location does not say, so - /// `WithFileSystem()` is refused here rather than ignored. - /// - /// There is no commit user: a format table keeps no snapshot to record one in. - /// - /// @param table The format table to commit to, as `Catalog::GetTable()` hands it back. - explicit CommitContextBuilder(const std::shared_ptr& table); - ~CommitContextBuilder(); /// Set a configuration options map to set some option entries which are not defined in the diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index 8a6eaad5c..b0fcb17e1 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -50,8 +50,9 @@ class CommitMessage; /// manages versions. `UseRESTCatalogCommit()` only prepares a request for the caller to send. class PAIMON_EXPORT FileStoreCommit { public: - /// Create an instance of `FileStoreCommit`. A context that already carries a loaded table goes - /// straight to that table's `NewCommit`; otherwise the table the context names is loaded first. + /// Create an instance of `FileStoreCommit`. The data table the context names is loaded from its + /// root path - a format table is refused, served by the table a catalog hands back through its + /// own `NewCommit` instead. /// /// @param context A unique pointer to the `CommitContext` used for commit operations. /// diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index 0778e5bb7..351b98b06 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -42,8 +42,9 @@ class WriteContext; /// Interface for write operations in a file store. class PAIMON_EXPORT FileStoreWrite { public: - /// Create an instance of `FileStoreWrite`. A context that already carries a loaded table goes - /// straight to that table's `NewWrite`; otherwise the table the context names is loaded first. + /// Create an instance of `FileStoreWrite`. The data table the context names is loaded from its + /// root path - a format table is refused, served by the table a catalog hands back through its + /// own `NewWrite` instead. /// /// @param context A unique pointer to the `WriteContext` used for write operations. /// diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index d01e28df5..568315ef6 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -38,7 +38,6 @@ namespace paimon { class Catalog; class Executor; struct FullTextSearch; -class FormatTable; class MemoryPool; class Predicate; class FileSystem; @@ -52,23 +51,25 @@ struct VectorSearch; /// @see ReadContextBuilder class PAIMON_EXPORT ReadContext { public: - ReadContext( - const std::string& path, const std::string& branch, - const std::vector& read_field_names, - const std::vector& read_field_ids, const std::shared_ptr& predicate, - const std::shared_ptr& vector_search, - const std::shared_ptr& full_text_search, bool enable_predicate_filter, - bool enable_prefetch, bool enable_late_materializing, uint32_t prefetch_batch_count, - uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch, - uint32_t row_to_batch_thread_number, const std::optional& table_schema, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& fs_scheme_to_identifier_map, - const std::shared_ptr& realtime_context, - const std::map& options, bool read_ahead_cache_enabled, - const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level, - const std::shared_ptr& catalog, const std::optional& identifier); + ReadContext(const std::string& path, const std::string& branch, + const std::vector& read_field_names, + const std::vector& read_field_ids, + const std::shared_ptr& predicate, + const std::shared_ptr& vector_search, + const std::shared_ptr& full_text_search, + bool enable_predicate_filter, bool enable_prefetch, bool enable_late_materializing, + uint32_t prefetch_batch_count, uint32_t prefetch_max_parallel_num, + bool enable_multi_thread_row_to_batch, uint32_t row_to_batch_thread_number, + const std::optional& table_schema, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor, + const std::shared_ptr& specific_file_system, + const std::map& fs_scheme_to_identifier_map, + const std::shared_ptr& realtime_context, + const std::map& options, bool read_ahead_cache_enabled, + const CacheConfig& cache_config, const std::shared_ptr& cache, + WarmupLevel warmup_level, const std::shared_ptr& catalog, + const std::optional& identifier); ~ReadContext(); @@ -164,12 +165,6 @@ class PAIMON_EXPORT ReadContext { return cache_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - WarmupLevel GetWarmupLevel() const { return warmup_level_; } @@ -225,7 +220,6 @@ class PAIMON_EXPORT ReadContext { bool read_ahead_cache_enabled_; CacheConfig cache_config_; std::shared_ptr cache_; - std::shared_ptr format_table_; WarmupLevel warmup_level_; std::shared_ptr catalog_; std::optional identifier_; @@ -240,15 +234,6 @@ class PAIMON_EXPORT ReadContextBuilder { /// @param path The root path of the table. explicit ReadContextBuilder(const std::string& path); - /// Constructs a `ReadContextBuilder` for a format table that is already loaded: the only way - /// to read one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `SetTableSchema()`, `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()` and a - /// branch are refused here rather than ignored. - /// - /// @param table The format table to read, as `Catalog::GetTable()` hands it back. - explicit ReadContextBuilder(const std::shared_ptr& table); - ~ReadContextBuilder(); ReadContextBuilder(ReadContextBuilder&&) noexcept; diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 2e3f8d9a8..99f5057eb 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -37,7 +37,6 @@ class ScanContextBuilder; class ScanFilter; class Catalog; class Executor; -class FormatTable; class MemoryPool; class Predicate; @@ -57,9 +56,7 @@ class PAIMON_EXPORT ScanContext { const std::shared_ptr& specific_file_system, const std::optional& table_schema, const std::map& options, - const std::shared_ptr& cache, - const std::shared_ptr& format_table, - const std::shared_ptr& catalog, + const std::shared_ptr& cache, const std::shared_ptr& catalog, const std::optional& identifier); ~ScanContext(); @@ -111,12 +108,6 @@ class PAIMON_EXPORT ScanContext { return cache_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - /// Returns the catalog supplying table metadata, or null if unset. const std::shared_ptr& GetCatalog() const { return catalog_; @@ -140,7 +131,6 @@ class PAIMON_EXPORT ScanContext { std::optional table_schema_; std::map options_; std::shared_ptr cache_; - std::shared_ptr format_table_; std::shared_ptr catalog_; std::optional identifier_; }; @@ -178,14 +168,6 @@ class PAIMON_EXPORT ScanContextBuilder { /// @param path The root path of the table. explicit ScanContextBuilder(const std::string& path); - /// Constructs a `ScanContextBuilder` for a format table that is already loaded: the only way - /// to scan one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `SetTableSchema()` and `WithFileSystem()` are refused here rather than ignored. - /// - /// @param table The format table to scan, as `Catalog::GetTable()` hands it back. - explicit ScanContextBuilder(const std::shared_ptr& table); - ~ScanContextBuilder(); /// If limit is not set, it defaults to unlimited. ScanContextBuilder& SetLimit(int32_t limit); diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index 5f8303fe3..fa1aef730 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -42,8 +42,9 @@ class PAIMON_EXPORT TableRead { public: virtual ~TableRead() = default; - /// Create an instance of `TableRead`. A context that already carries a loaded table goes - /// straight to that table's `NewRead`; otherwise the table the context names is loaded first. + /// Create an instance of `TableRead`. The data table the context names is loaded from its path + /// - a system table behind a `T$system` path and a format table are refused, served by the + /// table a catalog hands back through its own `NewRead` instead. /// /// @param context A unique pointer to the `ReadContext` used for read operations. /// @return A Result containing a unique pointer to the `TableRead` instance. diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h index 4944dafd2..edecb59c8 100644 --- a/include/paimon/table/source/table_scan.h +++ b/include/paimon/table/source/table_scan.h @@ -36,8 +36,9 @@ class ScanContext; /// A scanner interface for reading table's meta and create a plan. class PAIMON_EXPORT TableScan { public: - /// Create an instance of `TableScan`. A context that already carries a loaded table goes - /// straight to that table's `NewScan`; otherwise the table the context names is loaded first. + /// Create an instance of `TableScan`. The data table the context names is loaded from its path + /// - a system table behind a `T$system` path and a format table are refused, served by the + /// table a catalog hands back through its own `NewScan` instead. /// /// @param context A unique pointer to the `ScanContext` used for scan operations. /// @return A Result containing a unique pointer to the `TableScan` instance. diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index f1d3ac06e..ba55f37e8 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -33,7 +33,6 @@ namespace paimon { class Catalog; class Executor; -class FormatTable; class MemoryPool; /// `WriteContext` is some configuration for write operations. @@ -53,7 +52,6 @@ class PAIMON_EXPORT WriteContext { const std::map& fs_scheme_to_identifier_map, const std::shared_ptr& realtime_context, const std::map& options, - const std::shared_ptr& format_table, const std::shared_ptr& catalog = nullptr, const std::optional& identifier = std::nullopt); @@ -124,12 +122,6 @@ class PAIMON_EXPORT WriteContext { return realtime_context_; } - /// The format table this context was built from, or null when it names a table path and the - /// schema under that path says what kind of table it is. - const std::shared_ptr& GetFormatTable() const { - return format_table_; - } - /// Returns the catalog supplying table metadata, or null if unset. const std::shared_ptr& GetCatalog() const { return catalog_; @@ -157,7 +149,6 @@ class PAIMON_EXPORT WriteContext { std::map fs_scheme_to_identifier_map_; std::shared_ptr realtime_context_; std::map options_; - std::shared_ptr format_table_; std::shared_ptr catalog_; std::optional identifier_; }; @@ -170,17 +161,6 @@ class PAIMON_EXPORT WriteContextBuilder { /// @param commit_user The user identifier for commit operations. WriteContextBuilder(const std::string& root_path, const std::string& commit_user); - /// Constructs a `WriteContextBuilder` for a format table that is already loaded: the only way - /// to write one whose schema lives in a metastore rather than under its location, such as a - /// table a REST catalog serves. The table carries what such a location does not say, so - /// `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()`, `WithCatalog()` and a branch - /// are refused here rather than ignored. - /// - /// There is no commit user: a format table keeps no snapshot to record one in. - /// - /// @param table The format table to write to, as `Catalog::GetTable()` hands it back. - explicit WriteContextBuilder(const std::shared_ptr& table); - ~WriteContextBuilder(); /// Set a configuration options map to set some option entries which are not defined in the diff --git a/src/paimon/core/operation/commit_context.cpp b/src/paimon/core/operation/commit_context.cpp index e61ec950f..c632ac9c6 100644 --- a/src/paimon/core/operation/commit_context.cpp +++ b/src/paimon/core/operation/commit_context.cpp @@ -33,7 +33,6 @@ #include "paimon/memory/memory_pool.h" #include "paimon/result.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { @@ -43,12 +42,11 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table) + const std::map& options) : CommitContext(root_path, commit_user, ignore_empty_commit, use_rest_catalog_commit, /*catalog=*/nullptr, /*identifier=*/std::nullopt, /*table_id=*/std::nullopt, append_commit_check_conflict, memory_pool, executor, specific_file_system, - options, format_table) {} + options) {} CommitContext::CommitContext(const std::string& root_path, const std::string& commit_user, bool ignore_empty_commit, bool use_rest_catalog_commit, @@ -59,8 +57,7 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options, - const std::shared_ptr& format_table) + const std::map& options) : root_path_(root_path), commit_user_(commit_user), ignore_empty_commit_(ignore_empty_commit), @@ -72,8 +69,7 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co memory_pool_(memory_pool), executor_(executor), specific_file_system_(specific_file_system), - options_(options), - format_table_(format_table) {} + options_(options) {} CommitContext::~CommitContext() = default; @@ -96,10 +92,6 @@ class CommitContextBuilder::Impl { private: std::string root_path_; - /// Kept across `Reset()`, as `root_path_` is: both name the table this builder builds for, - /// rather than a setting of one commit to it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; std::string commit_user_; bool ignore_empty_commit_ = true; bool use_rest_catalog_commit_ = false; @@ -120,15 +112,6 @@ CommitContextBuilder::CommitContextBuilder(const std::string& root_path, impl_->commit_user_ = commit_user; } -CommitContextBuilder::CommitContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->root_path_ = table->Location(); - } -} - CommitContextBuilder::~CommitContextBuilder() = default; CommitContextBuilder& CommitContextBuilder::AddOption(const std::string& key, @@ -190,9 +173,6 @@ CommitContextBuilder& CommitContextBuilder::WithFileSystem( } Result> CommitContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot commit with null format table"); - } if (impl_->catalog_ == nullptr && impl_->identifier_) { return Status::Invalid("cannot commit through a null catalog"); } @@ -230,13 +210,6 @@ Result> CommitContextBuilder::Finish() { "a commit either goes to the catalog of WithCatalog() or is handed back as a request " "for the caller to send, which is what UseRESTCatalogCommit() asks for"); } - // The table already carries the file system it was loaded through, and from a source this - // cannot see behind, so a second answer is refused rather than silently dropped. - if (impl_->format_table_ != nullptr && impl_->specific_file_system_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "cannot be used with one"); - } PAIMON_ASSIGN_OR_RAISE(impl_->root_path_, PathUtil::NormalizePath(impl_->root_path_)); if (impl_->root_path_.empty()) { return Status::Invalid("root path is empty"); @@ -245,7 +218,7 @@ Result> CommitContextBuilder::Finish() { impl_->root_path_, impl_->commit_user_, impl_->ignore_empty_commit_, impl_->use_rest_catalog_commit_, impl_->catalog_, impl_->identifier_, impl_->table_id_, impl_->append_commit_check_conflict_, impl_->memory_pool_, impl_->executor_, - impl_->specific_file_system_, impl_->options_, impl_->format_table_); + impl_->specific_file_system_, impl_->options_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/commit_context_test.cpp b/src/paimon/core/operation/commit_context_test.cpp index 51fcab399..d86a79149 100644 --- a/src/paimon/core/operation/commit_context_test.cpp +++ b/src/paimon/core/operation/commit_context_test.cpp @@ -40,7 +40,7 @@ TEST(CommitContextTest, TestLegacyConstructor) { auto fs = std::make_shared(); const std::map options = {{"key", "value"}}; CommitContext context("table_root_path", "commit_user_1", false, true, true, pool, executor, fs, - options, nullptr); + options); ASSERT_EQ(context.GetRootPath(), "table_root_path"); ASSERT_EQ(context.GetCommitUser(), "commit_user_1"); ASSERT_FALSE(context.IgnoreEmptyCommit()); diff --git a/src/paimon/core/operation/file_store_commit_test.cpp b/src/paimon/core/operation/file_store_commit_test.cpp index a7f39628d..c67c761c5 100644 --- a/src/paimon/core/operation/file_store_commit_test.cpp +++ b/src/paimon/core/operation/file_store_commit_test.cpp @@ -99,8 +99,7 @@ TEST(FileStoreCommitTest, TestCreateWithCatalogRequiresIdentifier) { dir->Str(), "commit_user", /*ignore_empty_commit=*/true, /*use_rest_catalog_commit=*/false, catalog, /*identifier=*/std::nullopt, /*table_id=*/std::nullopt, /*append_commit_check_conflict=*/false, GetDefaultPool(), - CreateDefaultExecutor(), /*specific_file_system=*/nullptr, options, - /*format_table=*/nullptr); + CreateDefaultExecutor(), /*specific_file_system=*/nullptr, options); ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "a catalog commit requires a table identifier"); } diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index bc65b4661..40ed358de 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -32,7 +32,6 @@ #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/vector_search.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { class Predicate; @@ -50,8 +49,7 @@ ReadContext::ReadContext( const std::map& fs_scheme_to_identifier_map, const std::shared_ptr& realtime_context, const std::map& options, bool read_ahead_cache_enabled, - const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level, + const CacheConfig& cache_config, const std::shared_ptr& cache, WarmupLevel warmup_level, const std::shared_ptr& catalog, const std::optional& identifier) : path_(path), branch_(branch), @@ -77,7 +75,6 @@ ReadContext::ReadContext( read_ahead_cache_enabled_(read_ahead_cache_enabled), cache_config_(cache_config), cache_(cache), - format_table_(format_table), warmup_level_(warmup_level), catalog_(catalog), identifier_(identifier) {} @@ -135,10 +132,6 @@ class ReadContextBuilder::Impl { private: std::string path_; - /// Kept across `Reset()`, as `path_` is: both name the table this builder builds for, rather - /// than a setting of one read of it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; /// Unset until `WithBranch()` names one, so that a branch named only by the `branch` /// option is not overruled by the default. std::optional branch_; @@ -175,15 +168,6 @@ ReadContextBuilder::ReadContextBuilder(const std::string& path) impl_->path_ = path; } -ReadContextBuilder::ReadContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->path_ = table->Location(); - } -} - ReadContextBuilder::~ReadContextBuilder() = default; ReadContextBuilder::ReadContextBuilder(ReadContextBuilder&&) noexcept = default; @@ -359,34 +343,6 @@ Result> ReadContextBuilder::Finish() { return Status::Invalid("FullTextSearch field name and query must not be empty"); } } - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot read with null format table"); - } - if (impl_->format_table_ != nullptr) { - // The table already answers each of these, and from a source this cannot see behind, so a - // second answer is refused rather than silently dropped. - if (impl_->catalog_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithCatalog() " - "cannot be used with one"); - } - if (impl_->table_schema_) { - return Status::Invalid( - "a format table carries its own schema, so SetTableSchema() cannot be used with " - "one"); - } - if (impl_->specific_file_system_ != nullptr || - !impl_->fs_scheme_to_identifier_map_.empty()) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "and WithFileSystemSchemeToIdentifierMap() cannot be used with one"); - } - if (impl_->branch_ && - !BranchManager::IsMainBranch(BranchManager::NormalizeBranch(impl_->branch_.value()))) { - return Status::Invalid( - "a format table has no branches, so WithBranch() cannot be used with one"); - } - } PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_)); if (impl_->path_.empty()) { return Status::Invalid("cannot read with empty table path"); @@ -436,8 +392,8 @@ Result> ReadContextBuilder::Finish() { impl_->row_to_batch_thread_number_, impl_->table_schema_, impl_->memory_pool_, impl_->executor_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, impl_->options_, impl_->read_ahead_cache_enabled_, - impl_->cache_config_, impl_->cache_, impl_->format_table_, impl_->warmup_level_, - impl_->catalog_, impl_->identifier_); + impl_->cache_config_, impl_->cache_, impl_->warmup_level_, impl_->catalog_, + impl_->identifier_); if (impl_->read_schema_ && impl_->read_schema_->release) { ctx->SetReadSchema(std::move(impl_->read_schema_)); } diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index 89cccaa69..3fd859c5b 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -25,7 +25,6 @@ #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { class Predicate; @@ -39,8 +38,7 @@ ScanContext::ScanContext( const std::shared_ptr& specific_file_system, const std::optional& table_schema, const std::map& options, const std::shared_ptr& cache, - const std::shared_ptr& format_table, const std::shared_ptr& catalog, - const std::optional& identifier) + const std::shared_ptr& catalog, const std::optional& identifier) : path_(path), is_streaming_mode_(is_streaming_mode), limit_(limit), @@ -53,7 +51,6 @@ ScanContext::ScanContext( table_schema_(table_schema), options_(options), cache_(cache), - format_table_(format_table), catalog_(catalog), identifier_(identifier) {} @@ -83,10 +80,6 @@ class ScanContextBuilder::Impl { private: std::string path_; - /// Kept across `Reset()`, as `path_` is: both name the table this builder builds for, rather - /// than a setting of one scan of it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; bool is_streaming_mode_ = false; std::optional limit_; std::optional bucket_filter_; @@ -110,15 +103,6 @@ ScanContextBuilder::ScanContextBuilder(const std::string& path) impl_->path_ = path; } -ScanContextBuilder::ScanContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->path_ = table->Location(); - } -} - ScanContextBuilder::~ScanContextBuilder() = default; ScanContextBuilder& ScanContextBuilder::WithStreamingMode(bool is_streaming_mode) { impl_->is_streaming_mode_ = is_streaming_mode; @@ -205,28 +189,6 @@ ScanContextBuilder& ScanContextBuilder::WithCache(const std::shared_ptr& } Result> ScanContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot scan with null format table"); - } - if (impl_->format_table_ != nullptr) { - // The table already answers both, and from a source this cannot see behind, so a second - // answer is refused rather than silently dropped. - if (impl_->catalog_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithCatalog() " - "cannot be used with one"); - } - if (impl_->table_schema_) { - return Status::Invalid( - "a format table carries its own schema, so SetTableSchema() cannot be used with " - "one"); - } - if (impl_->specific_file_system_ != nullptr) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "cannot be used with one"); - } - } PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_)); if (impl_->path_.empty()) { return Status::Invalid("cannot scan with empty table path"); @@ -248,7 +210,7 @@ Result> ScanContextBuilder::Finish() { impl_->bucket_filter_), impl_->global_index_result_, impl_->realtime_context_, impl_->memory_pool_, executor, impl_->specific_file_system_, impl_->table_schema_, impl_->options_, impl_->cache_, - impl_->format_table_, impl_->catalog_, impl_->identifier_); + impl_->catalog_, impl_->identifier_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/write_context.cpp b/src/paimon/core/operation/write_context.cpp index e31d64ac4..2b890f934 100644 --- a/src/paimon/core/operation/write_context.cpp +++ b/src/paimon/core/operation/write_context.cpp @@ -30,7 +30,6 @@ #include "paimon/memory/memory_pool.h" #include "paimon/result.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" namespace paimon { @@ -43,8 +42,7 @@ WriteContext::WriteContext( const std::shared_ptr& specific_file_system, const std::map& fs_scheme_to_identifier_map, const std::shared_ptr& realtime_context, - const std::map& options, - const std::shared_ptr& format_table, const std::shared_ptr& catalog, + const std::map& options, const std::shared_ptr& catalog, const std::optional& identifier) : root_path_(root_path), commit_user_(commit_user), @@ -62,7 +60,6 @@ WriteContext::WriteContext( fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), realtime_context_(realtime_context), options_(options), - format_table_(format_table), catalog_(catalog), identifier_(identifier) {} @@ -93,10 +90,6 @@ class WriteContextBuilder::Impl { private: std::string root_path_; - /// Kept across `Reset()`, as `root_path_` is: both name the table this builder builds for, - /// rather than a setting of one write to it. - std::shared_ptr format_table_; - bool built_from_format_table_ = false; std::string commit_user_; /// Unset until `WithBranch()` names one, so that a branch named only by the identifier or /// the `branch` option is not overruled by the default. @@ -125,15 +118,6 @@ WriteContextBuilder::WriteContextBuilder(const std::string& root_path, impl_->commit_user_ = commit_user; } -WriteContextBuilder::WriteContextBuilder(const std::shared_ptr& table) - : impl_(std::make_unique()) { - impl_->format_table_ = table; - impl_->built_from_format_table_ = true; - if (table != nullptr) { - impl_->root_path_ = table->Location(); - } -} - WriteContextBuilder::~WriteContextBuilder() = default; WriteContextBuilder& WriteContextBuilder::AddOption(const std::string& key, @@ -226,27 +210,6 @@ WriteContextBuilder& WriteContextBuilder::WithCatalog(const std::shared_ptr> WriteContextBuilder::Finish() { - if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) { - return Status::Invalid("cannot write with null format table"); - } - if (impl_->format_table_ != nullptr) { - if (impl_->catalog_ != nullptr) { - return Status::Invalid("WithCatalog() requires a native table"); - } - // The table already answers each of these, and from a source this cannot see behind, so a - // second answer is refused rather than silently dropped. - if (impl_->specific_file_system_ != nullptr || - !impl_->fs_scheme_to_identifier_map_.empty()) { - return Status::Invalid( - "a format table carries the file system it was loaded through, so WithFileSystem() " - "and WithFileSystemSchemeToIdentifierMap() cannot be used with one"); - } - if (impl_->branch_ && - !BranchManager::IsMainBranch(BranchManager::NormalizeBranch(impl_->branch_.value()))) { - return Status::Invalid( - "a format table has no branches, so WithBranch() cannot be used with one"); - } - } PAIMON_ASSIGN_OR_RAISE(impl_->root_path_, PathUtil::NormalizePath(impl_->root_path_)); if (impl_->root_path_.empty()) { return Status::Invalid("root path is empty"); @@ -286,8 +249,7 @@ Result> WriteContextBuilder::Finish() { impl_->ignore_num_bucket_check_, impl_->ignore_previous_files_, enable_multi_thread_spill, impl_->write_id_, branch, impl_->write_schema_, impl_->memory_pool_, impl_->executor_, impl_->temp_directory_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, - impl_->realtime_context_, impl_->options_, impl_->format_table_, impl_->catalog_, - impl_->identifier_); + impl_->realtime_context_, impl_->options_, impl_->catalog_, impl_->identifier_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp index d5082ef73..5eca88fcf 100644 --- a/src/paimon/core/table/file_store_table_factory.cpp +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -29,7 +29,6 @@ #include "paimon/catalog/identifier.h" #include "paimon/catalog/table.h" #include "paimon/commit_context.h" -#include "paimon/common/utils/checked_cast.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/core_options.h" @@ -41,52 +40,17 @@ #include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/fs/file_system.h" -#include "paimon/predicate/predicate_utils.h" #include "paimon/read_context.h" #include "paimon/result.h" #include "paimon/scan_context.h" #include "paimon/schema/schema.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" #include "paimon/write_context.h" namespace paimon { namespace { -/// Recognises a format table behind `table_path`, or returns null when it is not one. The scan, -/// read, write and commit entry points below all dispatch on the same question, so one place -/// answers it and they cannot disagree. Null is also the answer when no schema was found under -/// the path at all: what to say about a table that is not there is the managed path's to decide. -/// -/// The table type is read from the schema, never from `options`: `type` is structural, and one -/// call's options must not decide what kind of table this is. -Result> TryLoadFormatTable( - const std::shared_ptr& file_system, const std::string& table_path, - const std::string& branch, const std::map& options, - const std::shared_ptr& table_schema) { - if (table_schema == nullptr) { - return std::shared_ptr(); - } - // From the schema alone: see `FormatTable::Create()`. - PAIMON_ASSIGN_OR_RAISE(bool is_format_table, - TableTypeDefine::IsFormatTable(table_schema->Options())); - if (!is_format_table) { - return std::shared_ptr(); - } - // A format table has no branches, so it is refused rather than served from the table path - // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. - if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { - return Status::Invalid( - fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", - table_path, branch)); - } - Identifier identifier = CatalogUtils::IdentifierFromPath(table_path, branch); - return FormatTable::Create(file_system, table_path, identifier, - checked_pointer_cast(table_schema), - /*location_carries_paimon_metadata=*/true, options); -} - /// The options a context carries, merged into the table the load hands back: a table is the /// product of the load, so the options speak there - the way `Copy` merges them - and `NewXxx` on /// the product serves them without merging again. A caller holding a table already changes it @@ -108,13 +72,13 @@ Result> WithContextOptions( return table; } -/// The tail every `Create` ends with: the schema one of them read says what the path names. A -/// format table is handed back as the `FormatTable` it is, everything else as the `FileStoreTable` +/// The tail every `Create` ends with: the schema one of them read says what the path names, and +/// only a data table is served by it - a format table is refused, the table a catalog hands back +/// serves one through its own `NewXxx`, and everything else is handed back as the `FileStoreTable` /// a managed table is, carrying the schema for its `NewXxx` to build from. A null schema is handed -/// over as it is - see `TryLoadFormatTable`. +/// over as it is, so the managed path below decides what to say about a table that is not there. /// -/// @param file_system The file system the schema above was read through, which a format table -/// is served by - it has no schema options of its own to name another one. +/// @param file_system The file system the schema above was read through. /// @param explicit_file_system The file system the context supplied, which a managed table is /// opened through when there is one. A null one leaves the table to resolve from its /// schema's own access configuration rather than being pinned to the file system this load @@ -126,10 +90,19 @@ Result> TableOfLoadedSchema( const std::shared_ptr& schema_manager, const std::shared_ptr& latest_schema, const std::optional& identifier, const std::string& uuid, const std::map& fs_scheme_map) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - TryLoadFormatTable(file_system, path, branch, options, latest_schema)); - if (format_table != nullptr) { - return std::shared_ptr
(std::move(format_table)); + if (latest_schema != nullptr) { + // The table type is read from the schema, never from `options`: `type` is structural, and + // one call's options must not decide what kind of table this is. See + // `FormatTable::Create()`. + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(latest_schema->Options())); + if (is_format_table) { + return Status::Invalid(fmt::format( + "the schema under {} names a format table, which is not loaded from a path: a " + "catalog hands the table back through GetTable, and its operators come from " + "table->NewScan/NewRead/NewWrite/NewCommit on it", + path)); + } } // The branch the schema was read on is folded into the object name. Identifier table_identifier = @@ -210,24 +183,21 @@ Result> CatalogMainBranchSchemaJson( } // namespace -/// The table a scan context names: the format table the builder was handed, a system table behind -/// a `T$system` path, or the table the schema under the context's path says is there. +/// The data table a scan context names: only a data table is served by a load from the context's +/// path - a system table behind a `T$system` path and a format table are refused, served by the +/// table a catalog hands back through its own `NewXxx` instead. Result> FileStoreTableFactory::Create(const ScanContext& ctx) { - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), {})); PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, SystemTableLoader::TryParsePath(ctx.GetPath())); if (system_table_path) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), - ctx.GetPath(), ctx.GetOptions())); - return std::shared_ptr
(std::move(system_table)); + return Status::Invalid(fmt::format( + "the path {} names a system table, which is not loaded from a path: a catalog " + "hands the table back through GetTable, and its operators come from " + "table->NewScan/NewRead on it", + ctx.GetPath())); } - // One `TableScan` interface serves a format table too, the way Java Paimon serves both - // through one `ReadBuilder`; a format table is planned by listing directories instead. const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), @@ -235,35 +205,23 @@ Result> FileStoreTableFactory::Create(const ScanContext& /*uuid=*/"", /*fs_scheme_map=*/{}); } -/// The table a read context names: the format table the builder was handed, a system table behind -/// a `T$system` path, or the table the schema under the context's path says is there. +/// The data table a read context names: only a data table is served by a load from the context's +/// path - a system table behind a `T$system` path and a format table are refused, served by the +/// table a catalog hands back through its own `NewXxx` instead. Result> FileStoreTableFactory::Create(const ReadContext& ctx) { - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), ctx.GetFileSystemSchemeToIdentifierMap())); PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, SystemTableLoader::TryParsePath(ctx.GetPath())); if (system_table_path) { - if (PredicateUtils::ContainsFullTextMatch(ctx.GetPredicate())) { - return Status::NotImplemented( - "FullTextMatch predicate execution is not implemented yet"); - } - if (ctx.HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for system tables"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), - ctx.GetPath(), ctx.GetOptions())); - return std::shared_ptr
(std::move(system_table)); + return Status::Invalid(fmt::format( + "the path {} names a system table, which is not loaded from a path: a catalog " + "hands the table back through GetTable, and its operators come from " + "table->NewScan/NewRead on it", + ctx.GetPath())); } - // A format table has no manifest and a file another engine wrote carries no field ids, so how - // a row is put back together is its own; how a file is opened is not - both paths go through - // `DataFileReaderFactory`. const std::string branch = ctx.GetBranch(); // The scheme map the schema was read through, so the table's data resolves custom schemes the // same way its metadata does. @@ -273,12 +231,10 @@ Result> FileStoreTableFactory::Create(const ReadContext& ctx.GetFileSystemSchemeToIdentifierMap()); } -/// The table a write context names: the format table the builder was handed, or the table the -/// schema under the context's root path says is there. +/// The data table a write context names: only a data table is served by a load from the context's +/// root path - a format table is refused, served by the table a catalog hands back through its own +/// `NewXxx` instead. Result> FileStoreTableFactory::Create(const WriteContext& ctx) { - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); if (specific_fs == nullptr && ctx.GetFileSystemSchemeToIdentifierMap().empty() && @@ -307,12 +263,10 @@ Result> FileStoreTableFactory::Create(const WriteContext& ctx.GetFileSystemSchemeToIdentifierMap()); } -/// The table a commit context names: the format table the builder was handed, or the table the -/// schema under the context's root path says is there. +/// The data table a commit context names: only a data table is served by a load from the context's +/// root path - a format table is refused, served by the table a catalog hands back through its own +/// `NewXxx` instead. Result> FileStoreTableFactory::Create(const CommitContext& ctx) { - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); // Use this table's catalog credentials unless the caller supplied a file system: a catalog // issuing per-table temporary ones only hands them out through GetTableFileSystem. diff --git a/src/paimon/core/table/file_store_table_factory.h b/src/paimon/core/table/file_store_table_factory.h index a1bb77062..e1090dc97 100644 --- a/src/paimon/core/table/file_store_table_factory.h +++ b/src/paimon/core/table/file_store_table_factory.h @@ -30,10 +30,11 @@ class ScanContext; class Table; class WriteContext; -/// Loads the table a context names, as Java Paimon's `FileStoreTableFactory` does: the format -/// table the builder was handed, the system table behind a `T$system` path, or the managed table -/// the schema under the context's path describes. The static `Create` of all four operators asks -/// the same question, so one place answers it and they cannot disagree. +/// Loads the data table a context names, as Java Paimon's `FileStoreTableFactory` does: the schema +/// under the context's path describes it. A system table behind a `T$system` path and a format +/// table are refused - a catalog hands those back through `GetTable`, and their operators come from +/// the table itself - so the static `Create` of all four operators answers the one question this +/// place asks and they cannot disagree. class FileStoreTableFactory { public: FileStoreTableFactory() = delete; diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index f14ac66d1..acb5f03e1 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -297,8 +297,8 @@ Result> FormatTable::NewWrite( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, CopySelf(context->GetOptions())); if (context->GetCatalog() != nullptr) { return Status::Invalid( - "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " - "format table"); + "WithCatalog() requires a native table; a format table takes a WriteContextBuilder " + "built from its location instead"); } if (context->IsStreamingMode()) { return Status::Invalid( diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index 84fbce351..af0686c6f 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -763,14 +763,14 @@ TEST(FormatTableTest, TestCatalogManagedPartitionsAreRejected) { ASSERT_TRUE(dynamic_table.status().IsNotImplemented()) << dynamic_table.status().ToString(); } -TEST(FormatTableTest, TestTheGenericEntryPointsValidateOptionsGivenAtTheCall) { - // An option a format table refuses has to be refused wherever it comes from. Every generic - // entry point merges what the call gave it over what the schema stored, so each runs the same - // checks over the merged result; otherwise a caller could set through a context what the - // schema would have rejected and have it silently dropped. +TEST(FormatTableTest, TestTheTableOperatorsValidateOptionsGivenAtTheCall) { + // An option a format table refuses has to be refused wherever it comes from. Every operator + // merges what the call gave it over what the schema stored, so each runs the same checks over + // the merged result; otherwise a caller could set through a context what the schema would have + // rejected and have it silently dropped. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); const std::map refused = { {Options::METASTORE_PARTITIONED_TABLE, "true"}}; @@ -778,24 +778,22 @@ TEST(FormatTableTest, TestTheGenericEntryPointsValidateOptionsGivenAtTheCall) { ScanContextBuilder scan_builder(dir->Str()); scan_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_context)), "metastore.partitioned-table"); ReadContextBuilder read_builder(dir->Str()); read_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), "metastore.partitioned-table"); WriteContextBuilder write_builder(dir->Str(), "test-user"); write_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), - "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_context)), "metastore.partitioned-table"); CommitContextBuilder commit_builder(dir->Str(), "test-user"); commit_builder.SetOptions(refused); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), - "metastore.partitioned-table"); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_context)), "metastore.partitioned-table"); } TEST(FormatTableTest, TestValueOnlyPartitionCannotBeNamedAfterThisTablesMetadata) { @@ -2667,19 +2665,20 @@ TEST(FormatTableTest, TestScanRejectsUnknownPartitionField) { ASSERT_TRUE(scan.status().IsInvalid()); } -TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { - // A caller holding a table path uses the interfaces it uses for every other table. Each one - // recognises a format table from its schema and dispatches to it, the way Java Paimon serves - // both kinds through one `ReadBuilder` and one `BatchWriteBuilder`. +TEST(FormatTableTest, TestAFormatTableIsWrittenCommittedAndReadBack) { + // A caller holding the table a catalog hands back drives it through the operators the table + // itself serves, with contexts built from its location the way they are for every other + // table - the C++ side of what Java serves through one `ReadBuilder` and one + // `BatchWriteBuilder` for both kinds of table. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); WriteContextBuilder write_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + table->NewWrite(std::move(write_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({1, 2}, {"alice", "bob"}, "20240101", {{"dt", "20240101"}})); ASSERT_OK(write->Write(std::move(batch))); @@ -2696,26 +2695,26 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { WriteContextBuilder write_id_builder(dir->Str(), "test-user"); write_id_builder.WithWriteId(3); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id_context, write_id_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_id_context)), + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_id_context)), "a write id would name nothing"); CommitContextBuilder empty_commit_builder(dir->Str(), "test-user"); empty_commit_builder.IgnoreEmptyCommit(false); ASSERT_OK_AND_ASSIGN(std::unique_ptr empty_commit_context, empty_commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(empty_commit_context)), + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(empty_commit_context)), "cannot record an empty commit"); CommitContextBuilder rest_builder(dir->Str(), "test-user"); rest_builder.UseRESTCatalogCommit(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr rest_context, rest_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(rest_context)), "rest catalog"); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(rest_context)), "rest catalog"); CommitContextBuilder table_id_builder(dir->Str(), "test-user"); table_id_builder.WithTableId("table-uuid"); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_id_context, table_id_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(table_id_context)), + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(table_id_context)), "a format table sends no commit table request"); auto format_catalog = std::make_shared(); @@ -2723,20 +2722,20 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { CommitContextBuilder catalog_builder(dir->Str(), "test-user"); catalog_builder.WithCatalog(format_catalog, Identifier("db", "fmt")); ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_context, catalog_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(catalog_context)), + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(catalog_context)), "keeps no snapshot for a catalog to take"); CommitContextBuilder conflict_builder(dir->Str(), "test-user"); conflict_builder.AppendCommitCheckConflict(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr conflict_context, conflict_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(conflict_context)), + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(conflict_context)), "no manifests to check a concurrent commit against"); CommitContextBuilder commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + table->NewCommit(std::move(commit_context))); ASSERT_OK(commit->Commit(messages)); // The same for the snapshot half of the commit interface. ASSERT_NOK_WITH_MSG(commit->Expire(), "no snapshots to expire"); @@ -2745,15 +2744,13 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { ScanContextBuilder scan_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1u); ReadContextBuilder read_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch read_batch, reader->NextBatch()); ASSERT_FALSE(BatchReader::IsEofBatch(read_batch)); @@ -2795,11 +2792,12 @@ TEST(FormatTableTest, TestACatalogInTheContextIsRefusedAlikeByWriteAndCommit) { ASSERT_TRUE(commit.status().IsInvalid()); } -TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { +TEST(FormatTableTest, TestALoadedFormatTableIsServedThroughItsOwnOperators) { // A table a catalog keeps the schema for, the way a rest catalog does. Nothing under its // location says that it is a format table, nor that everything below the location is data, so - // a path is not enough to reach it: the loaded table is handed to the context instead, which - // is what `FormatTable.newReadBuilder()` and `newBatchWriteBuilder()` do in Java. + // a path is not enough to reach it: the loaded table serves the operators itself, with a + // context built from its location, which is what `FormatTable.newReadBuilder()` and + // `newBatchWriteBuilder()` do in Java. std::unique_ptr schema_dir = UniqueTestDirectory::Create(); ASSERT_TRUE(schema_dir); SchemaManager schema_manager(schema_dir->GetFileSystem(), schema_dir->Str()); @@ -2826,11 +2824,11 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { // Options given at the call still win over the ones the schema stored, so a file is rolled // per row rather than one file holding all three. - WriteContextBuilder write_builder(external); + WriteContextBuilder write_builder(external->Location(), "test-user"); write_builder.SetOptions({{Options::TARGET_FILE_ROW_NUM, "1"}}); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + external->NewWrite(std::move(write_context))); for (int32_t i = 1; i <= 3; i++) { ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({i}, {"name"}, "20240101", {{"dt", "20240101"}})); @@ -2840,25 +2838,25 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { write->PrepareCommit()); ASSERT_EQ(messages.size(), 3u) << "target-file-row-num given at the call was ignored"; - CommitContextBuilder commit_builder(external); + CommitContextBuilder commit_builder(external->Location(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + external->NewCommit(std::move(commit_context))); ASSERT_OK(commit->Commit(messages)); ASSERT_OK(write->Close()); - ScanContextBuilder scan_builder(external); + ScanContextBuilder scan_builder(external->Location()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + external->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(PartitionList partitions, scan->ListPartitions()); ASSERT_EQ(partitions, (PartitionList{{{"dt", "20240101"}}})); - ReadContextBuilder read_builder(external); + ReadContextBuilder read_builder(external->Location()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, - TableRead::Create(std::move(read_context))); + external->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); int64_t rows = 0; while (true) { @@ -2936,8 +2934,9 @@ TEST(FormatTableTest, TestATableServesAFormatTableThroughItsOwnEntryPoints) { TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) { // A refusal is a promise about what was not done, so the table's own entry points must make - // the same promises the generic ones do. Each context is handed to both, and the answer has - // to come out the same whichever way in the caller takes. + // the same promises the generic ones do: a generic entry point refuses the format table the + // schema under the path names outright, while the table's operators name the setting they + // cannot honour, so a caller who reaches the table finds out exactly what is missing. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr format_table, @@ -2962,8 +2961,7 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) return builder.Finish(); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context, streaming_scan_context()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(streaming_context)), - "nothing for a streaming scan to follow"); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(streaming_context)), "names a format table"); ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context_again, streaming_scan_context()); ASSERT_NOK_WITH_MSG(table->NewScan(std::move(streaming_context_again)), @@ -2979,8 +2977,7 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) return builder.Finish(); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context, vector_read_context()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(vector_context)), - "not supported for format tables"); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(vector_context)), "names a format table"); ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context_again, vector_read_context()); ASSERT_NOK_WITH_MSG(table->NewRead(std::move(vector_context_again)), "not supported for format tables"); @@ -2994,20 +2991,18 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) return builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_write, catalog_write_context()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(catalog_write)), - "use WriteContextBuilder(FormatTable)"); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(catalog_write)), "names a format table"); ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_write_again, catalog_write_context()); ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(catalog_write_again)), - "use WriteContextBuilder(FormatTable)"); + "WithCatalog() requires a native table"); auto catalog_commit_context = [&dir, &catalog]() -> Result> { CommitContextBuilder builder(dir->Str(), "test-user"); return builder.WithCatalog(catalog, Identifier("db", "fmt")).Finish(); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_commit, catalog_commit_context()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(catalog_commit)), - "keeps no snapshot for a catalog to take"); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(catalog_commit)), "names a format table"); ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_commit_again, catalog_commit_context()); ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(catalog_commit_again)), @@ -3020,75 +3015,50 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) return builder.Finish(); }; ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id, write_id_context()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_id)), - "a write id would name nothing"); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_id)), "names a format table"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id_again, write_id_context()); ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_id_again)), "a write id would name nothing"); } -TEST(FormatTableTest, TestAContextBuiltFromAFormatTableRefusesASecondAnswer) { - // The table already carries its schema and the file system it was loaded through, from a - // source the context cannot see behind, so a second answer is refused rather than dropped. +TEST(FormatTableTest, TestTheGenericEntryPointsRefuseAFormatTablePath) { + // A load from a path is the data table's way in, so every generic entry point refuses the + // format table the schema under the path names rather than dispatching to it - and says where + // the table does come from, so a caller holding only the path is pointed at the catalog. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); - SchemaManager schema_manager(dir->GetFileSystem(), dir->Str()); - ASSERT_OK_AND_ASSIGN(std::optional> latest, - schema_manager.Latest()); - ASSERT_TRUE(latest.has_value()); - ASSERT_OK_AND_ASSIGN(std::string schema_json, (*latest)->GetJsonSchema()); - - ScanContextBuilder schema_builder(table); - schema_builder.SetTableSchema(schema_json); - ASSERT_NOK_WITH_MSG(schema_builder.Finish(), "carries its own schema"); - - ScanContextBuilder fs_builder(table); - fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(fs_builder.Finish(), "carries the file system"); - ReadContextBuilder branch_builder(table); - branch_builder.WithBranch("other"); - ASSERT_NOK_WITH_MSG(branch_builder.Finish(), "has no branches"); + ScanContextBuilder scan_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "names a format table"); - WriteContextBuilder write_fs_builder(table); - write_fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(write_fs_builder.Finish(), "carries the file system"); + ReadContextBuilder read_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "names a format table"); - auto catalog = std::make_shared(); - WriteContextBuilder write_catalog_builder(table); - ASSERT_NOK_WITH_MSG( - write_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() requires a native table"); - ReadContextBuilder read_catalog_builder(table); - ASSERT_NOK_WITH_MSG(read_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() cannot be used with one"); - ScanContextBuilder scan_catalog_builder(table); - ASSERT_NOK_WITH_MSG(scan_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), - "WithCatalog() cannot be used with one"); - catalog->SetTableSchema(latest.value()); - WriteContextBuilder path_catalog_builder(dir->Str(), "test-user"); - ASSERT_OK_AND_ASSIGN( - auto path_context, - path_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(path_context)), - "use WriteContextBuilder(FormatTable)"); + WriteContextBuilder write_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), "names a format table"); - CommitContextBuilder commit_fs_builder(table); - commit_fs_builder.WithFileSystem(dir->GetFileSystem()); - ASSERT_NOK_WITH_MSG(commit_fs_builder.Finish(), "carries the file system"); + CommitContextBuilder commit_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "names a format table"); - // A null table is the caller's mistake, and says so rather than complaining about a path it - // was never given. - ScanContextBuilder null_builder{std::shared_ptr()}; - ASSERT_NOK_WITH_MSG(null_builder.Finish(), "null format table"); + // The refusal points at the table's own operators, and those serve the same path-built + // contexts: this is the caller the message sends to `GetTable` and `NewXxx`. + ScanContextBuilder served_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr served_context, served_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr served_scan, + table->NewScan(std::move(served_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, served_scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 0u); } -TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) { - // `WithBranch()` is refused on a context built from a format table, and the `branch` option - // says the same thing by another name: a format table keeps no metadata to branch, and a read - // or a write would go to its one location whatever the option named. Refusing it is the only +TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedByTheTableOperators) { + // The `branch` option names a branch a format table keeps no metadata for, and a read or a + // write would go to its one location whatever the option named. Refusing it is the only // answer that is not silently the wrong one - an overwrite would delete the main branch's // data while claiming to replace another branch's. std::unique_ptr dir = UniqueTestDirectory::Create(); @@ -3105,27 +3075,27 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) FormatTable::Create(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"), branch), "has no branches"); - ScanContextBuilder scan_builder(table); + ScanContextBuilder scan_builder(dir->Str()); scan_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(scan_context)), "has no branches"); - ReadContextBuilder read_builder(table); + ReadContextBuilder read_builder(dir->Str()); read_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_context)), "has no branches"); - WriteContextBuilder write_builder(table); + WriteContextBuilder write_builder(dir->Str(), "test-user"); write_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_context)), "has no branches"); // The commit is the one that matters most: an overwrite clears what it replaces before it // publishes anything, so the refusal has to come before the deleting does. - CommitContextBuilder commit_builder(table); + CommitContextBuilder commit_builder(dir->Str(), "test-user"); commit_builder.SetOptions(branch); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(commit_context)), "has no branches"); ASSERT_OK_AND_ASSIGN( std::unique_ptr scan, @@ -3165,9 +3135,10 @@ TEST(FormatTableTest, TestATableCopiesItselfWithTheOptionsGivenAtTheCall) { TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFromTheTableRoot) { // A context that names a path takes its branch from `WithBranch()`, and the schema of that - // branch is what the entry point dispatches on. A format table's data is the files under the - // table path whichever branch named it, so serving one from there under another branch's name - // would answer a question about `dev` with the rows of `main`. + // branch decides what the entry point refuses. A format table's data is the files under the + // table path whichever branch named it, so the format table the branch's schema names is + // refused rather than served from there under another branch's name - the branch's table + // comes from the catalog, and its rows from that table's own operators. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -3186,29 +3157,30 @@ TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFr ReadContextBuilder read_builder(dir->Str()); read_builder.WithBranch("dev"); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "names a format table"); // A scan takes its branch from the option rather than from a `WithBranch()` of its own, which // is the same branch by another name and gets the same answer. ScanContextBuilder scan_builder(dir->Str()); scan_builder.SetOptions({{Options::BRANCH, "dev"}}); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "has no branches"); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "names a format table"); - // The main branch is untouched by any of it. + // The main branch is untouched by any of it: the table the catalog hands back for it serves + // the scan from its own operators. ScanContextBuilder main_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_context, main_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan, - TableScan::Create(std::move(main_context))); + table->NewScan(std::move(main_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, main_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::vector rows, ReadAll(table, plan->Splits())); ASSERT_EQ(rows, (std::vector{"1|alice|20240101"})); } -TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { +TEST(FormatTableTest, TestTheTableOperatorsCarryTheContextThrough) { // A context promises that options given at the call win over the ones the schema stored, and - // that a branch and a caller-held schema are honoured. Dispatching to a format table must not - // quietly drop any of that. + // that a branch is honoured. Serving the operators off the table itself must not quietly drop + // any of that. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -3220,7 +3192,7 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { write_builder.SetOptions({{Options::TARGET_FILE_ROW_NUM, "1"}}); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + table->NewWrite(std::move(write_context))); for (int32_t i = 0; i < 3; i++) { ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch({i}, {"name"}, "20240101", {})); @@ -3230,41 +3202,23 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { write->PrepareCommit()); ASSERT_EQ(messages.size(), 3u) << "target-file-row-num given at the call was ignored"; - // Published through the generic commit, or nothing below would see the rows: until the commit - // renames them the files sit in the hidden `_temporary` directory a scan skips. + // Published through the table's own commit, or nothing below would see the rows: until the + // commit renames them the files sit in the hidden `_temporary` directory a scan skips. CommitContextBuilder generic_commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr generic_commit_context, generic_commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr generic_commit, - FileStoreCommit::Create(std::move(generic_commit_context))); + table->NewCommit(std::move(generic_commit_context))); ASSERT_OK(generic_commit->Commit(messages)); ASSERT_OK(write->Close()); - // A branch this table has no schema on is a managed table as far as dispatch is concerned, so - // the format branch must not answer for it. + // A branch this table has no schema on names no table at all as far as the generic entry + // points are concerned, so the format branch must not answer for it. ReadContextBuilder branch_builder(dir->Str()); branch_builder.WithBranch("nosuchbranch"); ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_context, branch_builder.Finish()); ASSERT_NOK(TableRead::Create(std::move(branch_context))); - // A caller-held schema is used instead of reading one from under the path. - SchemaManager schema_manager(dir->GetFileSystem(), dir->Str()); - ASSERT_OK_AND_ASSIGN(std::optional> latest, - schema_manager.Latest()); - ASSERT_TRUE(latest.has_value()); - ASSERT_OK_AND_ASSIGN(std::string schema_json, (*latest)->GetJsonSchema()); - ScanContextBuilder seeded_builder(dir->Str()); - seeded_builder.SetTableSchema(schema_json); - ASSERT_OK_AND_ASSIGN(std::unique_ptr seeded_context, seeded_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr seeded_scan, - TableScan::Create(std::move(seeded_context))); - // Planning, not just dispatch: handing the schema over must not turn the `schema` directory - // the table keeps under its own path into data, which a plan would then read as a data file. - ASSERT_OK_AND_ASSIGN(std::shared_ptr seeded_plan, seeded_scan->CreatePlan()); - ASSERT_OK_AND_ASSIGN(std::vector seeded_rows, - ReadAll(table, seeded_plan->Splits())); - ASSERT_EQ(seeded_rows.size(), 3u); - // `type` is structural, so an option given at the call must not decide what kind of table // this is. Checked by reading the rows back rather than by counting splits: three small files // are packed into one split, so a split count says nothing about how many there are. @@ -3272,7 +3226,7 @@ TEST(FormatTableTest, TestTheGenericEntryPointsCarryTheContextThrough) { retyped_builder.SetOptions({{Options::TYPE, "table"}}); ASSERT_OK_AND_ASSIGN(std::unique_ptr retyped_context, retyped_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr retyped_scan, - TableScan::Create(std::move(retyped_context))); + table->NewScan(std::move(retyped_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr retyped_plan, retyped_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(std::vector retyped_rows, ReadAll(table, retyped_plan->Splits())); @@ -3307,7 +3261,7 @@ class CountingCache : public Cache { } // namespace -TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { +TEST(FormatTableTest, TestTheReadCarriesPrefetchAndTheCacheThrough) { // A format table opens its files through the same component the managed table path opens its // own with, so what a `ReadContext` asks for about opening a file applies here too. std::unique_ptr dir = UniqueTestDirectory::Create(); @@ -3321,13 +3275,11 @@ TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { ScanContextBuilder scan_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, - TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); - auto read_rows = [&plan](std::unique_ptr context) -> Result { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - TableRead::Create(std::move(context))); + auto read_rows = [&plan, &table](std::unique_ptr context) -> Result { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, table->NewRead(std::move(context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(plan->Splits())); int64_t rows = 0; @@ -3362,7 +3314,7 @@ TEST(FormatTableTest, TestTheGenericReadCarriesPrefetchAndTheCacheThrough) { ASSERT_GT(cache->gets, 0) << "the cache the read context carries never reached the file reader"; } -TEST(FormatTableTest, TestTheGenericEntryPointsRefuseWhatTheyCannotHonour) { +TEST(FormatTableTest, TestTheOperatorsRefuseWhatAFormatTableCannotHonour) { // Anything a context carries that a format table cannot do is a refusal naming the setting, // never a read or a write that quietly did something else. std::unique_ptr dir = UniqueTestDirectory::Create(); @@ -3377,14 +3329,14 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseWhatTheyCannotHonour) { read_schema_builder.SetReadSchema(std::move(projected_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_schema_context, read_schema_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_schema_context)), + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(read_schema_context)), "does not take a projected read schema"); ScanContextBuilder streaming_builder(dir->Str()); streaming_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context, streaming_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(streaming_context)), + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(streaming_context)), "nothing for a streaming scan to follow"); ScanContextBuilder predicate_builder(dir->Str()); @@ -3392,13 +3344,12 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseWhatTheyCannotHonour) { /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(1))); ASSERT_OK_AND_ASSIGN(std::unique_ptr predicate_context, predicate_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(predicate_context)), - "does not take a predicate"); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(predicate_context)), "does not take a predicate"); CommitContextBuilder commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + table->NewCommit(std::move(commit_context))); // A commit takes batch writes only. The rest of the snapshot half is // `TestTheGenericCommitRefusesEveryCallAboutSnapshots`. ASSERT_NOK_WITH_MSG(commit->Commit({}, /*commit_identifier=*/7), "commit identifier"); @@ -3419,12 +3370,12 @@ TEST(FormatTableTest, TestTheGenericCommitRefusesEveryCallAboutSnapshots) { // types then finds out at the call rather than from a table that did not change. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr table, + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); CommitContextBuilder commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + table->NewCommit(std::move(commit_context))); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress({}, BATCH_WRITE_COMMIT_IDENTIFIER, /*watermark=*/std::nullopt), @@ -3468,7 +3419,7 @@ TEST(FormatTableTest, TestTheGenericCommitOverwritesOnePartition) { WriteContextBuilder write_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, - FileStoreWrite::Create(std::move(write_context))); + table->NewWrite(std::move(write_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr replacement, MakeBatch({3}, {"carol"}, "20240101", {{"dt", "20240101"}})); ASSERT_OK(write->Write(std::move(replacement))); @@ -3481,7 +3432,7 @@ TEST(FormatTableTest, TestTheGenericCommitOverwritesOnePartition) { CommitContextBuilder commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + table->NewCommit(std::move(commit_context))); // Batch writes only, and refused before anything is cleared: an overwrite that failed on its // arguments must not have deleted the partition it named. ASSERT_NOK_WITH_MSG(commit->Overwrite({{"dt", "20240101"}}, messages, /*commit_identifier=*/7), diff --git a/src/paimon/core/table/source/table_scan_test.cpp b/src/paimon/core/table/source/table_scan_test.cpp index 435e2254f..17bf2e874 100644 --- a/src/paimon/core/table/source/table_scan_test.cpp +++ b/src/paimon/core/table/source/table_scan_test.cpp @@ -19,12 +19,15 @@ #include "paimon/table/source/table_scan.h" +#include #include #include #include #include #include "gtest/gtest.h" +#include "paimon/core/core_options.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/defs.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/metrics.h" @@ -164,12 +167,19 @@ TEST(TableScanTest, TestPkSchemaEvolutionScan) { TEST(TableScanTest, TestReadOptimizedPrimaryKeyStreamingScanUnsupported) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table$ro"; + // A system table is not served by the entry point a path names any more: the loader hands + // the table back the way a catalog's `GetTable` would, and the table refuses the scan. + std::map options = {{Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), path, options)); ScanContextBuilder builder(path); builder.AddOption(Options::FILE_FORMAT, "orc"); builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)), + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(context)), "read-optimized system table does not support streaming scan for primary " "key table"); } diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index d6333475b..cfd4f5591 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -56,6 +56,7 @@ #include "paimon/scan_context.h" #include "paimon/status.h" #include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_file_batch_reader.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" @@ -331,6 +332,21 @@ TEST(SystemTableTest, TestOperatorEntriesRefuseANullContext) { ASSERT_NOK_WITH_MSG(read_optimized.NewRead(nullptr), "read context is null pointer"); } +// The scan and read entry points serve a data table only: a `T$system` path is refused with a +// message pointing at the catalog that hands the table back, so a caller holding only the path is +// not told the table does not exist. +TEST(SystemTableTest, TestTheOperatorEntriesRefuseASystemTablePath) { + const std::string system_table_path = "/tmp/warehouse/db.db/tbl$options"; + + ScanContextBuilder scan_builder(system_table_path); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "names a system table"); + + ReadContextBuilder read_builder(system_table_path); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "names a system table"); +} + // The reads these views serve build a context of their own for the data table underneath, which // carries no search: a search the caller asked for is refused rather than quietly answered with // plain rows. diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 8b7b2da45..2b6be91a4 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -36,6 +36,7 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/file_store_table_factory.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/fallback_data_split.h" #include "paimon/defs.h" @@ -1289,22 +1290,18 @@ TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { // A context that names no catalog knows the table by its directory: the directory above // the table's names the database when it carries the `.db` suffix, the way the Java - // `identifierFromPath` reads it, so the table the load serves is named for both. The - // refusal a format table open speaks names the table - and the option the context - // carries is what drives the open to it, since the schema alone opens cleanly. + // `identifierFromPath` reads it, so the table the load serves is named for both. const std::string table_dir = dir->Str() + "/my_db.db/my_table"; ASSERT_OK(fs->Mkdirs(table_dir)); SchemaManager schema_manager(fs, table_dir); - ASSERT_OK_AND_ASSIGN( - [[maybe_unused]] std::unique_ptr format_schema, - schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, - {{"type", "format-table"}})); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr managed_schema, + schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, {{"file.format", "orc"}})); { ScanContextBuilder builder(table_dir); - builder.AddOption(Options::REALTIME_ENABLED, "true"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), - "cannot open format table my_db.my_table"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + ASSERT_EQ(table->FullName(), "my_db.my_table"); } // A table sitting directly under the root, with no `.db` directory above it, names no @@ -1315,13 +1312,12 @@ TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { ASSERT_OK_AND_ASSIGN( [[maybe_unused]] std::unique_ptr loose_schema, loose_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, - {{"type", "format-table"}})); + {{"file.format", "orc"}})); { ScanContextBuilder builder(loose_table_dir); - builder.AddOption(Options::REALTIME_ENABLED, "true"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), - "cannot open format table loose_table"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + ASSERT_EQ(table->FullName(), "loose_table"); } } diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index ef8e5c853..41e76b32d 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -49,6 +49,7 @@ #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/read_ahead_cache.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/core_options.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/snapshot.h" @@ -57,6 +58,7 @@ #include "paimon/core/table/source/deletion_file.h" #include "paimon/core/table/source/fallback_data_split.h" #include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/tag/tag.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/data/decimal.h" @@ -274,6 +276,13 @@ Result> ReadSystemTable( const std::string& system_table_path, const std::map& options, bool streaming_mode = false, const std::shared_ptr& predicate = nullptr, const std::vector& read_field_names = {}, bool read_next_plan = false) { + // A system table is not served by the operator entry points a path names any more: the loader + // hands the table back, and the table serves the scan and the read itself, the way a catalog's + // `GetTable` would. + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), system_table_path, options)); ScanContextBuilder scan_context_builder(system_table_path); scan_context_builder.SetOptions(options).WithStreamingMode(streaming_mode); if (predicate) { @@ -282,7 +291,7 @@ Result> ReadSystemTable( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + table->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); if (read_next_plan) { PAIMON_ASSIGN_OR_RAISE(plan, table_scan->CreatePlan()); @@ -299,7 +308,7 @@ Result> ReadSystemTable( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + table->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, @@ -650,17 +659,20 @@ TEST(SystemTableReadInteTest, TestReadOptionsSystemTable) { ArrowSchemaRelease(&schema); std::string system_table_path = PathUtil::JoinPath(dir->Str(), "warehouse/db1.db/tbl1$options"); + // A system table comes from the catalog, and the operators from the table it hands back - + // the path alone no longer reaches it. + ASSERT_OK_AND_ASSIGN(auto table, catalog->GetTable(Identifier("db1", "tbl1$options"))); ScanContextBuilder scan_context_builder(system_table_path); scan_context_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); ReadContextBuilder read_context_builder(system_table_path); read_context_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader))); ASSERT_TRUE(result); @@ -682,17 +694,21 @@ TEST(SystemTableReadInteTest, TestReadBranchOptionsSystemTable) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}}; std::string system_table_path = table_path + "$branch_rt$options"; + // No catalog here: the loader is the path-side half of what a catalog's `GetTable` does. + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN(auto table, SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), + system_table_path, options)); ScanContextBuilder scan_context_builder(system_table_path); scan_context_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); ASSERT_EQ(plan->Splits().size(), 1); ReadContextBuilder read_context_builder(system_table_path); read_context_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -1002,12 +1018,18 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP std::shared_ptr predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"k", FieldType::INT, Literal(2)); + // A system table is not served by the operator entry points a path names any more: the loader + // hands the table back, and the table serves the scan and the read itself. + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), table_path + "$ro", options)); ScanContextBuilder ro_scan_context_builder(table_path + "$ro"); ro_scan_context_builder.SetOptions(options).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_scan_context, ro_scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_table_scan, - TableScan::Create(std::move(ro_scan_context))); + table->NewScan(std::move(ro_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr ro_plan, ro_table_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(int64_t ro_file_count, CountDataFiles(ro_plan->Splits())); ASSERT_EQ(ro_file_count, 1); @@ -1018,7 +1040,7 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP read_context_builder.SetOptions(options).SetPredicate(predicate).SetReadFieldNames({"k", "v"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(ro_plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -1062,11 +1084,17 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(WriteAndFullCompact(std::move(batch), /*commit_identifier=*/0, helper.get())); + // A system table is not served by the operator entry points a path names any more: the loader + // hands the table back, and the table serves the scan and the read itself. + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), table_path + "$ro", options)); ScanContextBuilder scan_context_builder(table_path + "$ro"); scan_context_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); + table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); auto projected_schema = arrow::schema({ @@ -1079,7 +1107,7 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { read_context_builder.SetOptions(options).SetReadSchema(std::move(c_projected_schema)); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 7111f5302..1fd0ea9ab 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -61,6 +61,7 @@ #include "paimon/core/table/sink/commit_message_impl.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/realtime_split.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/data/shredding/map_shared_shredding_schema_utils.h" #include "paimon/data/variant.h" @@ -2918,10 +2919,16 @@ TEST_F(RealtimeWriteInteTest, TestPkRealtimeReadOptimizedScanUnsupported) { ASSERT_OK_AND_ASSIGN(std::vector disk_rows, ReadRows()); ASSERT_EQ(rows, disk_rows); + // A system table is not served by the entry point a path names any more: the loader hands + // the table back, and the table refuses the scan itself. + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), + table_path_ + "$ro", options_)); ScanContextBuilder scan_builder(table_path_ + "$ro"); scan_builder.SetOptions(options_).WithRealtimeContext(realtime_context).WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - Result> scan = TableScan::Create(std::move(scan_context)); + Result> scan = table->NewScan(std::move(scan_context)); ASSERT_TRUE(scan.status().IsNotImplemented()) << scan.status().ToString(); ASSERT_NE(std::string::npos, scan.status().ToString().find( "PK real-time union read does not support read-optimized")); From 1d7661b93f9629f32d06034d851c86c2d8f24da6 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Wed, 28 Oct 2026 10:58:47 +0800 Subject: [PATCH 19/19] fix --- src/paimon/core/catalog/catalog.cpp | 3 - .../core/catalog/file_system_catalog.cpp | 28 +- src/paimon/core/catalog/file_system_catalog.h | 2 +- .../core/catalog/file_system_catalog_test.cpp | 76 +++- src/paimon/core/catalog/snapshot_commit.cpp | 2 - .../core/operation/internal_read_context.cpp | 14 - .../core/operation/internal_read_context.h | 7 +- src/paimon/core/schema/schema_validation.cpp | 1 - src/paimon/core/schema/table_schema.cpp | 4 - .../table/append_only_file_store_table.cpp | 3 - src/paimon/core/table/file_store.cpp | 21 +- src/paimon/core/table/file_store.h | 11 +- src/paimon/core/table/file_store_table.cpp | 130 ++++--- src/paimon/core/table/file_store_table.h | 15 + .../core/table/file_store_table_factory.cpp | 66 ---- src/paimon/core/table/format/format_table.cpp | 7 - .../core/table/format/format_table_test.cpp | 61 ---- .../table/primary_key_file_store_table.cpp | 7 - .../core/table/source/table_read_test.cpp | 10 - .../core/table/source/table_scan_test.cpp | 2 - .../table/system/audit_log_system_table.cpp | 5 - .../table/system/global_system_tables.cpp | 8 - .../table/system/in_memory_system_table.cpp | 2 - .../system/read_optimized_system_table.cpp | 5 - src/paimon/core/table/system/system_table.cpp | 6 - .../core/table/system/system_table_test.cpp | 18 - src/paimon/core/table/table_test.cpp | 335 +++++++++++------- src/paimon/rest/rest_catalog.cpp | 15 - src/paimon/rest/rest_catalog_test.cpp | 12 - test/inte/read_inte_test.cpp | 10 - test/inte/realtime_write_inte_test.cpp | 2 - 31 files changed, 394 insertions(+), 494 deletions(-) diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index b14a53eab..717606612 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -57,9 +57,6 @@ Result> Catalog::Create( std::unique_ptr rest_catalog, RestCatalog::Create(root_path, options, file_system, RestHttpClient::Config(), fs_scheme_to_identifier_map)); - // Built from the derived type so that the `enable_shared_from_this` base a - // `RestCatalog` carries is initialized: a `sys` table keeps its catalog alive through - // `weak_from_this()`, which stays empty when the pointer is widened first. return std::shared_ptr(std::move(rest_catalog)); #else return Status::NotImplemented( diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index d061acbf2..d5365d2ab 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -287,8 +287,6 @@ Result> FileSystemCatalog::LoadTableSchema( if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } - // The table built here is transient: only its schema is read and nothing outlives the - // call. It builds no rows, so the context carries no catalog. GlobalSystemTableContext context; context.fs = fs_; context.warehouse = warehouse_; @@ -316,9 +314,27 @@ Result> FileSystemCatalog::LoadTableSchema( } Result> FileSystemCatalog::GetTable(const Identifier& identifier) const { + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, GlobalSystemTableLoader::IsSupported( + identifier.GetObjectName(), catalog_options_)); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::shared_ptr self = weak_from_this().lock(); + if (self == nullptr) { + return Status::Invalid( + "a global system table retains its catalog, so the catalog must be owned by " + "a std::shared_ptr before it is asked for one"); + } + GlobalSystemTableContext context; + context.catalog = std::move(self); + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = catalog_options_; + return GlobalSystemTableLoader::Load(identifier.GetObjectName(), context); + } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { - // A system table reads the data table's files through this catalog's file system. return LoadSystemTable(identifier); } // The schema is read on the table's own branch: `root/branch/branch-` below the @@ -330,14 +346,10 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); - // The schema's `type` says what the table is. A format table is the same directory read as - // plain data, so the schema read above is all it takes to serve one; no manifest is read. std::shared_ptr data_schema = checked_pointer_cast(*latest_schema); PAIMON_ASSIGN_OR_RAISE(bool is_format_table, TableTypeDefine::IsFormatTable(data_schema->Options())); if (is_format_table) { - // This catalog keeps a table's own metadata under the table path, so a `schema` or - // `branch` directory there is metadata rather than table content. return FormatTable::Create(fs_, table_path, identifier, data_schema, /*location_carries_paimon_metadata=*/true, /*dynamic_options=*/{}); @@ -347,7 +359,6 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide // table rather than at the first read. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); - // The branch the schema above was read on, so the table's managers read the same one. PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); return FileStoreTable::Create(fs_, table_path, checked_pointer_cast(*latest_schema), identifier, /*uuid=*/"", branch, /*schema_manager=*/nullptr, @@ -375,7 +386,6 @@ Result> FileSystemCatalog::LoadSystemTable( if (branch) { dynamic_options[Options::BRANCH] = branch.value(); } - // A system table reads the files of the data table it belongs to, so its root path is that one. PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); return SystemTableLoader::Load(system_table_name.value(), fs_, table_path, latest_schema.value(), dynamic_options); diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index 203e1b92e..0f3123238 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -39,7 +39,7 @@ class Identifier; class Logger; class SystemTable; -class FileSystemCatalog : public Catalog { +class FileSystemCatalog : public Catalog, public std::enable_shared_from_this { public: FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, const std::map& catalog_options); diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 294b7e9b9..cc8fea8d8 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -230,9 +230,6 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_FALSE(loaded_schema->field(1)->nullable()); ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(options_identifier)); - // A system table comes back as the `SystemTable` it is: it answers for itself with its own - // read-only schema and the root path of the data table it reads. Its name carries the data - // table it belongs to. ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); ASSERT_TRUE(std::dynamic_pointer_cast(system_table) == nullptr); ASSERT_EQ("tbl1$options", system_table->Name()); @@ -255,9 +252,6 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { "Cannot 'renameTable' for system table"); } -// A `T$system` identifier opens the read-only view it names, which serves scans and reads of -// the data table it belongs to through the same `Table` interface, and refuses the operations -// that would modify the data table. TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { std::map options; options[Options::FILE_SYSTEM] = "local"; @@ -276,7 +270,6 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - // A managed table with no `type` option still opens, as the managed table it is. ASSERT_OK_AND_ASSIGN(auto managed, catalog.GetTable(Identifier("db1", "tbl1"))); ASSERT_TRUE(std::dynamic_pointer_cast(managed) == nullptr); @@ -284,7 +277,6 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); ASSERT_EQ("tbl1$options", system_table->Name()); - // The scan and the read work from a context of the caller's, like every other table. ScanContextBuilder scan_builder(system_table->GetRootPath()); scan_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); @@ -298,7 +290,6 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(auto table_result, reader->NextBatch()); - // The options the table was created with come back, one row per key. auto& [c_array, c_schema] = table_result; std::shared_ptr batch; auto maybe_batch = arrow::ImportRecordBatch(c_array.get(), c_schema.get()); @@ -306,7 +297,6 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { batch = std::move(maybe_batch).ValueUnsafe(); ASSERT_GE(batch->num_rows(), 2); - // The operations that would modify the data table underneath are refused by name. WriteContextBuilder write_builder(system_table->GetRootPath(), "commit_user"); write_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); @@ -321,6 +311,70 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { "tbl1$options system table is read-only: committing for it is not supported"); } +TEST(FileSystemCatalogTest, TestGetTableServesTheSysDatabase) { + std::map options; + options[Options::FILE_SYSTEM] = "local"; + options[Options::FILE_FORMAT] = "orc"; + options[CatalogOptionsSystemTable::kEnabledOption] = "true"; + ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto catalog = + std::make_shared(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK(catalog->CreateDatabase("db1", options, /*ignore_if_exists=*/false)); + + auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32(), /*nullable=*/false)}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("db1", "tbl1"), &schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + { + std::map disabled = options; + disabled.erase(CatalogOptionsSystemTable::kEnabledOption); + auto disabled_catalog = + std::make_shared(core_options.GetFileSystem(), dir->Str(), disabled); + ASSERT_NOK_WITH_MSG(disabled_catalog->GetTable(Identifier("sys", "catalog_options")), + "not exist"); + } + ASSERT_OK_AND_ASSIGN(auto catalog_options, + catalog->GetTable(Identifier("sys", "catalog_options"))); + ASSERT_TRUE(std::dynamic_pointer_cast(catalog_options) != nullptr); + + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("sys", "nope")), "not exist"); + + ASSERT_OK_AND_ASSIGN(auto tables, catalog->GetTable(Identifier("sys", "tables"))); + ScanContextBuilder scan_builder("sys/tables"); + scan_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan, tables->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1); + + ReadContextBuilder read_builder("sys/tables"); + read_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto read, tables->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto table_result, reader->NextBatch()); + + auto& [c_array, c_schema] = table_result; + std::shared_ptr batch; + auto maybe_batch = arrow::ImportRecordBatch(c_array.get(), c_schema.get()); + ASSERT_TRUE(maybe_batch.ok()) << maybe_batch.status().ToString(); + batch = std::move(maybe_batch).ValueUnsafe(); + ASSERT_GE(batch->num_rows(), 1); + + std::weak_ptr weak = catalog; + catalog.reset(); + ASSERT_FALSE(weak.expired()) << "the global system table does not retain its catalog"; + + FileSystemCatalog stack_catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_NOK_WITH_MSG(stack_catalog.GetTable(Identifier("sys", "tables")), + "must be owned by a std::shared_ptr"); +} + TEST(FileSystemCatalogTest, TestBranchTableLoadsBranchSchema) { std::map options; options[Options::FILE_SYSTEM] = "local"; @@ -729,8 +783,6 @@ TEST(FileSystemCatalogTest, TestGetTableServesAFormatTable) { /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - // The table's `type` says what it is, so the one call that opens a table hands a format - // table back as the `FormatTable` it is, through the `Table` interface the two share. ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db1", "fmt"))); std::shared_ptr format_table = std::dynamic_pointer_cast(table); ASSERT_NE(nullptr, format_table); diff --git a/src/paimon/core/catalog/snapshot_commit.cpp b/src/paimon/core/catalog/snapshot_commit.cpp index 35efe767c..91e70a851 100644 --- a/src/paimon/core/catalog/snapshot_commit.cpp +++ b/src/paimon/core/catalog/snapshot_commit.cpp @@ -38,14 +38,12 @@ Result> SnapshotCommit::Create( if (!catalog_identifier) { return Status::Invalid("a catalog commit requires a table identifier"); } - // Reuse the caller's UUID so table recreation cannot redirect this commit. return std::shared_ptr( std::make_shared(catalog, catalog_identifier.value(), table_id)); } if (use_rest_catalog_commit) { return std::shared_ptr(std::make_shared(table_id)); } - // Catalogs without version management may still supply a table ID. if (table_id && catalog == nullptr) { return Status::Invalid( "a commit which publishes the snapshot by writing it to the table directory sends no " diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 3ee0eb755..e17cfbd4e 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -295,10 +295,6 @@ Result> InternalReadContext::CreateFromOpti const std::shared_ptr& context, const std::shared_ptr& table_schema, const std::map& options, const std::shared_ptr& file_system, std::optional path) { - // The file system the caller named, or null for the read context's own, which callers - // without a table hand over. The scheme map the context carries does not take part: a file - // system handed over wins and one missing falls to the resolving one, the same way a scan - // resolves its own. PAIMON_ASSIGN_OR_RAISE( CoreOptions core_options, CoreOptions::FromMap( @@ -328,21 +324,11 @@ Result> InternalReadContext::CreateForFallb } assert(table_schema); auto options = table_schema->Options(); - // The dynamic options the load merged into this table - and the ones its copies accumulated - - // speak for the read, and the fallback branch serves the same read: they are merged over the - // fallback's own schema options the way the load merged them over the table's, so the fallback - // reader recognises what the main one does. `type` stays the branch's own: neither a copy nor - // a fallback restyles the table. for (const auto& [key, value] : table_dynamic_options) { if (key != Options::TYPE) { options[key] = value; } } - for (const auto& [key, value] : context->GetOptions()) { - if (key != Options::TYPE) { - options[key] = value; - } - } if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { options[Options::BRANCH] = branch; } else { diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index 9e5d91ce0..a710ca15e 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -79,8 +79,11 @@ class InternalReadContext { /// The read context of a fallback branch, whose schema is not the table's own: it is read /// through a manager of that branch, and the read serves the options that schema was loaded /// with, the branch it was read from among them. A branch the context carries is not honoured - /// here. The C++ reading of the Java `FileStoreTableFactory#createOtherBranchTable`, which - /// loads the other branch's table the same way for the fallback reader to serve. + /// here, and neither are options the context adds on its own: the fallback reader serves the + /// table the way the main reader does, and a load merges the context's options into the + /// dynamic options handed in here. The C++ reading of the Java + /// `FileStoreTableFactory#createOtherBranchTable`, which loads the other branch's table the + /// same way for the fallback reader to serve. /// @param file_system The file system the table was opened with, in place of one resolved from /// the context. /// @param root_path The root path the table was opened with, in place of the one the context diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index c799c24f8..9fcb5f370 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -443,7 +443,6 @@ Status SchemaValidation::CheckImmutableOptions( return Status::Invalid(fmt::format("Change '{}' is not supported yet.", key)); } if (key == Options::IGNORE_DELETE) { - // An unset option is one that never ignored deletes, the same default they parse to. PAIMON_ASSIGN_OR_RAISE(bool old_ignore_delete, OptionsUtils::GetValueFromMap(schema_options, key, false)); PAIMON_ASSIGN_OR_RAISE(bool new_ignore_delete, diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index e6151d4c7..e8a3b7b65 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -203,16 +203,12 @@ Result> TableSchema::MergeOptions( auto merged = std::unique_ptr( new TableSchema(version_, id_, fields_, highest_field_id_, partition_keys_, primary_keys_, options, comment_, time_millis_)); - // The derived members follow the merged options: a copy the options restyle must serve the - // bucket keys and the bucket count it carries, not the ones the base schema carried. PAIMON_RETURN_NOT_OK(merged->DeriveBucketInfo()); return merged; } Status TableSchema::DeriveBucketInfo() { PAIMON_ASSIGN_OR_RAISE(std::vector keys, TrimmedPrimaryKeys()); - - // Try to validate bucket keys PAIMON_ASSIGN_OR_RAISE(std::vector bucket_keys, OriginalBucketKeys()); if (bucket_keys.empty()) { bucket_keys = keys; diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp index fb504ee4c..713c9160b 100644 --- a/src/paimon/core/table/append_only_file_store_table.cpp +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -58,7 +58,6 @@ Result> AppendOnlyFileStoreTable::CreateWrite( const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { - // append table const std::shared_ptr& schema = store_->GetSchema(); const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); @@ -177,8 +176,6 @@ CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& manifest_file, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const { - // The schema manager the table carries, which the scans this supplier hands out read the - // schema through. const std::shared_ptr& schema_manager = GetSchemaManager(); const std::shared_ptr& table_schema = store_->GetSchema(); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index fe8b092c7..649a7f980 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -49,10 +49,6 @@ namespace paimon { namespace { -/// Whether an option key takes part in file-system resolution: the backend `file-system` picks, -/// an `fs.` access key carries, or a native S3 key names - `s3.` and its `s3a.` spelling, the -/// prefixes the S3 file system consumes alongside `fs.s3.`/`fs.s3a.`. The scheme-map keys a -/// context carries are the same set. bool IsFileSystemResolutionKey(const std::string& key) { return key == Options::FILE_SYSTEM || key.rfind("fs.", 0) == 0 || key.rfind("s3.", 0) == 0 || key.rfind("s3a.", 0) == 0; @@ -69,10 +65,6 @@ Result> FileStore::Create( auto store = std::shared_ptr(new FileStore( file_system, root_path, schema, branch, schema_manager, snapshot_manager, fs_scheme_map)); if (schema != nullptr) { - // The options the schema was loaded with, on the branch this store serves: what reads - // through them - whose cache keys are scoped by the branch - reads the same one the - // managers point at. The scheme map is the one the schema was read through, so a scheme - // it mapped resolves here the way it resolved there. std::map options = schema->Options(); if (BranchManager::IsMainBranch(store->branch_)) { options.erase(Options::BRANCH); @@ -83,10 +75,6 @@ Result> FileStore::Create( CoreOptions::FromMap(options, file_system, fs_scheme_map)); store->core_options_ = std::move(core_options); } - // The file system the table is served through: the one the options resolve - a schema - // carrying access configuration names its own, which the file system the caller opened the - // table through wins over - and without a schema, the one handed over, whose absence leaves - // the default local one. if (store->core_options_.has_value()) { store->file_system_ = store->core_options_->GetFileSystem(); } else if (file_system != nullptr) { @@ -95,10 +83,6 @@ Result> FileStore::Create( PAIMON_ASSIGN_OR_RAISE(CoreOptions default_options, CoreOptions::FromMap({})); store->file_system_ = default_options.GetFileSystem(); } - // The managers read through the same file system: the metadata belongs to the data, and - // both were written under the options the schema carries. A manager handed over already - // points at `root_path` on `branch`; a missing one is built here, so the operators always - // read through the managers the store carries. if (store->schema_manager_ == nullptr) { store->schema_manager_ = std::make_shared(store->file_system_, root_path, store->branch_); @@ -161,14 +145,15 @@ FileStore::FileStore(const std::shared_ptr& file_system, const std:: snapshot_manager_(snapshot_manager) {} Result> FileStore::NewPathFactory( - const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, const std::string& file_format_identifier, const std::shared_ptr& memory_pool) const { PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, options.CreateGlobalIndexExternalPath()); return FileStorePathFactory::Create( - root_path_, arrow_schema, schema_->PartitionKeys(), options.GetPartitionDefaultName(), + root_path_, arrow_schema, partition_keys, options.GetPartitionDefaultName(), file_format_identifier, options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, options.IndexFileInDataFileDir(), memory_pool); } diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index 339dd3810..1dbdee7ef 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -23,6 +23,7 @@ #include #include #include +#include #include "paimon/catalog/identifier.h" #include "paimon/core/core_options.h" @@ -141,13 +142,15 @@ class FileStore { } /// Builds the path factory an operator names its metadata paths through, over this store's - /// root path and the partition keys of its schema. No schema change can alter the partition - /// keys, so an operator serving a schema read after the table was loaded still names the same - /// partitions. The caller names the file format identifier: a write names the format of its + /// root path. The caller names the partition keys of the schema the arrow schema was converted + /// from: an operator serving the table's own schema names this store's partition keys, while + /// a read of a fallback branch names the keys of that branch's schema, which is not this + /// store's. The caller names the file format identifier: a write names the format of its /// level-0 files, which the options may set per level, while a scan, a read and a commit name /// the table's format. Result> NewPathFactory( - const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, const std::string& file_format_identifier, const std::shared_ptr& memory_pool) const; diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index da86a590e..707a1d67d 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -33,6 +33,7 @@ #include "paimon/common/types/data_field.h" #include "paimon/common/utils/binary_row_partition_computer.h" #include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/path_util.h" #include "paimon/common/utils/sensitive_config_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_utils.h" @@ -107,10 +108,6 @@ Result> FileStoreTable::Create( const std::shared_ptr& snapshot_manager, const std::map& fs_scheme_map, const std::map& dynamic_options) { - // The kind the schema's primary keys name, which no schema change can add or drop, so the kind - // the table is built as is the kind every schema its root path ever held describes. A caller - // without a schema gets the append-only kind, whose operators report that none was found; a - // schema that is not a table schema - a system table's read-only one - is refused, not cast. std::shared_ptr table_schema; if (schema != nullptr) { table_schema = std::dynamic_pointer_cast(schema); @@ -119,7 +116,6 @@ Result> FileStoreTable::Create( fmt::format("{} is not a managed table schema", identifier.ToString())); } } - // The store parses the schema's options, so one that does not parse is refused here. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr store, FileStore::Create(file_system, root_path, table_schema, branch, schema_manager, snapshot_manager, fs_scheme_map)); @@ -139,29 +135,14 @@ Result> FileStoreTable::Copy( if (loaded_schema == nullptr) { return Status::Invalid(fmt::format("not found latest schema of {}", FullName())); } - // `type` is structural and read from the schema alone, the promise `Copy` makes: an option - // given at the call must not restyle the table. `branch` is the same promise - the copy serves - // the branch this table does, and a caller that means to move has `SwitchToBranch`. Both are - // dropped before the merge, so the copy's options never speak of a branch it does not serve. std::map overridable = dynamic_options; overridable.erase(Options::TYPE); overridable.erase(Options::BRANCH); - // The physical layout the data on the path was written by is as immovable as the style or the - // branch: the copy serves the same data. PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*loaded_schema, overridable)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); - // The managers are handed over, not rebuilt, when the copy resolves the same file system this - // table does: its operators then read the schema cache this table already warmed. The copy - // resolves from the file system the caller opened this table through, not the one it resolved - // to, so a file-system option it was given speaks; when one does change the resolution, fresh - // managers are built over it instead. The scheme map travels the same way. const bool fs_resolution_may_change = FileStore::FileSystemResolutionMayChange(*loaded_schema, overridable); - // The dynamic options the copy was given join the ones this table was loaded with - which a - // copy of a copy carries the same way - so a read's fallback reader serves what the copy's own - // reader does however many copies deep. The branch was dropped above, so none accumulates, and - // an option given again replaces the one carried, as the merge above does. std::map accumulated = dynamic_options_; for (const auto& [key, value] : overridable) { accumulated.insert_or_assign(key, value); @@ -177,31 +158,23 @@ Result> FileStoreTable::Copy( Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { const std::string target_branch = BranchManager::NormalizeBranch(branch); if (target_branch == CurrentBranch()) { - // The table already serves this branch, so a copy of it is the same table to the caller. return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, schema_, identifier_, CatalogUuid().value_or(""), CurrentBranch(), GetSchemaManager(), GetSnapshotManager(), store_->GetFsSchemeMap(), dynamic_options_); } - // The schema of the branch switched to, read from its own directory. auto schema_manager = std::make_shared(file_system_, root_path_, target_branch); PAIMON_ASSIGN_OR_RAISE(std::optional> latest, schema_manager->Latest()); if (!latest) { return Status::Invalid(fmt::format("branch {} does not exist", target_branch)); } - // A branch other than main is named in the options the schema carries - the way a read context - // names it - so what reads through them serves the branch the managers point at. std::map overridable; if (target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { overridable[Options::BRANCH] = target_branch; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr branch_schema, (*latest)->MergeOptions(overridable)); - // The branch's schema may name another file system than this table's options resolved - a - // recreation that changed the access configuration - and a manager kept through that - // disagreement would read the branch's metadata through the backend this table left, so the - // branch builds its own. The schema is in hand already, so only the cache is lost. const std::shared_ptr branch_schema_manager = (store_->GetSpecifiedFileSystem() == nullptr && (!store_->GetFsSchemeMap().empty() || @@ -209,8 +182,6 @@ Result> FileStoreTable::SwitchToBranch(const std::string& branch_schema->Options(), std::static_pointer_cast(schema_)->Options()))) ? nullptr : schema_manager; - // The identifier names the branch switched to, as Java's `newCatalogEnvironment` rebuilds it, - // so `Name()` and `FullName()` answer for the table served rather than the one left behind. PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, CatalogUtils::BranchIdentifier(identifier_, target_branch)); return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, @@ -244,7 +215,6 @@ Status FileStoreTable::CheckOptionsAgreeWithTable( for (const auto& [key, value] : context_options) { auto table_option = table_options.find(key); if (table_option != table_options.end() && table_option->second != value) { - // A value may be a credential and the error travels into logs, so it is redacted. return Status::Invalid( fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, SensitiveConfigUtils::RedactValue(key, value), @@ -254,6 +224,59 @@ Status FileStoreTable::CheckOptionsAgreeWithTable( return Status::OK(); } +Status FileStoreTable::CheckContextNamesThisTable( + const std::string& kind, const std::string& context_root_path, + const std::optional& context_identifier, + const std::optional& context_table_id) const { + if (!root_path_.empty()) { + PAIMON_ASSIGN_OR_RAISE(std::string context_path, + PathUtil::NormalizePath(context_root_path)); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, PathUtil::NormalizePath(root_path_)); + if (context_path != table_path) { + return Status::Invalid(fmt::format( + "the {} context names root path '{}' but this table serves '{}': the store below " + "reads and writes this table's files, so a context naming another path would " + "leave this table's metadata there and publish it as the other's", + kind, context_root_path, root_path_)); + } + } + if (context_identifier) { + PAIMON_ASSIGN_OR_RAISE(std::optional context_branch, + context_identifier.value().GetBranchName()); + if (context_branch && context_branch.value() != CurrentBranch()) { + return Status::Invalid(fmt::format( + "the {} context names branch '{}' of '{}' but this table serves branch '{}', " + "which is the one a {} of this table writes and publishes", + kind, context_branch.value(), context_identifier.value().ToString(), + CurrentBranch(), kind)); + } + PAIMON_ASSIGN_OR_RAISE( + Identifier context_table, + CatalogUtils::BranchIdentifier(context_identifier.value(), CurrentBranch())); + PAIMON_ASSIGN_OR_RAISE(Identifier served_table, + CatalogUtils::BranchIdentifier(identifier_, CurrentBranch())); + const bool names_this_table = + (served_table.GetDatabaseName() == Identifier::kUnknownDatabase || + context_table.GetDatabaseName() == served_table.GetDatabaseName()) && + context_table.GetObjectName() == served_table.GetObjectName(); + if (!names_this_table) { + return Status::Invalid(fmt::format( + "the {} context names table '{}' but this table is '{}': the store below serves " + "this table's files, so a context naming another table would write this table's " + "metadata and publish it as the other's", + kind, context_identifier.value().ToString(), identifier_.ToString())); + } + } + if (context_table_id && context_table_id != CatalogUuid()) { + return Status::Invalid(fmt::format( + "the {} context carries table id '{}' but this table's is '{}': the id tells a " + "recreated table from the one the changes were prepared for, so one this table does " + "not carry would publish this table's metadata as the recreation's", + kind, context_table_id.value(), CatalogUuid().value_or("none"))); + } + return Status::OK(); +} + std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr&& scan, const std::string& root_path) { scan->WithTablePath(root_path); @@ -263,14 +286,18 @@ std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr> FileStoreTable::NewWrite( std::unique_ptr ctx) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("write", ctx->GetRootPath(), + ctx->GetIdentifier(), + /*context_table_id=*/std::nullopt)); + if (AsVersionManaged(ctx->GetCatalog()) != nullptr) { + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); + } PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - // A catalog write names its table; the file system comes from the table's own credentials. if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { return Status::Invalid("a catalog write requires a table identifier"); } - // The write goes to the branch the table serves, not the context's own. PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); const std::shared_ptr& schema = store_->GetSchema(); @@ -292,7 +319,7 @@ Result> FileStoreTable::NewWrite( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr file_store_path_factory, - store_->NewPathFactory(arrow_schema, options, + store_->NewPathFactory(arrow_schema, schema->PartitionKeys(), options, options.GetWriteFileFormat(/*level=*/0)->Identifier(), ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( @@ -328,16 +355,13 @@ Result> FileStoreTable::NewScan( PAIMON_ASSIGN_OR_RAISE(bool read_optimized, OptionsUtils::GetValueFromMap(context->GetOptions(), kReadOptimizedScanOption, false)); - // The branch in the store's options is the table's, so one the context carries is not honoured. CoreOptions core_options = store_->GetOptions(); - // The cache is a runtime setting of this scan, not of the table. core_options.WithCache(context->GetCache()); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); PAIMON_RETURN_NOT_OK( RealtimeTableScan::ValidateScan(*table_schema, core_options, *context, read_optimized)); - // validate options if (core_options.GetBucket() == -1) { if (!table_schema->PrimaryKeys().empty()) { return Status::NotImplemented(fmt::format( @@ -349,17 +373,17 @@ Result> FileStoreTable::NewScan( fmt::format("do not support bucket={} in scan process", core_options.GetBucket())); } - // validate schema and scan filter auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( *arrow_schema, context->GetScanFilters()->GetPredicate(), /*validate_field_idx=*/false)); } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory, - store_->NewPathFactory(arrow_schema, core_options, - core_options.GetFileFormat()->Identifier(), - context->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + store_->NewPathFactory(arrow_schema, table_schema->PartitionKeys(), core_options, + core_options.GetFileFormat()->Identifier(), + context->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, @@ -438,11 +462,8 @@ Result> FileStoreTable::CreateTableRead( auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, - store_->NewPathFactory(arrow_schema, core_options, + store_->NewPathFactory(arrow_schema, table_schema->PartitionKeys(), core_options, core_options.GetFileFormat()->Identifier(), memory_pool)); - // The reader is selected by the schema the read serves, not by the table's kind: an - // append-only table can have a primary-key fallback branch and vice versa, so a reader typed - // after the main table would fail the fallback's splits. if (internal_context->GetPrimaryKeys().empty()) { return std::make_unique(path_factory, internal_context, memory_pool, executor); @@ -459,7 +480,6 @@ Result> FileStoreTable::NewRead( return Status::Invalid("not found latest schema"); } PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); - // The read reads through the branch the table serves, not the context's own. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, InternalReadContext::CreateForTable(read_context, loaded_schema, store_->GetOptions(), root_path_)); @@ -475,7 +495,6 @@ Result> FileStoreTable::NewRead( return std::move(table_read); } - // The fallback branch is not the table's, so its schema is read through its own manager. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fallback_context, InternalReadContext::CreateForFallbackBranch( read_context, /*branch=*/scan_fallback_branch.value(), file_system_, @@ -492,17 +511,18 @@ Result> FileStoreTable::NewRead( Result> FileStoreTable::NewCommit( std::unique_ptr ctx) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("commit", ctx->GetRootPath(), + ctx->GetIdentifier(), ctx->GetTableId())); + if (ctx->GetCatalog() != nullptr || ctx->UseRESTCatalogCommit()) { + PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch())); + } PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - // A catalog commit names its table; the file system comes from the table's own credentials. if (specific_fs == nullptr && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { return Status::Invalid("a catalog commit requires a table identifier"); } - // A commit commits to the root path and branch the table was loaded with, not the context's. PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); - // The id a snapshot records the catalog's current schema by, read through the loader below - // rather than trusted from a schema this commit will not re-read. FileStoreCommitImpl::SchemaIdLoader schema_id_loader; if (ctx->GetCatalog() != nullptr) { if (!catalog_identifier) { @@ -536,8 +556,6 @@ Result> FileStoreTable::NewCommit( assert(options.GetFileFormat()); PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path_)); - // Catalog commits do not require atomic rename on the object store. The inte-test marker is - // not a table option, so it is read from the context the commit was handed. if (is_object_store && !ctx->UseRESTCatalogCommit() && AsVersionManaged(ctx->GetCatalog()) == nullptr && ctx->GetOptions().find("enable-object-store-commit-in-inte-test") == @@ -550,8 +568,8 @@ Result> FileStoreTable::NewCommit( store_->NewPartitionComputer(arrow_schema, options, ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, - store_->NewPathFactory(arrow_schema, options, options.GetFileFormat()->Identifier(), - ctx->GetMemoryPool())); + store_->NewPathFactory(arrow_schema, schema->PartitionKeys(), options, + options.GetFileFormat()->Identifier(), ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr snapshot_manager, diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 04b088ab8..66e40ad59 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -134,6 +134,21 @@ class FileStoreTable : public Table { static Status CheckOptionsAgreeWithTable( const TableSchema& table_schema, const std::map& context_options); + /// Checks the identity a context carries - the path it names, the identifier it gives a + /// catalog, the table id it records - against this table's own. The store below serves this + /// table's files, so a context naming another table would write this table's metadata and + /// publish it as the other's; a field that contradicts the table is refused by name, the way + /// `CheckOptionsAgreeWithTable` refuses a contradicting option. A field the context does not + /// carry is no contradiction: a write or a commit from a bare path names no catalog object. + /// An identifier that names a branch this table does not serve names a branch of another + /// table, as the operator serves the branch the table was loaded with alone. + /// @param kind The operator the context was handed to, which the error names. + /// @param context_table_id The id `CommitContext` records for a catalog commit; a write + /// context carries none, so `NewWrite` passes null. + Status CheckContextNamesThisTable(const std::string& kind, const std::string& context_root_path, + const std::optional& context_identifier, + const std::optional& context_table_id) const; + /// Builds the read `NewRead` hands back, from the internal context it read the schema into. The /// path factory comes from the store. The reader is picked by the schema the read serves - the /// primary keys it carries - not by the table's own kind. diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp index 5eca88fcf..c0c5e7152 100644 --- a/src/paimon/core/table/file_store_table_factory.cpp +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -51,10 +51,6 @@ namespace paimon { namespace { -/// The options a context carries, merged into the table the load hands back: a table is the -/// product of the load, so the options speak there - the way `Copy` merges them - and `NewXxx` on -/// the product serves them without merging again. A caller holding a table already changes it -/// through `Copy` itself, and an option that contradicts the table that way is refused. Result> WithContextOptions( const std::shared_ptr
& table, const std::shared_ptr& schema, const std::map& options) { @@ -64,7 +60,6 @@ Result> WithContextOptions( const std::map& schema_options = schema->Options(); for (const auto& [key, value] : options) { auto schema_option = schema_options.find(key); - // A key the schema does not carry, or carries differently: the merge has something to do. if (schema_option == schema_options.end() || schema_option->second != value) { return table->Copy(options); } @@ -72,17 +67,6 @@ Result> WithContextOptions( return table; } -/// The tail every `Create` ends with: the schema one of them read says what the path names, and -/// only a data table is served by it - a format table is refused, the table a catalog hands back -/// serves one through its own `NewXxx`, and everything else is handed back as the `FileStoreTable` -/// a managed table is, carrying the schema for its `NewXxx` to build from. A null schema is handed -/// over as it is, so the managed path below decides what to say about a table that is not there. -/// -/// @param file_system The file system the schema above was read through. -/// @param explicit_file_system The file system the context supplied, which a managed table is -/// opened through when there is one. A null one leaves the table to resolve from its -/// schema's own access configuration rather than being pinned to the file system this load -/// happened to read the schema through. Result> TableOfLoadedSchema( const std::shared_ptr& file_system, const std::shared_ptr& explicit_file_system, const std::string& path, @@ -91,9 +75,6 @@ Result> TableOfLoadedSchema( const std::shared_ptr& latest_schema, const std::optional& identifier, const std::string& uuid, const std::map& fs_scheme_map) { if (latest_schema != nullptr) { - // The table type is read from the schema, never from `options`: `type` is structural, and - // one call's options must not decide what kind of table this is. See - // `FormatTable::Create()`. PAIMON_ASSIGN_OR_RAISE(bool is_format_table, TableTypeDefine::IsFormatTable(latest_schema->Options())); if (is_format_table) { @@ -104,15 +85,8 @@ Result> TableOfLoadedSchema( path)); } } - // The branch the schema was read on is folded into the object name. Identifier table_identifier = identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)); - // The manager was built over the file system the context's own options resolve, while the - // store resolves the schema's: a schema carrying access configuration names a backend the - // context's options never did, and a manager kept through that disagreement would read the - // table's metadata through the wrong one. The schema is in hand already, so dropping the - // manager costs its cache alone. A context that supplied a file system wins on both sides and - // keeps it; a scheme map only the store's resolution takes part in is the same disagreement. std::shared_ptr manager = schema_manager; if (manager != nullptr && latest_schema != nullptr && explicit_file_system == nullptr && (!fs_scheme_map.empty() || @@ -127,15 +101,6 @@ Result> TableOfLoadedSchema( return WithContextOptions(table, latest_schema, options); } -/// The managed table a context names behind a path: the schema is read through a manager of the -/// context's branch - handed over instead when the caller already read it - and what the schema -/// says is there decides the table `TableOfLoadedSchema` hands back. -/// -/// @param schema_json The schema of the table, when the caller already read it, or null when it -/// has to be read here. It is honoured on the main branch only; a branch publishes its own -/// under `branch/branch-`, which is where the read below goes. -/// @param explicit_file_system The file system the context supplied, which the managed table the -/// schema describes is opened through - see `TableOfLoadedSchema`. Result> ManagedTableOf( const std::shared_ptr& file_system, const std::shared_ptr& explicit_file_system, const std::string& path, @@ -145,8 +110,6 @@ Result> ManagedTableOf( auto schema_manager = std::make_shared(file_system, path, branch); std::shared_ptr latest_schema; if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. PAIMON_ASSIGN_OR_RAISE(latest_schema, TableSchema::CreateFromJson(schema_json.value())); } else { PAIMON_ASSIGN_OR_RAISE(std::optional> latest, @@ -159,10 +122,6 @@ Result> ManagedTableOf( schema_manager, latest_schema, identifier, uuid, fs_scheme_map); } -/// The schema a catalog may hold for the table, read from it on the main branch only and null -/// otherwise: a branch publishes its own under `branch/branch-`, which is where -/// `ManagedTableOf` reads instead, so a catalog answering for no branch schema does not fail the -/// `kind` operation to one. `kind` also names the operator the messages speak of. Result> CatalogMainBranchSchemaJson( const std::shared_ptr& catalog, const std::optional& catalog_identifier, const std::string& branch, const char* kind) { @@ -183,9 +142,6 @@ Result> CatalogMainBranchSchemaJson( } // namespace -/// The data table a scan context names: only a data table is served by a load from the context's -/// path - a system table behind a `T$system` path and a format table are refused, served by the -/// table a catalog hands back through its own `NewXxx` instead. Result> FileStoreTableFactory::Create(const ScanContext& ctx) { PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), {})); @@ -205,9 +161,6 @@ Result> FileStoreTableFactory::Create(const ScanContext& /*uuid=*/"", /*fs_scheme_map=*/{}); } -/// The data table a read context names: only a data table is served by a load from the context's -/// path - a system table behind a `T$system` path and a format table are refused, served by the -/// table a catalog hands back through its own `NewXxx` instead. Result> FileStoreTableFactory::Create(const ReadContext& ctx) { PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), @@ -223,17 +176,12 @@ Result> FileStoreTableFactory::Create(const ReadContext& } const std::string branch = ctx.GetBranch(); - // The scheme map the schema was read through, so the table's data resolves custom schemes the - // same way its metadata does. return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), ctx.GetIdentifier(), /*uuid=*/"", ctx.GetFileSystemSchemeToIdentifierMap()); } -/// The data table a write context names: only a data table is served by a load from the context's -/// root path - a format table is refused, served by the table a catalog hands back through its own -/// `NewXxx` instead. Result> FileStoreTableFactory::Create(const WriteContext& ctx) { PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); @@ -242,8 +190,6 @@ Result> FileStoreTableFactory::Create(const WriteContext& if (!ctx.GetIdentifier()) { return Status::Invalid("a catalog write requires a table identifier"); } - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the write uses this table's. PAIMON_ASSIGN_OR_RAISE(specific_fs, ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); } @@ -253,8 +199,6 @@ Result> FileStoreTableFactory::Create(const WriteContext& std::string branch = ctx.GetBranch(); PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), branch)); - // Only the main branch may have a current schema the catalog alone holds: a branch publishes - // its own under `branch/branch-`, which is where the load below reads it instead. PAIMON_ASSIGN_OR_RAISE( std::optional catalog_table_schema, CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "write")); @@ -263,13 +207,8 @@ Result> FileStoreTableFactory::Create(const WriteContext& ctx.GetFileSystemSchemeToIdentifierMap()); } -/// The data table a commit context names: only a data table is served by a load from the context's -/// root path - a format table is refused, served by the table a catalog hands back through its own -/// `NewXxx` instead. Result> FileStoreTableFactory::Create(const CommitContext& ctx) { PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); - // Use this table's catalog credentials unless the caller supplied a file system: a catalog - // issuing per-table temporary ones only hands them out through GetTableFileSystem. std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); if (specific_fs == nullptr && ctx.GetCatalog() != nullptr) { if (!ctx.GetIdentifier()) { @@ -281,19 +220,14 @@ Result> FileStoreTableFactory::Create(const CommitContext PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, CoreOptions::FromMap(ctx.GetOptions(), specific_fs)); const std::string& root_path = ctx.GetRootPath(); - // The branch the context was built for, already validated by `CommitContextBuilder`. PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( ctx.GetIdentifier(), ctx.GetOptions(), /*explicit_branch=*/std::nullopt, "commit")); PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), branch)); - // Only the main branch may have a current schema the catalog alone holds: a branch publishes - // its own under `branch/branch-`, which is where the load below reads it instead. PAIMON_ASSIGN_OR_RAISE( std::optional catalog_table_schema, CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "commit")); - // The managed path below reads the commit's branch, so this reads the same one: the two must - // not dispatch on different schemas. return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, root_path, branch, ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), ctx.GetTableId().value_or(""), /*fs_scheme_map=*/{}); diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index acb5f03e1..846842cb7 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -224,17 +224,12 @@ Result> FormatTable::GetArrowSchema() const { return schema_->GetArrowSchema(); } -// The refusals below use two codes. `Invalid` is a setting that clashes with what a format table -// is, and will never work: no snapshots, no buckets, no real-time store, no manifests, no -// catalog-side version. `NotImplemented` is a capability this code lacks today and could gain. Result> FormatTable::NewScan( std::unique_ptr context) const { std::shared_ptr shared_context = std::move(context); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(shared_context, "scan")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, CopySelf(shared_context->GetOptions())); - // Anything a `ScanContext` carries that a format table cannot honour is refused rather than - // silently dropped. if (shared_context->IsStreamingMode()) { return Status::Invalid( "a format table has no snapshots, so there is nothing for a streaming scan to follow"); @@ -313,8 +308,6 @@ Result> FormatTable::NewWrite( "a format table write takes the table's own columns; a write schema naming a " "subset of them is not supported yet"); } - // A write id prefixes a postpone-bucket writer's files so that one compaction reader can - // put them back in order. A format table has no buckets, so it would identify nothing. if (context->GetWriteId().has_value()) { return Status::Invalid("a format table has no buckets, so a write id would name nothing"); } diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index af0686c6f..c7456d005 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -764,10 +764,6 @@ TEST(FormatTableTest, TestCatalogManagedPartitionsAreRejected) { } TEST(FormatTableTest, TestTheTableOperatorsValidateOptionsGivenAtTheCall) { - // An option a format table refuses has to be refused wherever it comes from. Every operator - // merges what the call gave it over what the schema stored, so each runs the same checks over - // the merged result; otherwise a caller could set through a context what the schema would have - // rejected and have it silently dropped. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -2666,10 +2662,6 @@ TEST(FormatTableTest, TestScanRejectsUnknownPartitionField) { } TEST(FormatTableTest, TestAFormatTableIsWrittenCommittedAndReadBack) { - // A caller holding the table a catalog hands back drives it through the operators the table - // itself serves, with contexts built from its location the way they are for every other - // table - the C++ side of what Java serves through one `ReadBuilder` and one - // `BatchWriteBuilder` for both kinds of table. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -2764,10 +2756,6 @@ TEST(FormatTableTest, TestAFormatTableIsWrittenCommittedAndReadBack) { } TEST(FormatTableTest, TestACatalogInTheContextIsRefusedAlikeByWriteAndCommit) { - // `WithCatalog()` asks for a catalog-side version of the commit, which a format table has no - // way to produce. Both halves of the write path answer with the same code, so a caller reads - // one kind of failure out of them rather than "cannot be done" from one and "not yet" from - // the other. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -2793,11 +2781,6 @@ TEST(FormatTableTest, TestACatalogInTheContextIsRefusedAlikeByWriteAndCommit) { } TEST(FormatTableTest, TestALoadedFormatTableIsServedThroughItsOwnOperators) { - // A table a catalog keeps the schema for, the way a rest catalog does. Nothing under its - // location says that it is a format table, nor that everything below the location is data, so - // a path is not enough to reach it: the loaded table serves the operators itself, with a - // context built from its location, which is what `FormatTable.newReadBuilder()` and - // `newBatchWriteBuilder()` do in Java. std::unique_ptr schema_dir = UniqueTestDirectory::Create(); ASSERT_TRUE(schema_dir); SchemaManager schema_manager(schema_dir->GetFileSystem(), schema_dir->Str()); @@ -2873,11 +2856,6 @@ TEST(FormatTableTest, TestALoadedFormatTableIsServedThroughItsOwnOperators) { } TEST(FormatTableTest, TestATableServesAFormatTableThroughItsOwnEntryPoints) { - // `NewScan()` and its kin take the table they were called on as the one to serve, the way Java - // Paimon's `Table` serves both kinds through one `newReadBuilder()`, so a caller holding a - // format table as the `Table` a catalog hands out reaches it through them as well. The - // contexts name a path only and carry no format table, so it is the table itself that - // answers. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); std::shared_ptr
table; @@ -2918,8 +2896,6 @@ TEST(FormatTableTest, TestATableServesAFormatTableThroughItsOwnEntryPoints) { } ASSERT_OK_AND_ASSIGN(std::shared_ptr record_batch, ImportBatch(read_batch)); - // `_VALUE_KIND` first, then the table's own columns, exactly as the generic entry points - // give them. auto ids = checked_pointer_cast(record_batch->column(1)); auto names = checked_pointer_cast(record_batch->column(2)); auto dts = checked_pointer_cast(record_batch->column(3)); @@ -2933,18 +2909,12 @@ TEST(FormatTableTest, TestATableServesAFormatTableThroughItsOwnEntryPoints) { } TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) { - // A refusal is a promise about what was not done, so the table's own entry points must make - // the same promises the generic ones do: a generic entry point refuses the format table the - // schema under the path names outright, while the table's operators name the setting they - // cannot honour, so a caller who reaches the table finds out exactly what is missing. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr format_table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); std::shared_ptr
table = format_table; - // A context that never was is refused first of all, and by name, exactly as the generic - // entry points refuse one. ASSERT_NOK_WITH_MSG(TableScan::Create(nullptr), "scan context is null pointer"); ASSERT_NOK_WITH_MSG(table->NewScan(nullptr), "scan context is null pointer"); ASSERT_NOK_WITH_MSG(TableRead::Create(nullptr), "read context is null pointer"); @@ -2954,7 +2924,6 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(nullptr), "commit context is null pointer"); ASSERT_NOK_WITH_MSG(table->NewCommit(nullptr), "commit context is null pointer"); - // A streaming scan follows snapshots, which a format table keeps none of. auto streaming_scan_context = [&dir]() -> Result> { ScanContextBuilder builder(dir->Str()); builder.WithStreamingMode(true); @@ -2967,7 +2936,6 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) ASSERT_NOK_WITH_MSG(table->NewScan(std::move(streaming_context_again)), "nothing for a streaming scan to follow"); - // Vector search needs an index a format table never builds. auto vector_read_context = [&dir]() -> Result> { ReadContextBuilder builder(dir->Str()); builder.SetVectorSearch(std::make_shared( @@ -2982,7 +2950,6 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) ASSERT_NOK_WITH_MSG(table->NewRead(std::move(vector_context_again)), "not supported for format tables"); - // A catalog takes the snapshot a native table commits, and a format table has none to give. auto catalog = std::make_shared(); catalog->SetTableSchema(format_table->LatestSchema()); @@ -3008,7 +2975,6 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(catalog_commit_again)), "keeps no snapshot for a catalog to take"); - // A write id prefixes a postpone-bucket writer's files; a format table has no buckets. auto write_id_context = [&dir]() -> Result> { WriteContextBuilder builder(dir->Str(), "test-user"); builder.WithWriteId(3); @@ -3022,9 +2988,6 @@ TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) } TEST(FormatTableTest, TestTheGenericEntryPointsRefuseAFormatTablePath) { - // A load from a path is the data table's way in, so every generic entry point refuses the - // format table the schema under the path names rather than dispatching to it - and says where - // the table does come from, so a caller holding only the path is pointed at the catalog. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -3046,8 +3009,6 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseAFormatTablePath) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)), "names a format table"); - // The refusal points at the table's own operators, and those serve the same path-built - // contexts: this is the caller the message sends to `GetTable` and `NewXxx`. ScanContextBuilder served_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr served_context, served_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr served_scan, @@ -3057,10 +3018,6 @@ TEST(FormatTableTest, TestTheGenericEntryPointsRefuseAFormatTablePath) { } TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedByTheTableOperators) { - // The `branch` option names a branch a format table keeps no metadata for, and a read or a - // write would go to its one location whatever the option named. Refusing it is the only - // answer that is not silently the wrong one - an overwrite would delete the main branch's - // data while claiming to replace another branch's. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -3117,8 +3074,6 @@ TEST(FormatTableTest, TestATableCopiesItselfWithTheOptionsGivenAtTheCall) { ASSERT_OK_AND_ASSIGN(std::shared_ptr table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); - // The copy overlays what the call gives on the options the schema stored, the precedence - // `Copy()` promises: a probe option lands, `type` - structural - does not. ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::TYPE, "table"}, {"probe", "on-top"}})); std::shared_ptr copied_table = std::static_pointer_cast(copied); @@ -3128,17 +3083,10 @@ TEST(FormatTableTest, TestATableCopiesItselfWithTheOptionsGivenAtTheCall) { EXPECT_EQ(copied_table->Location(), table->Location()); EXPECT_EQ(copied_table->GetFileSystem(), table->GetFileSystem()); - // The refusal `Copy(table, ...)` reports is the one the table itself reports: the copy goes - // through the same entry point, not a shortcut around its checks. ASSERT_NOK_WITH_MSG(table->Copy({{Options::BRANCH, "dev"}}), "has no branches"); } TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFromTheTableRoot) { - // A context that names a path takes its branch from `WithBranch()`, and the schema of that - // branch decides what the entry point refuses. A format table's data is the files under the - // table path whichever branch named it, so the format table the branch's schema names is - // refused rather than served from there under another branch's name - the branch's table - // comes from the catalog, and its rows from that table's own operators. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -3166,8 +3114,6 @@ TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFr ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "names a format table"); - // The main branch is untouched by any of it: the table the catalog hands back for it serves - // the scan from its own operators. ScanContextBuilder main_builder(dir->Str()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_context, main_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan, @@ -3178,9 +3124,6 @@ TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFr } TEST(FormatTableTest, TestTheTableOperatorsCarryTheContextThrough) { - // A context promises that options given at the call win over the ones the schema stored, and - // that a branch is honoured. Serving the operators off the table itself must not quietly drop - // any of that. std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, @@ -3202,8 +3145,6 @@ TEST(FormatTableTest, TestTheTableOperatorsCarryTheContextThrough) { write->PrepareCommit()); ASSERT_EQ(messages.size(), 3u) << "target-file-row-num given at the call was ignored"; - // Published through the table's own commit, or nothing below would see the rows: until the - // commit renames them the files sit in the hidden `_temporary` directory a scan skips. CommitContextBuilder generic_commit_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr generic_commit_context, generic_commit_builder.Finish()); @@ -3212,8 +3153,6 @@ TEST(FormatTableTest, TestTheTableOperatorsCarryTheContextThrough) { ASSERT_OK(generic_commit->Commit(messages)); ASSERT_OK(write->Close()); - // A branch this table has no schema on names no table at all as far as the generic entry - // points are concerned, so the format branch must not answer for it. ReadContextBuilder branch_builder(dir->Str()); branch_builder.WithBranch("nosuchbranch"); ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_context, branch_builder.Finish()); diff --git a/src/paimon/core/table/primary_key_file_store_table.cpp b/src/paimon/core/table/primary_key_file_store_table.cpp index 9a5875c9f..90ed47a30 100644 --- a/src/paimon/core/table/primary_key_file_store_table.cpp +++ b/src/paimon/core/table/primary_key_file_store_table.cpp @@ -68,9 +68,6 @@ class MergeFunctionWrapper; namespace { -/// Whether a schema older than `current_schema_id` carried a primary key b-tree index, which a -/// write has to keep maintaining: the index files an earlier schema wrote are still there to be -/// read, and dropping the definition now would leave them orphaned. Result HasHistoricalPrimaryKeyBTreeDefinition( const std::shared_ptr& schema_manager, int64_t current_schema_id) { if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { @@ -106,7 +103,6 @@ Result> PrimaryKeyFileStoreTable::CreateWrite( const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { - // pk table const std::shared_ptr& schema = store_->GetSchema(); if (ctx.GetRealtimeContext()) { PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); @@ -142,7 +138,6 @@ Result> PrimaryKeyFileStoreTable::CreateWrite( PrimaryKeyTableUtils::CreateMergeFunction(arrow_schema, primary_keys, options, ctx.GetMemoryPool())); if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { - // don't wrap first row, it is already OK merge_function = std::make_unique(std::move(merge_function)); } std::shared_ptr> merge_function_wrapper = @@ -261,8 +256,6 @@ CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& manifest_file, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const { - // The schema manager the table carries, which the scans this supplier hands out read the - // schema through. const std::shared_ptr& schema_manager = GetSchemaManager(); const std::shared_ptr& table_schema = store_->GetSchema(); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 3e79c537d..bd608bd74 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -262,9 +262,6 @@ TEST(TableReadTest, TestCreateAppendOnlyTableRead) { TEST(TableReadTest, TestMergeOptions) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; { - // The options the schema carries stand in the table the load builds, and a setting the - // context adds on top - the table was loaded with neither of these keys - is served as - // given. ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); context_builder.AddOption("read.batch-size", "2"); @@ -287,10 +284,6 @@ TEST(TableReadTest, TestMergeOptions) { ASSERT_EQ(expected_options, core_options.ToMap()); } { - // An option the schema also carries speaks at the load: the table the context is - // handed back is built with the context's value over the schema's, and the read serves - // the merged options. A caller holding a table already - a load that is over - is the - // one that gets a refusal instead, which `TableTest` covers. ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); context_builder.AddOption("bucket", "10"); @@ -334,9 +327,6 @@ TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { expected_options[read_via_option] = explicit_header ? "read-view" : "table-view"; ASSERT_EQ(dependency_context->root_path, "warehouse"); ASSERT_EQ(dependency_context->options, expected_options); - // The load merged the read options into the table it built - a table is the product of - // the load - so the schema the read serves carries the explicit header over the one the - // schema was written with, the same value the dependency context sees. ASSERT_EQ(internal_context->GetTableSchema()->Options(), expected_options); } } diff --git a/src/paimon/core/table/source/table_scan_test.cpp b/src/paimon/core/table/source/table_scan_test.cpp index 17bf2e874..4fd0f9873 100644 --- a/src/paimon/core/table/source/table_scan_test.cpp +++ b/src/paimon/core/table/source/table_scan_test.cpp @@ -167,8 +167,6 @@ TEST(TableScanTest, TestPkSchemaEvolutionScan) { TEST(TableScanTest, TestReadOptimizedPrimaryKeyStreamingScanUnsupported) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table$ro"; - // A system table is not served by the entry point a path names any more: the loader hands - // the table back the way a catalog's `GetTable` would, and the table refuses the scan. std::map options = {{Options::FILE_FORMAT, "orc"}}; ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); ASSERT_OK_AND_ASSIGN( diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 0bf82ead3..671081e54 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -400,9 +400,6 @@ Result> AuditLogSystemTable::NewScan( .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - // The view's own file system when the context names none, rather than the default backend. - // A `ScanContext` carries no scheme-to-identifier map, so unlike `CreateDataReadContext` - // below there is no map here to resolve through instead. .WithFileSystem(context->GetSpecificFileSystem() != nullptr ? context->GetSpecificFileSystem() : GetFileSystem()) @@ -465,8 +462,6 @@ Result> AuditLogSystemTable::CreateDataReadContext( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr base_read_schema, BaseReadSchema()); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); ReadContextBuilder builder(table_path_); - // The view's own file system, but only when the context names neither one of its own nor a - // scheme map to resolve through - a file system and a scheme map cannot both be named. std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { data_table_fs = GetFileSystem(); diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 92f10c49d..050964d62 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -126,8 +126,6 @@ Result IsEnabled(const GlobalSystemTableRegistryEntry& entry, CatalogOptionsSystemTable::kEnabledOption, false); } -// Whether a table reads its rows by enumerating what the catalog holds, and so cannot be built -// without one. bool RequiresCatalog(const std::string& table_name) { std::string normalized = StringUtils::ToLowerCase(table_name); for (const auto& entry : GlobalSystemTableRegistry()) { @@ -146,7 +144,6 @@ struct CatalogTableInfo { // Match Java CatalogUtils::listAllTables: tolerate databases or tables removed concurrently, but // propagate all other catalog and schema errors instead of returning incomplete system-table rows. -// `table_name` names the table asking, for the refusal a missing catalog reports. Result> LoadAllDataTables( const std::shared_ptr& catalog, const std::string& table_name) { if (catalog == nullptr) { @@ -306,8 +303,6 @@ Result GlobalSystemTableLoader::IsSupported( Result> GlobalSystemTableLoader::Load( const std::string& table_name, const GlobalSystemTableContext& context) { - // Refused here rather than at the first read: a table handed back to a caller that can never - // build its rows is a failure the caller finds out about late and far from its cause. if (context.catalog == nullptr && RequiresCatalog(table_name)) { return Status::NotImplemented("global system table requires catalog context: ", table_name); } @@ -325,9 +320,6 @@ Result> GlobalSystemTableLoader::LoadForSchema( } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, entry.factory(context)); - // The table is handed out through the `Table` interface, which serves the schema - // the base holds, so its own read-only schema is adopted before it leaves the - // loader. PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); return system_table; } diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index 87fdd1d29..a2c69195a 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -119,8 +119,6 @@ Result> InMemorySystemTable::NewRead( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); - // The rows this view serves are its own, read straight out of memory: there is no search - // over them, so a search the caller asked for would be quietly answered with plain rows. PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return std::make_unique( checked_pointer_cast(shared_from_this()), diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 2c4123f2f..89f0455b0 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -69,9 +69,6 @@ Result> ReadOptimizedSystemTable::NewScan( .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - // The view's own file system when the context names none, rather than the default backend. - // A `ScanContext` carries no scheme-to-identifier map, so unlike `CreateDataReadContext` - // below there is no map here to resolve through instead. .WithFileSystem(context->GetSpecificFileSystem() != nullptr ? context->GetSpecificFileSystem() : GetFileSystem()) @@ -106,8 +103,6 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon branch = branch_iter->second; } ReadContextBuilder builder(table_path_); - // The view's own file system, but only when the context names neither one of its own nor a - // scheme map to resolve through - a file system and a scheme map cannot both be named. std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { data_table_fs = GetFileSystem(); diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 7a8146534..872f6e502 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -198,9 +198,6 @@ Status SystemTable::CheckNoSearch(const std::shared_ptr& context) { return Status::NotImplemented( "VectorSearch or FullTextSearch does not support system-table reads yet"); } - // A `FullTextMatch` predicate is not a TopK request, so the check above does not see it. The - // factory refuses one for a system table with the same message. An in-memory view serves rows - // of its own and never evaluates a predicate, so without this it answers with all of them. if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); } @@ -241,8 +238,6 @@ Result> SystemTableLoader::Load( if (factory) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, factory.value()(fs, table_path, table_schema, dynamic_options)); - // The table is handed out through the `Table` interface, which serves the schema the - // base holds, so its own read-only schema is adopted before it leaves the loader. PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); return system_table; } @@ -295,7 +290,6 @@ Result> SystemTableLoader::LoadFromPath( // The warehouse is the grandparent of the sys/
path context.warehouse = PathUtil::GetParentDirPath(PathUtil::GetParentDirPath(path)); context.catalog_options = dynamic_options; - // Note: context.catalog is intentionally left empty here. // Global tables loaded from path do not have a Catalog reference and // cannot enumerate databases/tables. Only tables that don't require // catalog enumeration (e.g. catalog_options) will work in this path. diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index cfd4f5591..9ea5f3389 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -227,8 +227,6 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, "/tmp/table", table_schema, /*dynamic_options=*/{})); - // The name carries the data table it belongs to, the way Java Paimon's - // `ReadOptimizedTable.name()` does. ASSERT_EQ(system_table->Name(), "table$ro"); ASSERT_OK_AND_ASSIGN(std::shared_ptr arrow_schema, system_table->ArrowSchema()); @@ -238,8 +236,6 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { } TEST(SystemTableTest, TestCopyIsRefused) { - // A system table is a read-only view over the data table underneath it, and the schema it - // exposes is not one a copy reconfigures. The data table is copied through its own `Table`. std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}}; ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, @@ -311,9 +307,6 @@ TEST(SystemTableTest, TestGlobalSystemTableWithoutCatalogReturnsNotImplemented) "global system table requires catalog context: tables"); } -// A system table reached through `NewXxx` directly - one a catalog handed out - is checked the -// way the static factories check the contexts they take: a null context is refused rather than -// dereferenced. TEST(SystemTableTest, TestOperatorEntriesRefuseANullContext) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}}; @@ -332,9 +325,6 @@ TEST(SystemTableTest, TestOperatorEntriesRefuseANullContext) { ASSERT_NOK_WITH_MSG(read_optimized.NewRead(nullptr), "read context is null pointer"); } -// The scan and read entry points serve a data table only: a `T$system` path is refused with a -// message pointing at the catalog that hands the table back, so a caller holding only the path is -// not told the table does not exist. TEST(SystemTableTest, TestTheOperatorEntriesRefuseASystemTablePath) { const std::string system_table_path = "/tmp/warehouse/db.db/tbl$options"; @@ -347,9 +337,6 @@ TEST(SystemTableTest, TestTheOperatorEntriesRefuseASystemTablePath) { ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "names a system table"); } -// The reads these views serve build a context of their own for the data table underneath, which -// carries no search: a search the caller asked for is refused rather than quietly answered with -// plain rows. TEST(SystemTableTest, TestNewReadRefusesASearch) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}}; @@ -382,9 +369,6 @@ TEST(SystemTableTest, TestNewReadRefusesASearch) { "VectorSearch or FullTextSearch does not support system-table reads yet"); } -// `SetPredicate` carries a search too, as a `FullTextMatch` leaf. An in-memory view reads rows -// of its own and never evaluates a predicate, so without the check it would answer with all of -// them where `TableRead::Create` refuses the same request. TEST(SystemTableTest, TestNewReadRefusesAFullTextMatchPredicate) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}}; @@ -447,8 +431,6 @@ TEST(SystemTableTest, TestGlobalPartitionsUsesTheFileSystemOfEachTable) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - // Held by a `shared_ptr` because a global system table keeps shared ownership of the catalog - // it reads its rows through. std::shared_ptr catalog = std::make_shared(core_options.GetFileSystem(), dir->Str(), options); diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 2b6be91a4..4f321f17b 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -62,8 +62,6 @@ namespace paimon::test { namespace { -// Creates a managed table directory and a `Table` over it, holding the schema written to -// `branch`'s directory under `identifier` with `options`. Result> CreateManagedTable( const std::shared_ptr& file_system, const std::string& path, const Identifier& identifier, const std::string& branch = "main", @@ -80,7 +78,6 @@ Result> CreateManagedTable( /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); } -/// Writes two rows through the table's own write and commit, so a scan below has data to find. void WriteAndCommitTwoRows(const std::shared_ptr
& table, const std::string& root_path) { WriteContextBuilder write_builder(root_path, "commit-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); @@ -108,8 +105,6 @@ void WriteAndCommitTwoRows(const std::shared_ptr
& table, const std::strin ASSERT_OK(write->Close()); } -/// Writes one `id`/`v` row through the table's own write and commit, so a scan below has data -/// to find. void WriteAndCommitRow(const std::shared_ptr
& table, const std::string& root_path, int32_t id, int32_t v) { WriteContextBuilder write_builder(root_path, "commit-user"); @@ -157,14 +152,12 @@ TEST(TableTest, TestTableCarriesManagersOfItsBranch) { /*branch=*/"dev")); auto file_store_table = std::dynamic_pointer_cast(table); ASSERT_TRUE(file_store_table != nullptr); - // The branch the table was built for is the one its managers point at. EXPECT_EQ(file_store_table->CurrentBranch(), "dev"); ASSERT_TRUE(file_store_table->GetSchemaManager() != nullptr); ASSERT_TRUE(file_store_table->GetSnapshotManager() != nullptr); EXPECT_EQ(file_store_table->GetSnapshotManager()->Branch(), "dev"); EXPECT_EQ(file_store_table->GetSnapshotManager()->RootPath(), path); EXPECT_EQ(file_store_table->GetSnapshotManager()->Fs(), fs); - // A copy serves the same branch through the same managers, whose cache it shares. ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({})); auto file_store_copy = std::dynamic_pointer_cast(copy); ASSERT_TRUE(file_store_copy != nullptr); @@ -182,10 +175,6 @@ TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"main", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // A copy handed a `branch` option is the table it was copied from: the branch is the - // table's own - `SwitchToBranch` is the way to change it - so the option moves neither the - // copy nor its options, and an operator asked for the branch the copy serves - `main` - is - // served rather than refused by options claiming a `dev` the copy does not serve. ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({{Options::BRANCH, "dev"}})); auto file_store_copy = std::dynamic_pointer_cast(copy); ASSERT_TRUE(file_store_copy != nullptr); @@ -198,8 +187,6 @@ TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { file_store_copy->NewScan(std::move(ctx))); } - // The same the other way round: a copy of a branch table serves that branch still, so - // naming the branch it serves - `dev` - is agreed with rather than refused. ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/branch_tbl", identifier, /*branch=*/"dev", @@ -225,7 +212,6 @@ TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { const std::string path = dir->Str() + "/tbl"; const Identifier identifier("db", "tbl"); - // A primary-key table whose reads fall back to the `fb` branch, and that branch's schema. SchemaManager schema_manager(fs, path, "main"); auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); ASSERT_OK_AND_ASSIGN( @@ -248,11 +234,6 @@ TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); - // The copy turns the sequence-number column on, and a read projects it through a context - // that carries no options of its own. The main reader serves the copy's merged options, so - // it knows the column; the fallback serves the `fb` branch's schema, whose options do not - // name it, so the copy's own dynamic options are what let the fallback reader know it too - - // without them the read as a whole fails to build. ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); ReadContextBuilder read_builder(path); @@ -269,7 +250,6 @@ TEST(TableTest, TestACopyOfACopyServesTheLatestValueOfARepeatedDynamicOption) { const std::string path = dir->Str() + "/tbl"; const Identifier identifier("db", "tbl"); - // A primary-key table whose reads fall back to the `fb` branch, and that branch's schema. SchemaManager schema_manager(fs, path, "main"); auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); ASSERT_OK_AND_ASSIGN( @@ -292,11 +272,6 @@ TEST(TableTest, TestACopyOfACopyServesTheLatestValueOfARepeatedDynamicOption) { identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); - // The option is turned off and then on again by the copy the read serves, so the copy's - // schema carries the latest value. The fallback branch's schema does not carry the option - // at all, so what lets its reader know the column is the copy's accumulated dynamic - // options - which must speak the same latest value the schema does, not the older one a - // first copy left, or the fallback reader fails to build while the main one succeeds. ASSERT_OK_AND_ASSIGN(std::shared_ptr
off, table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "false"}})); ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, @@ -315,9 +290,6 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { const std::string path = dir->Str() + "/tbl"; const Identifier identifier("db", "tbl"); - // The main branch is an append-only table whose reads fall back to the `fb` branch, and - // that branch is laid out as a primary-key table: a second write of the key the first - // wrote is a merge, not a duplicate. SchemaManager main_schema_manager(fs, path, "main"); auto row_schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), arrow::field("v", arrow::int32())}); @@ -351,11 +323,6 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/10); WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); - // The read of the main table carries its fallback reader, which reads the `fb` branch's - // data through the `fb` branch's own schema - so it is the primary-key read that schema's - // layout asks for, not the append-only one the main table's kind would pick. Its splits - // are the `fb` branch's own, scanned through a table that serves `fb` and read as - // fallback ones, which is how a fallback reader is asked for its rows. ReadContextBuilder read_builder(path); read_builder.SetReadFieldNames({"id", "v"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); @@ -376,10 +343,6 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, read->CreateReader(fallback_splits)); - // The one merged row the `fb` branch's primary-key layout leaves behind - at the value - // the second write carried - is what the fallback reader answers with; the append-only - // reader the main table's kind would have picked answers with both rows, or refuses the - // split outright. ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); ASSERT_OK_AND_ASSIGN(std::shared_ptr array, ReadResultCollector::GetArray(std::move(batch))); @@ -401,6 +364,129 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { EXPECT_TRUE(BatchReader::IsEofBatch(eof_batch)); } +TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnPartitionKeys) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + SchemaManager main_schema_manager(fs, path, "main"); + auto partitioned_schema = + arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32()), arrow::field("dt", arrow::utf8())}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_created, + main_schema_manager.CreateTable(partitioned_schema, + /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, + {{Options::SCAN_FALLBACK_BRANCH, "fb"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
main, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(main_created))), + identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); + + SchemaManager fallback_schema_manager(fs, path, "fb"); + auto row_schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32())}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr fallback_created, + fallback_schema_manager.CreateTable(row_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
fallback, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(fallback_created))), + identifier, /*uuid=*/"", /*branch=*/"fb", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); + WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); + + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "v"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, main->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(path).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fallback_scan, + fallback->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fallback_plan, fallback_scan->CreatePlan()); + ASSERT_FALSE(fallback_plan->Splits().empty()); + std::vector> fallback_splits; + for (const std::shared_ptr& split : fallback_plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split != nullptr); + fallback_splits.push_back( + std::make_shared(data_split, /*is_fallback=*/true)); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(fallback_splits)); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_EQ(array->length(), 1); + auto struct_array = std::dynamic_pointer_cast(array); + ASSERT_TRUE(struct_array != nullptr); + // Every batch reader leads with `_VALUE_KIND`, ahead of the fields the read named. + EXPECT_EQ(struct_array->num_fields(), 3); + auto value_kind_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto id_array = std::dynamic_pointer_cast(struct_array->field(1)); + auto v_array = std::dynamic_pointer_cast(struct_array->field(2)); + ASSERT_TRUE(value_kind_array != nullptr); + ASSERT_TRUE(id_array != nullptr); + ASSERT_TRUE(v_array != nullptr); + EXPECT_EQ(value_kind_array->Value(0), 0); + EXPECT_EQ(id_array->Value(0), 1); + EXPECT_EQ(v_array->Value(0), 20); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof_batch, batch_reader->NextBatch()); + EXPECT_TRUE(BatchReader::IsEofBatch(eof_batch)); +} + +TEST(TableTest, TestAFallbackReaderIgnoresReadContextOptionsAbsentFromTheTable) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + + auto make_table = + [&](const std::string& path, + const std::map& options) -> Result> { + SchemaManager schema_manager(fs, path, "main"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options)); + if (options.count(Options::SCAN_FALLBACK_BRANCH) != 0) { + SchemaManager fallback_schema_manager(fs, path, "fb"); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr fallback_created, + fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, {})); + } + return FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(created))), + Identifier("db", "tbl"), /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); + }; + ASSERT_OK_AND_ASSIGN(std::shared_ptr
plain, make_table(dir->Str() + "/plain", {})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
with_fallback, + make_table(dir->Str() + "/fallback", {{Options::SCAN_FALLBACK_BRANCH, "fb"}})); + + for (const std::shared_ptr
& table : {plain, with_fallback}) { + SCOPED_TRACE(table->GetRootPath()); + ReadContextBuilder read_builder(table->GetRootPath()); + read_builder.AddOption(Options::FILE_FORMAT, "bogus"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + table->NewRead(std::move(read_context))); + } +} + TEST(TableTest, TestACopyThatMovesAnS3OptionRebuildsItsManagers) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -415,10 +501,6 @@ TEST(TableTest, TestACopyThatMovesAnS3OptionRebuildsItsManagers) { auto file_store_table = std::dynamic_pointer_cast(table); ASSERT_TRUE(file_store_table != nullptr); - // A native `s3.` option names the file system the S3 backend resolves from, the way an - // `fs.` one does: a copy that moves it re-resolves, so the managers are rebuilt rather - // than inherited - otherwise the copy would read its metadata through the old endpoint - // while its data goes through the new one. ASSERT_OK_AND_ASSIGN(std::shared_ptr
moved, table->Copy({{"s3.endpoint", "http://new"}})); auto moved_file_store_table = std::dynamic_pointer_cast(moved); @@ -426,8 +508,6 @@ TEST(TableTest, TestACopyThatMovesAnS3OptionRebuildsItsManagers) { EXPECT_NE(moved_file_store_table->GetSchemaManager(), file_store_table->GetSchemaManager()); EXPECT_NE(moved_file_store_table->GetSnapshotManager(), file_store_table->GetSnapshotManager()); - // A copy whose `s3.` option says what the schema already says resolves the same file - // system, so the managers are the ones this table already holds. ASSERT_OK_AND_ASSIGN(std::shared_ptr
kept, table->Copy({{"s3.endpoint", "http://old"}})); auto kept_file_store_table = std::dynamic_pointer_cast(kept); ASSERT_TRUE(kept_file_store_table != nullptr); @@ -442,14 +522,12 @@ TEST(TableTest, TestSwitchToBranch) { const std::string path = dir->Str() + "/tbl"; const Identifier identifier(Identifier::kUnknownDatabase, "tbl"); ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, CreateManagedTable(fs, path, identifier)); - // A branch of its own, with a schema of its own. SchemaManager dev_schema_manager(fs, path, "dev"); ASSERT_OK_AND_ASSIGN( [[maybe_unused]] std::unique_ptr dev_schema, dev_schema_manager.CreateTable(arrow::schema({arrow::field("c0", arrow::int32())}), /*partition_keys=*/{}, /*primary_keys=*/{}, {})); - // Switching to a branch serves that branch's schema and managers. auto main_file_store_table = std::dynamic_pointer_cast(table); ASSERT_TRUE(main_file_store_table != nullptr); ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_table, @@ -457,8 +535,6 @@ TEST(TableTest, TestSwitchToBranch) { auto dev_file_store_table = std::dynamic_pointer_cast(dev_table); ASSERT_TRUE(dev_file_store_table != nullptr); EXPECT_EQ("dev", dev_file_store_table->CurrentBranch()); - // The identifier names the branch switched to, so `Name()` and `FullName()` answer for the - // table served, and a caller that reloads through `FullName()` lands on the same branch. EXPECT_EQ("tbl$branch_dev", dev_table->Name()); EXPECT_EQ("tbl$branch_dev", dev_table->FullName()); ASSERT_EQ((std::vector{"c0"}), dev_table->LatestSchema()->FieldNames()); @@ -469,7 +545,6 @@ TEST(TableTest, TestSwitchToBranch) { ASSERT_TRUE(dev_file_store_table->GetSnapshotManager() != nullptr); EXPECT_EQ("dev", dev_file_store_table->GetSnapshotManager()->Branch()); - // Switching to the branch already served hands back the same table, managers shared. ASSERT_OK_AND_ASSIGN(std::shared_ptr
same_table, dev_file_store_table->SwitchToBranch("dev")); auto same_file_store_table = std::dynamic_pointer_cast(same_table); @@ -479,13 +554,11 @@ TEST(TableTest, TestSwitchToBranch) { EXPECT_EQ(same_file_store_table->GetSnapshotManager(), dev_file_store_table->GetSnapshotManager()); - // Switching back serves the main branch's schema again. ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_again, dev_file_store_table->SwitchToBranch("main")); auto main_again_file_store_table = std::dynamic_pointer_cast(main_again); ASSERT_TRUE(main_again_file_store_table != nullptr); EXPECT_EQ("main", main_again_file_store_table->CurrentBranch()); - // Switching back leaves no branch in the name either, the way the main branch is addressed. EXPECT_EQ("tbl", main_again->Name()); EXPECT_EQ("tbl", main_again->FullName()); ASSERT_EQ((std::vector{"id"}), main_again->LatestSchema()->FieldNames()); @@ -493,7 +566,6 @@ TEST(TableTest, TestSwitchToBranch) { ASSERT_TRUE(main_again_schema != nullptr); EXPECT_EQ(0, main_again_schema->Options().count(Options::BRANCH)); - // A branch with no schema does not exist. ASSERT_NOK_WITH_MSG(main_file_store_table->SwitchToBranch("missing"), "branch"); } @@ -506,8 +578,6 @@ TEST(TableTest, TestNewWriteServesTheBranchTheTableWasLoadedWith) { CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"dev", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // A context that names no branch is the one a caller hands straight to the table's write, - // which serves the branch the table was loaded with rather than the context's own main. WriteContextBuilder write_builder(dir->Str(), "commit-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, @@ -528,8 +598,6 @@ TEST(TableTest, TestNewWriteServesTheBranchTheTableWasLoadedWith) { write->PrepareCommit()); ASSERT_OK(write->Close()); - // The files land under the branch's own directory, which a write of the main branch - the - // branch the context alone would name - would never touch. std::vector branch_entries; ASSERT_OK(fs->ListDir(BranchManager::BranchPath(dir->Str(), "dev"), &branch_entries)); EXPECT_FALSE(branch_entries.empty()); @@ -544,8 +612,6 @@ TEST(TableTest, TestOperatorsServeTheBranchTheTableWasLoadedWith) { CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"dev", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // The write and the commit below carry no branch, and both serve the branch the table was - // loaded with, so the data they publish lands on it. WriteContextBuilder write_builder(dir->Str(), "commit-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, @@ -571,15 +637,12 @@ TEST(TableTest, TestOperatorsServeTheBranchTheTableWasLoadedWith) { table->NewCommit(std::move(commit_context))); ASSERT_OK(commit->Commit(messages)); - // The scan finds what they wrote, reading through the table's own managers on its branch. ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, ScanContextBuilder(dir->Str()).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); EXPECT_FALSE(plan->Splits().empty()); - // A branch the scan context carries does not win either: the table's own does, so the data - // it finds is still the branch's. ScanContextBuilder main_scan_builder(dir->Str()); main_scan_builder.AddOption(Options::BRANCH, "main"); ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan_ctx, main_scan_builder.Finish()); @@ -687,8 +750,6 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { {Identifier("db", "tbl"), Identifier("db", "tbl$branch_main"), Identifier("db", "tbl$branch_MAIN")}) { SCOPED_TRACE(main_identifier.ToString()); - // A catalog that loaded the schema itself builds the table through the factory, which a - // rest catalog is shaped like. The alias resolves to the main branch as well. ASSERT_OK_AND_ASSIGN(std::string branch, main_identifier.GetBranchNameOrDefault()); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, FileStoreTable::Create(fs, dir->Str(), schema, main_identifier, @@ -713,7 +774,6 @@ TEST(TableTest, TestCurrentBranch) { ASSERT_TRUE(main_file_store_table != nullptr); EXPECT_EQ(main_file_store_table->CurrentBranch(), "main"); - // The branch a table was created on is the one it carries. ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/branch_tbl", Identifier("db", "tbl", "dev"), /*branch=*/"dev")); @@ -732,9 +792,6 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { auto catalog = std::make_shared(); catalog->SetTableSchema(table->LatestSchema()); - // A catalog that hands out no per-table file system opens the table through the one its - // schema's options resolve - never through none, the default local one when they name no - // other - and one that does opens it with the file system issued for that table. ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog->GetTable(identifier)); EXPECT_NE(opened->GetFileSystem(), nullptr); @@ -746,8 +803,6 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { } TEST(TableTest, TestGetTableServesEachKindOfTableAsWhatItIs) { - // The table's `type` option says what it is, so the one call that opens a table hands each - // kind back as what it is, through the `Table` interface the two kinds share. auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); @@ -763,7 +818,6 @@ TEST(TableTest, TestGetTableServesEachKindOfTableAsWhatItIs) { format_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {{"type", "format-table"}, {"file.format", "parquet"}})); - // A format table comes back as the `FormatTable` it is. ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog.GetTable(Identifier("db", "format"))); ASSERT_NE(nullptr, std::dynamic_pointer_cast(opened)); @@ -831,8 +885,6 @@ TEST(TableTest, TestNewScanServesTheSchemaTheTableWasLoadedWith) { ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, CreateManagedTable(fs, dir->Str(), identifier)); - // With the schema file gone the table still serves a scan: it carries the schema it was - // loaded with, which is what a scan is built from. SchemaManager schema_manager(fs, dir->Str()); ASSERT_OK(fs->Delete(schema_manager.ToSchemaPath(0), /*recursive=*/false)); ASSERT_OK_AND_ASSIGN(std::unique_ptr held_ctx, @@ -840,8 +892,6 @@ TEST(TableTest, TestNewScanServesTheSchemaTheTableWasLoadedWith) { ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, table->NewScan(std::move(held_ctx))); - // A context without a table still reads the schema from the directory, which no longer has - // one. ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ScanContextBuilder(dir->Str()).Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); } @@ -855,8 +905,6 @@ TEST(TableTest, TestNewScanServesTheFileSystemTheTableWasOpenedWith) { CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // A file system is only observable once a scan has data to read, so two rows are written and - // committed through the table's own write and commit. WriteContextBuilder write_builder(dir->Str(), "commit-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, @@ -882,8 +930,6 @@ TEST(TableTest, TestNewScanServesTheFileSystemTheTableWasOpenedWith) { ASSERT_OK(commit->Commit(messages)); ASSERT_OK(write->Close()); - // The table was opened with the local file system, so a scan serves its data through that - // one even when the context hands over a mock that can read nothing. ASSERT_OK_AND_ASSIGN( std::unique_ptr held_ctx, ScanContextBuilder(dir->Str()).WithFileSystem(std::make_shared()).Finish()); @@ -891,8 +937,6 @@ TEST(TableTest, TestNewScanServesTheFileSystemTheTableWasOpenedWith) { ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_FALSE(plan->Splits().empty()); - // A context without a table resolves the file system itself, which is the mock and reads no - // schema at all: the file system counterpart of the schema case above. ASSERT_OK_AND_ASSIGN( std::unique_ptr ctx, ScanContextBuilder(dir->Str()).WithFileSystem(std::make_shared()).Finish()); @@ -908,8 +952,6 @@ TEST(TableTest, TestNewScanServesTheRootPathTheTableWasOpenedWith) { CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // Two rows are written and committed through the table's own write and commit, so the scan - // below has data to find. WriteContextBuilder write_builder(dir->Str(), "commit-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, @@ -935,16 +977,12 @@ TEST(TableTest, TestNewScanServesTheRootPathTheTableWasOpenedWith) { ASSERT_OK(commit->Commit(messages)); ASSERT_OK(write->Close()); - // The table was opened with this directory as its root path, so a scan serves its data from - // there even when the context names a different one. ASSERT_OK_AND_ASSIGN(std::unique_ptr held_ctx, ScanContextBuilder(dir->Str() + "/not-the-table").Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(held_ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_FALSE(plan->Splits().empty()); - // A context without a table reads the schema from the directory it names, which is not a - // table at all: the root path counterpart of the schema case above. ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ScanContextBuilder(dir->Str() + "/not-the-table").Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); @@ -1004,6 +1042,89 @@ TEST(TableTest, TestNewCommitCreatesAFileStoreCommit) { table->NewCommit(std::move(ctx))); } +TEST(TableTest, TestNewWriteAndCommitRefuseAContextThatNamesAnotherTable) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto other_dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(other_dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier)); + + { + WriteContextBuilder write_builder(other_dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(ctx)), "names root path"); + } + { + CommitContextBuilder commit_builder(other_dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), "names root path"); + } + + auto catalog = std::make_shared(); + catalog->SetFileSystem(fs); + catalog->SetTableSchema(table->LatestSchema()); + { + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + write_builder.WithCatalog(catalog, Identifier("db", "other")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(ctx)), "names table"); + } + { + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.WithCatalog(catalog, Identifier("db", "other")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), "names table"); + } + { + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.WithCatalog(catalog, Identifier("db", "tbl$branch_dev")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), "names branch"); + } + { + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.UseRESTCatalogCommit(true).WithTableId("another-table"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), "table id"); + } + + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr commit, + table->NewCommit(std::move(ctx))); +} + +TEST(TableTest, TestNewCommitRefusesABranchACatalogCannotAddress) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"dev$x")); + + CommitContextBuilder fs_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fs_ctx, fs_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr fs_commit, + table->NewCommit(std::move(fs_ctx))); + + CommitContextBuilder rest_builder(dir->Str(), "commit-user"); + rest_builder.UseRESTCatalogCommit(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr rest_ctx, rest_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(rest_ctx)), "cannot contain"); + + auto catalog = std::make_shared(); + catalog->SetFileSystem(fs); + catalog->SetTableSchema(table->LatestSchema()); + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + write_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_ctx, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_ctx)), "cannot contain"); +} + TEST(TableTest, TestNewScanReadWriteCommitRejectANullContext) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -1028,9 +1149,6 @@ TEST(TableTest, TestOperatorsRefuseAnOptionThatContradictsTheTable) { CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"main", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // The table was loaded with `bucket` = 1, so every operator refuses a context that names - // another value for it: the table serves the options it was loaded with, and a caller that - // means to change them copies the table. { ScanContextBuilder scan_builder(dir->Str()); scan_builder.AddOption(Options::BUCKET, "2"); @@ -1060,7 +1178,6 @@ TEST(TableTest, TestOperatorsRefuseAnOptionThatContradictsTheTable) { "option 'bucket' is '2' in the context but '1' on the table"); } - // The copy made with the option in question agrees with the context, so it serves it. ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "2"}})); ScanContextBuilder agreeing_scan_builder(dir->Str()); agreeing_scan_builder.AddOption(Options::BUCKET, "2"); @@ -1079,10 +1196,6 @@ TEST(TableTest, TestOperatorsIgnoreARuntimeKeyTheTableWasNotLoadedWith) { {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); WriteAndCommitTwoRows(table, dir->Str()); - // `scan.mode` and `scan.snapshot-id` name no option the table was loaded with, so they name - // a runtime setting the scan does not take: the scan serves the table's own options and - // reads its latest snapshot - a merged context would have gone looking for snapshot 999, - // which was never committed. ScanContextBuilder scan_builder(dir->Str()); scan_builder.AddOption(Options::SCAN_MODE, "from-snapshot") .AddOption(Options::SCAN_SNAPSHOT_ID, "999"); @@ -1092,8 +1205,6 @@ TEST(TableTest, TestOperatorsIgnoreARuntimeKeyTheTableWasNotLoadedWith) { ASSERT_FALSE(plan->Splits().empty()); ASSERT_EQ(plan->SnapshotId(), std::optional(1)); - // A read takes a runtime key the same way: the key names nothing the table carries, so the - // read is served rather than refused. ReadContextBuilder read_builder(dir->Str()); read_builder.AddOption(Options::SCAN_MODE, "from-snapshot") .AddOption(Options::SCAN_SNAPSHOT_ID, "999"); @@ -1101,9 +1212,6 @@ TEST(TableTest, TestOperatorsIgnoreARuntimeKeyTheTableWasNotLoadedWith) { ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, table->NewRead(std::move(read_ctx))); - // The runtime setting a caller means to serve is the copy's: `Table::Copy` derives the table - // whose options carry it, and that table's scan reads the snapshot it names - which is not - // there, as this table only ever committed snapshot 1. ASSERT_OK_AND_ASSIGN( std::shared_ptr
copied, table->Copy({{Options::SCAN_MODE, "from-snapshot"}, {Options::SCAN_SNAPSHOT_ID, "999"}})); @@ -1117,9 +1225,6 @@ TEST(TableTest, TestOperatorsIgnoreARuntimeKeyTheTableWasNotLoadedWith) { TEST(TableTest, TestCreateFailsWhenTheSchemaOptionsDoNotParse) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - // The schema itself carries no validation, so a schema written with options no - // `CoreOptions` accepts surfaces when the store parses them - at the load, not at the first - // operator built on the table. ASSERT_OK_AND_ASSIGN( std::unique_ptr created, TableSchema::Create( @@ -1146,10 +1251,6 @@ TEST(TableTest, TestTheCacheAScanContextCarriesReachesTheScan) { {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); WriteAndCommitTwoRows(table, dir->Str()); - // The cache is a runtime setting the scan serves from the context rather than from the - // table's own options, so a scan built with one reads its manifests through it. A scan reads - // the manifest list of a snapshot on its executor's threads, so the counter the cache keeps - // is guarded rather than a plain field. auto cache = std::make_shared( std::map{{CacheKind::MANIFEST, 64 * 1024 * 1024}, {CacheKind::SNAPSHOT_LIVE_MANIFEST, 64 * 1024 * 1024}}); @@ -1172,8 +1273,6 @@ TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // Two rows are written and committed through the table's own write and commit, so the copy - // below has data to find. WriteContextBuilder write_builder(dir->Str(), "commit-user"); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr write, @@ -1199,16 +1298,12 @@ TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { ASSERT_OK(commit->Commit(messages)); ASSERT_OK(write->Close()); - // The copy overlays what the call gives on the options the schema stored; the schema itself - // and the identity the table was opened with are copied as they are. ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "2"}, {Options::TYPE, "format-table"}})); std::shared_ptr copied_schema = std::dynamic_pointer_cast(copied->LatestSchema()); ASSERT_TRUE(copied_schema != nullptr); EXPECT_EQ(copied_schema->Options().at(Options::BUCKET), "2"); - // `type` is structural and read from the schema alone, so the copy carries none of it: the - // option given at the call does not restyle a managed table. EXPECT_EQ(copied_schema->Options().count(Options::TYPE), 0); EXPECT_EQ(copied_schema->Options().at(Options::BUCKET_KEY), "id"); EXPECT_EQ(copied_schema->Id(), @@ -1216,12 +1311,10 @@ TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { EXPECT_EQ(copied->FullName(), table->FullName()); EXPECT_EQ(copied->GetRootPath(), table->GetRootPath()); EXPECT_EQ(copied->GetFileSystem(), table->GetFileSystem()); - // The original keeps the options it was loaded with. EXPECT_EQ( std::dynamic_pointer_cast(table->LatestSchema())->Options().at(Options::BUCKET), "1"); - // The copy is a table like the one it came from, and serves its data as one. ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, ScanContextBuilder(dir->Str()).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, copied->NewScan(std::move(scan_ctx))); @@ -1238,14 +1331,6 @@ TEST(TableTest, TestCopyRefusesThePhysicalOptionsTheDataWasLaidOutWith) { /*branch=*/"main", {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); - // The data on the path was hashed by the `bucket-key` it was written with: a copy that - // renamed the key would prune by a hash its rows never matched, dropping them silently, so - // the change is refused - at `Copy` and at the load a context drives alike, which merges - // through `Copy` - the way the Java `checkImmutability` refuses it. The same goes for the - // other options that name the physical layout, which a key the schema does not carry - // changes as much as one it carries differently. `merge-engine`, `bucket`, - // `row-tracking.enabled` and `data-evolution.enabled` are the options the Java immutable - // set holds that stay servable here. ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_KEY, "v"}}), "Change 'bucket-key' is not supported yet."); ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_FUNCTION_TYPE, "hash"}}), @@ -1260,9 +1345,6 @@ TEST(TableTest, TestCopyRefusesThePhysicalOptionsTheDataWasLaidOutWith) { "Change 'bucket-key' is not supported yet."); } - // An option that says what the schema already says moves nothing, and a bucket count is - // the physical option a rescale legitimately rewrites, so a copy may move it. An - // `ignore-delete` may only be turned on, never back off. ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
unchanged, table->Copy({{Options::BUCKET_KEY, "id"}, {Options::BUCKET, "1"}})); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
rescaled, @@ -1288,9 +1370,6 @@ TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), arrow::field("dt", arrow::utf8())}); - // A context that names no catalog knows the table by its directory: the directory above - // the table's names the database when it carries the `.db` suffix, the way the Java - // `identifierFromPath` reads it, so the table the load serves is named for both. const std::string table_dir = dir->Str() + "/my_db.db/my_table"; ASSERT_OK(fs->Mkdirs(table_dir)); SchemaManager schema_manager(fs, table_dir); @@ -1304,8 +1383,6 @@ TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { ASSERT_EQ(table->FullName(), "my_db.my_table"); } - // A table sitting directly under the root, with no `.db` directory above it, names no - // database: the table is known by its directory alone, the way it was before. const std::string loose_table_dir = dir->Str() + "/loose_table"; ASSERT_OK(fs->Mkdirs(loose_table_dir)); SchemaManager loose_schema_manager(fs, loose_table_dir); diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 0752af6f7..872c3c465 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -383,8 +383,6 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } - // Only the schema is read here and the table does not outlive the call, so it needs no - // catalog and this one hands over no ownership of itself. GlobalSystemTableContext context; context.fs = fs_; context.warehouse = warehouse_; @@ -427,8 +425,6 @@ Result> RestCatalog::LoadSystemTable( if (branch) { dynamic_options[Options::BRANCH] = branch.value(); } - // The system table reads the data table's own files, so it is served by that table's own - // credentials - with data tokens the only file system that can open them. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); return SystemTableLoader::Load(system_table_name.value(), table_file_system, data_table_path, @@ -436,9 +432,6 @@ Result> RestCatalog::LoadSystemTable( } Result> RestCatalog::GetTable(const Identifier& identifier) const { - // The global system tables of the "sys" database expose the catalog itself rather than any - // table the server stands behind, so they are served locally and never reach the managed-table - // load below, whose schema cast a read-only system table schema would not survive. if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { PAIMON_ASSIGN_OR_RAISE(bool supported, GlobalSystemTableLoader::IsSupported(identifier.GetObjectName(), @@ -446,9 +439,6 @@ Result> RestCatalog::GetTable(const Identifier& identifie if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } - // A global system table builds its rows lazily through this catalog, so the table handed - // back has to keep it alive or the pointer dangles at the first `BuildRows`. A catalog - // not owned by a `std::shared_ptr` cannot be kept, so it is refused by name. std::shared_ptr self = weak_from_this().lock(); if (self == nullptr) { return Status::Invalid( @@ -473,15 +463,11 @@ Result> RestCatalog::GetTable(const Identifier& identifie std::string table_path; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier, &uuid, &table_path)); - // The schema the same response carried says what the table is: a format table is handed back - // as the `FormatTable` it is rather than refused as the managed table it is not. std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); if (data_schema != nullptr) { PAIMON_ASSIGN_OR_RAISE(bool is_format_table, TableTypeDefine::IsFormatTable(data_schema->Options())); if (is_format_table) { - // A rest catalog holds the schema itself, so everything below the location is data. - // The catalog-wide file system, as a format table reads no metadata of its own. return FormatTable::Create(fs_, table_path, identifier, data_schema, /*location_carries_paimon_metadata=*/false, /*dynamic_options=*/{}); @@ -493,7 +479,6 @@ Result> RestCatalog::GetTable(const Identifier& identifie // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); - // The branch the schema and path above were read on, which the table's managers then serve. PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); return FileStoreTable::Create( table_file_system, table_path, schema, identifier, uuid, diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index cc197b45d..3f130028a 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -1003,8 +1003,6 @@ TEST_F(RestCatalogTest, SystemTableSchema) { requests_before = state_->get_table_requests; } ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(system_identifier)); - // A system table comes back as the read-only `SystemTable` it is, naming itself after the - // data table it belongs to. std::shared_ptr system_table = std::dynamic_pointer_cast(table); ASSERT_NE(nullptr, system_table); ASSERT_EQ("t1$options", system_table->Name()); @@ -1226,7 +1224,6 @@ TEST_F(RestCatalogTest, FormatTableIsLoadedInOneRequest) { // directory named "schema" there is a partition value, not metadata to skip. ASSERT_FALSE(table->LocationCarriesPaimonMetadata()); - // A managed table comes back through the same call, as the managed table it is. ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); ASSERT_OK_AND_ASSIGN(std::shared_ptr
managed, catalog->GetTable(Identifier("db1", "t1"))); @@ -1478,9 +1475,6 @@ TEST_F(RestCatalogTest, SystemDatabase) { } TEST_F(RestCatalogTest, GlobalSystemTableRetainsItsCatalog) { - // A global system table builds its rows lazily through the catalog that served it, so the - // table handed back keeps that catalog alive: reading it after the caller let its own - // reference go still answers, rather than reading through a catalog already destroyed. ASSERT_OK_AND_ASSIGN(std::unique_ptr created, CreateRestCatalog()); std::shared_ptr catalog = std::move(created); ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); @@ -1492,8 +1486,6 @@ TEST_F(RestCatalogTest, GlobalSystemTableRetainsItsCatalog) { catalog.reset(); ASSERT_FALSE(weak.expired()) << "the global system table does not retain its catalog"; - // The rows are built through the retained catalog, so they still answer once the caller's - // own reference is gone: the one table the catalog stands behind shows up in them. ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, ScanContextBuilder("sys/tables").Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); @@ -1512,10 +1504,6 @@ TEST_F(RestCatalogTest, GlobalSystemTableRetainsItsCatalog) { } TEST_F(RestCatalogTest, GlobalSystemTableWorksThroughThePublicCatalogFactory) { - // `Catalog::Create()` is the factory a caller outside paimon uses, and it hands back a - // `Catalog` rather than a `RestCatalog`. A global system table keeps its catalog alive - // through that pointer, so this path has to work too - a catalog whose shared ownership - // was not set up would fail here with the refusal naming the missing ownership. ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(kWarehouse, options_)); ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 41e76b32d..5aa2f22c3 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -276,9 +276,6 @@ Result> ReadSystemTable( const std::string& system_table_path, const std::map& options, bool streaming_mode = false, const std::shared_ptr& predicate = nullptr, const std::vector& read_field_names = {}, bool read_next_plan = false) { - // A system table is not served by the operator entry points a path names any more: the loader - // hands the table back, and the table serves the scan and the read itself, the way a catalog's - // `GetTable` would. PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr table, @@ -659,8 +656,6 @@ TEST(SystemTableReadInteTest, TestReadOptionsSystemTable) { ArrowSchemaRelease(&schema); std::string system_table_path = PathUtil::JoinPath(dir->Str(), "warehouse/db1.db/tbl1$options"); - // A system table comes from the catalog, and the operators from the table it hands back - - // the path alone no longer reaches it. ASSERT_OK_AND_ASSIGN(auto table, catalog->GetTable(Identifier("db1", "tbl1$options"))); ScanContextBuilder scan_context_builder(system_table_path); scan_context_builder.SetOptions(options); @@ -694,7 +689,6 @@ TEST(SystemTableReadInteTest, TestReadBranchOptionsSystemTable) { std::map options = {{Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}}; std::string system_table_path = table_path + "$branch_rt$options"; - // No catalog here: the loader is the path-side half of what a catalog's `GetTable` does. ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); ASSERT_OK_AND_ASSIGN(auto table, SystemTableLoader::LoadFromPath(core_options.GetFileSystem(), system_table_path, options)); @@ -1018,8 +1012,6 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP std::shared_ptr predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"k", FieldType::INT, Literal(2)); - // A system table is not served by the operator entry points a path names any more: the loader - // hands the table back, and the table serves the scan and the read itself. ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); ASSERT_OK_AND_ASSIGN( std::shared_ptr table, @@ -1084,8 +1076,6 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { /*partition_map=*/{}, /*bucket=*/0, {})); ASSERT_OK(WriteAndFullCompact(std::move(batch), /*commit_identifier=*/0, helper.get())); - // A system table is not served by the operator entry points a path names any more: the loader - // hands the table back, and the table serves the scan and the read itself. ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); ASSERT_OK_AND_ASSIGN( std::shared_ptr table, diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 1fd0ea9ab..7c4110257 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -2919,8 +2919,6 @@ TEST_F(RealtimeWriteInteTest, TestPkRealtimeReadOptimizedScanUnsupported) { ASSERT_OK_AND_ASSIGN(std::vector disk_rows, ReadRows()); ASSERT_EQ(rows, disk_rows); - // A system table is not served by the entry point a path names any more: the loader hands - // the table back, and the table refuses the scan itself. ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, SystemTableLoader::LoadFromPath(core_options.GetFileSystem(),